From 60a51a4fa4108d07b556c516e169939a312f8ef2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:28:24 +0100 Subject: [PATCH 01/16] feat(native): resolve column references with rename spans --- .../src/bindings/_helpers/functions.rs | 7 + .../column_references/_helpers/references.rs | 593 ++++++++++++++++++ .../src/column_references/main/analyze.rs | 3 + .../src/column_references/main/mod.rs | 1 + .../src/column_references/mod.rs | 6 + .../src/column_references/tests.rs | 6 + .../src/column_references/tests/helpers.rs | 58 ++ .../tests/test_column_references.rs | 182 ++++++ .../src/column_references/tests/test_types.rs | 14 + crates/sqlbuild-rules-native/src/lib.rs | 1 + src/sqlbuild/_native.pyi | 1 + 11 files changed, 872 insertions(+) create mode 100644 crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs create mode 100644 crates/sqlbuild-rules-native/src/column_references/main/analyze.rs create mode 100644 crates/sqlbuild-rules-native/src/column_references/main/mod.rs create mode 100644 crates/sqlbuild-rules-native/src/column_references/mod.rs create mode 100644 crates/sqlbuild-rules-native/src/column_references/tests.rs create mode 100644 crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs create mode 100644 crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs create mode 100644 crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 186c8ab787..a61408c6cc 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -154,6 +154,12 @@ fn analyze_sql_uses_json(py: Python<'_>, request_json: &str) -> PyResult .map_err(value_error) } +#[pyfunction] +fn analyze_column_references_json(py: Python<'_>, request_json: &str) -> PyResult { + py.compiler_detach(|| crate::column_references::main::analyze::analyze_json(request_json)) + .map_err(value_error) +} + #[pyfunction] fn analyze_queries_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| crate::query_analysis::main::analyze::analyze_json(request_json)) @@ -429,6 +435,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?; module.add_function(wrap_pyfunction!(schema_validations_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_sql_uses_json, module)?)?; + module.add_function(wrap_pyfunction!(analyze_column_references_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_queries_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_project_queries_json, module)?)?; module.add_function(wrap_pyfunction!( diff --git a/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs b/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs new file mode 100644 index 0000000000..10cc97f6ff --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs @@ -0,0 +1,593 @@ +//! Resolve every reference to one relation column, with the spans a rename rewrites. + +use polyglot_sql::{ + Dialect, DialectType, Expression, MappingSchema, Resolver, Scope, ValidationSchema, + build_scope, mapping_schema_from_validation_schema_with_dialect, walk_in_scope, +}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; + +#[derive(Deserialize)] +struct ReferenceRequest { + sql: String, + #[serde(default = "default_dialect")] + dialect: String, + schema: ValidationSchema, + column: String, + #[serde(default)] + target_tables: Vec, + #[serde(default)] + target_ctes: Vec, + #[serde(default)] + output_ctes: Vec, +} + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct ReferenceResponse { + parsed: bool, + references: Vec, + stars: Vec, + joins: Vec, + unresolved: Vec, + outputs: Vec, + star_outputs: Vec, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ReferenceSite { + start: usize, + end: usize, + name_start: usize, + name_end: usize, + scope: String, + projection: Option, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ProjectionSite { + alias: Option, + alias_start: Option, + alias_end: Option, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct StarSite { + start: Option, + end: Option, + scope: String, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct SpanSite { + start: Option, + end: Option, + scope: String, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct OutputSite { + scope: String, + start: Option, + end: Option, + alias_start: Option, + alias_end: Option, +} + +const ROOT_SCOPE: &str = "root"; +const DERIVED_SCOPE: &str = "derived"; +const SUBQUERY_SCOPE: &str = "subquery"; +const CTE_SCOPE_PREFIX: &str = "cte:"; +const STAR: &str = "*"; + +struct Analysis<'a> { + schema: &'a MappingSchema, + column: String, + target_tables: HashSet, + target_ctes: HashSet, + output_ctes: HashSet, + response: ReferenceResponse, + seen: HashSet<(usize, usize)>, +} + +fn default_dialect() -> String { + "generic".to_string() +} + +pub(crate) fn analyze_json(request_json: &str) -> Result { + let request: ReferenceRequest = + serde_json::from_str(request_json).map_err(|error| error.to_string())?; + let dialect: DialectType = request.dialect.parse().map_err(|error| { + format!( + "unsupported Polyglot dialect '{}': {error}", + request.dialect + ) + })?; + let Ok(statements) = Dialect::get(dialect).parse(&request.sql) else { + return serde_json::to_string(&ReferenceResponse::default()) + .map_err(|error| error.to_string()); + }; + let schema = mapping_schema_from_validation_schema_with_dialect(&request.schema, dialect); + let scopes: Vec = statements.iter().map(build_scope).collect(); + let ctes: Vec<&polyglot_sql::expressions::Cte> = scopes.iter().flat_map(scope_ctes).collect(); + let target_ctes: HashSet = star_derived_ctes( + &ctes, + &lowered(&request.target_tables), + lowered(&request.target_ctes), + ); + let mut analysis = Analysis { + schema: &schema, + column: request.column.to_lowercase(), + target_tables: lowered(&request.target_tables), + target_ctes, + output_ctes: lowered(&request.output_ctes), + response: ReferenceResponse { + parsed: true, + ..ReferenceResponse::default() + }, + seen: HashSet::new(), + }; + for scope in &scopes { + analysis.visit(scope, ROOT_SCOPE.to_string(), &[]); + } + serde_json::to_string(&analysis.response).map_err(|error| error.to_string()) +} + +fn lowered(names: &[String]) -> HashSet { + names.iter().map(|name| name.to_lowercase()).collect() +} + +fn scope_ctes(scope: &Scope) -> Vec<&polyglot_sql::expressions::Cte> { + let mut ctes: Vec<&polyglot_sql::expressions::Cte> = Vec::new(); + if let Expression::Cte(cte) = &scope.expression { + ctes.push(cte); + } + for child in scope + .cte_scopes + .iter() + .chain(scope.derived_table_scopes.iter()) + .chain(scope.subquery_scopes.iter()) + .chain(scope.udtf_scopes.iter()) + .chain(scope.union_scopes.iter()) + { + ctes.extend(scope_ctes(child)); + } + ctes +} + +/// CTEs whose output passes the target column through a star become targets themselves. +fn star_derived_ctes( + ctes: &[&polyglot_sql::expressions::Cte], + target_tables: &HashSet, + seed: HashSet, +) -> HashSet { + let mut derived: HashSet = seed; + loop { + let before = derived.len(); + for cte in ctes { + let name = cte.alias.name.to_lowercase(); + if !derived.contains(&name) + && selects_star_over_targets(&cte.this, target_tables, &derived) + { + derived.insert(name); + } + } + if derived.len() == before { + return derived; + } + } +} + +fn selects_star_over_targets( + body: &Expression, + target_tables: &HashSet, + target_ctes: &HashSet, +) -> bool { + branch_selects(body) + .iter() + .any(|select| select_stars_over_targets(select, target_tables, target_ctes)) +} + +fn select_stars_over_targets( + select: &polyglot_sql::expressions::Select, + target_tables: &HashSet, + target_ctes: &HashSet, +) -> bool { + let relations: HashMap = select_relation_names(select); + for item in &select.expressions { + let covered: bool = match star_table(item) { + Some(None) => relations + .values() + .any(|name| is_target_name(name, target_tables, target_ctes)), + Some(Some(qualifier)) => relations + .get(&qualifier) + .is_some_and(|name| is_target_name(name, target_tables, target_ctes)), + None => false, + }; + if covered { + return true; + } + } + false +} + +fn is_target_name( + name: &str, + target_tables: &HashSet, + target_ctes: &HashSet, +) -> bool { + target_tables.contains(name) || target_ctes.contains(name) +} + +fn branch_selects(body: &Expression) -> Vec<&polyglot_sql::expressions::Select> { + match body { + Expression::Select(select) => vec![select], + Expression::Union(union) => { + [branch_selects(&union.left), branch_selects(&union.right)].concat() + } + Expression::Intersect(intersect) => [ + branch_selects(&intersect.left), + branch_selects(&intersect.right), + ] + .concat(), + Expression::Except(except) => { + [branch_selects(&except.left), branch_selects(&except.right)].concat() + } + Expression::Subquery(subquery) => branch_selects(&subquery.this), + Expression::Paren(paren) => branch_selects(&paren.this), + _ => Vec::new(), + } +} + +/// Map each visible alias of a SELECT's FROM and JOIN tables to its lower-cased table name. +fn select_relation_names(select: &polyglot_sql::expressions::Select) -> HashMap { + let mut names: HashMap = HashMap::new(); + let tables = select + .from + .iter() + .flat_map(|from| from.expressions.iter()) + .chain(select.joins.iter().map(|join| &join.this)); + for table in tables { + if let Expression::Table(table) = table { + let name = table.name.name.to_lowercase(); + let visible = table + .alias + .as_ref() + .map_or_else(|| name.clone(), |alias| alias.name.to_lowercase()); + names.insert(visible, name); + } + } + names +} + +/// `Some(None)` for `*`, `Some(Some(qualifier))` for `t.*`, `None` otherwise. +fn star_table(item: &Expression) -> Option> { + match item { + Expression::Star(star) => Some(star.table.as_ref().map(|table| table.name.to_lowercase())), + Expression::Column(column) if column.name.name == STAR => { + Some(column.table.as_ref().map(|table| table.name.to_lowercase())) + } + _ => None, + } +} + +fn star_span(item: &Expression) -> (Option, Option) { + match item { + Expression::Star(star) => ( + star.span.map(|span| span.start), + star.span.map(|span| span.end), + ), + Expression::Column(column) => ( + column.span.map(|span| span.start), + column.span.map(|span| span.end), + ), + _ => (None, None), + } +} + +fn scope_select(expression: &Expression) -> Option<&polyglot_sql::expressions::Select> { + match expression { + Expression::Select(select) => Some(select), + Expression::Cte(cte) => scope_select(&cte.this), + Expression::Subquery(subquery) => scope_select(&subquery.this), + Expression::Paren(paren) => scope_select(&paren.this), + _ => None, + } +} + +fn select_body(expression: &Expression) -> Option<&Expression> { + match expression { + Expression::Select(_) => Some(expression), + Expression::Cte(cte) => select_body(&cte.this), + Expression::Subquery(subquery) => select_body(&subquery.this), + Expression::Paren(paren) => select_body(&paren.this), + _ => None, + } +} + +impl Analysis<'_> { + fn visit(&mut self, scope: &Scope, role: String, outer: &[&Scope]) { + if let Some(select) = scope_select(&scope.expression) { + let local: Scope = local_scope(scope, select); + let mut chain: Vec<&Scope> = vec![&local]; + chain.extend(outer.iter().copied()); + self.visit_select(select, &scope.expression, &chain, &role); + } + for child in &scope.cte_scopes { + let name = match &child.expression { + Expression::Cte(cte) => cte.alias.name.clone(), + _ => String::new(), + }; + self.visit(child, format!("{CTE_SCOPE_PREFIX}{name}"), &[]); + } + for child in &scope.derived_table_scopes { + self.visit(child, DERIVED_SCOPE.to_string(), &[]); + } + let mut nested: Vec<&Scope> = vec![scope]; + nested.extend(outer.iter().copied()); + for child in scope.subquery_scopes.iter().chain(scope.udtf_scopes.iter()) { + self.visit(child, SUBQUERY_SCOPE.to_string(), &nested); + } + for child in &scope.union_scopes { + self.visit(child, role.clone(), outer); + } + } + + fn visit_select( + &mut self, + select: &polyglot_sql::expressions::Select, + expression: &Expression, + chain: &[&Scope], + role: &str, + ) { + self.record_outputs(select, role); + self.record_stars(select, chain[0], role); + self.record_joins(select, chain[0], role); + let Some(body) = select_body(expression) else { + return; + }; + for node in walk_in_scope(body, false) { + let Expression::Column(column) = node else { + continue; + }; + if column.name.name == STAR || column.name.name.to_lowercase() != self.column { + continue; + } + self.record_column(column, select, chain, role); + } + } + + fn record_column( + &mut self, + column: &polyglot_sql::expressions::Column, + select: &polyglot_sql::expressions::Select, + chain: &[&Scope], + role: &str, + ) { + let (Some(span), Some(name_span)) = (column.span, column.name.span) else { + return; + }; + if !self.seen.insert((span.start, span.end)) { + return; + } + match self.resolve(column, chain) { + Resolution::Target => self.response.references.push(ReferenceSite { + start: span.start, + end: span.end, + name_start: name_span.start, + name_end: name_span.end, + scope: role.to_string(), + projection: projection_site(select, span.start), + }), + Resolution::Other => {} + Resolution::Unknown => { + if chain.iter().any(|scope| self.has_target_source(scope)) { + self.response.unresolved.push(SpanSite { + start: Some(span.start), + end: Some(span.end), + scope: role.to_string(), + }); + } + } + } + } + + fn resolve(&self, column: &polyglot_sql::expressions::Column, chain: &[&Scope]) -> Resolution { + if let Some(qualifier) = &column.table { + for scope in chain { + if let Some(alias) = source_key(scope, &qualifier.name) { + return self.classify(scope, &alias); + } + } + return Resolution::Unknown; + } + for scope in chain { + let mut resolver = Resolver::new(scope, self.schema, false); + if resolver.is_ambiguous(&column.name.name) { + return Resolution::Unknown; + } + if let Some(alias) = resolver.get_table(&column.name.name) { + return self.classify(scope, &alias); + } + } + Resolution::Unknown + } + + fn classify(&self, scope: &Scope, alias: &str) -> Resolution { + match scope.sources.get(alias) { + Some(source) if self.is_target(&source.expression) => Resolution::Target, + Some(_) => Resolution::Other, + None => Resolution::Unknown, + } + } + + fn is_target(&self, expression: &Expression) -> bool { + match expression { + Expression::Table(table) => { + table.schema.is_none() + && self.target_tables.contains(&table.name.name.to_lowercase()) + } + Expression::Cte(cte) => self.target_ctes.contains(&cte.alias.name.to_lowercase()), + _ => false, + } + } + + fn has_target_source(&self, scope: &Scope) -> bool { + scope + .sources + .values() + .any(|source| self.is_target(&source.expression)) + } + + fn record_stars( + &mut self, + select: &polyglot_sql::expressions::Select, + scope: &Scope, + role: &str, + ) { + for item in &select.expressions { + let covers_target = match star_table(item) { + Some(None) => self.has_target_source(scope), + Some(Some(qualifier)) => source_key(scope, &qualifier) + .and_then(|alias| scope.sources.get(&alias)) + .is_some_and(|source| self.is_target(&source.expression)), + None => false, + }; + if covers_target { + let (start, end) = star_span(item); + self.response.stars.push(StarSite { + start, + end, + scope: role.to_string(), + }); + } + } + } + + fn record_joins( + &mut self, + select: &polyglot_sql::expressions::Select, + scope: &Scope, + role: &str, + ) { + if !self.has_target_source(scope) { + return; + } + for join in &select.joins { + let natural = format!("{:?}", join.kind).starts_with("Natural"); + let using = join + .using + .iter() + .find(|identifier| identifier.name.to_lowercase() == self.column); + if natural || using.is_some() { + let span = using.and_then(|identifier| identifier.span); + self.response.joins.push(SpanSite { + start: span.map(|value| value.start), + end: span.map(|value| value.end), + scope: role.to_string(), + }); + } + } + } + + fn record_outputs(&mut self, select: &polyglot_sql::expressions::Select, role: &str) { + let name = role.strip_prefix(CTE_SCOPE_PREFIX).unwrap_or(role); + if !self.output_ctes.contains(&name.to_lowercase()) { + return; + } + for item in &select.expressions { + match item { + Expression::Alias(alias) if alias.alias.name.to_lowercase() == self.column => { + self.response.outputs.push(OutputSite { + scope: role.to_string(), + start: None, + end: None, + alias_start: alias.alias.span.map(|span| span.start), + alias_end: alias.alias.span.map(|span| span.end), + }); + } + Expression::Column(column) if column.name.name.to_lowercase() == self.column => { + self.response.outputs.push(OutputSite { + scope: role.to_string(), + start: column.span.map(|span| span.start), + end: column.span.map(|span| span.end), + alias_start: None, + alias_end: None, + }); + } + _ if star_table(item).is_some() => { + self.response.star_outputs.push(role.to_string()); + } + _ => {} + } + } + } +} + +enum Resolution { + Target, + Other, + Unknown, +} + +/// Keep only the CTEs a SELECT reads; Polyglot also lists every visible CTE as a source. +fn local_scope(scope: &Scope, select: &polyglot_sql::expressions::Select) -> Scope { + let read: HashSet = select_relation_names(select).into_keys().collect(); + let mut local: Scope = Scope::new(scope.expression.clone()); + local.sources = scope + .sources + .iter() + .filter(|(name, source)| { + !matches!(*source.expression, Expression::Cte(_)) || read.contains(&name.to_lowercase()) + }) + .map(|(name, source)| (name.clone(), source.clone())) + .collect(); + local.lateral_sources = scope.lateral_sources.clone(); + local +} + +fn source_key(scope: &Scope, name: &str) -> Option { + if scope.sources.contains_key(name) { + return Some(name.to_string()); + } + scope + .sources + .keys() + .find(|key| key.eq_ignore_ascii_case(name)) + .cloned() +} + +fn projection_site( + select: &polyglot_sql::expressions::Select, + start: usize, +) -> Option { + for item in &select.expressions { + match item { + Expression::Column(column) if column.span.map(|span| span.start) == Some(start) => { + return Some(ProjectionSite { + alias: None, + alias_start: None, + alias_end: None, + }); + } + Expression::Alias(alias) => { + if let Expression::Column(column) = &alias.this + && column.span.map(|span| span.start) == Some(start) + { + return Some(ProjectionSite { + alias: Some(alias.alias.name.clone()), + alias_start: alias.alias.span.map(|span| span.start), + alias_end: alias.alias.span.map(|span| span.end), + }); + } + } + _ => {} + } + } + None +} diff --git a/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs b/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs new file mode 100644 index 0000000000..d1334fae40 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs @@ -0,0 +1,3 @@ +pub(crate) fn analyze_json(request_json: &str) -> Result { + crate::column_references::references::analyze_json(request_json) +} diff --git a/crates/sqlbuild-rules-native/src/column_references/main/mod.rs b/crates/sqlbuild-rules-native/src/column_references/main/mod.rs new file mode 100644 index 0000000000..135b61fdc6 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod analyze; diff --git a/crates/sqlbuild-rules-native/src/column_references/mod.rs b/crates/sqlbuild-rules-native/src/column_references/mod.rs new file mode 100644 index 0000000000..c776b9c603 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod main; +#[path = "_helpers/references.rs"] +mod references; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/column_references/tests.rs b/crates/sqlbuild-rules-native/src/column_references/tests.rs new file mode 100644 index 0000000000..0b84c98315 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_column_references.rs"] +mod column_references; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs b/crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs new file mode 100644 index 0000000000..37991a44a9 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs @@ -0,0 +1,58 @@ +use crate::column_references::main::analyze::analyze_json; +use serde_json::{Value, json}; + +pub(crate) fn analyze(sql: &str, output_ctes: &[&str]) -> Result { + let request: Value = json!({ + "sql": sql, + "dialect": "duckdb", + "column": "amount", + "schema": {"strict": false, "tables": [ + {"name": "orders_fact", "columns": [ + {"name": "order_id", "type": "INTEGER"}, + {"name": "amount", "type": "INTEGER"} + ]}, + {"name": "customers", "columns": [ + {"name": "customer_id", "type": "INTEGER"}, + {"name": "order_id", "type": "INTEGER"} + ]} + ]}, + "target_tables": ["orders_fact"], + "output_ctes": output_ctes, + }); + serde_json::from_str(&analyze_json(&request.to_string())?).map_err(|error| error.to_string()) +} + +pub(crate) fn slices(sql: &str, sites: &Value, start: &str, end: &str) -> Vec { + let chars: Vec = sql.chars().collect(); + sites + .as_array() + .map(|items| { + items + .iter() + .filter_map(|item| span_text(&chars, item, start, end)) + .collect() + }) + .unwrap_or_default() +} + +pub(crate) fn strings(sites: &Value, key: &str) -> Vec { + sites + .as_array() + .map(|items| { + items + .iter() + .filter_map(|item| item[key].as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default() +} + +pub(crate) fn count(sites: &Value) -> usize { + sites.as_array().map_or(0, Vec::len) +} + +fn span_text(chars: &[char], item: &Value, start: &str, end: &str) -> Option { + let start: usize = usize::try_from(item[start].as_u64()?).ok()?; + let end: usize = usize::try_from(item[end].as_u64()?).ok()?; + Some(chars[start..end].iter().collect()) +} diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs b/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs new file mode 100644 index 0000000000..eb358a8b03 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs @@ -0,0 +1,182 @@ +use crate::column_references::tests::helpers::{analyze, count, slices, strings}; +use crate::column_references::tests::test_types::ColumnReferenceTestCase; +use serde_json::Value; + +#[test] +fn given_column_uses_when_analyzing_then_reports_rename_sites() -> Result<(), String> { + let test_cases = [ + ColumnReferenceTestCase { + description: "qualified and unqualified uses across clauses", + sql: "SELECT f.amount, SUM(amount) AS total FROM orders_fact f \ + JOIN customers c ON c.order_id = f.order_id WHERE amount > 0 GROUP BY f.amount", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount", "amount", "amount", "amount"], + expected_scopes: &["root", "root", "root", "root"], + expected_bare_projections: 1, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "same-named column of another relation is not the target", + sql: "SELECT c.amount FROM customers c JOIN orders_fact f ON c.order_id = f.order_id", + output_ctes: &[], + expected_parsed: true, + expected_names: &[], + expected_scopes: &[], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "pass-through inside a CTE is reported in the CTE scope", + sql: "WITH x AS (SELECT amount FROM orders_fact) SELECT amount FROM x", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount"], + expected_scopes: &["cte:x"], + expected_bare_projections: 1, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "a star CTE over the target makes its column a target too", + sql: "WITH x AS (SELECT * FROM orders_fact) SELECT x.amount AS a FROM x", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount"], + expected_scopes: &["root"], + expected_bare_projections: 0, + expected_aliases: &["a"], + expected_star_scopes: &["cte:x"], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "a USING join on the column is reported", + sql: "SELECT order_id FROM orders_fact JOIN customers USING (amount)", + output_ctes: &[], + expected_parsed: true, + expected_names: &[], + expected_scopes: &[], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 1, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "output scopes report aliased and bare projections of every branch", + sql: "WITH fixture AS (SELECT 1 AS amount UNION ALL SELECT 2 AS amount) \ + SELECT amount FROM orders_fact", + output_ctes: &["fixture", "root"], + expected_parsed: true, + expected_names: &["amount"], + expected_scopes: &["root"], + expected_bare_projections: 1, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &["amount", "amount"], + expected_outputs: 3, + }, + ColumnReferenceTestCase { + description: "unparseable SQL reports that it was not analysed", + sql: "SELECT FROM WHERE (", + output_ctes: &[], + expected_parsed: false, + expected_names: &[], + expected_scopes: &[], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ]; + for test_case in &test_cases { + let response: Value = analyze(test_case.sql, test_case.output_ctes)?; + let references: &Value = &response["references"]; + let projections: Vec<&Value> = references + .as_array() + .map(|items| items.iter().map(|item| &item["projection"]).collect()) + .unwrap_or_default(); + let bare: usize = projections + .iter() + .filter(|projection| projection.is_object() && projection["alias"].is_null()) + .count(); + let aliases: Vec = projections + .iter() + .filter_map(|projection| projection["alias"].as_str().map(str::to_string)) + .collect(); + assert_eq!( + response["parsed"], test_case.expected_parsed, + "{}", + test_case.description + ); + assert_eq!( + slices(test_case.sql, references, "nameStart", "nameEnd"), + test_case.expected_names, + "{}", + test_case.description + ); + assert_eq!( + strings(references, "scope"), + test_case.expected_scopes, + "{}", + test_case.description + ); + assert_eq!( + bare, test_case.expected_bare_projections, + "{}", + test_case.description + ); + assert_eq!( + aliases, test_case.expected_aliases, + "{}", + test_case.description + ); + assert_eq!( + strings(&response["stars"], "scope"), + test_case.expected_star_scopes, + "{}", + test_case.description + ); + assert_eq!( + count(&response["joins"]), + test_case.expected_joins, + "{}", + test_case.description + ); + assert_eq!( + slices( + test_case.sql, + &response["outputs"], + "aliasStart", + "aliasEnd" + ), + test_case.expected_output_aliases, + "{}", + test_case.description + ); + assert_eq!( + count(&response["outputs"]), + test_case.expected_outputs, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs b/crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs new file mode 100644 index 0000000000..0032a0ffae --- /dev/null +++ b/crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs @@ -0,0 +1,14 @@ +pub(crate) struct ColumnReferenceTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) output_ctes: &'static [&'static str], + pub(crate) expected_parsed: bool, + pub(crate) expected_names: &'static [&'static str], + pub(crate) expected_scopes: &'static [&'static str], + pub(crate) expected_bare_projections: usize, + pub(crate) expected_aliases: &'static [&'static str], + pub(crate) expected_star_scopes: &'static [&'static str], + pub(crate) expected_joins: usize, + pub(crate) expected_output_aliases: &'static [&'static str], + pub(crate) expected_outputs: usize, +} diff --git a/crates/sqlbuild-rules-native/src/lib.rs b/crates/sqlbuild-rules-native/src/lib.rs index b37d85783b..056f1dd5f9 100644 --- a/crates/sqlbuild-rules-native/src/lib.rs +++ b/crates/sqlbuild-rules-native/src/lib.rs @@ -1,6 +1,7 @@ #![forbid(unsafe_code)] pub mod bindings; +mod column_references; mod compiler; mod configuration; mod constants; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f4a8b177bd..37bd4c9921 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -8,6 +8,7 @@ class LintPreparationRequest(TypedDict): prior_sites: list[int] dialect: str +def analyze_column_references_json(request_json: str) -> str: ... def catalogue_json() -> str: ... def evaluate_json(request_json: str) -> str: ... def finalize_rule_findings_json(request_json: str) -> str: ... From f28553bcf678a0f8980e86c5dfaedc379dee5377 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:00:56 +0100 Subject: [PATCH 02/16] refactor(native): derive the column-reference scope select from its body --- .../src/column_references/_helpers/references.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs b/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs index 10cc97f6ff..3a2a7fcf23 100644 --- a/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs +++ b/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs @@ -291,11 +291,8 @@ fn star_span(item: &Expression) -> (Option, Option) { } fn scope_select(expression: &Expression) -> Option<&polyglot_sql::expressions::Select> { - match expression { + match select_body(expression)? { Expression::Select(select) => Some(select), - Expression::Cte(cte) => scope_select(&cte.this), - Expression::Subquery(subquery) => scope_select(&subquery.this), - Expression::Paren(paren) => scope_select(&paren.this), _ => None, } } From f5c7e0be298810b729c2b817efb84a4caae02246 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:01:02 +0100 Subject: [PATCH 03/16] feat(cli): add sqb rename and sqb mv for compiler-verified refactors --- Makefile | 1 + .../cli/commands/_helpers/entry/dispatch.py | 19 + .../commands/_helpers/entry/lazy_handlers.py | 5 + .../cli/commands/_helpers/entry/parser.py | 49 ++ .../commands/_helpers/refactor/__init__.py | 1 + .../cli/commands/_helpers/refactor/command.py | 231 +++++++++ .../_helpers/refactor/compile_facts.py | 79 +++ .../cli/commands/_helpers/refactor/output.py | 231 +++++++++ .../cli/commands/classes/cli_namespace.py | 10 + .../cli/commands/main/project/_refactor.py | 10 + src/sqlbuild/cli/commands/models.py | 26 + src/sqlbuild/cli/commands/types.py | 2 + src/sqlbuild/cli/entry/models.py | 1 + src/sqlbuild/compiler/refactoring/__init__.py | 1 + .../compiler/refactoring/_helpers/__init__.py | 1 + .../refactoring/_helpers/column_planning.py | 465 ++++++++++++++++++ .../refactoring/_helpers/column_references.py | 398 +++++++++++++++ .../refactoring/_helpers/header_edits.py | 391 +++++++++++++++ .../refactoring/_helpers/model_migration.py | 131 +++++ .../refactoring/_helpers/model_planning.py | 239 +++++++++ .../refactoring/_helpers/model_references.py | 180 +++++++ .../refactoring/_helpers/project_files.py | 182 +++++++ .../refactoring/_helpers/sql_sites.py | 142 ++++++ .../refactoring/_helpers/text_edits.py | 193 ++++++++ .../refactoring/_helpers/workspace.py | 174 +++++++ .../compiler/refactoring/constants.py | 105 ++++ .../compiler/refactoring/exceptions.py | 27 + .../compiler/refactoring/main/__init__.py | 1 + .../refactoring/main/commit_refactor_plan.py | 16 + .../refactoring/main/plan_column_rename.py | 31 ++ .../refactoring/main/plan_model_refactor.py | 29 ++ .../refactoring/main/stage_refactor_plan.py | 24 + .../refactoring/main/with_model_migration.py | 80 +++ src/sqlbuild/compiler/refactoring/models.py | 295 +++++++++++ src/sqlbuild/compiler/refactoring/types.py | 65 +++ .../lint/main/scan_interpolation_sites.py | 12 + .../cli/commands/main/refactor/__init__.py | 0 .../cli/commands/main/refactor/_test_types.py | 61 +++ .../cli/commands/main/refactor/helpers.py | 235 +++++++++ .../main/refactor/test_rename_columns.py | 209 ++++++++ .../main/refactor/test_rename_models.py | 209 ++++++++ .../sqlbuild/compiler/refactoring/__init__.py | 0 .../compiler/refactoring/_helpers/__init__.py | 0 .../refactoring/_helpers/_test_types.py | 55 +++ .../compiler/refactoring/_helpers/helpers.py | 108 ++++ .../_helpers/test_column_references.py | 104 ++++ .../refactoring/_helpers/test_header_edits.py | 130 +++++ .../refactoring/_helpers/test_text_edits.py | 92 ++++ .../refactoring/_helpers/test_workspace.py | 76 +++ 49 files changed, 5126 insertions(+) create mode 100644 src/sqlbuild/cli/commands/_helpers/refactor/__init__.py create mode 100644 src/sqlbuild/cli/commands/_helpers/refactor/command.py create mode 100644 src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py create mode 100644 src/sqlbuild/cli/commands/_helpers/refactor/output.py create mode 100644 src/sqlbuild/cli/commands/main/project/_refactor.py create mode 100644 src/sqlbuild/compiler/refactoring/__init__.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/__init__.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/column_planning.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/column_references.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/header_edits.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/model_migration.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/model_planning.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/model_references.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/project_files.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/sql_sites.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/text_edits.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/workspace.py create mode 100644 src/sqlbuild/compiler/refactoring/constants.py create mode 100644 src/sqlbuild/compiler/refactoring/exceptions.py create mode 100644 src/sqlbuild/compiler/refactoring/main/__init__.py create mode 100644 src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py create mode 100644 src/sqlbuild/compiler/refactoring/main/plan_column_rename.py create mode 100644 src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py create mode 100644 src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py create mode 100644 src/sqlbuild/compiler/refactoring/main/with_model_migration.py create mode 100644 src/sqlbuild/compiler/refactoring/models.py create mode 100644 src/sqlbuild/compiler/refactoring/types.py create mode 100644 src/sqlbuild/lint/main/scan_interpolation_sites.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/__init__.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/_helpers/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py diff --git a/Makefile b/Makefile index 5e1dfc37ec..3f16e3e5ad 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,7 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/playground \ tests/e2e/src/sqlbuild/cli/commands/main/postgres \ tests/e2e/src/sqlbuild/cli/commands/main/query \ + tests/e2e/src/sqlbuild/cli/commands/main/refactor \ tests/e2e/src/sqlbuild/cli/commands/main/scope \ tests/e2e/src/sqlbuild/cli/commands/main/seed \ tests/e2e/src/sqlbuild/cli/commands/main/skills \ diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 200ed070e9..6ccfd72b83 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -422,6 +422,25 @@ def _dispatch_compile_or_contract( project_dir: Path | None, select: tuple[str, ...], ) -> int | None: + if args.command in {CliCommand.RENAME, CliCommand.MV}: + from sqlbuild.cli.commands.models import RefactorCommandRequest + + if handlers.run_refactor is None or args.refactor_target is None: + raise CliUserError(f"{args.command} command handler is unavailable", code="C950") + return handlers.run_refactor( + request=RefactorCommandRequest( + command=str(args.command), + target=args.refactor_target, + new_name=args.refactor_new_name, + destination=args.refactor_destination, + project_dir=project_dir, + cascade=args.refactor_cascade, + allow_manual=args.refactor_allow_manual, + dry_run=args.dry_run, + json_output=args.json, + no_color=args.no_color, + ) + ) if args.command == CliCommand.CONTRACT: from sqlbuild.cli.commands.models import ContractCommandRequest diff --git a/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py b/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py index 261ede787a..386aa017f6 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py @@ -89,6 +89,10 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: module_name="sqlbuild.cli.commands.main.inspection._scope", function_name="run_scope", ), + "refactor": _lazy_handler( + module_name="sqlbuild.cli.commands.main.project._refactor", + function_name="run_refactor", + ), "contract": _lazy_handler( module_name="sqlbuild.cli.commands.main.inspection._contract", function_name="run_contract", @@ -220,6 +224,7 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: run_rules=lazy["rules"], run_scope=lazy["scope"], run_contract=lazy["contract"], + run_refactor=lazy["refactor"], ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index 23a39ed79d..58998b2623 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -68,6 +68,7 @@ def build_cli_parser(*, use_color: bool = False) -> argparse.ArgumentParser: _add_data_parsers(subparsers) _add_inspection_parsers(subparsers) _add_contract_parser(subparsers) + _add_refactor_parsers(subparsers) _add_maintenance_parsers(subparsers) _add_workspace_parsers(subparsers) _add_dbt_parsers(subparsers) @@ -106,6 +107,54 @@ def _add_contract_parser( action_parser.add_argument("--overwrite", action="store_true") +def _add_refactor_parsers( + subparsers: argparse._SubParsersAction[argparse.ArgumentParser], +) -> None: + rename_parser: argparse.ArgumentParser = subparsers.add_parser( + CliCommand.RENAME, help="rename a model or column and update every reference" + ) + rename_parser.add_argument( + "refactor_target", metavar="TARGET", help="model: or column:." + ) + rename_parser.add_argument("refactor_new_name", metavar="NEW_NAME") + rename_parser.add_argument( + "--cascade", + dest="refactor_cascade", + action="store_true", + default=False, + help="rename a column through every downstream model that passes it through", + ) + move_parser: argparse.ArgumentParser = subparsers.add_parser( + CliCommand.MV, help="move a model file, optionally renaming it, and update references" + ) + move_parser.add_argument("refactor_target", metavar="TARGET", help="model:") + move_parser.add_argument( + "refactor_destination", metavar="DESTINATION", help="new .sql path or folder/" + ) + parser: argparse.ArgumentParser + for parser in (rename_parser, move_parser): + parser.add_argument( + "--allow-manual", + dest="refactor_allow_manual", + action="store_true", + default=False, + help="apply the safe edits and leave the listed locations to edit by hand", + ) + parser.add_argument( + "--dry-run", + dest="dry_run", + action="store_true", + default=False, + help="print every edit and verify it compiles, without writing files", + ) + parser.add_argument( + "--json", + action="store_true", + default=False, + help="print the result as JSON on stdout; progress goes to stderr", + ) + + def _add_compile_and_dag_parsers( subparsers: argparse._SubParsersAction[argparse.ArgumentParser], ) -> None: diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py b/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py new file mode 100644 index 0000000000..d3461aba0d --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/__init__.py @@ -0,0 +1 @@ +"""Helpers for the rename and mv commands.""" diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/command.py b/src/sqlbuild/cli/commands/_helpers/refactor/command.py new file mode 100644 index 0000000000..c163468b02 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/command.py @@ -0,0 +1,231 @@ +"""Run `sqb rename` and `sqb mv`: plan from compiler facts, verify, then write.""" + +from __future__ import annotations + +import sys +import tempfile +import time +from dataclasses import replace +from pathlib import Path +from typing import TextIO + +from sqlbuild.cli.commands._helpers.refactor.compile_facts import compile_for_refactor +from sqlbuild.cli.commands._helpers.refactor.output import ( + render_refactor_json, + render_refactor_text, +) +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.cli.commands.models import RefactorCommandRequest, RefactorCompile +from sqlbuild.cli.commands.types import CliCommand +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.refactoring.constants import ( + COLUMN_KIND_PREFIX, + COLUMN_TARGET_SEPARATOR, + MODEL_KIND_PREFIX, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorInputError +from sqlbuild.compiler.refactoring.main.commit_refactor_plan import commit_refactor_plan +from sqlbuild.compiler.refactoring.main.plan_column_rename import plan_column_rename +from sqlbuild.compiler.refactoring.main.plan_model_refactor import plan_model_refactor +from sqlbuild.compiler.refactoring.main.stage_refactor_plan import stage_refactor_plan +from sqlbuild.compiler.refactoring.main.with_model_migration import with_model_migration +from sqlbuild.compiler.refactoring.models import RefactorPlan, RefactorRequest +from sqlbuild.compiler.refactoring.types import RefactorOperation, RefactorStatus +from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild.presentation.main.supports_color import supports_color + +_STAGING_PREFIX: str = "sqb-refactor-" + + +def run_refactor_command(*, request: RefactorCommandRequest) -> int: + """Plan, verify in a scratch copy, and write one rename or move.""" + + project_dir: Path = (request.project_dir or Path.cwd()).resolve() + stream: TextIO = sys.stderr if request.json_output else sys.stdout + use_color: bool = not request.no_color and supports_color() + status: TransientStatusReporter = TransientStatusReporter(stream=stream, use_color=use_color) + try: + return _run(request=request, project_dir=project_dir, status=status, use_color=use_color) + except RefactorInputError as error: + status.close() + raise CliUserError(error.message, code=error.code, help=error.help) from error + finally: + status.close() + + +def _run( + *, + request: RefactorCommandRequest, + project_dir: Path, + status: TransientStatusReporter, + use_color: bool, +) -> int: + started: float = time.monotonic() + status.start("Compiling project...") + before: RefactorCompile = compile_for_refactor(project_dir=project_dir) + if before.project is None or before.errors: + status.error("Project does not compile; fix it before renaming (sqb compile)") + _write_errors(errors=before.errors) + return 1 + status.complete(message=f"Compiled project. ({time.monotonic() - started:.2f}s)") + refactor_request: RefactorRequest = _refactor_request(request=request) + status.start("Planning edits...") + plan: RefactorPlan = ( + plan_column_rename(project=before.project, request=refactor_request) + if refactor_request.operation == RefactorOperation.RENAME_COLUMN + else plan_model_refactor(project=before.project, request=refactor_request) + ) + status.complete(message=f"Planned edits to {len(plan.changes)} files.") + if plan.blocking or (plan.manual and not request.allow_manual): + status.error("Refused: some references cannot be rewritten safely.") + return _finish( + request=request, + plan=plan, + status=RefactorStatus.REFUSED, + diagnostics=(), + use_color=use_color, + ) + with tempfile.TemporaryDirectory(prefix=_STAGING_PREFIX) as staging: + staging_dir: Path = Path(staging) + status.start("Verifying the edited project compiles...") + originals: dict[str, str] = stage_refactor_plan( + project_dir=project_dir, staging_dir=staging_dir, plan=plan + ) + after: RefactorCompile = compile_for_refactor(project_dir=staging_dir, no_cache=True) + if ( + after.project is not None + and not after.errors + and refactor_request.operation != (RefactorOperation.RENAME_COLUMN) + ): + migrated: RefactorPlan = with_model_migration( + plan=plan, + before=before.project.graph.project, + after=after.project.graph.project, + originals=originals, + ) + if migrated.blocking: + status.error("Refused: the renamed model's history cannot be kept.") + return _finish( + request=request, + plan=migrated, + status=RefactorStatus.REFUSED, + diagnostics=(), + use_color=use_color, + ) + if migrated.migrations != plan.migrations: + plan = migrated + originals = stage_refactor_plan( + project_dir=project_dir, staging_dir=staging_dir, plan=plan, copy_inputs=False + ) + after = compile_for_refactor(project_dir=staging_dir, no_cache=True) + if after.errors: + status.error("Verification failed: the edited project does not compile.") + return _finish( + request=request, + plan=plan, + status=RefactorStatus.COMPILE_FAILED, + diagnostics=_relative(errors=after.errors, staging_dir=staging_dir), + use_color=use_color, + ) + status.complete(message="Verified: the edited project compiles.") + if request.dry_run: + return _finish( + request=request, + plan=plan, + status=RefactorStatus.DRY_RUN, + diagnostics=(), + use_color=use_color, + ) + status.start(f"Writing {len(plan.changes)} files...") + commit_refactor_plan(project_dir=project_dir, originals=originals, plan=plan) + status.complete(message=f"Wrote {len(plan.changes)} files.") + return _finish( + request=request, + plan=plan, + status=RefactorStatus.APPLIED, + diagnostics=(), + use_color=use_color, + ) + + +def _finish( + *, + request: RefactorCommandRequest, + plan: RefactorPlan, + status: RefactorStatus, + diagnostics: tuple[CompilerDiagnostic, ...], + use_color: bool, +) -> int: + output: str = ( + render_refactor_json(plan=plan, status=status, diagnostics=diagnostics) + if request.json_output + else render_refactor_text( + plan=plan, status=status, diagnostics=diagnostics, use_color=use_color + ) + ) + _ = sys.stdout.write(output) + return 0 if status in {RefactorStatus.APPLIED, RefactorStatus.DRY_RUN} else 1 + + +def _refactor_request(*, request: RefactorCommandRequest) -> RefactorRequest: + target: str = request.target.strip() + if request.command == CliCommand.MV: + if not target.startswith(MODEL_KIND_PREFIX): + raise CliUserError( + f"sqb mv moves models; got '{target}'", + code="C954", + help="write the target as model:", + ) + return RefactorRequest( + operation=RefactorOperation.MOVE_MODEL, + model_name=target.removeprefix(MODEL_KIND_PREFIX), + new_name="", + destination=request.destination or "", + ) + new_name: str = (request.new_name or "").strip() + if target.startswith(MODEL_KIND_PREFIX): + if request.cascade: + raise CliUserError("--cascade applies to column renames only", code="C954") + return RefactorRequest( + operation=RefactorOperation.RENAME_MODEL, + model_name=target.removeprefix(MODEL_KIND_PREFIX), + new_name=new_name, + ) + if target.startswith(COLUMN_KIND_PREFIX) and COLUMN_TARGET_SEPARATOR in target: + model_name: str + column_name: str + model_name, column_name = target.removeprefix(COLUMN_KIND_PREFIX).rsplit( + COLUMN_TARGET_SEPARATOR, 1 + ) + return RefactorRequest( + operation=RefactorOperation.RENAME_COLUMN, + model_name=model_name, + column_name=column_name, + new_name=new_name, + cascade=request.cascade, + ) + raise CliUserError( + f"cannot rename '{target}'", + code="C954", + help="write the target as model: or column:.", + ) + + +def _relative( + *, errors: tuple[CompilerDiagnostic, ...], staging_dir: Path +) -> tuple[CompilerDiagnostic, ...]: + return tuple( + replace(error, path=error.path.relative_to(staging_dir)) + if error.path is not None + and error.path.is_absolute() + and error.path.is_relative_to(staging_dir) + else error + for error in errors + ) + + +def _write_errors(*, errors: tuple[CompilerDiagnostic, ...]) -> None: + error: CompilerDiagnostic + for error in errors: + location: str = f"{error.path.as_posix()}: " if error.path is not None else "" + print(f"error[{error.code}] {location}{error.message}", file=sys.stderr) diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py new file mode 100644 index 0000000000..5bc0c1e0a1 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py @@ -0,0 +1,79 @@ +"""Offline compiles that feed and verify a refactoring.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.cli.commands._helpers.compile.pipeline import analyze_compile_project +from sqlbuild.cli.commands._helpers.compile.target_writer import ( + static_sql_test_planning_diagnostics, +) +from sqlbuild.cli.commands.models import RefactorCompile +from sqlbuild.cli.commands.types import CompileLineageMode +from sqlbuild.cli.compile.models import CompileAnalysis, CompileProfileFlags +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic +from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.refactoring.models import RefactorProject +from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.spec.contracts.exceptions import SpecConfigError + + +def compile_for_refactor(*, project_dir: Path, no_cache: bool = False) -> RefactorCompile: + """Compile a project the way `sqb compile` does and collect every error.""" + + try: + analysis: CompileAnalysis = analyze_compile_project( + project_dir=project_dir, + no_sql_validation=False, + selected_target=None, + lineage_mode=CompileLineageMode.NONE, + cli_vars=None, + profile_flags=CompileProfileFlags(), + analysis_selection=CompileAnalysisSelection(no_cache=no_cache), + status=None, + ) + except (ValueError, RulesError, SpecConfigError) as error: + return RefactorCompile( + project=None, + errors=( + CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=str(getattr(error, "code", "E001")), + message=str(getattr(error, "message", str(error))), + ), + ), + ) + errors: list[CompilerDiagnostic] = [ + diagnostic for diagnostic in analysis.diagnostics if diagnostic.is_error + ] + if not errors: + try: + errors.extend( + diagnostic + for diagnostic in static_sql_test_planning_diagnostics( + target_dir=project_dir / "target", + adapter=analysis.adapter, + project=analysis.graph.project, + ) + if diagnostic.is_error + ) + except (CompileInputError, PlannerInputError) as error: + errors.append( + CompilerDiagnostic( + phase=DiagnosticPhase.TEST, + severity=DiagnosticSeverity.ERROR, + code=error.code, + message=str(error), + ) + ) + return RefactorCompile( + project=RefactorProject( + project_dir=project_dir, + graph=analysis.graph, + discovered=analysis.discovered_inputs, + ), + errors=tuple(errors), + ) diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/output.py b/src/sqlbuild/cli/commands/_helpers/refactor/output.py new file mode 100644 index 0000000000..fc7d7becdb --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/output.py @@ -0,0 +1,231 @@ +"""Human and JSON output for `sqb rename` and `sqb mv`.""" + +from __future__ import annotations + +import json + +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.refactoring.constants import OPERATION_TITLES +from sqlbuild.compiler.refactoring.models import ( + FileChange, + ManualLocation, + MigrationDeclaration, + RefactorPlan, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import RefactorOperation, RefactorStatus +from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild.presentation.main.tree_connector import tree_connector + +_CONTINUE: str = "\u2502 " +_BLANK: str = " " +_POSITION_WIDTH: int = 8 +_KIND_WIDTH: int = 10 +_EXCERPT_LIMIT: int = 60 + + +def render_refactor_text( + *, + plan: RefactorPlan, + status: RefactorStatus, + diagnostics: tuple[CompilerDiagnostic, ...], + use_color: bool, +) -> str: + """Render the edit tree, manual locations, and one terminal status line.""" + + style: CliStyle = CliStyle(use_color=use_color) + lines: list[str] = [style.title(_heading(plan))] + blocked: bool = status == RefactorStatus.REFUSED + if not blocked or not (plan.blocking or plan.manual): + lines.extend(_change_lines(style=style, changes=plan.changes)) + if plan.migrations and not blocked: + lines.append(style.section("Migrations")) + lines.extend(_migration_lines(style=style, migrations=plan.migrations)) + if blocked: + lines.append(style.error_strong(f"Cannot {_verb(plan)} {_target(plan)} automatically")) + lines.extend(_location_lines(style=style, locations=(*plan.blocking, *plan.manual))) + if plan.help and plan.manual and not plan.blocking: + lines.append(plan.help) + elif plan.manual: + lines.append(style.warning_strong("Edit by hand")) + lines.extend(_location_lines(style=style, locations=plan.manual)) + if diagnostics: + lines.append(style.error_strong("Compile errors in the edited project")) + lines.extend(_diagnostic_lines(style=style, diagnostics=diagnostics)) + lines.append(_status_line(style=style, plan=plan, status=status)) + return "\n".join(lines) + "\n" + + +def render_refactor_json( + *, + plan: RefactorPlan, + status: RefactorStatus, + diagnostics: tuple[CompilerDiagnostic, ...], +) -> str: + """Render the same facts as stable JSON.""" + + payload: dict[str, object] = { + "operation": plan.request.operation.value, + "target": _target(plan), + "new_name": plan.request.new_name, + "destination": plan.request.destination, + "cascade": plan.request.cascade, + "status": status.value, + "files_changed": len(plan.changes) if status == RefactorStatus.APPLIED else 0, + "files": [_file_payload(change) for change in plan.changes], + "migrations": [ + { + "model": migration.model_name, + "declaration": migration.declaration, + "reason": migration.reason, + } + for migration in plan.migrations + ], + "renamed_columns": [ + {"model": model, "from": old, "to": new} for model, old, new in plan.renamed_columns + ], + "manual": [_location_payload(item) for item in plan.manual], + "blocking": [_location_payload(item) for item in plan.blocking], + "compile": { + "ok": not diagnostics, + "errors": [ + { + "code": diagnostic.code, + "message": diagnostic.message, + "path": diagnostic.path.as_posix() if diagnostic.path is not None else None, + "line": diagnostic.line, + } + for diagnostic in diagnostics + ], + }, + } + return json.dumps(payload, indent=2) + "\n" + + +def _file_payload(change: FileChange) -> dict[str, object]: + return { + "path": change.path, + "from_path": change.original_path if change.moved else None, + "edits": [ + _edit_payload(edit) for edit in sorted(change.edits, key=lambda item: item.start) + ], + } + + +def _edit_payload(edit: TextEdit) -> dict[str, object]: + return { + "line": edit.line, + "column": edit.column, + "kind": edit.kind.value, + "before": edit.before, + "after": edit.after, + } + + +def _heading(plan: RefactorPlan) -> str: + title: str = OPERATION_TITLES[plan.request.operation] + if plan.request.operation == RefactorOperation.RENAME_COLUMN: + return ( + f"{title} {plan.request.model_name}.{plan.request.column_name} -> " + f"{plan.request.new_name}" + ) + if plan.request.operation == RefactorOperation.MOVE_MODEL: + return f"{title} {plan.request.model_name} -> {plan.request.destination}" + return f"{title} {plan.request.model_name} -> {plan.request.new_name}" + + +def _verb(plan: RefactorPlan) -> str: + return "move" if plan.request.operation == RefactorOperation.MOVE_MODEL else "rename" + + +def _target(plan: RefactorPlan) -> str: + if plan.request.operation == RefactorOperation.RENAME_COLUMN: + return f"column:{plan.request.model_name}.{plan.request.column_name}" + return f"model:{plan.request.model_name}" + + +def _change_lines(*, style: CliStyle, changes: tuple[FileChange, ...]) -> list[str]: + lines: list[str] = [] + index: int + change: FileChange + for index, change in enumerate(changes): + last: bool = index == len(changes) - 1 + moved: str = style.muted(f" (moved from {change.original_path})") if change.moved else "" + lines.append(f"{tree_connector(style=style, last=last)} {change.path}{moved}") + prefix: str = _BLANK if last else _CONTINUE + edits: list[TextEdit] = sorted(change.edits, key=lambda item: item.start) + edit_index: int + edit: TextEdit + for edit_index, edit in enumerate(edits): + connector: str = tree_connector(style=style, last=edit_index == len(edits) - 1) + position: str = f"{edit.line}:{edit.column}".ljust(_POSITION_WIDTH) + lines.append( + f"{style.muted(prefix)}{connector} {position}{edit.kind.value.ljust(_KIND_WIDTH)}" + f"{_excerpt(edit.before)} -> {_excerpt(edit.after)}" + if edit.before + else f"{style.muted(prefix)}{connector} {position}" + f"{edit.kind.value.ljust(_KIND_WIDTH)}+ {_excerpt(edit.after)}" + ) + return lines + + +def _migration_lines(*, style: CliStyle, migrations: tuple[MigrationDeclaration, ...]) -> list[str]: + return [ + f"{tree_connector(style=style, last=index == len(migrations) - 1)} " + f"{style.object_name(item.model_name)} {item.declaration} {style.muted(item.reason)}" + for index, item in enumerate(migrations) + ] + + +def _location_lines(*, style: CliStyle, locations: tuple[ManualLocation, ...]) -> list[str]: + return [ + f"{tree_connector(style=style, last=index == len(locations) - 1)} " + f"{_position(item)} {item.reason}" + for index, item in enumerate(locations) + ] + + +def _diagnostic_lines(*, style: CliStyle, diagnostics: tuple[CompilerDiagnostic, ...]) -> list[str]: + return [ + f"{tree_connector(style=style, last=index == len(diagnostics) - 1)} " + f"{diagnostic.code} " + + ( + f"{diagnostic.path.as_posix()}:{diagnostic.line or ''} " + if diagnostic.path is not None + else "" + ) + + diagnostic.message.splitlines()[0] + for index, diagnostic in enumerate(diagnostics) + ] + + +def _status_line(*, style: CliStyle, plan: RefactorPlan, status: RefactorStatus) -> str: + count: int = len(plan.changes) + files: str = f"{count} file{'s' if count != 1 else ''}" + if status == RefactorStatus.APPLIED: + return style.success_strong(f"Compiled: ok, {files} changed") + if status == RefactorStatus.DRY_RUN: + return style.success_strong(f"Dry run: compiles, {files} would change; nothing written") + if status == RefactorStatus.COMPILE_FAILED: + return style.error_strong("Compiled: failed; no files changed") + return style.error_strong("Refused; no files changed") + + +def _position(location: ManualLocation) -> str: + if location.line is None: + return location.path + return f"{location.path}:{location.line}:{location.column or 1}" + + +def _location_payload(location: ManualLocation) -> dict[str, object]: + return { + "path": location.path, + "line": location.line, + "column": location.column, + "reason": location.reason, + } + + +def _excerpt(text: str) -> str: + single: str = " ".join(text.split()) + return single if len(single) <= _EXCERPT_LIMIT else single[: _EXCERPT_LIMIT - 1] + "\u2026" diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index e4cac9dbda..4c1fd9afb8 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -160,6 +160,11 @@ "scope_page_size": 100, "scope_after": None, "scope_paths": "relative", + "refactor_target": None, + "refactor_new_name": None, + "refactor_destination": None, + "refactor_cascade": False, + "refactor_allow_manual": False, } @@ -238,6 +243,11 @@ class CliNamespace: query_limit: int | None query_no_limit: bool cost_selector: str + refactor_target: str | None + refactor_new_name: str | None + refactor_destination: str | None + refactor_cascade: bool + refactor_allow_manual: bool cost_limit: int | None cost_no_limit: bool cost_sort: str | None diff --git a/src/sqlbuild/cli/commands/main/project/_refactor.py b/src/sqlbuild/cli/commands/main/project/_refactor.py new file mode 100644 index 0000000000..9752d45334 --- /dev/null +++ b/src/sqlbuild/cli/commands/main/project/_refactor.py @@ -0,0 +1,10 @@ +"""CLI entry for `sqb rename` and `sqb mv`.""" + +from sqlbuild.cli.commands._helpers.refactor.command import run_refactor_command +from sqlbuild.cli.commands.models import RefactorCommandRequest + + +def run_refactor(*, request: RefactorCommandRequest) -> int: + """Run one rename or move.""" + + return run_refactor_command(request=request) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 74abf7baf5..ab622e8694 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -52,6 +52,7 @@ CompiledObjectKey, CompiledProject, CompiledSqlScenario, + CompilerDiagnostic, ) from sqlbuild.compiler.discovery.models import ( DiscoveredCheckFunction, @@ -69,6 +70,7 @@ ) from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan +from sqlbuild.compiler.refactoring.models import RefactorProject from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus from sqlbuild.cost.types import CostStatus from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult @@ -1052,6 +1054,30 @@ class TestCommandRequest: inspect: bool = False +@dataclass(frozen=True) +class RefactorCommandRequest: + """CLI inputs for one `sqb rename` or `sqb mv` invocation.""" + + command: str + target: str + new_name: str | None = None + destination: str | None = None + project_dir: Path | None = None + cascade: bool = False + allow_manual: bool = False + dry_run: bool = False + json_output: bool = False + no_color: bool = False + + +@dataclass(frozen=True) +class RefactorCompile: + """One offline compile for a refactoring: its facts, or the errors that stopped it.""" + + project: RefactorProject | None + errors: tuple[CompilerDiagnostic, ...] + + @dataclass(frozen=True) class ScopeCommandRequest: """CLI inputs for one offline declaration-scope inspection.""" diff --git a/src/sqlbuild/cli/commands/types.py b/src/sqlbuild/cli/commands/types.py index 57203149cf..f9adb82b2d 100644 --- a/src/sqlbuild/cli/commands/types.py +++ b/src/sqlbuild/cli/commands/types.py @@ -53,6 +53,8 @@ class CliCommand(StrEnum): RULES = "rules" SCOPE = "scope" CONTRACT = "contract" + RENAME = "rename" + MV = "mv" class ScopeCommandHandler(Protocol): diff --git a/src/sqlbuild/cli/entry/models.py b/src/sqlbuild/cli/entry/models.py index c0b4ec4faa..371fafc465 100644 --- a/src/sqlbuild/cli/entry/models.py +++ b/src/sqlbuild/cli/entry/models.py @@ -78,3 +78,4 @@ class CliEntrypointHandlers: run_rules: Callable[..., int] run_scope: ScopeCommandHandler run_contract: Callable[..., int] | None = None + run_refactor: Callable[..., int] | None = None diff --git a/src/sqlbuild/compiler/refactoring/__init__.py b/src/sqlbuild/compiler/refactoring/__init__.py new file mode 100644 index 0000000000..c2d3b86586 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/__init__.py @@ -0,0 +1 @@ +"""Compiler-driven project refactorings: model renames, moves, and column renames.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/__init__.py new file mode 100644 index 0000000000..355dcd1846 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/__init__.py @@ -0,0 +1 @@ +"""Internal refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/column_planning.py b/src/sqlbuild/compiler/refactoring/_helpers/column_planning.py new file mode 100644 index 0000000000..97b94d5e3c --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/column_planning.py @@ -0,0 +1,465 @@ +"""Walk a column rename from its owner model through every downstream reader.""" + +from __future__ import annotations + +from collections import deque + +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.refactoring._helpers.column_references import ( + analyze_column, + consumer_edits, + output_edits, + resource_columns, +) +from sqlbuild.compiler.refactoring._helpers.header_edits import ( + add_column_entry_edit, + column_config_edits, + column_entry_edits, + consumer_column_header_edits, + header_tokens, + unhandled_word_offsets, +) +from sqlbuild.compiler.refactoring._helpers.model_migration import needs_column_migration +from sqlbuild.compiler.refactoring._helpers.model_planning import find_model, validate_identifier +from sqlbuild.compiler.refactoring._helpers.project_files import ( + authored_bodies, + project_sql_files, + python_string_locations, +) +from sqlbuild.compiler.refactoring._helpers.sql_sites import ( + analysis_sql, + authored_offset, + authored_span, + model_body, +) +from sqlbuild.compiler.refactoring._helpers.text_edits import manual_at, merge_parts, path_edits +from sqlbuild.compiler.refactoring.constants import ( + EXPECTED_FIXTURE_PREFIX, + FIXTURE_ROLES, + GENERIC_DIALECT, + MIGRATE_FROM_KEY, + REF_FIXTURE_PREFIX, + REF_KIND, + ROOT_SCOPE, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorInputError +from sqlbuild.compiler.refactoring.models import ( + AnalysisSql, + AuthoredBody, + BodyContext, + BodyEdits, + ColumnFacts, + ColumnQuery, + ColumnRenameContext, + HeaderToken, + ManualLocation, + MigrationDeclaration, + ModelBody, + ProjectSqlFile, + RefactorParts, + RefactorProject, + RefactorRequest, + TextEdit, +) +from sqlbuild.spec.contracts.models import SchemaColumn + +type _ModelContext = tuple[BodyContext | None, str, tuple[ManualLocation, ...]] + + +def column_rename_context( + *, project: RefactorProject, request: RefactorRequest +) -> ColumnRenameContext: + """Validate a column rename and collect everything its walk reads.""" + + owner: CompiledModel = find_model(project=project.graph.project, name=request.model_name) + old: str = request.column_name or "" + new: str = request.new_name + validate_identifier(name=new, noun="column") + files: tuple[ProjectSqlFile, ...] = project_sql_files(discovered=project.discovered) + context: ColumnRenameContext = ColumnRenameContext( + project=project, + owner=owner, + dialect=project.graph.project.sql_analysis_dialect or GENERIC_DIALECT, + columns=resource_columns(project=project.graph.project), + contents={item.relative_path: item.contents for item in files}, + bodies=authored_bodies(discovered=project.discovered), + old=old, + new=new, + cascade=request.cascade, + ) + if old.lower() not in _output_names(context=context): + raise RefactorInputError( + f"model:{owner.name} has no output column '{old}'", + code="C956", + help="column names match case-insensitively; check the spelling", + ) + if old.lower() == new.lower(): + raise RefactorInputError(f"column {old} already has that name", code="C953") + return context + + +def column_rename_parts(*, context: ColumnRenameContext) -> RefactorParts: + """Return every edit of the rename; `cascaded` lists each model whose column is renamed.""" + + parts: list[RefactorParts] = [_collision(context=context), _owner_output(context=context)] + queue: deque[CompiledModel] = deque([context.owner]) + seen: set[str] = set() + while queue: + model: CompiledModel = queue.popleft() + if model.name in seen: + continue + seen.add(model.name) + consumers: RefactorParts + cascaded: tuple[CompiledModel, ...] + consumers, cascaded = _consumers(context=context, model=model) + parts.extend( + ( + RefactorParts(cascaded=(model.name,)), + _declarations(context=context, model=model), + consumers, + _authored_body_parts(context=context, model=model), + ) + ) + queue.extend(cascaded) + merged: RefactorParts = merge_parts(parts=tuple(parts)) + return merge_parts(parts=(merged, _python_locations(context=context, merged=merged))) + + +def _output_names(*, context: ColumnRenameContext) -> frozenset[str]: + return frozenset( + name.lower() for name in context.columns.get((REF_KIND, context.owner.name), ()) + ) + + +def _collision(*, context: ColumnRenameContext) -> RefactorParts: + if context.new.lower() not in _output_names(context=context): + return RefactorParts() + return RefactorParts( + blocking=( + ManualLocation( + path=context.owner.relative_path.as_posix(), + line=None, + column=None, + reason=f"model:{context.owner.name} already has a column named {context.new}", + ), + ) + ) + + +def _owner_output(*, context: ColumnRenameContext) -> RefactorParts: + body: BodyContext | None + compiled_sql: str + unmapped: tuple[ManualLocation, ...] + body, compiled_sql, unmapped = _model_context(context=context, model=context.owner) + if body is None: + return RefactorParts(manual=unmapped) + facts: ColumnFacts = analyze_column( + analysis=analysis_sql(text=compiled_sql, dialect=context.dialect), + dialect=context.dialect, + columns=context.columns, + query=ColumnQuery( + column=context.old, + target_tables=frozenset(), + output_scopes=frozenset({ROOT_SCOPE}), + ), + ) + result: BodyEdits = output_edits( + facts=facts, + context=body, + scopes=frozenset({ROOT_SCOPE}), + old=context.old, + new=context.new, + ) + missing: tuple[ManualLocation, ...] = ( + () + if result.edits or result.manual + else ( + ManualLocation( + path=body.path, + line=None, + column=None, + reason=f"cannot find where model:{context.owner.name} produces {context.old}", + ), + ) + ) + return RefactorParts( + edits=path_edits(path=body.path, edits=result.edits), manual=(*result.manual, *missing) + ) + + +def _declarations(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: + path: str = model.relative_path.as_posix() + declared: SchemaColumn | None = next( + ( + column + for column in (model.schema_entry.columns if model.schema_entry is not None else ()) + if column.name.lower() == context.old.lower() + ), + None, + ) + if declared is not None and declared.migrate_from is not None: + return RefactorParts( + blocking=( + ManualLocation( + path=path, + line=None, + column=None, + reason=( + f"column {context.old} of model:{model.name} still declares " + "migrate_from; build it on every target and remove migrate_from before " + "renaming it again" + ), + ), + ) + ) + shared: tuple[ManualLocation, ...] = ( + ( + ManualLocation( + path=declared.location.path.as_posix(), + line=declared.location.line, + column=declared.location.column, + reason=( + f"column {context.old} of model:{model.name} is declared in a shared schema" + ), + ), + ) + if declared is not None + and declared.location is not None + and declared.location.path.as_posix() != path + else () + ) + return merge_parts( + parts=(RefactorParts(manual=shared), _header_parts(context=context, model=model)) + ) + + +def _header_parts(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: + path: str = model.relative_path.as_posix() + contents: str = context.contents.get(path, "") + tokens: tuple[HeaderToken, ...] = header_tokens(contents=contents) or () + migrate: bool = needs_column_migration(model=model) + entry_edits: tuple[TextEdit, ...] | None = column_entry_edits( + contents=contents, tokens=tokens, old=context.old, new=context.new, migrate=migrate + ) + added: TextEdit | None = ( + add_column_entry_edit(contents=contents, tokens=tokens, new=context.new, old=context.old) + if entry_edits is None and migrate + else None + ) + edits: tuple[TextEdit, ...] = ( + *column_config_edits(contents=contents, tokens=tokens, old=context.old, new=context.new), + *(entry_edits or ()), + *((added,) if added is not None else ()), + ) + handled: frozenset[int] = frozenset(edit.start for edit in edits) + return RefactorParts( + edits=path_edits(path=path, edits=edits), + manual=tuple( + manual_at( + path=path, + text=contents, + offset=offset, + reason=f"the MODEL header mentions {context.old} here; update it by hand", + ) + for offset in unhandled_word_offsets( + contents=contents, tokens=tokens, word=context.old, handled=handled + ) + ), + migrations=( + MigrationDeclaration( + model_name=model.name, + declaration=f"{context.new} ({MIGRATE_FROM_KEY} {context.old})", + reason="keeps the column's history in place", + ), + ) + if migrate + else (), + ) + + +def _consumers( + *, context: ColumnRenameContext, model: CompiledModel +) -> tuple[RefactorParts, tuple[CompiledModel, ...]]: + parts: list[RefactorParts] = [] + cascaded: list[CompiledModel] = [] + consumer: CompiledModel + for consumer in context.project.graph.project.models: + if not any( + dep.resource_type == CompiledResourceType.MODEL and dep.name == model.name + for dep in consumer.deps + ): + continue + consumer_parts: RefactorParts + passes_through: bool + consumer_parts, passes_through = _consumer( + context=context, upstream=model, consumer=consumer + ) + parts.append(consumer_parts) + if passes_through: + cascaded.append(consumer) + return merge_parts(parts=tuple(parts)), tuple(cascaded) + + +def _consumer( + *, context: ColumnRenameContext, upstream: CompiledModel, consumer: CompiledModel +) -> tuple[RefactorParts, bool]: + body: BodyContext | None + compiled_sql: str + unmapped: tuple[ManualLocation, ...] + body, compiled_sql, unmapped = _model_context(context=context, model=consumer) + if body is None: + return RefactorParts(manual=unmapped), False + analysis: AnalysisSql = analysis_sql(text=compiled_sql, dialect=context.dialect) + facts: ColumnFacts = analyze_column( + analysis=analysis, + dialect=context.dialect, + columns=context.columns, + query=ColumnQuery( + column=context.old, + target_tables=analysis.tables.get((REF_KIND, upstream.name), frozenset()), + ), + ) + result: BodyEdits = consumer_edits( + facts=facts, + context=body, + old=context.old, + new=context.new, + cascade_root=context.cascade, + root_stars_pass=context.cascade, + ) + header: tuple[TextEdit, ...] = consumer_column_header_edits( + contents=body.contents, upstream=upstream.name, old=context.old, new=context.new + ) + return ( + RefactorParts( + edits=path_edits(path=body.path, edits=(*result.edits, *header)), + manual=result.manual, + ), + result.passes_through, + ) + + +def _authored_body_parts(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: + return merge_parts( + parts=tuple( + _authored_body(context=context, model=model, body=body) + for body in context.bodies + if model.name.lower() in body.text.lower() + ) + ) + + +def _authored_body( + *, context: ColumnRenameContext, model: CompiledModel, body: AuthoredBody +) -> RefactorParts: + lowered: str = body.text.lower() + analysis: AnalysisSql = analysis_sql(text=body.text, dialect=context.dialect) + targets: frozenset[str] = analysis.tables.get((REF_KIND, model.name), frozenset()) + fixture_scopes: frozenset[str] = ( + frozenset( + name + for name in ( + f"{EXPECTED_FIXTURE_PREFIX}{model.name}", + f"{REF_FIXTURE_PREFIX}{model.name}", + ) + if name.lower() in lowered + ) + if body.role in FIXTURE_ROLES + else frozenset() + ) + if not targets and not fixture_scopes: + return RefactorParts() + facts: ColumnFacts = analyze_column( + analysis=analysis, + dialect=context.dialect, + columns=context.columns, + query=ColumnQuery( + column=context.old, + target_tables=targets, + target_ctes=frozenset( + name for name in fixture_scopes if name.startswith(REF_FIXTURE_PREFIX) + ), + output_scopes=fixture_scopes, + ), + ) + located: BodyContext = BodyContext( + path=body.path, + contents=body.contents, + map_span=lambda start, end: (body.start + start, body.start + end), + locate=lambda offset: body.start + offset, + fallback_offset=body.start, + ) + outputs: BodyEdits = output_edits( + facts=facts, context=located, scopes=fixture_scopes, old=context.old, new=context.new + ) + consumers: BodyEdits = consumer_edits( + facts=facts, + context=located, + old=context.old, + new=context.new, + cascade_root=False, + root_stars_pass=True, + ) + return RefactorParts( + edits=path_edits(path=body.path, edits=(*outputs.edits, *consumers.edits)), + manual=(*outputs.manual, *consumers.manual), + ) + + +def _python_locations(*, context: ColumnRenameContext, merged: RefactorParts) -> RefactorParts: + edited: frozenset[str] = frozenset(path for path, _ in merged.edits) + return RefactorParts( + manual=tuple( + location + for location in python_string_locations( + project_dir=context.project.project_dir, + discovered=context.project.discovered, + names=(context.old,), + reason=( + f"Python SQL may read column {context.old} of {context.owner.name}; check it " + "by hand" + ), + context=context.owner.name, + ) + if location.path not in edited + ) + ) + + +def _model_context(*, context: ColumnRenameContext, model: CompiledModel) -> _ModelContext: + path: str = model.relative_path.as_posix() + contents: str | None = context.contents.get(path) + if contents is None: + return None, "", () + body: ModelBody | None = model_body( + project=context.project.graph.project, + project_dir=context.project.project_dir, + model=model, + contents=contents, + ) + if body is None: + return ( + None, + "", + ( + ManualLocation( + path=path, + line=None, + column=None, + reason="the compiled SQL of this model cannot be mapped back to the file", + ), + ), + ) + mapped: ModelBody = body + return ( + BodyContext( + path=path, + contents=contents, + map_span=lambda start, end: authored_span(body=mapped, start=start, end=end), + locate=lambda offset: authored_offset(body=mapped, offset=offset)[0], + fallback_offset=body.body_start, + ), + body.compiled_sql, + (), + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/column_references.py b/src/sqlbuild/compiler/refactoring/_helpers/column_references.py new file mode 100644 index 0000000000..46b691d32d --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/column_references.py @@ -0,0 +1,398 @@ +"""Native column-reference facts for one SQL body, and the edits they imply.""" + +from __future__ import annotations + +import json +from typing import Any, cast + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.refactoring._helpers.text_edits import manual_at, text_edit +from sqlbuild.compiler.refactoring.constants import ( + CTE_SCOPE_PREFIX, + QUOTE_CHARACTERS, + REF_KIND, + ROOT_SCOPE, + SEED_KIND, + SOURCE_KIND, + UNKNOWN_COLUMN_TYPE, +) +from sqlbuild.compiler.refactoring.models import ( + AnalysisSql, + BodyContext, + BodyEdits, + ColumnFacts, + ColumnQuery, + ColumnReference, + ColumnSite, + ManualLocation, + OutputColumn, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind, NativeColumnReferences, ResourceColumns + + +def resource_columns(*, project: CompiledProject) -> ResourceColumns: + """Return the known output columns of every model, source, and seed.""" + + columns: ResourceColumns = {} + for model in project.models: + names: list[str] = [column.name for column in model.inferred_columns or ()] + names.extend( + column.name + for column in (model.schema_entry.columns if model.schema_entry is not None else ()) + if column.name not in names + ) + columns[(REF_KIND, model.name)] = tuple(names) + for source in project.sources: + columns[(SOURCE_KIND, source.name)] = tuple( + column.name for column in source.source_entry.columns + ) + for seed in project.seeds: + columns[(SEED_KIND, seed.name)] = tuple(column.name for column in seed.schema_entry.columns) + return columns + + +def analyze_column( + *, analysis: AnalysisSql, dialect: str, columns: ResourceColumns, query: ColumnQuery +) -> ColumnFacts: + """Resolve every use of one column of the target relations in one SQL body.""" + + request: dict[str, object] = { + "sql": analysis.sql, + "dialect": dialect, + "column": query.column, + "schema": {"strict": False, "tables": _schema_tables(analysis=analysis, columns=columns)}, + "target_tables": sorted(query.target_tables), + "target_ctes": sorted(query.target_ctes), + "output_ctes": sorted(query.output_scopes), + } + payload: dict[str, Any] = json.loads( + cast(NativeColumnReferences, _native).analyze_column_references_json(json.dumps(request)) + ) + return ColumnFacts( + parsed=bool(payload.get("parsed")), + references=tuple(_reference(item) for item in payload.get("references", ())), + stars=tuple(_site(item) for item in payload.get("stars", ())), + joins=tuple(_site(item) for item in payload.get("joins", ())), + unresolved=tuple(_site(item) for item in payload.get("unresolved", ())), + outputs=tuple(_output(item) for item in payload.get("outputs", ())), + star_outputs=tuple(str(item) for item in payload.get("starOutputs", ())), + ) + + +def _schema_tables(*, analysis: AnalysisSql, columns: ResourceColumns) -> list[dict[str, object]]: + tables: list[dict[str, object]] = [] + key: tuple[str, str] + placeholders: frozenset[str] + for key, placeholders in analysis.tables.items(): + known: list[dict[str, str]] = [ + {"name": name, "type": UNKNOWN_COLUMN_TYPE} for name in columns.get(key, ()) + ] + name: str + for name in sorted(placeholders): + table: dict[str, object] = {"name": name, "columns": known} + tables.append(table) + return tables + + +def _reference(item: dict[str, Any]) -> ColumnReference: + projection: dict[str, Any] | None = item.get("projection") + return ColumnReference( + start=int(item["start"]), + end=int(item["end"]), + name_start=int(item["nameStart"]), + name_end=int(item["nameEnd"]), + scope=str(item["scope"]), + projected=projection is not None, + alias=projection.get("alias") if projection is not None else None, + alias_start=projection.get("aliasStart") if projection is not None else None, + alias_end=projection.get("aliasEnd") if projection is not None else None, + ) + + +def _site(item: dict[str, Any]) -> ColumnSite: + return ColumnSite(scope=str(item["scope"]), start=item.get("start"), end=item.get("end")) + + +def _output(item: dict[str, Any]) -> OutputColumn: + return OutputColumn( + scope=str(item["scope"]), + start=item.get("start"), + end=item.get("end"), + alias_start=item.get("aliasStart"), + alias_end=item.get("aliasEnd"), + ) + + +def consumer_edits( + *, + facts: ColumnFacts, + context: BodyContext, + old: str, + new: str, + cascade_root: bool, + root_stars_pass: bool, +) -> BodyEdits: + """Rename every reference; keep consumer output names unless the root passes it on.""" + + if not facts.parsed: + return BodyEdits( + manual=(_manual(context=context, offset=None, reason="SQL could not be analysed"),) + ) + return combine_body_edits( + results=( + *( + _reference_edits( + reference=reference, + context=context, + old=old, + new=new, + cascade_root=cascade_root, + ) + for reference in facts.references + ), + *( + _star_edits(star=star, context=context, old=old, root_stars_pass=root_stars_pass) + for star in facts.stars + ), + *( + _finding( + context=context, + site=site, + reason=f"USING or NATURAL join on {old}; rewrite it as an ON condition", + ) + for site in facts.joins + ), + *( + _finding( + context=context, + site=site, + reason=f"cannot tell whether {old} here is the renamed column; qualify it", + ) + for site in facts.unresolved + ), + ) + ) + + +def combine_body_edits(*, results: tuple[BodyEdits, ...]) -> BodyEdits: + """Concatenate the edits of several analyses of one body.""" + + edits: list[TextEdit] = [] + manual: list[ManualLocation] = [] + result: BodyEdits + for result in results: + edits.extend(result.edits) + manual.extend(result.manual) + return BodyEdits( + edits=tuple(edits), + manual=tuple(manual), + passes_through=any(result.passes_through for result in results), + ) + + +def _reference_edits( + *, reference: ColumnReference, context: BodyContext, old: str, new: str, cascade_root: bool +) -> BodyEdits: + name_span: tuple[int, int] | None = context.map_span(reference.name_start, reference.name_end) + if name_span is None: + return BodyEdits( + manual=( + _manual( + context=context, + offset=context.locate(reference.name_start), + reason=f"column {old} is referenced in SQL a macro generates; update the " + "macro call by hand", + ), + ) + ) + authored_name: str = context.contents[name_span[0] : name_span[1]] + renamed: TextEdit = _rename_span(context=context, span=name_span, new=new) + if not reference.projected: + return BodyEdits(edits=(renamed,)) + same_name_alias: bool = reference.alias is not None and reference.alias.lower() == old.lower() + if ( + reference.scope == ROOT_SCOPE + and cascade_root + and (reference.alias is None or same_name_alias) + ): + alias_span: tuple[int, int] | None = ( + context.map_span(reference.alias_start, reference.alias_end) + if same_name_alias + and reference.alias_start is not None + and reference.alias_end is not None + else None + ) + alias_edits: tuple[TextEdit, ...] = ( + (_rename_span(context=context, span=alias_span, new=new),) + if alias_span is not None + else () + ) + return BodyEdits(edits=(renamed, *alias_edits), passes_through=True) + if reference.alias is not None: + return BodyEdits(edits=(renamed,)) + end_span: tuple[int, int] | None = context.map_span(reference.start, reference.end) + if end_span is None: + return BodyEdits( + manual=( + _manual( + context=context, + offset=name_span[0], + reason=f"column {old} is projected inside a macro call", + ), + ) + ) + return BodyEdits( + edits=( + renamed, + text_edit( + text=context.contents, + start=end_span[1], + end=end_span[1], + replacement=f" AS {authored_name}", + kind=EditKind.COLUMN, + before="", + after=f"AS {authored_name}", + ), + ) + ) + + +def _star_edits( + *, star: ColumnSite, context: BodyContext, old: str, root_stars_pass: bool +) -> BodyEdits: + if star.scope.startswith(CTE_SCOPE_PREFIX): + return BodyEdits() + if star.scope == ROOT_SCOPE and root_stars_pass: + return BodyEdits(passes_through=True) + return _finding( + context=context, + site=star, + reason=( + f"SELECT * passes {old} through, so this model's output column would be renamed " + "too; rerun with --cascade or list the columns" + if star.scope == ROOT_SCOPE + else f"SELECT * in a subquery passes {old} through; list the columns" + ), + ) + + +def _finding(*, context: BodyContext, site: ColumnSite, reason: str) -> BodyEdits: + return BodyEdits( + manual=( + _manual( + context=context, offset=_site_offset(context=context, site=site), reason=reason + ), + ) + ) + + +def _rename_span(*, context: BodyContext, span: tuple[int, int], new: str) -> TextEdit: + return text_edit( + text=context.contents, + start=span[0], + end=span[1], + replacement=_respell(authored=context.contents[span[0] : span[1]], name=new), + kind=EditKind.COLUMN, + ) + + +def output_edits( + *, facts: ColumnFacts, context: BodyContext, scopes: frozenset[str], old: str, new: str +) -> BodyEdits: + """Rename the projection that names the column in the given output scopes.""" + + edits: list[TextEdit] = [] + manual: list[ManualLocation] = [] + found: set[str] = set() + output: OutputColumn + for output in facts.outputs: + found.add(output.scope) + if output.alias_start is not None and output.alias_end is not None: + alias_span: tuple[int, int] | None = context.map_span( + output.alias_start, output.alias_end + ) + if alias_span is None: + manual.append( + _manual( + context=context, + offset=context.locate(output.alias_start), + reason=f"column {old} is named inside a macro call", + ) + ) + continue + edits.append( + text_edit( + text=context.contents, + start=alias_span[0], + end=alias_span[1], + replacement=_respell( + authored=context.contents[alias_span[0] : alias_span[1]], name=new + ), + kind=EditKind.COLUMN, + ) + ) + continue + if output.start is None or output.end is None: + continue + column_span: tuple[int, int] | None = context.map_span(output.start, output.end) + if column_span is None: + manual.append( + _manual( + context=context, + offset=context.locate(output.start), + reason=f"column {old} is produced inside a macro call", + ) + ) + continue + edits.append( + text_edit( + text=context.contents, + start=column_span[1], + end=column_span[1], + replacement=f" AS {new}", + kind=EditKind.COLUMN, + before="", + after=f"AS {new}", + ) + ) + missing: frozenset[str] = frozenset( + scope for scope in scopes if _scope_role(scope) not in found + ) + scope: str + for scope in sorted(missing): + if _scope_role(scope) in facts.star_outputs: + manual.append( + _manual( + context=context, + offset=None, + reason=f"{old} comes from SELECT * in {scope}; list the columns explicitly", + ) + ) + return BodyEdits(edits=tuple(edits), manual=tuple(manual)) + + +def _scope_role(scope: str) -> str: + return scope if scope == ROOT_SCOPE else f"{CTE_SCOPE_PREFIX}{scope}" + + +def _respell(*, authored: str, name: str) -> str: + if authored and authored[0] in QUOTE_CHARACTERS: + return f"{authored[0]}{name}{QUOTE_CHARACTERS[authored[0]]}" + return name + + +def _site_offset(*, context: BodyContext, site: ColumnSite) -> int | None: + if site.start is None: + return None + return context.locate(site.start) + + +def _manual(*, context: BodyContext, offset: int | None, reason: str) -> ManualLocation: + return manual_at( + path=context.path, + text=context.contents, + offset=context.fallback_offset if offset is None else offset, + reason=reason, + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py new file mode 100644 index 0000000000..14d7f40aa0 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py @@ -0,0 +1,391 @@ +"""Token-exact MODEL header edits for renamed models and columns.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery.main._model_header_spans import get_model_header_spans +from sqlbuild.compiler.discovery.models import ModelHeaderSpans +from sqlbuild.compiler.refactoring._helpers.text_edits import text_edit, whole_word_offsets +from sqlbuild.compiler.refactoring.constants import ( + COLUMN_VALUED_CONFIG_KEYS, + COLUMNS_KEY, + CURSOR_INPUTS_KEY, + HEADER_CLOSERS, + HEADER_DESCRIPTION_KEY, + HEADER_ESCAPE, + HEADER_INDENT, + HEADER_KEY_AND_VALUE_TOKENS, + HEADER_OPEN_PAREN, + HEADER_OPENERS, + HEADER_SEPARATOR, + IDENTIFIER_PATTERN, + NATIVE_HEADER_TOKEN_KINDS, + PARENTHESIZED_EMPTY_TOKENS, + RELATIONSHIPS_AUDIT, + RELATIONSHIPS_FIELD_KEY, + RELATIONSHIPS_TO_KEY, +) +from sqlbuild.compiler.refactoring.models import HeaderEntry, HeaderToken, TextEdit +from sqlbuild.compiler.refactoring.types import EditKind, HeaderTokenKind + +_VALUE_KINDS: frozenset[HeaderTokenKind] = frozenset({HeaderTokenKind.WORD, HeaderTokenKind.STRING}) + + +def header_tokens(*, contents: str) -> tuple[HeaderToken, ...] | None: + """Tokenize the MODEL header body in file offsets, or None without a header.""" + + spans: ModelHeaderSpans = get_model_header_spans(contents=contents) + if spans.body is None: + return None + body_start: int = spans.body[0] + body: str = contents[spans.body[0] : spans.body[1]] + tokens: list[HeaderToken] = [] + depth: int = 0 + raw_kind: int + value: str + position: int + for raw_kind, value, position in _native.tokenize_model_header(body): + kind: HeaderTokenKind = NATIVE_HEADER_TOKEN_KINDS[raw_kind] + if kind == HeaderTokenKind.END: + continue + start: int = body_start + position + end: int = _token_end(contents=contents, kind=kind, value=value, start=start) + if kind == HeaderTokenKind.SYMBOL and value in HEADER_CLOSERS: + depth -= 1 + tokens.append(HeaderToken(kind=kind, value=value, start=start, end=end, depth=depth)) + if kind == HeaderTokenKind.SYMBOL and value in HEADER_OPENERS: + depth += 1 + return tuple(tokens) + + +def header_entries(*, tokens: tuple[HeaderToken, ...]) -> tuple[HeaderEntry, ...]: + """Split header tokens into top-level entries keyed by their first word.""" + + return tuple( + HeaderEntry(key=first.value, tokens=(first, *rest)) + for first, rest in _nested_entries(tokens=tokens, depth=0) + if first.kind == HeaderTokenKind.WORD + ) + + +def rename_value_token( + *, contents: str, token: HeaderToken, new_value: str, kind: EditKind +) -> TextEdit: + """Replace one word or string value, keeping string quoting.""" + + replacement: str = ( + new_value + if token.kind == HeaderTokenKind.WORD + else f"{contents[token.start]}{new_value}{contents[token.start]}" + ) + return text_edit( + text=contents, + start=token.start, + end=token.end, + replacement=replacement, + kind=kind, + ) + + +def is_value(*, token: HeaderToken, value: str) -> bool: + """Return whether a word or string token spells a name, ignoring case.""" + + return token.kind in _VALUE_KINDS and token.value.lower() == value.lower() + + +def cursor_input_tokens( + *, tokens: tuple[HeaderToken, ...] +) -> tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...]: + """Return (input name token, value tokens) for every cursor_inputs entry.""" + + entry: HeaderEntry + for entry in header_entries(tokens=tokens): + if entry.key != CURSOR_INPUTS_KEY: + continue + return _nested_entries(tokens=entry.tokens[1:], depth=1) + return () + + +def relationship_tokens( + *, tokens: tuple[HeaderToken, ...] +) -> tuple[tuple[HeaderToken | None, HeaderToken | None], ...]: + """Return the (to, field) value tokens of every relationships audit.""" + + found: list[tuple[HeaderToken | None, HeaderToken | None]] = [] + index: int + token: HeaderToken + for index, token in enumerate(tokens): + if not ( + token.kind == HeaderTokenKind.WORD + and token.value == RELATIONSHIPS_AUDIT + and index + 1 < len(tokens) + and tokens[index + 1].value == HEADER_OPEN_PAREN + ): + continue + depth: int = tokens[index + 1].depth + 1 + values: dict[str, HeaderToken] = {} + cursor: int = index + 2 + while cursor < len(tokens) and tokens[cursor].depth >= depth: + candidate: HeaderToken = tokens[cursor] + if ( + candidate.depth == depth + and candidate.kind == HeaderTokenKind.WORD + and candidate.value in {RELATIONSHIPS_TO_KEY, RELATIONSHIPS_FIELD_KEY} + and cursor + 1 < len(tokens) + ): + values[candidate.value] = tokens[cursor + 1] + cursor += 1 + found.append((values.get(RELATIONSHIPS_TO_KEY), values.get(RELATIONSHIPS_FIELD_KEY))) + return tuple(found) + + +def column_config_edits( + *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename a column in column-valued config such as cursor or unique_key.""" + + edits: list[TextEdit] = [] + entry: HeaderEntry + for entry in header_entries(tokens=tokens): + if entry.key not in COLUMN_VALUED_CONFIG_KEYS: + continue + token: HeaderToken + for token in entry.tokens[1:]: + if is_value(token=token, value=old): + edits.append( + rename_value_token( + contents=contents, token=token, new_value=new, kind=EditKind.HEADER + ) + ) + return tuple(edits) + + +def column_entry_edits( + *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str, migrate: bool +) -> tuple[TextEdit, ...] | None: + """Rename a `columns` entry and add `migrate_from` if needed; None without an entry.""" + + entry: HeaderEntry + for entry in header_entries(tokens=tokens): + if entry.key != COLUMNS_KEY: + continue + nested: tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...] = _nested_entries( + tokens=entry.tokens[1:], depth=1 + ) + name_token: HeaderToken + values: tuple[HeaderToken, ...] + for name_token, values in nested: + if not is_value(token=name_token, value=old): + continue + edits: list[TextEdit] = [ + rename_value_token( + contents=contents, token=name_token, new_value=new, kind=EditKind.HEADER + ) + ] + if migrate: + edits.append( + _column_migration_edit( + contents=contents, name=name_token, values=values, old=old + ) + ) + return tuple(edits) + return None + + +def add_column_entry_edit( + *, contents: str, tokens: tuple[HeaderToken, ...], new: str, old: str +) -> TextEdit | None: + """Declare `new (migrate_from old)` in the columns block, creating it when absent.""" + + declaration: str = f"{new} (migrate_from {old})" + entry: HeaderEntry + for entry in header_entries(tokens=tokens): + if entry.key != COLUMNS_KEY or len(entry.tokens) < HEADER_KEY_AND_VALUE_TOKENS: + continue + opener: HeaderToken = entry.tokens[1] + indentation: str = _line_indent(contents=contents, offset=entry.tokens[0].start) + return text_edit( + text=contents, + start=opener.end, + end=opener.end, + replacement=f"\n{indentation}{HEADER_INDENT}{declaration},", + kind=EditKind.MIGRATION, + before="", + after=declaration, + ) + return insert_header_entry_edit( + contents=contents, entry=f"columns ({declaration})", display=declaration + ) + + +def insert_header_entry_edit(*, contents: str, entry: str, display: str) -> TextEdit | None: + """Insert one top-level entry as the first MODEL header line.""" + + spans: ModelHeaderSpans = get_model_header_spans(contents=contents) + if spans.body is None: + return None + start: int = spans.body[0] + body: str = contents[spans.body[0] : spans.body[1]] + if body.startswith("\n"): + line_end: int = body.find("\n", 1) + first_line: str = body[1:] if line_end < 0 else body[1:line_end] + indentation: str = first_line[: len(first_line) - len(first_line.lstrip())] or HEADER_INDENT + replacement: str = f"\n{indentation}{entry}," + elif body.strip(): + replacement = f"{entry}, " + else: + replacement = f"\n{HEADER_INDENT}{entry},\n" + return text_edit( + text=contents, + start=start, + end=start, + replacement=replacement, + kind=EditKind.MIGRATION, + before="", + after=display, + ) + + +def unhandled_word_offsets( + *, contents: str, tokens: tuple[HeaderToken, ...], word: str, handled: frozenset[int] +) -> tuple[int, ...]: + """Return header offsets that still mention a word outside descriptions.""" + + offsets: list[int] = [] + previous: HeaderToken | None = None + token: HeaderToken + for token in tokens: + described: bool = previous is not None and previous.value == HEADER_DESCRIPTION_KEY + previous = token + if token.kind == HeaderTokenKind.SYMBOL or described or token.start in handled: + continue + text: str = contents[token.start : token.end] + offsets.extend(token.start + offset for offset in whole_word_offsets(text=text, word=word)) + return tuple(offsets) + + +def model_name_header_edits(*, contents: str, old: str, new: str) -> tuple[TextEdit, ...]: + """Rename a model in cursor_inputs keys and relationships `to` values.""" + + tokens: tuple[HeaderToken, ...] | None = header_tokens(contents=contents) + if tokens is None: + return () + names: list[HeaderToken] = [ + name for name, _ in cursor_input_tokens(tokens=tokens) if is_value(token=name, value=old) + ] + names.extend( + to_token + for to_token, _ in relationship_tokens(tokens=tokens) + if to_token is not None and is_value(token=to_token, value=old) + ) + return tuple( + rename_value_token(contents=contents, token=name, new_value=new, kind=EditKind.HEADER) + for name in names + ) + + +def consumer_column_header_edits( + *, contents: str, upstream: str, old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename an upstream column in cursor_inputs values and relationships `field` values.""" + + tokens: tuple[HeaderToken, ...] | None = header_tokens(contents=contents) + if tokens is None: + return () + values: list[HeaderToken] = [] + name: HeaderToken + inputs: tuple[HeaderToken, ...] + for name, inputs in cursor_input_tokens(tokens=tokens): + if is_value(token=name, value=upstream): + values.extend(value for value in inputs if is_value(token=value, value=old)) + to_token: HeaderToken | None + field_token: HeaderToken | None + for to_token, field_token in relationship_tokens(tokens=tokens): + if ( + to_token is not None + and field_token is not None + and is_value(token=to_token, value=upstream) + and is_value(token=field_token, value=old) + ): + values.append(field_token) + return tuple( + rename_value_token(contents=contents, token=value, new_value=new, kind=EditKind.HEADER) + for value in values + ) + + +def quoted_name(name: str) -> str: + """Return a header value spelling for a name.""" + + return name if IDENTIFIER_PATTERN.match(name) else f'"{name}"' + + +def _column_migration_edit( + *, contents: str, name: HeaderToken, values: tuple[HeaderToken, ...], old: str +) -> TextEdit: + declaration: str = f"migrate_from {old}" + if values and values[0].value == HEADER_OPEN_PAREN: + opener: HeaderToken = values[0] + has_metadata: bool = len(values) > PARENTHESIZED_EMPTY_TOKENS + return text_edit( + text=contents, + start=opener.end, + end=opener.end, + replacement=f"{declaration}, " if has_metadata else declaration, + kind=EditKind.MIGRATION, + before="", + after=declaration, + ) + return text_edit( + text=contents, + start=name.end, + end=name.end, + replacement=f" ({declaration})", + kind=EditKind.MIGRATION, + before="", + after=declaration, + ) + + +def _nested_entries( + *, tokens: tuple[HeaderToken, ...], depth: int +) -> tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...]: + """Split the contents of one bracketed value into (first token, rest) entries.""" + + groups: list[list[HeaderToken]] = [[]] + token: HeaderToken + for token in tokens: + if token.depth < depth: + continue + if token.depth == depth and _is_separator(token): + groups.append([]) + continue + groups[-1].append(token) + return tuple( + (group[0], tuple(group[1:])) for group in groups if group and group[0].kind in _VALUE_KINDS + ) + + +def _is_separator(token: HeaderToken) -> bool: + return token.kind == HeaderTokenKind.SYMBOL and token.value == HEADER_SEPARATOR + + +def _token_end(*, contents: str, kind: HeaderTokenKind, value: str, start: int) -> int: + if kind != HeaderTokenKind.STRING: + return start + len(value) + quote: str = contents[start] + index: int = start + 1 + while index < len(contents): + if contents[index] == HEADER_ESCAPE: + index += 2 + continue + if contents[index] == quote: + return index + 1 + index += 1 + return len(contents) + + +def _line_indent(*, contents: str, offset: int) -> str: + line_start: int = contents.rfind("\n", 0, offset) + 1 + prefix: str = contents[line_start:offset] + return prefix if not prefix.strip() else HEADER_INDENT diff --git a/src/sqlbuild/compiler/refactoring/_helpers/model_migration.py b/src/sqlbuild/compiler/refactoring/_helpers/model_migration.py new file mode 100644 index 0000000000..d0dbb0accd --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/model_migration.py @@ -0,0 +1,131 @@ +"""Decide whether a renamed or moved model needs an explicit migrate_from.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( + MigrationFingerprintCache, +) +from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( + build_function_local_hashes, +) +from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( + build_model_version_identity_metadata_json, +) +from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities +from sqlbuild.compiler.refactoring.constants import ( + HISTORY_MATERIALIZATIONS, + MATERIALIZED_KEY, + MIGRATABLE_MATERIALIZATIONS, + MIGRATE_FROM_KEY, +) +from sqlbuild.compiler.refactoring.models import ModelMigrationDecision +from sqlbuild.spec.contracts.main.get_config_str import get_config_str + + +def decide_model_migration( + *, before: CompiledProject, after: CompiledProject, old: str, new: str +) -> ModelMigrationDecision: + """Compare the old and new compile the way automatic rename discovery would.""" + + old_model: CompiledModel | None = _model(project=before, name=old) + new_model: CompiledModel | None = _model(project=after, name=new) + if old_model is None or new_model is None: + return ModelMigrationDecision(needed=False, reason="model not compiled") + materialized: str | None = get_config_str(values=new_model.config.values, key=MATERIALIZED_KEY) + if materialized not in MIGRATABLE_MATERIALIZATIONS: + return ModelMigrationDecision( + needed=False, reason=f"'{materialized}' models have no migration" + ) + if _location_key(old_model) == _location_key(new_model): + return ModelMigrationDecision(needed=False, reason="relation name is unchanged") + old_database: str = (old_model.destination.database or "").lower() + new_database: str = (new_model.destination.database or "").lower() + if old_database and new_database and old_database != new_database: + return ModelMigrationDecision( + needed=False, + blocked=True, + reason=( + f"moves from database {old_model.destination.database} to " + f"{new_model.destination.database}; migrations cannot cross databases" + ), + ) + old_schema: str = (old_model.destination.schema or "").lower() + project_schemas: frozenset[str] = frozenset( + (model.destination.schema or "").lower() for model in after.models + ) + if old_schema not in project_schemas: + return ModelMigrationDecision( + needed=False, + blocked=True, + reason=( + f"no model is left in schema {old_model.destination.schema}, so neither " + f"automatic discovery nor migrate_from {old} can find the old relation; add a " + "schema-qualified migrate_from for each target" + ), + ) + old_materialized: str | None = get_config_str( + values=old_model.config.values, key=MATERIALIZED_KEY + ) + if (old_materialized in HISTORY_MATERIALIZATIONS) != (materialized in HISTORY_MATERIALIZATIONS): + return ModelMigrationDecision( + needed=True, reason=f"materialization changes from {old_materialized}" + ) + before_fingerprint: str | None = _fingerprint(project=before, model=old_model) + after_fingerprint: str | None = _fingerprint(project=after, model=new_model) + if before_fingerprint is None: + return ModelMigrationDecision( + needed=True, + reason="automatic discovery cannot fingerprint its SQL, so it would not find the " + "old relation", + ) + if before_fingerprint != after_fingerprint: + return ModelMigrationDecision( + needed=True, + reason="its definition changes at the new location, so automatic discovery " + "cannot match it", + ) + return ModelMigrationDecision( + needed=False, reason="automatic rename discovery matches the unchanged definition" + ) + + +def needs_column_migration(*, model: CompiledModel) -> bool: + """Return whether renaming a column of this model must declare migrate_from.""" + + materialized: str | None = get_config_str(values=model.config.values, key=MATERIALIZED_KEY) + if materialized in HISTORY_MATERIALIZATIONS: + return True + return ( + materialized in MIGRATABLE_MATERIALIZATIONS + and model.config.values.get(MIGRATE_FROM_KEY) is not None + ) + + +def _fingerprint(*, project: CompiledProject, model: CompiledModel) -> str | None: + metadata_json: str = build_model_version_identity_metadata_json( + model=model, + function_local_hashes=build_function_local_hashes(functions=project.functions), + hook_version_hashes={ + name: identity.version_hash + for name, identity in build_hook_identities(project.hook_functions).items() + }, + ) + return MigrationFingerprintCache().fingerprint( + query_sql=model.query_sql, + metadata_json=metadata_json, + ref_identities={}, + dialect=project.sql_analysis_dialect, + ) + + +def _model(*, project: CompiledProject, name: str) -> CompiledModel | None: + return next((model for model in project.models if model.name == name), None) + + +def _location_key(model: CompiledModel) -> tuple[str, str, str]: + return ( + (model.destination.database or "").lower(), + (model.destination.schema or "").lower(), + model.destination.name.lower(), + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/model_planning.py b/src/sqlbuild/compiler/refactoring/_helpers/model_planning.py new file mode 100644 index 0000000000..1d2c35cece --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/model_planning.py @@ -0,0 +1,239 @@ +"""Resolve a model rename or move and find everything that blocks it.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path, PurePosixPath + +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.refactoring._helpers.header_edits import header_tokens, is_value +from sqlbuild.compiler.refactoring._helpers.model_references import ( + macro_reference_locations, + model_reference_edits, +) +from sqlbuild.compiler.refactoring._helpers.project_files import ( + project_sql_files, + python_string_locations, +) +from sqlbuild.compiler.refactoring._helpers.text_edits import manual_at +from sqlbuild.compiler.refactoring.constants import ( + GENERIC_DIALECT, + IDENTIFIER_PATTERN, + MIGRATE_FROM_KEY, + MODEL_KIND_PREFIX, + PATH_SEPARATOR, + SQL_SUFFIX, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorInputError +from sqlbuild.compiler.refactoring.models import ( + HeaderToken, + ManualLocation, + ModelTarget, + ProjectSqlFile, + RefactorParts, + RefactorProject, + RefactorRequest, +) +from sqlbuild.compiler.refactoring.types import RefactorOperation +from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup +from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move +from sqlbuild.compiler.scopes.models import DeclarationReport, MovePreview, ScopeDiagnostic + + +def find_model(*, project: CompiledProject, name: str) -> CompiledModel: + """Return the compiled model a kind-qualified target names.""" + + model: CompiledModel | None = next((item for item in project.models if item.name == name), None) + if model is None: + raise RefactorInputError( + f"{MODEL_KIND_PREFIX}{name} is not a model in this project", + code="C951", + help="list models with sqb dag --json or sqb scope --browse models", + ) + return model + + +def validate_identifier(*, name: str, noun: str) -> None: + """Refuse a new name that is not a plain SQL identifier.""" + + if not IDENTIFIER_PATTERN.match(name): + raise RefactorInputError( + f"'{name}' is not a valid {noun} name", + code="C952", + help="use letters, digits, and underscores, starting with a letter or underscore", + ) + + +def model_target(*, project: RefactorProject, request: RefactorRequest) -> ModelTarget: + """Resolve the model, its new name, and its destination file.""" + + model: CompiledModel = find_model(project=project.graph.project, name=request.model_name) + source_path: str = model.relative_path.as_posix() + destination: str = ( + _resolve_destination( + project_dir=project.project_dir, + raw=request.destination or "", + file_name=model.relative_path.name, + ) + if request.operation == RefactorOperation.MOVE_MODEL + else PurePosixPath(source_path).with_name(f"{request.new_name}{SQL_SUFFIX}").as_posix() + ) + new: str = ( + PurePosixPath(destination).stem + if request.operation == RefactorOperation.MOVE_MODEL + else request.new_name + ) + validate_identifier(name=new, noun="model") + if new == model.name and destination == source_path: + raise RefactorInputError( + f"model:{model.name} already has that name and location", code="C953" + ) + return ModelTarget( + model=model, + request=replace(request, new_name=new, destination=destination), + source_path=source_path, + ) + + +def model_parts(*, project: RefactorProject, target: ModelTarget) -> RefactorParts: + """Return every edit, manual location, and blocker of one model rename or move.""" + + files: tuple[ProjectSqlFile, ...] = project_sql_files(discovered=project.discovered) + dialect: str = project.graph.project.sql_analysis_dialect or GENERIC_DIALECT + old: str = target.model.name + new: str = target.request.new_name + return RefactorParts( + edits=model_reference_edits(files=files, old=old, new=new, dialect=dialect) + if new != old + else (), + manual=( + *macro_reference_locations(project=project, files=files, old=old, dialect=dialect), + *python_string_locations( + project_dir=project.project_dir, + discovered=project.discovered, + names=(old,), + reason=f"Python code names model {old}; update it by hand", + ), + ), + blocking=( + *_collisions(project=project, target=target), + *_scope_losses(project=project, target=target), + *_pending_migration(files=files, target=target), + ), + ) + + +def _resolve_destination(*, project_dir: Path, raw: str, file_name: str) -> str: + path: Path = Path(raw) + absolute: Path = path if path.is_absolute() else project_dir / path + if raw.endswith(PATH_SEPARATOR) or absolute.is_dir(): + absolute = absolute / file_name + resolved: Path = absolute.resolve() + if not resolved.is_relative_to(project_dir): + raise RefactorInputError(f"destination '{raw}' is outside the project", code="C955") + if resolved.suffix != SQL_SUFFIX: + raise RefactorInputError( + f"destination '{raw}' must be a .sql file or a folder ending in /", code="C955" + ) + return resolved.relative_to(project_dir).as_posix() + + +def _collisions(*, project: RefactorProject, target: ModelTarget) -> tuple[ManualLocation, ...]: + new: str = target.request.new_name + found: list[ManualLocation] = [ + ManualLocation( + path=other.relative_path.as_posix(), + line=None, + column=None, + reason=f"model:{new} already exists", + ) + for other in project.graph.project.models + if other.name.lower() == new.lower() and other.name != target.model.name + ] + if ( + target.destination != target.source_path + and (project.project_dir / PurePosixPath(target.destination)).exists() + ): + found.append( + ManualLocation( + path=target.destination, + line=None, + column=None, + reason="destination file already exists", + ) + ) + return tuple(found) + + +def _scope_losses(*, project: RefactorProject, target: ModelTarget) -> tuple[ManualLocation, ...]: + if PurePosixPath(target.destination).parent == PurePosixPath(target.source_path).parent: + return () + move: MovePreview | None + diagnostics: tuple[ScopeDiagnostic, ...] + move, diagnostics = preview_scope_move( + lookup=build_scope_lookup(index=project.graph.project.scope_index), + resource=f"{MODEL_KIND_PREFIX}{target.model.name}", + destination=target.destination, + ) + if move is None: + return tuple( + ManualLocation(path=target.destination, line=None, column=None, reason=item.message) + for item in diagnostics + ) or ( + ManualLocation( + path=target.destination, + line=None, + column=None, + reason="declaration scopes at the destination could not be checked", + ), + ) + lost: dict[str, DeclarationReport] = {report.identity: report for report in move.lost} + return tuple( + _scope_loss(identity=identity, report=lost.get(identity), target=target) + for identity in move.invalidated_usages + ) + + +def _scope_loss( + *, identity: str, report: DeclarationReport | None, target: ModelTarget +) -> ManualLocation: + reason: str = ( + f"{identity} used by model:{target.model.name} is not visible from {target.destination}" + ) + if report is None: + return ManualLocation(path=target.source_path, line=None, column=None, reason=reason) + return ManualLocation( + path=report.definition.path, + line=report.definition.line, + column=report.definition.column, + reason=reason, + ) + + +def _pending_migration( + *, files: tuple[ProjectSqlFile, ...], target: ModelTarget +) -> tuple[ManualLocation, ...]: + if MIGRATE_FROM_KEY not in target.model.config.values: + return () + contents: str = next( + (item.contents for item in files if item.relative_path == target.source_path), "" + ) + token: HeaderToken | None = next( + ( + item + for item in header_tokens(contents=contents) or () + if item.depth == 0 and is_value(token=item, value=MIGRATE_FROM_KEY) + ), + None, + ) + return ( + manual_at( + path=target.source_path, + text=contents, + offset=token.start if token is not None else None, + reason=( + f"model:{target.model.name} still declares migrate_from; build it on every " + "target and remove migrate_from before renaming it again" + ), + ), + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/model_references.py b/src/sqlbuild/compiler/refactoring/_helpers/model_references.py new file mode 100644 index 0000000000..40632f051a --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/model_references.py @@ -0,0 +1,180 @@ +"""Every authored reference to a model by name, and the ones a rename cannot rewrite.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.refactoring._helpers.header_edits import model_name_header_edits +from sqlbuild.compiler.refactoring._helpers.sql_sites import ( + authored_offset, + model_body, + resource_sites, +) +from sqlbuild.compiler.refactoring._helpers.text_edits import ( + identifier_sites, + manual_at, + path_edits, + text_edit, +) +from sqlbuild.compiler.refactoring.constants import ( + EXPECTED_FIXTURE_PREFIX, + FIXTURE_ROLES, + REF_FIXTURE_PREFIX, + REF_KIND, +) +from sqlbuild.compiler.refactoring.models import ( + ManualLocation, + ModelBody, + ProjectSqlFile, + RefactorProject, + ResourceSite, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind, SqlFileRole + + +def model_reference_edits( + *, files: tuple[ProjectSqlFile, ...], old: str, new: str, dialect: str +) -> tuple[tuple[str, TextEdit], ...]: + """Rewrite `__ref`, fixture CTE names, and header references to a renamed model.""" + + edits: list[tuple[str, TextEdit]] = [] + item: ProjectSqlFile + for item in files: + edits.extend( + path_edits( + path=item.relative_path, + edits=_ref_edits(item=item, old=old, new=new, dialect=dialect), + ) + ) + if item.role in FIXTURE_ROLES: + edits.extend( + path_edits( + path=item.relative_path, edits=_fixture_edits(item=item, old=old, new=new) + ) + ) + if item.role == SqlFileRole.MODEL: + edits.extend( + path_edits( + path=item.relative_path, + edits=model_name_header_edits(contents=item.contents, old=old, new=new), + ) + ) + return tuple(edits) + + +def macro_reference_locations( + *, project: RefactorProject, files: tuple[ProjectSqlFile, ...], old: str, dialect: str +) -> tuple[ManualLocation, ...]: + """Locate `__ref` calls to the model that macros produce, which no edit can reach.""" + + compiled: CompiledProject = project.graph.project + contents: dict[str, str] = {item.relative_path: item.contents for item in files} + locations: list[ManualLocation] = [] + model: CompiledModel + for model in compiled.models: + text: str | None = contents.get(model.relative_path.as_posix()) + if text is None or not _depends_on(model=model, name=old): + continue + locations.extend( + _generated_references( + compiled=compiled, project=project, model=model, text=text, old=old, dialect=dialect + ) + ) + return tuple(locations) + + +def _ref_edits(*, item: ProjectSqlFile, old: str, new: str, dialect: str) -> tuple[TextEdit, ...]: + return tuple( + text_edit( + text=item.contents, + start=site.name_start, + end=site.name_end, + replacement=new, + kind=EditKind.REFERENCE, + before=item.contents[site.start : site.end], + after=( + item.contents[site.start : site.name_start] + + new + + item.contents[site.name_end : site.end] + ), + ) + for site in _ref_sites(text=item.contents, name=old, dialect=dialect) + ) + + +def _fixture_edits(*, item: ProjectSqlFile, old: str, new: str) -> tuple[TextEdit, ...]: + fixture_names: dict[str, str] = { + f"{REF_FIXTURE_PREFIX}{old}": f"{REF_FIXTURE_PREFIX}{new}", + f"{EXPECTED_FIXTURE_PREFIX}{old}": f"{EXPECTED_FIXTURE_PREFIX}{new}", + } + return tuple( + text_edit( + text=item.contents, + start=start, + end=end, + replacement=fixture_names[name], + kind=EditKind.FIXTURE, + ) + for start, end, name in identifier_sites(text=item.contents, names=frozenset(fixture_names)) + ) + + +def _generated_references( + *, + compiled: CompiledProject, + project: RefactorProject, + model: CompiledModel, + text: str, + old: str, + dialect: str, +) -> tuple[ManualLocation, ...]: + relative: str = model.relative_path.as_posix() + compiled_sites: tuple[ResourceSite, ...] = _ref_sites( + text=model.query_sql, name=old, dialect=dialect + ) + authored_sites: tuple[ResourceSite, ...] = _ref_sites( + text=model.authored_query_sql, name=old, dialect=dialect + ) + if len(compiled_sites) <= len(authored_sites): + return () + body: ModelBody | None = model_body( + project=compiled, project_dir=project.project_dir, model=model, contents=text + ) + if body is None: + return ( + manual_at( + path=relative, + text=text, + offset=None, + reason=f"reference to {old} produced by a macro", + ), + ) + offsets: tuple[tuple[int, bool], ...] = tuple( + authored_offset(body=body, offset=site.start) for site in compiled_sites + ) + return tuple( + manual_at( + path=relative, + text=text, + offset=offset, + reason=f"reference to {old} produced by a macro; pass the model to the macro as an " + "argument", + ) + for offset, generated in offsets + if generated + ) + + +def _ref_sites(*, text: str, name: str, dialect: str) -> tuple[ResourceSite, ...]: + return tuple( + site + for site in resource_sites(text=text, dialect=dialect) + if site.kind == REF_KIND and site.name == name + ) + + +def _depends_on(*, model: CompiledModel, name: str) -> bool: + return any( + dep.resource_type == CompiledResourceType.MODEL and dep.name == name for dep in model.deps + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project_files.py b/src/sqlbuild/compiler/refactoring/_helpers/project_files.py new file mode 100644 index 0000000000..1e87b63dfd --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/project_files.py @@ -0,0 +1,182 @@ +"""Authored project files and SQL bodies a refactoring reads and edits.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +from sqlbuild.compiler.discovery.models import ( + DiscoveredAuditBlock, + DiscoveredProjectInputs, + DiscoveredSqlTestBlock, +) +from sqlbuild.compiler.refactoring._helpers.text_edits import whole_word_offsets +from sqlbuild.compiler.refactoring.constants import SQL_LIKE_PATTERN +from sqlbuild.compiler.refactoring.models import AuthoredBody, ManualLocation, ProjectSqlFile +from sqlbuild.compiler.refactoring.types import SqlFileRole + +type _AuthoredFile = tuple[SqlFileRole, Path, str, tuple[str, ...]] + + +def project_sql_files(*, discovered: DiscoveredProjectInputs) -> tuple[ProjectSqlFile, ...]: + """Return every discovered SQL file with its authored contents.""" + + files: dict[str, ProjectSqlFile] = {} + role: SqlFileRole + relative_path: Path + contents: str + for role, relative_path, contents, _ in ( + *_model_files(discovered=discovered), + *_authored_files(discovered=discovered), + ): + relative: str = relative_path.as_posix() + files.setdefault( + relative, ProjectSqlFile(relative_path=relative, contents=contents, role=role) + ) + return tuple(files[path] for path in sorted(files)) + + +def authored_bodies(*, discovered: DiscoveredProjectInputs) -> tuple[AuthoredBody, ...]: + """Locate every non-model SQL body in its file, in file order.""" + + bodies: list[AuthoredBody] = [] + role: SqlFileRole + relative_path: Path + contents: str + texts: tuple[str, ...] + for role, relative_path, contents, texts in _authored_files(discovered=discovered): + cursor: int = 0 + text: str + for text in texts: + start: int = contents.find(text, cursor) if text.strip() else -1 + if start < 0: + continue + cursor = start + len(text) + bodies.append( + AuthoredBody( + path=relative_path.as_posix(), + role=role, + contents=contents, + start=start, + text=text, + ) + ) + return tuple(bodies) + + +def python_string_locations( + *, + project_dir: Path, + discovered: DiscoveredProjectInputs, + names: tuple[str, ...], + reason: str, + context: str | None = None, +) -> tuple[ManualLocation, ...]: + """Flag Python strings that name a model or column; with context, only SQL naming it too.""" + + locations: list[ManualLocation] = [] + relative_path: Path + for relative_path in _python_paths(discovered=discovered): + locations.extend( + _string_locations( + project_dir=project_dir, + relative_path=relative_path, + names=names, + reason=reason, + context=context, + ) + ) + return tuple(locations) + + +def _model_files(*, discovered: DiscoveredProjectInputs) -> tuple[_AuthoredFile, ...]: + return tuple( + (SqlFileRole.MODEL, item.relative_path, item.contents, ()) + for item in discovered.model_files + ) + + +def _authored_files(*, discovered: DiscoveredProjectInputs) -> tuple[_AuthoredFile, ...]: + files: list[_AuthoredFile] = [] + files.extend( + (SqlFileRole.TEST, item.relative_path, item.contents, _test_bodies(item.blocks)) + for item in discovered.test_files + ) + files.extend( + (SqlFileRole.SCENARIO, item.relative_path, item.contents, (item.sql_body,)) + for item in discovered.scenario_files + ) + files.extend( + (SqlFileRole.AUDIT, item.relative_path, item.contents, _audit_bodies(item.blocks)) + for item in discovered.audit_files + ) + files.extend( + (SqlFileRole.HOOK, item.relative_path, item.contents, (item.sql_body,)) + for item in discovered.sql_hook_files + ) + files.extend( + (SqlFileRole.FUNCTION, item.relative_path, item.contents, (item.body_sql,)) + for item in discovered.sql_function_files + ) + return tuple(files) + + +def _test_bodies(blocks: tuple[DiscoveredSqlTestBlock, ...]) -> tuple[str, ...]: + return tuple(block.sql_body for block in blocks) + + +def _audit_bodies(blocks: tuple[DiscoveredAuditBlock, ...]) -> tuple[str, ...]: + return tuple(block.sql_body for block in blocks) + + +def _python_paths(*, discovered: DiscoveredProjectInputs) -> tuple[Path, ...]: + paths: set[Path] = set() + paths.update(item.relative_path for item in discovered.loader_functions) + paths.update(item.relative_path for item in discovered.task_functions) + paths.update(item.relative_path for item in discovered.asset_functions) + paths.update(item.relative_path for item in discovered.check_functions) + paths.update(item.relative_path for item in discovered.hook_functions) + paths.update(item.relative_path for item in discovered.audit_factories) + paths.update(item.relative_path for item in discovered.python_function_files) + paths.update(item.relative_path for item in discovered.materialization_files) + return tuple(sorted(paths)) + + +def _string_locations( + *, + project_dir: Path, + relative_path: Path, + names: tuple[str, ...], + reason: str, + context: str | None, +) -> tuple[ManualLocation, ...]: + try: + tree: ast.Module = ast.parse((project_dir / relative_path).read_text(encoding="utf-8")) + except (OSError, SyntaxError, UnicodeError): + return () + locations: list[ManualLocation] = [] + node: ast.AST + for node in ast.walk(tree): + if not isinstance(node, ast.Constant) or not isinstance(node.value, str): + continue + if _flags(value=node.value, names=names, context=context): + locations.append( + ManualLocation( + path=relative_path.as_posix(), + line=node.lineno, + column=node.col_offset + 1, + reason=reason, + ) + ) + return tuple(locations) + + +def _flags(*, value: str, names: tuple[str, ...], context: str | None) -> bool: + sql_like: bool = bool(SQL_LIKE_PATTERN.search(value)) + if context is not None and not (sql_like and whole_word_offsets(text=value, word=context)): + return False + return any( + value.strip().lower() == name.lower() + or (sql_like and bool(whole_word_offsets(text=value, word=name))) + for name in names + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/sql_sites.py b/src/sqlbuild/compiler/refactoring/_helpers/sql_sites.py new file mode 100644 index 0000000000..876158c1ff --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/sql_sites.py @@ -0,0 +1,142 @@ +"""Reference call sites, offset-preserving analysis SQL, and compiled-to-authored offsets.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from sqlbuild.compiler.compile.main.map_expanded_offset import map_expanded_offset +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledProject, + CompiledSqlExpansion, + ExpansionSpan, + MappedOffset, +) +from sqlbuild.compiler.refactoring.constants import ( + GENERIC_PLACEHOLDER_KIND, + PLACEHOLDER_BASE, + PLACEHOLDER_PAD, + RESOURCE_CALL_PATTERN, +) +from sqlbuild.compiler.refactoring.models import AnalysisSql, ModelBody, ResourceSite +from sqlbuild.lint.main.scan_interpolation_sites import scan_interpolation_sites +from sqlbuild.lint.models import InterpolationSite + + +def resource_sites(*, text: str, dialect: str) -> tuple[ResourceSite, ...]: + """Return resource calls outside comments and strings, with their name spans.""" + + sites: list[ResourceSite] = [] + site: InterpolationSite + for site in scan_interpolation_sites(body=text, dialect=dialect): + match: re.Match[str] | None = RESOURCE_CALL_PATTERN.match(site.original_text) + if match is None: + continue + sites.append( + ResourceSite( + kind=match.group("kind"), + name=match.group("name"), + start=site.original_start, + end=site.original_end, + name_start=site.original_start + match.start("name"), + name_end=site.original_start + match.end("name"), + ) + ) + return tuple(sites) + + +def analysis_sql(*, text: str, dialect: str) -> AnalysisSql: + """Replace every interpolation site with an identifier of the same length.""" + + pieces: list[str] = [] + tables: dict[tuple[str, str], set[str]] = {} + copied_to: int = 0 + index: int + site: InterpolationSite + for index, site in enumerate(scan_interpolation_sites(body=text, dialect=dialect)): + match: re.Match[str] | None = RESOURCE_CALL_PATTERN.match(site.original_text) + kind: str = match.group("kind")[0] if match is not None else GENERIC_PLACEHOLDER_KIND + placeholder: str = _placeholder( + kind=kind, index=index, length=site.original_end - site.original_start + ) + if match is not None: + tables.setdefault((match.group("kind"), match.group("name")), set()).add(placeholder) + pieces.extend((text[copied_to : site.original_start], placeholder)) + copied_to = site.original_end + pieces.append(text[copied_to:]) + return AnalysisSql( + sql="".join(pieces), + tables={key: frozenset(value) for key, value in tables.items()}, + ) + + +def model_body( + *, project: CompiledProject, project_dir: Path, model: CompiledModel, contents: str +) -> ModelBody | None: + """Return the mappable compiled body of a model, or None when it cannot be mapped.""" + + body_start: int = contents.rfind(model.authored_query_sql) if model.authored_query_sql else -1 + if body_start < 0: + return None + expansion: CompiledSqlExpansion | None = _expansion( + project=project, project_dir=project_dir, model=model + ) + passes: tuple[tuple[ExpansionSpan, ...], ...] = () + if expansion is not None: + if expansion.expanded_sql != model.query_sql: + return None + passes = expansion.passes + elif model.query_sql != model.authored_query_sql: + return None + return ModelBody( + relative_path=model.relative_path.as_posix(), + contents=contents, + body_start=body_start, + compiled_sql=model.query_sql, + passes=passes, + ) + + +def authored_offset(*, body: ModelBody, offset: int) -> tuple[int, bool]: + """Return the file offset of a compiled offset and whether a macro generated it.""" + + mapped: MappedOffset = map_expanded_offset(offset=offset, passes=body.passes) + return body.body_start + mapped.offset, mapped.generated + + +def authored_span(*, body: ModelBody, start: int, end: int) -> tuple[int, int] | None: + """Return the file span of compiled text written by the author, or None if generated.""" + + mapped_start: int + start_generated: bool + mapped_start, start_generated = authored_offset(body=body, offset=start) + if start_generated: + return None + if end <= start: + return mapped_start, mapped_start + mapped_last: int + last_generated: bool + mapped_last, last_generated = authored_offset(body=body, offset=end - 1) + if last_generated or mapped_last - mapped_start != end - 1 - start: + return None + return mapped_start, mapped_last + 1 + + +def _expansion( + *, project: CompiledProject, project_dir: Path, model: CompiledModel +) -> CompiledSqlExpansion | None: + target: Path = (project_dir / model.relative_path).resolve() + path: Path + expansion: CompiledSqlExpansion + for path, expansion in project.sql_expansions.items(): + if path.resolve() == target: + return expansion + return None + + +def _placeholder(*, kind: str, index: int, length: int) -> str: + base: str = PLACEHOLDER_BASE.format(kind=kind, index=index) + if len(base) > length: + return PLACEHOLDER_PAD * (length - 1) + kind + return base + PLACEHOLDER_PAD * (length - len(base)) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text_edits.py new file mode 100644 index 0000000000..a82cd19f21 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text_edits.py @@ -0,0 +1,193 @@ +"""Offsets, positions, and edit application for authored text.""" + +from __future__ import annotations + +import re +from functools import cache + +from sqlbuild.compiler.refactoring.constants import IDENTIFIER_CHARACTERS, NON_CODE_PATTERN +from sqlbuild.compiler.refactoring.exceptions import RefactorEditError +from sqlbuild.compiler.refactoring.models import ( + FileChange, + ManualLocation, + MigrationDeclaration, + RefactorParts, + RefactorPlan, + RefactorRequest, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind + + +def line_column(*, text: str, offset: int) -> tuple[int, int]: + """Return the 1-based line and column of an offset.""" + + line_start: int = text.rfind("\n", 0, offset) + 1 + return text.count("\n", 0, offset) + 1, offset - line_start + 1 + + +def text_edit( + *, + text: str, + start: int, + end: int, + replacement: str, + kind: EditKind, + before: str | None = None, + after: str | None = None, +) -> TextEdit: + """Build one edit with its position and display text.""" + + line: int + column: int + line, column = line_column(text=text, offset=start) + return TextEdit( + start=start, + end=end, + replacement=replacement, + kind=kind, + line=line, + column=column, + before=text[start:end] if before is None else before, + after=replacement.strip() if after is None else after, + ) + + +def manual_at(*, path: str, text: str, offset: int | None, reason: str) -> ManualLocation: + """Build a manual location at an optional offset.""" + + if offset is None: + return ManualLocation(path=path, line=None, column=None, reason=reason) + line: int + column: int + line, column = line_column(text=text, offset=offset) + return ManualLocation(path=path, line=line, column=column, reason=reason) + + +def apply_text_edits(*, text: str, edits: tuple[TextEdit, ...]) -> str: + """Apply non-overlapping edits back to front; identical duplicates apply once.""" + + unique: list[TextEdit] = sorted( + {(edit.start, edit.end, edit.replacement): edit for edit in edits}.values(), + key=lambda edit: (edit.start, edit.end), + reverse=True, + ) + result: str = text + previous_start: int = len(text) + 1 + edit: TextEdit + for edit in unique: + if edit.end > previous_start: + raise RefactorEditError(f"overlapping refactoring edits at offset {edit.start}") + result = result[: edit.start] + edit.replacement + result[edit.end :] + previous_start = edit.start + return result + + +def merge_parts(*, parts: tuple[RefactorParts, ...]) -> RefactorParts: + """Concatenate planning parts in order.""" + + edits: list[tuple[str, TextEdit]] = [] + manual: list[ManualLocation] = [] + blocking: list[ManualLocation] = [] + migrations: list[MigrationDeclaration] = [] + cascaded: list[str] = [] + part: RefactorParts + for part in parts: + edits.extend(part.edits) + manual.extend(part.manual) + blocking.extend(part.blocking) + migrations.extend(part.migrations) + cascaded.extend(part.cascaded) + return RefactorParts( + edits=tuple(edits), + manual=tuple(manual), + blocking=tuple(blocking), + migrations=tuple(migrations), + cascaded=tuple(cascaded), + ) + + +def path_edits(*, path: str, edits: tuple[TextEdit, ...]) -> tuple[tuple[str, TextEdit], ...]: + """Attach a file path to each edit.""" + + return tuple((path, edit) for edit in edits) + + +def file_changes( + *, edits: tuple[tuple[str, TextEdit], ...], moves: dict[str, str] +) -> tuple[FileChange, ...]: + """Group edits by file, move files, and order the result by final path.""" + + grouped: dict[str, list[TextEdit]] = {path: [] for path in moves} + path: str + edit: TextEdit + for path, edit in edits: + grouped.setdefault(path, []).append(edit) + return tuple( + sorted( + ( + FileChange(path=moves.get(path, path), original_path=path, edits=tuple(items)) + for path, items in grouped.items() + ), + key=lambda change: change.path, + ) + ) + + +def build_plan( + *, + request: RefactorRequest, + parts: RefactorParts, + moves: dict[str, str], + renamed_columns: tuple[tuple[str, str, str], ...] = (), + help: str | None = None, +) -> RefactorPlan: + """Assemble a plan from merged parts; help applies only when manual locations exist.""" + + return RefactorPlan( + request=request, + changes=file_changes(edits=parts.edits, moves=moves), + manual=parts.manual, + blocking=parts.blocking, + migrations=parts.migrations, + renamed_columns=renamed_columns, + help=help if parts.manual else None, + ) + + +def identifier_sites(*, text: str, names: frozenset[str]) -> tuple[tuple[int, int, str], ...]: + """Return unquoted identifier tokens matching names, outside comments and strings.""" + + if not names: + return () + lowered: dict[str, str] = {name.lower(): name for name in names} + sites: list[tuple[int, int, str]] = [] + match: re.Match[str] + for match in _identifier_pattern(frozenset(lowered)).finditer(text): + token: str | None = match.group("identifier") + if token is not None: + sites.append( + (match.start("identifier"), match.end("identifier"), lowered[token.lower()]) + ) + return tuple(sites) + + +def whole_word_offsets(*, text: str, word: str) -> tuple[int, ...]: + """Return the offsets of a case-insensitive whole identifier word anywhere in text.""" + + pattern: re.Pattern[str] = re.compile( + rf"(? re.Pattern[str]: + ordered: list[str] = sorted(names, key=lambda name: (-len(name), name)) + alternatives: str = "|".join(re.escape(name) for name in ordered) + return re.compile( + NON_CODE_PATTERN + + rf"|(?{alternatives})" + + rf"(?![{IDENTIFIER_CHARACTERS}])", + re.I, + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/workspace.py b/src/sqlbuild/compiler/refactoring/_helpers/workspace.py new file mode 100644 index 0000000000..ee629c72ac --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/workspace.py @@ -0,0 +1,174 @@ +"""Stage refactoring edits in a scratch copy, then commit them with rollback.""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path, PurePosixPath + +from sqlbuild.compiler.refactoring._helpers.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring.constants import ( + COPIED_PROJECT_SUFFIXES, + HIDDEN_PREFIX, + IGNORED_PROJECT_DIRECTORIES, + PYCACHE_DIRECTORY, + TEXT_ENCODING, +) +from sqlbuild.compiler.refactoring.exceptions import RefactorWriteError +from sqlbuild.compiler.refactoring.models import FileChange +from sqlbuild.lint.main.write_atomically import write_atomically + + +def copy_project_inputs(*, project_dir: Path, staging_dir: Path) -> int: + """Copy every file compilation can read, skipping build output and hidden directories.""" + + copied: int = 0 + root: str + directories: list[str] + names: list[str] + for root, directories, names in os.walk(project_dir): + current: Path = Path(root) + directories[:] = sorted( + name + for name in directories + if not name.startswith(HIDDEN_PREFIX) + and not (current == project_dir and name in IGNORED_PROJECT_DIRECTORIES) + and name != PYCACHE_DIRECTORY + ) + name: str + for name in names: + source: Path = current / name + if source.suffix not in COPIED_PROJECT_SUFFIXES or name.startswith(HIDDEN_PREFIX): + continue + destination: Path = staging_dir / source.relative_to(project_dir) + destination.parent.mkdir(parents=True, exist_ok=True) + _ = shutil.copyfile(source, destination) + copied += 1 + return copied + + +def edited_contents( + *, originals: dict[str, str], changes: tuple[FileChange, ...] +) -> dict[str, str]: + """Return the new text of every changed file, keyed by its final path.""" + + return { + change.path: apply_text_edits(text=originals[change.original_path], edits=change.edits) + for change in changes + } + + +def read_originals(*, project_dir: Path, changes: tuple[FileChange, ...]) -> dict[str, str]: + """Read the current text of every file a plan changes.""" + + return { + change.original_path: (project_dir / PurePosixPath(change.original_path)).read_text( + encoding=TEXT_ENCODING + ) + for change in changes + } + + +def write_staged_changes( + *, staging_dir: Path, originals: dict[str, str], changes: tuple[FileChange, ...] +) -> tuple[Path, ...]: + """Apply a plan to the scratch copy and return the written paths.""" + + contents: dict[str, str] = edited_contents(originals=originals, changes=changes) + change: FileChange + for change in changes: + if change.moved: + (staging_dir / PurePosixPath(change.original_path)).unlink(missing_ok=True) + written: list[Path] = [] + for change in changes: + target: Path = staging_dir / PurePosixPath(change.path) + target.parent.mkdir(parents=True, exist_ok=True) + _ = target.write_text(contents[change.path], encoding=TEXT_ENCODING) + written.append(target) + return tuple(written) + + +def commit_changes( + *, project_dir: Path, originals: dict[str, str], changes: tuple[FileChange, ...] +) -> tuple[Path, ...]: + """Write a verified plan into the project, restoring every file if any write fails.""" + + current: dict[str, str] = read_originals(project_dir=project_dir, changes=changes) + if current != originals: + changed: list[str] = sorted( + path for path in originals if current.get(path) != originals[path] + ) + raise RefactorWriteError( + f"files changed while the refactoring ran: {', '.join(changed)}", + help="nothing was written; run the command again", + ) + contents: dict[str, str] = edited_contents(originals=originals, changes=changes) + created_directories: list[Path] = [] + written: list[Path] = [] + try: + change: FileChange + for change in changes: + target: Path = project_dir / PurePosixPath(change.path) + created_directories.extend(_missing_directories(path=target.parent)) + target.parent.mkdir(parents=True, exist_ok=True) + written.append(target) + _write_file( + path=target, + contents=contents[change.path], + mode_source=project_dir / PurePosixPath(change.original_path), + ) + for change in changes: + if change.moved: + (project_dir / PurePosixPath(change.original_path)).unlink() + return tuple(written) + except BaseException: + _rollback( + project_dir=project_dir, + originals=originals, + written=written, + created_directories=created_directories, + ) + raise + + +def _rollback( + *, + project_dir: Path, + originals: dict[str, str], + written: list[Path], + created_directories: list[Path], +) -> None: + original_paths: frozenset[Path] = frozenset( + project_dir / PurePosixPath(path) for path in originals + ) + path: Path + for path in written: + if path not in original_paths: + path.unlink(missing_ok=True) + relative: str + text: str + for relative, text in originals.items(): + restored: Path = project_dir / PurePosixPath(relative) + if restored.exists(): + write_atomically(path=restored, contents=text) + else: + _ = restored.write_text(text, encoding=TEXT_ENCODING) + directory: Path + for directory in reversed(created_directories): + if directory.is_dir() and not any(directory.iterdir()): + directory.rmdir() + + +def _write_file(*, path: Path, contents: str, mode_source: Path) -> None: + if not path.exists(): + _ = shutil.copy(mode_source, path) + write_atomically(path=path, contents=contents) + + +def _missing_directories(*, path: Path) -> list[Path]: + missing: list[Path] = [] + current: Path = path + while not current.exists(): + missing.append(current) + current = current.parent + return list(reversed(missing)) diff --git a/src/sqlbuild/compiler/refactoring/constants.py b/src/sqlbuild/compiler/refactoring/constants.py new file mode 100644 index 0000000000..d16844106a --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/constants.py @@ -0,0 +1,105 @@ +"""Constants for project refactorings.""" + +from __future__ import annotations + +import re + +from sqlbuild.compiler.refactoring.types import HeaderTokenKind, RefactorOperation, SqlFileRole + +MODEL_KIND_PREFIX: str = "model:" +COLUMN_KIND_PREFIX: str = "column:" +SQL_SUFFIX: str = ".sql" +PATH_SEPARATOR: str = "/" +IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +REF_FUNCTION: str = "__ref" +REF_CALL_PATTERN: re.Pattern[str] = re.compile( + r"^__ref\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" +) +RESOURCE_CALL_PATTERN: re.Pattern[str] = re.compile( + r"^__(?Pref|source|seed)\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" +) +REF_FIXTURE_PREFIX: str = "__ref__" +EXPECTED_FIXTURE_PREFIX: str = "__expected__" +ASSERT_FIXTURE_PREFIX: str = "__assert__" +NON_CODE_PATTERN: str = ( + r"--[^\n]*(?:\n|\Z)" + r"|/\*[\s\S]*?(?:\*/|\Z)" + r"|'(?:''|[^'])*(?:'|\Z)" + r'|"(?:""|[^"])*(?:"|\Z)' +) +IDENTIFIER_CHARACTERS: str = r"A-Za-z0-9_$" +PLACEHOLDER_PAD: str = "_" +GENERIC_PLACEHOLDER_KIND: str = "x" +ROOT_SCOPE: str = "root" +CTE_SCOPE_PREFIX: str = "cte:" +MIGRATE_FROM_KEY: str = "migrate_from" +MATERIALIZED_KEY: str = "materialized" +CURSOR_INPUTS_KEY: str = "cursor_inputs" +COLUMNS_KEY: str = "columns" +RELATIONSHIPS_AUDIT: str = "relationships" +RELATIONSHIPS_TO_KEY: str = "to" +RELATIONSHIPS_FIELD_KEY: str = "field" +COLUMN_VALUED_CONFIG_KEYS: frozenset[str] = frozenset( + { + "cursor", + "unique_key", + "updated_at", + "check_columns", + "merge_exclude_columns", + "partition_by", + "cluster_by", + "observed_at", + "row_diff_exclude_columns", + } +) +HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset({"incremental", "snapshot"}) +MIGRATABLE_MATERIALIZATIONS: frozenset[str] = frozenset( + {"table", "view", "incremental", "snapshot"} +) +COPIED_PROJECT_SUFFIXES: frozenset[str] = frozenset( + {".sql", ".py", ".yml", ".yaml", ".toml", ".csv", ".json"} +) +IGNORED_PROJECT_DIRECTORIES: frozenset[str] = frozenset( + {"target", "logs", "venv", "node_modules", "__pycache__"} +) +SQL_LIKE_PATTERN: re.Pattern[str] = re.compile(r"\b(select|from|join|insert|update)\b", re.I) +OPERATION_TITLES: dict[RefactorOperation, str] = { + RefactorOperation.RENAME_MODEL: "Rename model", + RefactorOperation.MOVE_MODEL: "Move model", + RefactorOperation.RENAME_COLUMN: "Rename column", +} +PLACEHOLDER_BASE: str = "_q{kind}{index}" +QUOTE_CHARACTERS: dict[str, str] = {'"': '"', "`": "`", "[": "]"} +COLUMN_TARGET_SEPARATOR: str = "." +HEADER_OPENERS: frozenset[str] = frozenset({"(", "[", "{"}) +HEADER_CLOSERS: frozenset[str] = frozenset({")", "]", "}"}) +HEADER_SEPARATOR: str = "," +HEADER_OPEN_PAREN: str = "(" +HEADER_ESCAPE: str = "\\" +HEADER_DESCRIPTION_KEY: str = "description" +HEADER_INDENT: str = " " +PARENTHESIZED_EMPTY_TOKENS: int = 2 +REF_KIND: str = "ref" +FIXTURE_ROLES: frozenset[SqlFileRole] = frozenset({SqlFileRole.TEST, SqlFileRole.SCENARIO}) +HIDDEN_PREFIX: str = "." +TEXT_ENCODING: str = "utf-8" +GENERIC_DIALECT: str = "generic" +NATIVE_HEADER_TOKEN_KINDS: dict[int, HeaderTokenKind] = { + 0: HeaderTokenKind.END, + 1: HeaderTokenKind.WORD, + 2: HeaderTokenKind.STRING, + 3: HeaderTokenKind.SYMBOL, +} +HEADER_KEY_AND_VALUE_TOKENS: int = 2 +PYCACHE_DIRECTORY: str = "__pycache__" +SOURCE_KIND: str = "source" +SEED_KIND: str = "seed" +UNKNOWN_COLUMN_TYPE: str = "UNKNOWN" +MODEL_MANUAL_HELP: str = ( + "Pass the model into the macro as an argument, or edit the listed locations, then run the " + "command again; --allow-manual applies the safe edits and leaves them to you." +) +COLUMN_MANUAL_HELP: str = ( + "Edit the listed locations, or rerun with --allow-manual to apply the safe edits and finish " + "them by hand." +) diff --git a/src/sqlbuild/compiler/refactoring/exceptions.py b/src/sqlbuild/compiler/refactoring/exceptions.py new file mode 100644 index 0000000000..4226649f3f --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/exceptions.py @@ -0,0 +1,27 @@ +"""Expected refactoring errors.""" + +from __future__ import annotations + + +class RefactorInputError(ValueError): + """A rename or move request that cannot be planned at all.""" + + code: str = "C950" + + def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: + super().__init__(message) + self.message = message + self.code = code if code is not None else self.code + self.help = help + + +class RefactorEditError(RefactorInputError): + """Two planned edits overlap, which a correct plan never produces.""" + + code: str = "C957" + + +class RefactorWriteError(RefactorInputError): + """Project files changed between planning and writing.""" + + code: str = "C958" diff --git a/src/sqlbuild/compiler/refactoring/main/__init__.py b/src/sqlbuild/compiler/refactoring/main/__init__.py new file mode 100644 index 0000000000..fe1ccd557d --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/__init__.py @@ -0,0 +1 @@ +"""Refactoring entry modules.""" diff --git a/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py b/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py new file mode 100644 index 0000000000..88cf306a77 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py @@ -0,0 +1,16 @@ +"""Write a verified refactoring plan into the project.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.refactoring._helpers.workspace import commit_changes +from sqlbuild.compiler.refactoring.models import RefactorPlan + + +def commit_refactor_plan( + *, project_dir: Path, originals: dict[str, str], plan: RefactorPlan +) -> tuple[Path, ...]: + """Write every change, restoring all files if any write fails, and return written paths.""" + + return commit_changes(project_dir=project_dir, originals=originals, changes=plan.changes) diff --git a/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py b/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py new file mode 100644 index 0000000000..67080f0b0d --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py @@ -0,0 +1,31 @@ +"""Plan a column rename from compiler facts, one step or cascading through pass-throughs.""" + +from __future__ import annotations + +from sqlbuild.compiler.refactoring._helpers.column_planning import ( + column_rename_context, + column_rename_parts, +) +from sqlbuild.compiler.refactoring._helpers.text_edits import build_plan +from sqlbuild.compiler.refactoring.constants import COLUMN_MANUAL_HELP +from sqlbuild.compiler.refactoring.models import ( + ColumnRenameContext, + RefactorParts, + RefactorPlan, + RefactorProject, + RefactorRequest, +) + + +def plan_column_rename(*, project: RefactorProject, request: RefactorRequest) -> RefactorPlan: + """Plan every edit that renames one output column and its downstream references.""" + + context: ColumnRenameContext = column_rename_context(project=project, request=request) + parts: RefactorParts = column_rename_parts(context=context) + return build_plan( + request=request, + parts=parts, + moves={}, + renamed_columns=tuple((model, context.old, context.new) for model in parts.cascaded), + help=COLUMN_MANUAL_HELP, + ) diff --git a/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py new file mode 100644 index 0000000000..6a244337c8 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py @@ -0,0 +1,29 @@ +"""Plan a model rename or move from compiler facts.""" + +from __future__ import annotations + +from sqlbuild.compiler.refactoring._helpers.model_planning import model_parts, model_target +from sqlbuild.compiler.refactoring._helpers.text_edits import build_plan +from sqlbuild.compiler.refactoring.constants import MODEL_MANUAL_HELP +from sqlbuild.compiler.refactoring.models import ( + ModelTarget, + RefactorParts, + RefactorPlan, + RefactorProject, + RefactorRequest, +) + + +def plan_model_refactor(*, project: RefactorProject, request: RefactorRequest) -> RefactorPlan: + """Plan every edit that renames or moves one model, without touching files.""" + + target: ModelTarget = model_target(project=project, request=request) + parts: RefactorParts = model_parts(project=project, target=target) + return build_plan( + request=target.request, + parts=parts, + moves={target.source_path: target.destination} + if target.destination != target.source_path + else {}, + help=MODEL_MANUAL_HELP, + ) diff --git a/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py b/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py new file mode 100644 index 0000000000..b1c499a716 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py @@ -0,0 +1,24 @@ +"""Stage a refactoring plan in a scratch copy of the project.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.refactoring._helpers.workspace import ( + copy_project_inputs, + read_originals, + write_staged_changes, +) +from sqlbuild.compiler.refactoring.models import RefactorPlan + + +def stage_refactor_plan( + *, project_dir: Path, staging_dir: Path, plan: RefactorPlan, copy_inputs: bool = True +) -> dict[str, str]: + """Copy the project inputs once, apply the plan there, and return the original texts.""" + + if copy_inputs: + _ = copy_project_inputs(project_dir=project_dir, staging_dir=staging_dir) + originals: dict[str, str] = read_originals(project_dir=project_dir, changes=plan.changes) + _ = write_staged_changes(staging_dir=staging_dir, originals=originals, changes=plan.changes) + return originals diff --git a/src/sqlbuild/compiler/refactoring/main/with_model_migration.py b/src/sqlbuild/compiler/refactoring/main/with_model_migration.py new file mode 100644 index 0000000000..09f146fc7c --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/with_model_migration.py @@ -0,0 +1,80 @@ +"""Add model migrate_from to a staged rename exactly where discovery would miss it.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.refactoring._helpers.header_edits import insert_header_entry_edit +from sqlbuild.compiler.refactoring._helpers.model_migration import decide_model_migration +from sqlbuild.compiler.refactoring.constants import MIGRATE_FROM_KEY +from sqlbuild.compiler.refactoring.models import ( + FileChange, + ManualLocation, + MigrationDeclaration, + ModelMigrationDecision, + RefactorPlan, + TextEdit, +) + + +def with_model_migration( + *, + plan: RefactorPlan, + before: CompiledProject, + after: CompiledProject, + originals: dict[str, str], +) -> RefactorPlan: + """Return the plan with `migrate_from ` on the model when history would be lost.""" + + old: str = plan.request.model_name + new: str = plan.request.new_name + decision: ModelMigrationDecision = decide_model_migration( + before=before, after=after, old=old, new=new + ) + change: FileChange | None = next( + (item for item in plan.changes if item.path == plan.request.destination), None + ) + if decision.blocked: + return replace( + plan, + blocking=( + *plan.blocking, + ManualLocation( + path=plan.request.destination or "", + line=None, + column=None, + reason=decision.reason, + ), + ), + ) + if not decision.needed or change is None: + return plan + declaration: str = f"{MIGRATE_FROM_KEY} {old}" + edit: TextEdit | None = insert_header_entry_edit( + contents=originals[change.original_path], entry=declaration, display=declaration + ) + if edit is None: + return replace( + plan, + manual=( + *plan.manual, + ManualLocation( + path=change.path, + line=1, + column=1, + reason=f"add {declaration} to the MODEL header to keep history", + ), + ), + ) + return replace( + plan, + changes=tuple( + replace(item, edits=(*item.edits, edit)) if item is change else item + for item in plan.changes + ), + migrations=( + *plan.migrations, + MigrationDeclaration(model_name=new, declaration=declaration, reason=decision.reason), + ), + ) diff --git a/src/sqlbuild/compiler/refactoring/models.py b/src/sqlbuild/compiler/refactoring/models.py new file mode 100644 index 0000000000..e79594b157 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/models.py @@ -0,0 +1,295 @@ +"""Immutable models for planned and applied project refactorings.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from sqlbuild.compiler.compile.models import CompiledModel, ExpansionSpan +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.refactoring.types import ( + EditKind, + HeaderTokenKind, + OffsetLocator, + RefactorOperation, + ResourceColumns, + SpanMapper, + SqlFileRole, +) + + +@dataclass(frozen=True) +class RefactorProject: + """Compiler facts of one project, as the refactoring planner reads them.""" + + project_dir: Path + graph: ProjectGraph + discovered: DiscoveredProjectInputs + + +@dataclass(frozen=True) +class RefactorRequest: + """One requested rename or move.""" + + operation: RefactorOperation + model_name: str + new_name: str + column_name: str | None = None + destination: str | None = None + cascade: bool = False + + +@dataclass(frozen=True) +class ModelTarget: + """The model a request names, and the request with its name and destination resolved.""" + + model: CompiledModel + request: RefactorRequest + source_path: str + + @property + def destination(self) -> str: + return self.request.destination or self.source_path + + +@dataclass(frozen=True) +class TextEdit: + """One replacement in an authored file, in offsets of the file before any edit.""" + + start: int + end: int + replacement: str + kind: EditKind + line: int + column: int + before: str + after: str + + +@dataclass(frozen=True) +class FileChange: + """Every edit to one file, and where the file lives afterwards.""" + + path: str + original_path: str + edits: tuple[TextEdit, ...] = field(default_factory=tuple) + + @property + def moved(self) -> bool: + return self.path != self.original_path + + +@dataclass(frozen=True) +class ManualLocation: + """A location a refactoring cannot rewrite safely.""" + + path: str + line: int | None + column: int | None + reason: str + + +@dataclass(frozen=True) +class MigrationDeclaration: + """A migrate_from declaration the refactoring adds to keep history.""" + + model_name: str + declaration: str + reason: str + + +@dataclass(frozen=True) +class RefactorPlan: + """Every edit, manual location, and migration of one refactoring.""" + + request: RefactorRequest + changes: tuple[FileChange, ...] = field(default_factory=tuple) + manual: tuple[ManualLocation, ...] = field(default_factory=tuple) + blocking: tuple[ManualLocation, ...] = field(default_factory=tuple) + migrations: tuple[MigrationDeclaration, ...] = field(default_factory=tuple) + renamed_columns: tuple[tuple[str, str, str], ...] = field(default_factory=tuple) + help: str | None = None + + +@dataclass(frozen=True) +class RefactorParts: + """Edits and findings from one planning step, merged into a plan at the end.""" + + edits: tuple[tuple[str, TextEdit], ...] = field(default_factory=tuple) + manual: tuple[ManualLocation, ...] = field(default_factory=tuple) + blocking: tuple[ManualLocation, ...] = field(default_factory=tuple) + migrations: tuple[MigrationDeclaration, ...] = field(default_factory=tuple) + cascaded: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class ProjectSqlFile: + """One authored SQL file and the role discovery gave it.""" + + relative_path: str + contents: str + role: SqlFileRole + + +@dataclass(frozen=True) +class AuthoredBody: + """One SQL body inside an authored non-model file.""" + + path: str + role: SqlFileRole + contents: str + start: int + text: str + + +@dataclass(frozen=True) +class ResourceSite: + """One `__ref`, `__source`, or `__seed` call written in authored SQL.""" + + kind: str + name: str + start: int + end: int + name_start: int + name_end: int + + +@dataclass(frozen=True) +class AnalysisSql: + """SQL whose interpolation sites were replaced by same-length identifiers.""" + + sql: str + tables: dict[tuple[str, str], frozenset[str]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ModelBody: + """A model's compiled query and how it maps onto the authored file.""" + + relative_path: str + contents: str + body_start: int + compiled_sql: str + passes: tuple[tuple[ExpansionSpan, ...], ...] + + +@dataclass(frozen=True) +class HeaderToken: + """One MODEL header token in file offsets, with its nesting depth.""" + + kind: HeaderTokenKind + value: str + start: int + end: int + depth: int + + +@dataclass(frozen=True) +class HeaderEntry: + """One top-level `key value` entry of a MODEL header.""" + + key: str + tokens: tuple[HeaderToken, ...] + + +@dataclass(frozen=True) +class ColumnReference: + """One column expression that resolves to the renamed column.""" + + start: int + end: int + name_start: int + name_end: int + scope: str + projected: bool + alias: str | None + alias_start: int | None + alias_end: int | None + + +@dataclass(frozen=True) +class ColumnSite: + """A scoped construct at an optional span.""" + + scope: str + start: int | None + end: int | None + + +@dataclass(frozen=True) +class OutputColumn: + """One projection that names the column in a requested output scope.""" + + scope: str + start: int | None + end: int | None + alias_start: int | None + alias_end: int | None + + +@dataclass(frozen=True) +class ColumnFacts: + """Every rename-relevant fact about one column in one SQL body.""" + + parsed: bool + references: tuple[ColumnReference, ...] = field(default_factory=tuple) + stars: tuple[ColumnSite, ...] = field(default_factory=tuple) + joins: tuple[ColumnSite, ...] = field(default_factory=tuple) + unresolved: tuple[ColumnSite, ...] = field(default_factory=tuple) + outputs: tuple[OutputColumn, ...] = field(default_factory=tuple) + star_outputs: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class ColumnQuery: + """Which column of which relations one analysis resolves.""" + + column: str + target_tables: frozenset[str] + target_ctes: frozenset[str] = frozenset() + output_scopes: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class BodyContext: + """Where one analysed SQL body lives, and how its offsets map onto the file.""" + + path: str + contents: str + map_span: SpanMapper + locate: OffsetLocator + fallback_offset: int + + +@dataclass(frozen=True) +class BodyEdits: + """Edits, manual locations, and root pass-throughs found in one SQL body.""" + + edits: tuple[TextEdit, ...] = field(default_factory=tuple) + manual: tuple[ManualLocation, ...] = field(default_factory=tuple) + passes_through: bool = False + + +@dataclass(frozen=True) +class ColumnRenameContext: + """Everything a column rename reads while it walks downstream models.""" + + project: RefactorProject + owner: CompiledModel + dialect: str + columns: ResourceColumns + contents: dict[str, str] + bodies: tuple[AuthoredBody, ...] + old: str + new: str + cascade: bool + + +@dataclass(frozen=True) +class ModelMigrationDecision: + """Whether a model needs migrate_from, and why, or why none can be written.""" + + needed: bool + reason: str + blocked: bool = False diff --git a/src/sqlbuild/compiler/refactoring/types.py b/src/sqlbuild/compiler/refactoring/types.py new file mode 100644 index 0000000000..ccf89da446 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/types.py @@ -0,0 +1,65 @@ +"""Enums and type aliases for project refactorings.""" + +from __future__ import annotations + +from collections.abc import Callable +from enum import StrEnum +from typing import Protocol + + +class RefactorOperation(StrEnum): + """One supported refactoring.""" + + RENAME_MODEL = "rename_model" + MOVE_MODEL = "move_model" + RENAME_COLUMN = "rename_column" + + +class RefactorStatus(StrEnum): + """Terminal state of one refactoring command.""" + + APPLIED = "applied" + DRY_RUN = "dry_run" + REFUSED = "refused" + COMPILE_FAILED = "compile_failed" + + +class EditKind(StrEnum): + """What one text edit changes, for output grouping.""" + + REFERENCE = "reference" + FIXTURE = "fixture" + HEADER = "header" + COLUMN = "column" + MIGRATION = "migration" + + +class SqlFileRole(StrEnum): + """What an authored SQL file declares.""" + + MODEL = "model" + TEST = "test" + SCENARIO = "scenario" + AUDIT = "audit" + HOOK = "hook" + FUNCTION = "function" + + +class HeaderTokenKind(StrEnum): + """MODEL header token kinds.""" + + END = "end" + WORD = "word" + STRING = "string" + SYMBOL = "symbol" + + +type ResourceColumns = dict[tuple[str, str], tuple[str, ...]] +type SpanMapper = Callable[[int, int], tuple[int, int] | None] +type OffsetLocator = Callable[[int], int] + + +class NativeColumnReferences(Protocol): + """Native column-reference analysis boundary.""" + + def analyze_column_references_json(self, request_json: str) -> str: ... diff --git a/src/sqlbuild/lint/main/scan_interpolation_sites.py b/src/sqlbuild/lint/main/scan_interpolation_sites.py new file mode 100644 index 0000000000..0540c6608a --- /dev/null +++ b/src/sqlbuild/lint/main/scan_interpolation_sites.py @@ -0,0 +1,12 @@ +"""Public lexical scan for SQLBuild interpolation sites in authored SQL.""" + +from __future__ import annotations + +from sqlbuild.lint._helpers.sqlbuild_tokens import neutralize_interpolation +from sqlbuild.lint.models import InterpolationSite + + +def scan_interpolation_sites(*, body: str, dialect: str) -> tuple[InterpolationSite, ...]: + """Return every reference call, macro, and template site outside comments and strings.""" + + return neutralize_interpolation(body=body, dialect=dialect)[1] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py new file mode 100644 index 0000000000..6d6cf0e07c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -0,0 +1,61 @@ +"""Test case types for `sqb rename` and `sqb mv` e2e tests.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class ModelRefactorE2ETestCase: + description: str + command: tuple[str, ...] + expected_removed: str + expected_file: str + expected_fragments: dict[str, tuple[str, ...]] + expected_relation: str + extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ModelMigrationE2ETestCase: + description: str + materialized: str + udf: bool + expected_declaration: str + expected_migrate_from_count: int + expected_old_name_type: str + expected_order_ids: tuple[int, ...] + + +@dataclass(frozen=True) +class ColumnRenameE2ETestCase: + description: str + command: tuple[str, ...] + extra_files: dict[str, str] = field(default_factory=dict) + expected_fragments: dict[str, tuple[str, ...]] = field(default_factory=dict) + expected_columns: dict[str, tuple[str, ...]] = field(default_factory=dict) + expected_manual: tuple[str, ...] = () + + +@dataclass(frozen=True) +class RefusedRefactorE2ETestCase: + description: str + command: tuple[str, ...] + extra_files: dict[str, str] + expected_status: str + expected_reason: str + + +@dataclass(frozen=True) +class ColumnMigrationE2ETestCase: + description: str + expected_declaration: str + expected_columns: tuple[str, ...] + expected_order_ids: tuple[int, ...] + + +@dataclass(frozen=True) +class DryRunE2ETestCase: + description: str + command: tuple[str, ...] + expected_paths: frozenset[str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py new file mode 100644 index 0000000000..a17df157c0 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -0,0 +1,235 @@ +"""Project builders and warehouse readers for `sqb rename` and `sqb mv` e2e tests.""" + +from __future__ import annotations + +import json +import subprocess +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +DATABASE_FILE: str = "orders.duckdb" +SCHEMA: str = "analytics" +_PROJECT_TOML: str = ( + 'name = "orders_project"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + f'[connection]\ndatabase = "{DATABASE_FILE}"\n\n' + f'[targets.dev]\nschema = "{SCHEMA}"\n\n' + "[janitor]\nenabled = true\n" +) +_SOURCES_YML: str = ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: order_id\n" + " - name: customer_id\n" + " - name: amount\n" + " - name: order_date\n" +) +STG_ORDERS: str = ( + "MODEL (\n materialized view,\n);\n\n" + "SELECT\n order_id,\n customer_id,\n amount,\n order_date\n" + 'FROM __source("raw_orders")\n' +) +FACT_ORDERS: str = ( + "MODEL (\n materialized table,\n);\n\n" + "SELECT\n o.order_id,\n o.customer_id,\n o.amount,\n o.order_date\n" + 'FROM __ref("stg_orders") AS o\n' +) +CUSTOMER_TOTALS: str = ( + "MODEL (\n materialized table,\n);\n\n" + "SELECT\n customer_id,\n SUM(amount) AS total_amount\n" + 'FROM __ref("fact_orders")\nGROUP BY customer_id\n' +) +FACT_ORDERS_TEST: str = ( + "TEST();\n\n" + "WITH\n" + "__ref__stg_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, 25 AS amount, " + "CAST('2026-01-01' AS TIMESTAMP) AS order_date\n" + "),\n" + "__expected__fact_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, 25 AS amount, " + "CAST('2026-01-01' AS TIMESTAMP) AS order_date\n" + ")\n" + "SELECT 1\n" +) +BASE_FILES: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": _SOURCES_YML, + "models/staging/stg_orders.sql": STG_ORDERS, + "models/marts/fact_orders.sql": FACT_ORDERS, + "models/marts/customer_totals.sql": CUSTOMER_TOTALS, + "tests/unit/test_fact_orders.sql": FACT_ORDERS_TEST, +} +UDF_FILE: tuple[str, str] = ( + "functions/sql/udf__is_large_order.sql", + "FUNCTION (\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount >= 100\n", +) +CENTS_MACRO: dict[str, str] = { + "models/staging/_sqlbuild/_macros/cents.py": ( + 'def to_cents(expression: str) -> str:\n return f"({expression}) * 100"\n' + ), + "models/staging/stg_order_cents.sql": ( + "MODEL (\n materialized view,\n);\n\n" + 'SELECT\n order_id,\n @to_cents("amount") AS amount_cents\nFROM __ref("stg_orders")\n' + ), +} +ORDER_EXPORT: dict[str, str] = { + "models/marts/order_export.sql": ( + 'MODEL (\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' + ), +} +_INCREMENTAL_HEADER: str = ( + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + " cursor_inputs (\n stg_orders order_date,\n ),\n" +) + + +_HISTORY_HEADERS: dict[str, str] = { + "table": " materialized table,\n", + "view": " materialized view,\n", + "incremental": _INCREMENTAL_HEADER, +} +_UDF_COLUMN: dict[bool, str] = { + True: ',\n __udf("udf__is_large_order")(amount) AS is_large', + False: "", +} +_UDF_FILES: dict[bool, dict[str, str]] = {True: {UDF_FILE[0]: UDF_FILE[1]}, False: {}} + + +def order_history_files(*, materialized: str, udf: bool) -> dict[str, str]: + """Return an order_history model, optionally calling a UDF discovery cannot fingerprint.""" + + return { + "models/marts/order_history.sql": ( + f"MODEL (\n{_HISTORY_HEADERS[materialized]});\n\n" + f"SELECT\n order_id,\n amount,\n order_date{_UDF_COLUMN[udf]}\n" + 'FROM __ref("stg_orders")\n' + ), + **_UDF_FILES[udf], + } + + +def write_orders_project( + *, tmp_path: Path, files: Mapping[str, str] | None = None, raw_order_count: int = 3 +) -> Path: + """Write the base orders project plus overrides and load raw orders.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="orders_project", repo_files={**BASE_FILES, **(files or {})} + ) + load_raw_orders(project_dir=project_dir, order_ids=tuple(range(1, raw_order_count + 1))) + return project_dir + + +def load_raw_orders(*, project_dir: Path, order_ids: tuple[int, ...]) -> None: + """Replace the raw orders with one order per id, dated January of that day.""" + + values: str = ", ".join( + f"({order_id}, {10 + order_id % 2}, {order_id * 50}, TIMESTAMP '2026-01-{order_id:02d}')" + for order_id in order_ids + ) + execute_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "CREATE OR REPLACE TABLE main.raw_orders AS SELECT * FROM (VALUES " + f"{values}) AS t(order_id, customer_id, amount, order_date)" + ), + ) + + +def sqb(project_dir: Path, *args: str) -> subprocess.CompletedProcess[str]: + """Run one sqb command without colour.""" + + return run_sqb(command=("--no-color", *args), project_dir=project_dir) + + +def sqb_json(project_dir: Path, *args: str) -> tuple[int, dict[str, Any]]: + """Run one sqb command with --json and parse its stdout.""" + + result: subprocess.CompletedProcess[str] = sqb(project_dir, *args, "--json") + return result.returncode, json.loads(result.stdout) + + +def project_text(project_dir: Path) -> dict[str, str]: + """Return every authored project file keyed by its relative path.""" + + paths: tuple[Path, ...] = ( + project_dir / "sqlbuild_project.toml", + *project_dir.glob("sources/*.yml"), + *project_dir.glob("models/**/*.sql"), + *project_dir.glob("models/**/*.py"), + *project_dir.glob("functions/**/*.sql"), + *project_dir.glob("tests/**/*.sql"), + ) + return { + path.relative_to(project_dir).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(paths) + } + + +def fragments_present(*, files: dict[str, str], expected: dict[str, tuple[str, ...]]) -> bool: + """Return whether every expected fragment appears in its file.""" + + found: list[bool] = [] + path: str + fragments: tuple[str, ...] + for path, fragments in expected.items(): + found.extend(fragment in files[path] for fragment in fragments) + return all(found) + + +def relation_type(*, project_dir: Path, name: str) -> str | None: + """Return the information_schema table type of one relation, or None.""" + + rows: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT table_type FROM information_schema.tables " + f"WHERE table_schema = '{SCHEMA}' AND table_name = '{name}'" + ), + ) + return next((str(row[0]) for row in rows), None) + + +def relation_columns(*, project_dir: Path, name: str) -> tuple[str, ...]: + """Return the column names of one relation in ordinal order.""" + + return tuple( + str(row[0]) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT column_name FROM information_schema.columns " + f"WHERE table_schema = '{SCHEMA}' AND table_name = '{name}' " + "ORDER BY ordinal_position" + ), + ) + ) + + +def order_ids(*, project_dir: Path, name: str) -> tuple[int, ...]: + """Return the order ids stored in one relation.""" + + return tuple( + int(row[0]) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=f"SELECT order_id FROM {SCHEMA}.{name} ORDER BY order_id", + ) + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py new file mode 100644 index 0000000000..6de1d682bb --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py @@ -0,0 +1,209 @@ +"""CLI e2e coverage for `sqb rename column:` and for refactorings that must not write.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + ColumnMigrationE2ETestCase, + ColumnRenameE2ETestCase, + RefusedRefactorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + CENTS_MACRO, + ORDER_EXPORT, + fragments_present, + load_raw_orders, + order_history_files, + order_ids, + project_text, + relation_columns, + sqb, + sqb_json, + write_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnRenameE2ETestCase( + description="one step keeps downstream output names", + command=("rename", "column:stg_orders.amount", "revenue"), + expected_fragments={ + "models/staging/stg_orders.sql": (" amount AS revenue,",), + "models/marts/fact_orders.sql": (" o.revenue AS amount,",), + "tests/unit/test_fact_orders.sql": ("25 AS revenue",), + }, + expected_columns={ + "stg_orders": ("order_id", "customer_id", "revenue", "order_date"), + "fact_orders": ("order_id", "customer_id", "amount", "order_date"), + }, + ), + ColumnRenameE2ETestCase( + description="cascade renames pass-through outputs downstream", + command=("rename", "column:stg_orders.amount", "revenue", "--cascade"), + expected_fragments={ + "models/marts/fact_orders.sql": (" o.revenue,",), + "models/marts/customer_totals.sql": ("SUM(revenue) AS total_amount",), + }, + expected_columns={ + "fact_orders": ("order_id", "customer_id", "revenue", "order_date"), + "customer_totals": ("customer_id", "total_amount"), + }, + ), + ColumnRenameE2ETestCase( + description="cascade follows SELECT star consumers", + command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), + extra_files=ORDER_EXPORT, + expected_fragments={"models/marts/customer_totals.sql": ("SUM(revenue)",)}, + expected_columns={ + "fact_orders": ("order_id", "customer_id", "revenue", "order_date"), + "order_export": ("order_id", "customer_id", "revenue", "order_date"), + }, + ), + ColumnRenameE2ETestCase( + description="allow manual applies safe edits and lists the rest", + command=("rename", "column:fact_orders.amount", "revenue", "--allow-manual"), + extra_files=ORDER_EXPORT, + expected_fragments={"models/marts/fact_orders.sql": (" o.amount AS revenue,",)}, + expected_columns={ + "order_export": ("order_id", "customer_id", "revenue", "order_date"), + }, + expected_manual=("models/marts/order_export.sql",), + ), + ], + ids=lambda case: case.description, +) +def test_given_column_rename_when_applied_then_project_tests_and_builds( + tmp_path: Path, test_case: ColumnRenameE2ETestCase +) -> None: + """Column references, fixtures, and outputs change together, and the project still runs.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path, files=test_case.extra_files) + + code: int + payload: dict[str, Any] + code, payload = sqb_json(project_dir, *test_case.command) + tested: subprocess.CompletedProcess[str] = sqb(project_dir, "test") + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert code == 0, payload + assert payload["status"] == "applied" + assert tuple(item["path"] for item in payload["manual"]) == test_case.expected_manual + files: dict[str, str] = project_text(project_dir) + assert fragments_present(files=files, expected=test_case.expected_fragments), files + assert tested.returncode == 0, tested.stdout + tested.stderr + assert built.returncode == 0, built.stdout + built.stderr + assert { + name: relation_columns(project_dir=project_dir, name=name) + for name in test_case.expected_columns + } == test_case.expected_columns + + +@pytest.mark.parametrize( + "test_case", + [ + RefusedRefactorE2ETestCase( + description="move out of a macro scope", + command=("mv", "model:stg_order_cents", "models/marts/"), + extra_files=CENTS_MACRO, + expected_status="refused", + expected_reason="macro:to_cents used by model:stg_order_cents is not visible", + ), + RefusedRefactorE2ETestCase( + description="rename onto an existing model", + command=("rename", "model:stg_orders", "fact_orders"), + extra_files={}, + expected_status="refused", + expected_reason="model:fact_orders already exists", + ), + RefusedRefactorE2ETestCase( + description="column used inside macro-generated SQL", + command=("rename", "column:stg_orders.amount", "revenue"), + extra_files=CENTS_MACRO, + expected_status="refused", + expected_reason="referenced in SQL a macro generates", + ), + RefusedRefactorE2ETestCase( + description="SELECT star consumer without cascade", + command=("rename", "column:fact_orders.amount", "revenue"), + extra_files=ORDER_EXPORT, + expected_status="refused", + expected_reason="rerun with --cascade", + ), + RefusedRefactorE2ETestCase( + description="allow manual that leaves the project broken", + command=("rename", "column:stg_orders.amount", "revenue", "--allow-manual"), + extra_files=CENTS_MACRO, + expected_status="compile_failed", + expected_reason="Unknown column 'amount'", + ), + ], + ids=lambda case: case.description, +) +def test_given_unsafe_refactor_when_running_then_no_file_changes( + tmp_path: Path, test_case: RefusedRefactorE2ETestCase +) -> None: + """Refusals and failed verification exit 1, explain why, and leave every file untouched.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path, files=test_case.extra_files) + before: dict[str, str] = project_text(project_dir) + + code: int + payload: dict[str, Any] + code, payload = sqb_json(project_dir, *test_case.command) + text: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.command) + + assert code == 1 + assert payload["status"] == test_case.expected_status + assert test_case.expected_reason in json.dumps(payload) + assert text.returncode == 1 + assert "no files changed" in text.stdout + assert project_text(project_dir) == before + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnMigrationE2ETestCase( + description="incremental column renamed in place", + expected_declaration="revenue (migrate_from amount)", + expected_columns=("order_id", "revenue", "order_date"), + expected_order_ids=(1, 2, 3, 4), + ) + ], + ids=lambda case: case.description, +) +def test_given_built_incremental_when_renaming_column_then_build_keeps_history( + tmp_path: Path, test_case: ColumnMigrationE2ETestCase +) -> None: + """The rename declares the column migration, so the next build renames it in place.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, files=order_history_files(materialized="incremental", udf=False) + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + result: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "column:order_history.amount", "revenue" + ) + header: str = (project_dir / "models/marts/order_history.sql").read_text(encoding="utf-8") + load_raw_orders(project_dir=project_dir, order_ids=(3, 4)) + second: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert first.returncode == 0, first.stdout + first.stderr + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_declaration in header, header + assert second.returncode == 0, second.stdout + second.stderr + assert relation_columns(project_dir=project_dir, name="order_history") == ( + test_case.expected_columns + ) + assert order_ids(project_dir=project_dir, name="order_history") == ( + test_case.expected_order_ids + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py new file mode 100644 index 0000000000..2d69fd9418 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py @@ -0,0 +1,209 @@ +"""CLI e2e coverage for `sqb rename model:` and `sqb mv` on a real DuckDB project.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + DryRunE2ETestCase, + ModelMigrationE2ETestCase, + ModelRefactorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + fragments_present, + load_raw_orders, + order_history_files, + order_ids, + project_text, + relation_type, + sqb, + sqb_json, + write_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ModelRefactorE2ETestCase( + description="rename keeps the folder and rewrites refs and fixtures", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={ + "models/marts/fact_orders.sql": ('__ref("stg_order_lines")',), + "tests/unit/test_fact_orders.sql": ("__ref__stg_order_lines AS (",), + }, + expected_relation="stg_order_lines", + ), + ModelRefactorE2ETestCase( + description="rename rewrites cursor_inputs keys of incremental consumers", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={ + "models/marts/order_history.sql": ( + "stg_order_lines order_date,", + '__ref("stg_order_lines")', + ), + }, + expected_relation="order_history", + extra_files=order_history_files(materialized="incremental", udf=False), + ), + ModelRefactorE2ETestCase( + description="move to a new file renames the model after the file", + command=("mv", "model:fact_orders", "models/reporting/order_facts.sql"), + expected_removed="models/marts/fact_orders.sql", + expected_file="models/reporting/order_facts.sql", + expected_fragments={ + "models/marts/customer_totals.sql": ('__ref("order_facts")',), + "tests/unit/test_fact_orders.sql": ("__expected__order_facts AS (",), + }, + expected_relation="order_facts", + ), + ModelRefactorE2ETestCase( + description="move into a folder keeps the model name", + command=("mv", "model:customer_totals", "models/reporting/"), + expected_removed="models/marts/customer_totals.sql", + expected_file="models/reporting/customer_totals.sql", + expected_fragments={}, + expected_relation="customer_totals", + ), + ], + ids=lambda case: case.description, +) +def test_given_model_refactor_when_applied_then_project_tests_and_builds( + tmp_path: Path, test_case: ModelRefactorE2ETestCase +) -> None: + """Every reference and fixture follows the model, and the edited project really runs.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path, files=test_case.extra_files) + + result: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.command) + tested: subprocess.CompletedProcess[str] = sqb(project_dir, "test") + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert result.returncode == 0, result.stdout + result.stderr + assert "Compiled: ok" in result.stdout + assert not (project_dir / test_case.expected_removed).exists() + assert (project_dir / test_case.expected_file).is_file() + files: dict[str, str] = project_text(project_dir) + assert fragments_present(files=files, expected=test_case.expected_fragments), files + assert tested.returncode == 0, tested.stdout + tested.stderr + assert built.returncode == 0, built.stdout + built.stderr + assert relation_type(project_dir=project_dir, name=test_case.expected_relation) is not None + + +@pytest.mark.parametrize( + "test_case", + [ + DryRunE2ETestCase( + description="model rename", + command=("rename", "model:stg_orders", "stg_order_lines", "--dry-run"), + expected_paths=frozenset( + { + "models/staging/stg_order_lines.sql", + "models/marts/fact_orders.sql", + "tests/unit/test_fact_orders.sql", + } + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_dry_run_json_when_refactoring_then_nothing_is_written( + tmp_path: Path, test_case: DryRunE2ETestCase +) -> None: + """A verified dry run reports its edits as JSON on stdout and progress on stderr.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path) + before: dict[str, str] = project_text(project_dir) + + result: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.command, "--json") + code: int + payload: dict[str, Any] + code, payload = sqb_json(project_dir, *test_case.command) + + assert result.returncode == 0, result.stderr + assert "Compiling project" in result.stderr + assert code == 0 + assert payload["status"] == "dry_run" + assert payload["files_changed"] == 0 + assert payload["compile"]["ok"] is True + assert {item["path"] for item in payload["files"]} == test_case.expected_paths + assert project_text(project_dir) == before + + +@pytest.mark.parametrize( + "test_case", + [ + ModelMigrationE2ETestCase( + description="unfingerprintable table gains migrate_from", + materialized="table", + udf=True, + expected_declaration="migrate_from order_history", + expected_migrate_from_count=1, + expected_old_name_type="VIEW", + expected_order_ids=(3, 4), + ), + ModelMigrationE2ETestCase( + description="unfingerprintable view gains migrate_from", + materialized="view", + udf=True, + expected_declaration="migrate_from order_history", + expected_migrate_from_count=1, + expected_old_name_type="VIEW", + expected_order_ids=(3, 4), + ), + ModelMigrationE2ETestCase( + description="unfingerprintable incremental keeps its history", + materialized="incremental", + udf=True, + expected_declaration="migrate_from order_history", + expected_migrate_from_count=1, + expected_old_name_type="VIEW", + expected_order_ids=(1, 2, 3, 4), + ), + ModelMigrationE2ETestCase( + description="unchanged definition is left to automatic discovery", + materialized="incremental", + udf=False, + expected_declaration="", + expected_migrate_from_count=0, + expected_old_name_type="VIEW", + expected_order_ids=(1, 2, 3, 4), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_model_when_renamed_then_build_migrates_its_relation( + tmp_path: Path, test_case: ModelMigrationE2ETestCase +) -> None: + """The rename declares migrate_from exactly when discovery would miss the old relation.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, + files=order_history_files(materialized=test_case.materialized, udf=test_case.udf), + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + result: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:order_history", "order_ledger" + ) + header: str = (project_dir / "models/marts/order_ledger.sql").read_text(encoding="utf-8") + load_raw_orders(project_dir=project_dir, order_ids=(3, 4)) + second: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert first.returncode == 0, first.stdout + first.stderr + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_declaration in header + assert header.count("migrate_from") == test_case.expected_migrate_from_count, header + assert second.returncode == 0, second.stdout + second.stderr + assert relation_type(project_dir=project_dir, name="order_history") == ( + test_case.expected_old_name_type + ) + assert order_ids(project_dir=project_dir, name="order_ledger") == test_case.expected_order_ids diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/__init__.py b/tests/unit/src/sqlbuild/compiler/refactoring/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/__init__.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py new file mode 100644 index 0000000000..8a1b6f9020 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ColumnEditTestCase: + description: str + sql: str + cascade: bool + expected_sql: str + expected_manual: str = "" + expected_passes_through: bool = False + + +@dataclass(frozen=True) +class HeaderEditTestCase: + description: str + contents: str + expected_contents: str + + +@dataclass(frozen=True) +class IdentifierSitesTestCase: + description: str + text: str + expected_offsets: tuple[int, ...] + + +@dataclass(frozen=True) +class ApplyEditsTestCase: + description: str + text: str + spans: tuple[tuple[int, int, str], ...] + expected_text: str + + +@dataclass(frozen=True) +class ApplyEditsErrorTestCase: + description: str + text: str + spans: tuple[tuple[int, int, str], ...] + expected_error: type[Exception] + + +@dataclass(frozen=True) +class CommitChangesTestCase: + description: str + files: dict[str, str] + current_files: dict[str, str] + locked_directory: str + spans: tuple[tuple[int, int, str], ...] + edited_path: str + moved_from: str + moved_to: str + expected_error: type[Exception] + expected_files: dict[str, str] diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py new file mode 100644 index 0000000000..93d55e84ac --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py @@ -0,0 +1,108 @@ +"""Shared builders for refactoring helper tests.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.refactoring._helpers.column_references import ( + analyze_column, + consumer_edits, +) +from sqlbuild.compiler.refactoring._helpers.header_edits import ( + add_column_entry_edit, + column_config_edits, + column_entry_edits, + header_tokens, +) +from sqlbuild.compiler.refactoring._helpers.sql_sites import analysis_sql +from sqlbuild.compiler.refactoring._helpers.text_edits import text_edit +from sqlbuild.compiler.refactoring.models import ( + AnalysisSql, + BodyContext, + BodyEdits, + ColumnFacts, + ColumnQuery, + HeaderToken, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import EditKind + +DIALECT: str = "duckdb" +COLUMNS: dict[tuple[str, str], tuple[str, ...]] = { + ("ref", "fact_orders"): ("order_id", "customer_id", "amount"), + ("ref", "customers"): ("customer_id", "amount"), +} + + +def plan_consumer(*, sql: str, cascade: bool) -> BodyEdits: + """Plan renaming fact_orders.amount to revenue in one consumer query.""" + + analysis: AnalysisSql = analysis_sql(text=sql, dialect=DIALECT) + facts: ColumnFacts = analyze_column( + analysis=analysis, + dialect=DIALECT, + columns=COLUMNS, + query=ColumnQuery(column="amount", target_tables=analysis.tables[("ref", "fact_orders")]), + ) + return consumer_edits( + facts=facts, + context=BodyContext( + path="models/consumer.sql", + contents=sql, + map_span=lambda start, end: (start, end), + locate=lambda offset: offset, + fallback_offset=0, + ), + old="amount", + new="revenue", + cascade_root=cascade, + root_stars_pass=cascade, + ) + + +def migrated_column_edits(contents: str) -> tuple[TextEdit, ...]: + """Plan the header edits that rename amount to revenue with a column migration.""" + + tokens: tuple[HeaderToken, ...] = header_tokens(contents=contents) or () + added: tuple[TextEdit, ...] = tuple( + filter( + None, + (add_column_entry_edit(contents=contents, tokens=tokens, new="revenue", old="amount"),), + ) + ) + return ( + *column_config_edits(contents=contents, tokens=tokens, old="amount", new="revenue"), + *( + column_entry_edits( + contents=contents, tokens=tokens, old="amount", new="revenue", migrate=True + ) + or added + ), + ) + + +def span_edits(*, text: str, spans: tuple[tuple[int, int, str], ...]) -> tuple[TextEdit, ...]: + """Build column edits for (start, end, replacement) spans.""" + + return tuple( + text_edit(text=text, start=start, end=end, replacement=replacement, kind=EditKind.COLUMN) + for start, end, replacement in spans + ) + + +def write_tree(*, root: Path, files: dict[str, str]) -> None: + """Write project-relative files under a root.""" + + relative: str + contents: str + for relative, contents in files.items(): + _ = (root / relative).write_text(contents, encoding="utf-8") + + +def read_tree(*, root: Path) -> dict[str, str]: + """Read every file under a root, keyed by its relative path.""" + + return { + path.relative_to(root).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(root.rglob("*.sql")) + } diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py new file mode 100644 index 0000000000..d8dc0fc580 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py @@ -0,0 +1,104 @@ +"""Tests for column-rename edit planning inside one SQL body.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.refactoring._helpers.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring.models import BodyEdits +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ColumnEditTestCase +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import plan_consumer + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnEditTestCase( + description="bare projection keeps its output name", + sql='SELECT o.amount FROM __ref("fact_orders") AS o', + cascade=False, + expected_sql='SELECT o.revenue AS amount FROM __ref("fact_orders") AS o', + ), + ColumnEditTestCase( + description="cascade renames a bare projection's output", + sql='SELECT o.amount FROM __ref("fact_orders") AS o', + cascade=True, + expected_sql='SELECT o.revenue FROM __ref("fact_orders") AS o', + expected_passes_through=True, + ), + ColumnEditTestCase( + description="aliased projection keeps its alias", + sql='SELECT o.amount AS total FROM __ref("fact_orders") AS o', + cascade=True, + expected_sql='SELECT o.revenue AS total FROM __ref("fact_orders") AS o', + ), + ColumnEditTestCase( + description="filter reference is renamed", + sql='SELECT order_id FROM __ref("fact_orders") WHERE amount > 0', + cascade=False, + expected_sql='SELECT order_id FROM __ref("fact_orders") WHERE revenue > 0', + ), + ColumnEditTestCase( + description="same-named column of another relation is untouched", + sql=( + 'SELECT c.amount FROM __ref("fact_orders") AS o ' + 'JOIN __ref("customers") AS c ON o.customer_id = c.customer_id' + ), + cascade=False, + expected_sql=( + 'SELECT c.amount FROM __ref("fact_orders") AS o ' + 'JOIN __ref("customers") AS c ON o.customer_id = c.customer_id' + ), + ), + ColumnEditTestCase( + description="star through a CTE is followed", + sql='WITH c AS (SELECT * FROM __ref("fact_orders")) SELECT c.amount FROM c', + cascade=False, + expected_sql=( + 'WITH c AS (SELECT * FROM __ref("fact_orders")) SELECT c.revenue AS amount FROM c' + ), + ), + ColumnEditTestCase( + description="root star needs cascade", + sql='SELECT * FROM __ref("fact_orders")', + cascade=False, + expected_sql='SELECT * FROM __ref("fact_orders")', + expected_manual="rerun with --cascade", + ), + ColumnEditTestCase( + description="root star passes through under cascade", + sql='SELECT * FROM __ref("fact_orders")', + cascade=True, + expected_sql='SELECT * FROM __ref("fact_orders")', + expected_passes_through=True, + ), + ColumnEditTestCase( + description="USING join is left for the author", + sql=( + 'SELECT o.order_id FROM __ref("fact_orders") AS o ' + 'JOIN __ref("customers") AS c USING (amount)' + ), + cascade=False, + expected_sql=( + 'SELECT o.order_id FROM __ref("fact_orders") AS o ' + 'JOIN __ref("customers") AS c USING (amount)' + ), + expected_manual="USING or NATURAL join", + ), + ], + ids=lambda case: case.description, +) +def test_given_consumer_sql_when_planning_column_rename_then_edits_match( + test_case: ColumnEditTestCase, +) -> None: + result: BodyEdits = plan_consumer(sql=test_case.sql, cascade=test_case.cascade) + + assert apply_text_edits(text=test_case.sql, edits=result.edits) == test_case.expected_sql + assert result.passes_through == test_case.expected_passes_through + reasons: tuple[str, ...] = tuple(location.reason for location in result.manual) + assert bool(reasons) == bool(test_case.expected_manual), reasons + assert all(test_case.expected_manual in reason for reason in reasons), reasons + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py new file mode 100644 index 0000000000..397a8827e2 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py @@ -0,0 +1,130 @@ +"""Tests for token-exact MODEL header edits.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.refactoring._helpers.header_edits import ( + insert_header_entry_edit, + model_name_header_edits, +) +from sqlbuild.compiler.refactoring._helpers.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring.models import TextEdit +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import HeaderEditTestCase +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import migrated_column_edits + +_BODY: str = "SELECT order_id, amount FROM orders\n" + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderEditTestCase( + description="declared column gains migrate_from before its metadata", + contents=f"MODEL (\n columns (\n amount (nullable false),\n ),\n);\n{_BODY}", + expected_contents=( + "MODEL (\n columns (\n revenue (migrate_from amount, nullable false),\n" + f" ),\n);\n{_BODY}" + ), + ), + HeaderEditTestCase( + description="bare declared column gains a migrate_from block", + contents=f"MODEL (\n columns (amount),\n);\n{_BODY}", + expected_contents=f"MODEL (\n columns (revenue (migrate_from amount)),\n);\n{_BODY}", + ), + HeaderEditTestCase( + description="undeclared column gets a new columns entry", + contents=f"MODEL (\n materialized incremental,\n);\n{_BODY}", + expected_contents=( + "MODEL (\n columns (revenue (migrate_from amount)),\n" + f" materialized incremental,\n);\n{_BODY}" + ), + ), + HeaderEditTestCase( + description="column-valued config follows the column", + contents=f"MODEL (\n unique_key [order_id, amount],\n);\n{_BODY}", + expected_contents=( + "MODEL (\n columns (revenue (migrate_from amount)),\n" + f" unique_key [order_id, revenue],\n);\n{_BODY}" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_header_when_renaming_migrated_column_then_header_is_rewritten( + test_case: HeaderEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = migrated_column_edits(test_case.contents) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderEditTestCase( + description="cursor_inputs keys and relationships targets follow the model", + contents=( + "MODEL (\n cursor_inputs (\n fact_orders ordered_at,\n ),\n" + " columns (\n order_id (audits [relationships (to fact_orders, " + "field order_id)]),\n ),\n);\n" + f"{_BODY}" + ), + expected_contents=( + "MODEL (\n cursor_inputs (\n order_facts ordered_at,\n ),\n" + " columns (\n order_id (audits [relationships (to order_facts, " + "field order_id)]),\n ),\n);\n" + f"{_BODY}" + ), + ), + HeaderEditTestCase( + description="descriptions naming the model are left alone", + contents=f'MODEL (\n description "fact_orders summary",\n);\n{_BODY}', + expected_contents=f'MODEL (\n description "fact_orders summary",\n);\n{_BODY}', + ), + ], + ids=lambda case: case.description, +) +def test_given_header_when_renaming_model_then_references_follow( + test_case: HeaderEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = model_name_header_edits( + contents=test_case.contents, old="fact_orders", new="order_facts" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + HeaderEditTestCase( + description="multi-line header gets its own line", + contents=f"MODEL (\n materialized table,\n);\n{_BODY}", + expected_contents=( + f"MODEL (\n migrate_from fact_orders,\n materialized table,\n);\n{_BODY}" + ), + ), + HeaderEditTestCase( + description="empty header is expanded", + contents=f"MODEL ();\n{_BODY}", + expected_contents=f"MODEL (\n migrate_from fact_orders,\n);\n{_BODY}", + ), + ], + ids=lambda case: case.description, +) +def test_given_header_when_adding_model_migration_then_entry_is_inserted( + test_case: HeaderEditTestCase, +) -> None: + edit: TextEdit | None = insert_header_entry_edit( + contents=test_case.contents, + entry="migrate_from fact_orders", + display="migrate_from fact_orders", + ) + + assert edit is not None + assert apply_text_edits(text=test_case.contents, edits=(edit,)) == test_case.expected_contents + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py new file mode 100644 index 0000000000..a9448ea4b5 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py @@ -0,0 +1,92 @@ +"""Tests for authored-text edit application and identifier matching.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.refactoring._helpers.text_edits import apply_text_edits, identifier_sites +from sqlbuild.compiler.refactoring.exceptions import RefactorEditError +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( + ApplyEditsErrorTestCase, + ApplyEditsTestCase, + IdentifierSitesTestCase, +) +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import span_edits + + +@pytest.mark.parametrize( + "test_case", + [ + ApplyEditsTestCase( + description="identical duplicate edits apply once", + text="SELECT amount FROM orders", + spans=((7, 13, "revenue"), (7, 13, "revenue")), + expected_text="SELECT revenue FROM orders", + ), + ApplyEditsTestCase( + description="insertions and replacements apply back to front", + text="SELECT amount FROM orders", + spans=((7, 13, "revenue"), (13, 13, " AS amount")), + expected_text="SELECT revenue AS amount FROM orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_edits_when_applying_then_text_is_rewritten(test_case: ApplyEditsTestCase) -> None: + assert ( + apply_text_edits( + text=test_case.text, edits=span_edits(text=test_case.text, spans=test_case.spans) + ) + == test_case.expected_text + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ApplyEditsErrorTestCase( + description="overlapping replacements", + text="SELECT amount FROM orders", + spans=((7, 13, "revenue"), (10, 18, "x")), + expected_error=RefactorEditError, + ) + ], + ids=lambda case: case.description, +) +def test_given_overlapping_edits_when_applying_then_refuses( + test_case: ApplyEditsErrorTestCase, +) -> None: + with pytest.raises(test_case.expected_error): + _ = apply_text_edits( + text=test_case.text, edits=span_edits(text=test_case.text, spans=test_case.spans) + ) + + +@pytest.mark.parametrize( + "test_case", + [ + IdentifierSitesTestCase( + description="matches whole unquoted identifiers only", + text="__ref__orders AS (SELECT 1), __ref__orders_daily AS (SELECT 2)", + expected_offsets=(0,), + ), + IdentifierSitesTestCase( + description="skips comments and string literals", + text="-- __ref__orders\nSELECT '__ref__orders', __ref__orders", + expected_offsets=(41,), + ), + ], + ids=lambda case: case.description, +) +def test_given_text_when_finding_identifiers_then_code_sites_match( + test_case: IdentifierSitesTestCase, +) -> None: + sites: tuple[tuple[int, int, str], ...] = identifier_sites( + text=test_case.text, names=frozenset({"__ref__orders"}) + ) + + assert tuple(start for start, _, _ in sites) == test_case.expected_offsets + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py new file mode 100644 index 0000000000..cb29945bb3 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py @@ -0,0 +1,76 @@ +"""Tests for committing verified refactoring edits into a project.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.refactoring._helpers.workspace import commit_changes +from sqlbuild.compiler.refactoring.exceptions import RefactorWriteError +from sqlbuild.compiler.refactoring.models import FileChange +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( + CommitChangesTestCase, +) +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import ( + read_tree, + span_edits, + write_tree, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CommitChangesTestCase( + description="a failed write restores every file", + files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, + current_files={}, + locked_directory="locked", + spans=((7, 13, "revenue"),), + edited_path="a.sql", + moved_from="m.sql", + moved_to="locked/m.sql", + expected_error=PermissionError, + expected_files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, + ), + CommitChangesTestCase( + description="a file edited since planning stops the write", + files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, + current_files={"a.sql": "SELECT amount, 2"}, + locked_directory="open", + spans=((7, 13, "revenue"),), + edited_path="a.sql", + moved_from="m.sql", + moved_to="open/m.sql", + expected_error=RefactorWriteError, + expected_files={"a.sql": "SELECT amount, 2", "m.sql": "SELECT 1"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_commit_when_writing_then_project_is_unchanged( + tmp_path: Path, test_case: CommitChangesTestCase +) -> None: + write_tree(root=tmp_path, files={**test_case.files, **test_case.current_files}) + (tmp_path / test_case.locked_directory).mkdir() + (tmp_path / "locked").mkdir(exist_ok=True) + (tmp_path / "locked").chmod(0o500) + changes: tuple[FileChange, ...] = ( + FileChange( + path=test_case.edited_path, + original_path=test_case.edited_path, + edits=span_edits(text=test_case.files[test_case.edited_path], spans=test_case.spans), + ), + FileChange(path=test_case.moved_to, original_path=test_case.moved_from), + ) + + with pytest.raises(test_case.expected_error): + _ = commit_changes(project_dir=tmp_path, originals=test_case.files, changes=changes) + (tmp_path / "locked").chmod(0o700) + + assert read_tree(root=tmp_path) == test_case.expected_files + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 515bb34c915d35bbd754a6a5c6500a8f4dd740f9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:00:56 +0100 Subject: [PATCH 04/16] feat(plan): point automatically discovered renames at sqb rename --- src/sqlbuild/compiler/planner/_helpers/migrations/display.py | 3 +++ src/sqlbuild/compiler/planner/constants.py | 4 ++++ .../commands/main/model_migrations/test_identity_handover.py | 2 ++ .../planner/main/pre_build/test_pre_build_work_display.py | 4 +++- 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py index 8982d3f1b9..95b8c93128 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py @@ -11,6 +11,7 @@ MigrationDiscovery, OldNameViewAction, ) +from sqlbuild.compiler.planner.constants import MANUAL_RENAME_HINT from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanEntry, ModelMigrationPlanEntry, @@ -240,6 +241,8 @@ def _migration_lines( ) if entry.discovery != MigrationDiscovery.MANUAL: rows.append(_property_row(label="discovery", value=entry.discovery.value, style=style)) + if entry.discovery == MigrationDiscovery.AUTOMATIC and entry.completed_at is None: + rows.append(_property_row(label="hint", value=MANUAL_RENAME_HINT, style=style)) if entry.completed_at is not None: rows.append( _property_row( diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 991de00b28..008dbee04a 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -83,3 +83,7 @@ SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str = "valid_to" SOURCE_INPUT_FUNCTION: str = "__source" REF_INPUT_FUNCTION: str = "__ref" +MANUAL_RENAME_HINT: str = ( + "matched by unchanged definition; sqb rename model: also rewrites references " + "and adds migrate_from when discovery cannot match" +) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py index e9aaf3069b..e20b441280 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py @@ -153,6 +153,8 @@ def test_given_renamed_chain_with_full_replay_when_building_then_migrated_histor "├── customer_orders_enriched migrate main.orders_enriched -> " "main.customer_orders_enriched\n ├── transfer recreate (view)\n" " ├── discovery automatic\n" + " ├── hint matched by unchanged definition; sqb rename model: " + "also rewrites references and adds migrate_from when discovery cannot match\n" " └── old name main.orders_enriched\n" " └── left for janitor old_name_view false" ), diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index 583a7ca25a..9c3588affa 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -138,7 +138,9 @@ def test_given_migration_decision_when_formatting_then_applies_colour_roles( "└── daily_revenue migrate prod.revenue -> prod.daily_revenue\n" " ├── compatibility compatible\n" " ├── transfer physical copy, promote by transactional rename\n" - " └── discovery automatic" + " ├── discovery automatic\n" + " └── hint matched by unchanged definition; sqb rename model: " + "also rewrites references and adds migrate_from when discovery cannot match" ), ), ), From a3198a7853f1a8bd3b845b733b5b947d15e35a8a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:00:56 +0100 Subject: [PATCH 05/16] docs: document sqb rename and sqb mv for users and agents --- src/sqlbuild/.agents/skills/sqlbuild/SKILL.md | 13 ++- .../sqlbuild/references/docs/CONTENTS.md | 1 + .../sqlbuild/references/docs/cli/rename.md | 105 ++++++++++++++++++ .../docs/concepts/models/column-migrations.md | 2 + .../docs/concepts/models/migrations.md | 4 +- website/src/content/docs/docs/cli/rename.mdx | 102 +++++++++++++++++ .../concepts/models/column-migrations.mdx | 2 + .../docs/docs/concepts/models/migrations.mdx | 4 +- website/src/sidebar.json | 1 + 9 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md create mode 100644 website/src/content/docs/docs/cli/rename.mdx diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index 19c6124540..dc4bffac36 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -1,6 +1,6 @@ --- name: sqlbuild -description: ALWAYS load this skill when doing ANY SQLBuild work - models, sources, seeds, macros, enums, constants, tests, audits, scenarios, incremental and microbatch models, configuration, CLI, adapters, or dbt interop. Covers the verify loop (compile, plan, build), comparing data or two queries with sqb diff, inspecting macro/enum/constant visibility with sqb scope, impact analysis with sqb lineage, and where to find exact syntax. +description: ALWAYS load this skill when doing ANY SQLBuild work - models, sources, seeds, macros, enums, constants, tests, audits, scenarios, incremental and microbatch models, configuration, CLI, adapters, or dbt interop. Covers the verify loop (compile, plan, build), comparing data or two queries with sqb diff, inspecting macro/enum/constant visibility with sqb scope, impact analysis with sqb lineage, renaming or moving models and columns with sqb rename and sqb mv, and where to find exact syntax. --- # SQLBuild @@ -78,6 +78,7 @@ the trigger applies. | Know what a model can see: which macros, enums and constants are visible, used, or unavailable | `sqb scope model:` | | Fix "unknown macro/enum/constant", or decide where a declaration should live | `sqb scope model: --explain macro:` | | Preview whether moving a file breaks declaration visibility | `sqb scope model: --as-path ` | +| Rename a model, move a model file, or rename a column | `sqb rename model: `, `sqb mv model: `, `sqb rename column:. ` | | Find what a model depends on, or what breaks if it changes | `sqb lineage --direction both` | | Trace where a column comes from, or who consumes it before renaming or dropping it | `sqb lineage . --direction downstream` | | See which intermediate models a SQL test will really execute | `sqb test --select --inspect` | @@ -142,6 +143,16 @@ functions, but not in ordinary `MODEL()` config fields. Placement decides visibi **Snapshots** (SCD type 2) use `timestamp` or `check` strategies; full refreshes are policy-gated. +**Renaming and moving.** Never rename or move a model or column by editing files by hand. `sqb +rename` and `sqb mv` rewrite every `__ref`, fixture CTE, header reference and column use from +compiler facts, add `migrate_from` exactly where the warehouse history would otherwise be lost, +verify the edited project compiles in a scratch copy, and write nothing if any step fails. Run +with `--dry-run` first. A column rename is one step by default (downstream models keep their output +names via `AS `); `--cascade` renames pass-through outputs downstream too. When the command +refuses, fix the listed locations (usually a macro call or `SELECT *`) and rerun; use +`--allow-manual` only when the user agrees to finish those locations by hand. Then `sqb plan` to +see the migration. Read [references/docs/cli/rename.md](references/docs/cli/rename.md). + **Rules** are compile-time project checks (`sqb rules list`, `sqb rules show `). `sqb format` rewrites sources deterministically; `sqb format --check` only reports. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 0e700532a3..db76c4d019 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -111,6 +111,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb format](cli/format.md) (`cli/format`) - Apply canonical SQLBuild SQL formatting. - [sqb contract](cli/contract.md) (`cli/contract`) - Compare or generate repository contracts from physical warehouse schemas. - [scope](cli/scope.md) (`cli/scope`) - Inspect declaration visibility, usage, placement, and move impact offline. +- [rename and mv](cli/rename.md) (`cli/rename`) - Rename or move a model, or rename a column, and update every reference in one verified step. - [sqb rules](cli/rules.md) (`cli/rules`) - List, inspect, or run compiler-integrated Rules and generate project guidance. - [sqb plan](cli/plan.md) (`cli/plan`) - Preview what SQLBuild will do before executing. - [sqb build](cli/build.md) (`cli/build`) - Compile, plan, and execute the selected build lifecycle. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md new file mode 100644 index 0000000000..5fe4bb8ff8 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -0,0 +1,105 @@ + + +# rename and mv + +> Rename or move a model, or rename a column, and update every reference in one verified step. + +Online: https://sqlbuild.com/docs/cli/rename/ + +`sqb rename` and `sqb mv` change a name everywhere the compiler knows it is used, instead of leaving you to find every `__ref`, fixture, and header by hand. Every edit comes from compiled facts, not text search: a column rename follows the columns each query actually binds, so a same-named column of another table is left alone. + +Nothing is written until the edited project compiles. The command applies its edits to a scratch copy of the project, compiles it the way `sqb compile` does, and only then writes the files. If anything cannot be rewritten safely, the command lists those locations and writes nothing. + +## Usage + +```bash +sqb rename model: [flags] +sqb mv model: [flags] +sqb rename column:. [--cascade] [flags] +``` + +```bash +sqb rename model:stg_orders stg_order_lines +sqb mv model:fact_orders models/reporting/ # keeps the name +sqb mv model:fact_orders models/reporting/order_facts.sql # moves and renames +sqb rename column:stg_orders.amount revenue +sqb rename column:stg_orders.amount revenue --cascade +``` + +A model rename keeps the file in its folder and renames it to `.sql`. `sqb mv` takes a new file path, which also sets the model name, or a folder ending in `/`, which keeps it. + +## What changes + +Renaming or moving a model rewrites: + +- every `__ref("")` in models, tests, scenarios, audits, hooks, and SQL functions; +- `__ref__` and `__expected__` fixture CTEs in tests and scenarios; +- `cursor_inputs` keys and `relationships (to )` audits in `MODEL` headers. + +Renaming a column rewrites the model that produces it and every query that reads it: + +- the producing projection, as `amount AS revenue`, or the existing alias; +- every downstream reference, including filters, joins, grouping, and ordering; +- column declarations and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; +- fixture columns of `__ref__` and `__expected__` CTEs. + +By default a column rename is one step: downstream models keep their output names, so `SELECT o.amount` becomes `SELECT o.revenue AS amount`. With `--cascade`, a downstream model that passes the column through unchanged, as a bare column, a same-named alias, or `SELECT *`, renames its output too, and the rename continues through its own consumers. + +## History + +A renamed model keeps its warehouse history through a [model migration](../concepts/models/migrations.md). When SQLBuild's automatic rename discovery would match the renamed model by its unchanged definition, nothing is added. When it would not, for example because the model calls a UDF or its materialization family changes, the command adds `migrate_from ` to the model header. The next `sqb build` migrates the relation and leaves a compatibility view at the old name. + +A renamed column of an incremental or snapshot model gets ` (migrate_from )` in its `columns` block, so the next build renames it in place as a [column migration](../concepts/models/column-migrations.md). Table and view models are rebuilt with the new column anyway; they get the declaration only when the model itself declares `migrate_from`. + +The command refuses a rename while the model, or the column, still declares `migrate_from` from an earlier rename. Build it on every target and remove the declaration first. + +## Refusals + +The command refuses, and writes nothing, when: + +- the new name already belongs to another model or column, or the destination file exists; +- a move would take the model out of the scope of a macro, enum, or constant it uses (see [`sqb scope`](scope.md)); +- a move would leave no model in the old schema, so the old relation could not be found; +- a reference is produced by a macro, or a column is read inside SQL that a macro generates; +- a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; +- a downstream model's `SELECT *` would silently rename its output, without `--cascade`; +- Python code or Python node SQL strings name the model or column. + +Each location is listed with its file, line, and column. Edit those locations, or pass the model or column to the macro as an argument, and run the command again. + +`--allow-manual` applies the safe edits and lists the remaining locations for you to finish. The edited project must still compile; if the remaining locations break it, the command writes nothing and prints the compile errors. + +## Flags + +| Flag | Description | +|------|-------------| +| `--cascade` | For column renames, rename the output of every downstream model that passes the column through. | +| `--allow-manual` | Apply the safe edits even when some locations must be edited by hand. | +| `--dry-run` | Print every edit and verify the edited project compiles, without writing files. | +| `--json` | Print the result as JSON on stdout. Progress goes to stderr. | + +## Output + +```text +Rename model stg_orders -> stg_order_lines +├── models/marts/fact_orders.sql +│ └── 24:13 reference __ref("stg_orders") -> __ref("stg_order_lines") +├── models/staging/stg_order_lines.sql (moved from models/staging/stg_orders.sql) +└── tests/unit/test_fact_orders.sql + └── 4:1 fixture __ref__stg_orders -> __ref__stg_order_lines +Compiled: ok, 3 files changed +``` + +With `--json`, the result has `operation`, `target`, `new_name`, `destination`, `status` (`applied`, `dry_run`, `refused`, or `compile_failed`), `files` with every edit, `migrations`, `renamed_columns`, `manual`, `blocking`, and `compile.errors`. + +## Exit codes + +| Code | Meaning | +|------|---------| +| `0` | Applied, or a dry run that compiles. | +| `1` | Refused, the edited project did not compile, or the request was invalid. No files were changed. | + +## Limitations + +- `relationships` audits declared in source or seed YAML files, descriptions, and comments are not rewritten. +- Queries outside the project that read the old relation keep working through the old-name compatibility view until it expires. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md index 85eeb0161a..184babe8c6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md @@ -21,6 +21,8 @@ Column migrations apply to incremental models, including microbatch models, and ## Declaring a rename +[`sqb rename column:.`](../../cli/rename.md) renames the column in the model and every query that reads it, and adds the `migrate_from` declaration below for you. + Add `migrate_from` to the renamed column in the model header, naming the old column: ```sql diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index ed7efd0806..c6b898e9bc 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -23,7 +23,9 @@ Table and view models can declare a migration too. They hold no history, so they ## Declaring a migration -Add `migrate_from` to the renamed model's header, naming the old model: +[`sqb rename`](../../cli/rename.md) renames a model and every reference to it, and adds `migrate_from` only when [automatic discovery](#automatic-discovery) would not find the old relation. + +To declare a migration by hand, add `migrate_from` to the renamed model's header, naming the old model: ```sql MODEL ( diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx new file mode 100644 index 0000000000..e51828e58f --- /dev/null +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -0,0 +1,102 @@ +--- +title: "rename and mv" +description: "Rename or move a model, or rename a column, and update every reference in one verified step." +--- + +`sqb rename` and `sqb mv` change a name everywhere the compiler knows it is used, instead of leaving you to find every `__ref`, fixture, and header by hand. Every edit comes from compiled facts, not text search: a column rename follows the columns each query actually binds, so a same-named column of another table is left alone. + +Nothing is written until the edited project compiles. The command applies its edits to a scratch copy of the project, compiles it the way `sqb compile` does, and only then writes the files. If anything cannot be rewritten safely, the command lists those locations and writes nothing. + +## Usage + +```bash +sqb rename model: [flags] +sqb mv model: [flags] +sqb rename column:. [--cascade] [flags] +``` + +```bash +sqb rename model:stg_orders stg_order_lines +sqb mv model:fact_orders models/reporting/ # keeps the name +sqb mv model:fact_orders models/reporting/order_facts.sql # moves and renames +sqb rename column:stg_orders.amount revenue +sqb rename column:stg_orders.amount revenue --cascade +``` + +A model rename keeps the file in its folder and renames it to `.sql`. `sqb mv` takes a new file path, which also sets the model name, or a folder ending in `/`, which keeps it. + +## What changes + +Renaming or moving a model rewrites: + +- every `__ref("")` in models, tests, scenarios, audits, hooks, and SQL functions; +- `__ref__` and `__expected__` fixture CTEs in tests and scenarios; +- `cursor_inputs` keys and `relationships (to )` audits in `MODEL` headers. + +Renaming a column rewrites the model that produces it and every query that reads it: + +- the producing projection, as `amount AS revenue`, or the existing alias; +- every downstream reference, including filters, joins, grouping, and ordering; +- column declarations and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; +- fixture columns of `__ref__` and `__expected__` CTEs. + +By default a column rename is one step: downstream models keep their output names, so `SELECT o.amount` becomes `SELECT o.revenue AS amount`. With `--cascade`, a downstream model that passes the column through unchanged, as a bare column, a same-named alias, or `SELECT *`, renames its output too, and the rename continues through its own consumers. + +## History + +A renamed model keeps its warehouse history through a [model migration](/docs/concepts/models/migrations). When SQLBuild's automatic rename discovery would match the renamed model by its unchanged definition, nothing is added. When it would not, for example because the model calls a UDF or its materialization family changes, the command adds `migrate_from ` to the model header. The next `sqb build` migrates the relation and leaves a compatibility view at the old name. + +A renamed column of an incremental or snapshot model gets ` (migrate_from )` in its `columns` block, so the next build renames it in place as a [column migration](/docs/concepts/models/column-migrations). Table and view models are rebuilt with the new column anyway; they get the declaration only when the model itself declares `migrate_from`. + +The command refuses a rename while the model, or the column, still declares `migrate_from` from an earlier rename. Build it on every target and remove the declaration first. + +## Refusals + +The command refuses, and writes nothing, when: + +- the new name already belongs to another model or column, or the destination file exists; +- a move would take the model out of the scope of a macro, enum, or constant it uses (see [`sqb scope`](/docs/cli/scope)); +- a move would leave no model in the old schema, so the old relation could not be found; +- a reference is produced by a macro, or a column is read inside SQL that a macro generates; +- a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; +- a downstream model's `SELECT *` would silently rename its output, without `--cascade`; +- Python code or Python node SQL strings name the model or column. + +Each location is listed with its file, line, and column. Edit those locations, or pass the model or column to the macro as an argument, and run the command again. + +`--allow-manual` applies the safe edits and lists the remaining locations for you to finish. The edited project must still compile; if the remaining locations break it, the command writes nothing and prints the compile errors. + +## Flags + +| Flag | Description | +|------|-------------| +| `--cascade` | For column renames, rename the output of every downstream model that passes the column through. | +| `--allow-manual` | Apply the safe edits even when some locations must be edited by hand. | +| `--dry-run` | Print every edit and verify the edited project compiles, without writing files. | +| `--json` | Print the result as JSON on stdout. Progress goes to stderr. | + +## Output + +```text +Rename model stg_orders -> stg_order_lines +├── models/marts/fact_orders.sql +│ └── 24:13 reference __ref("stg_orders") -> __ref("stg_order_lines") +├── models/staging/stg_order_lines.sql (moved from models/staging/stg_orders.sql) +└── tests/unit/test_fact_orders.sql + └── 4:1 fixture __ref__stg_orders -> __ref__stg_order_lines +Compiled: ok, 3 files changed +``` + +With `--json`, the result has `operation`, `target`, `new_name`, `destination`, `status` (`applied`, `dry_run`, `refused`, or `compile_failed`), `files` with every edit, `migrations`, `renamed_columns`, `manual`, `blocking`, and `compile.errors`. + +## Exit codes + +| Code | Meaning | +|------|---------| +| `0` | Applied, or a dry run that compiles. | +| `1` | Refused, the edited project did not compile, or the request was invalid. No files were changed. | + +## Limitations + +- `relationships` audits declared in source or seed YAML files, descriptions, and comments are not rewritten. +- Queries outside the project that read the old relation keep working through the old-name compatibility view until it expires. diff --git a/website/src/content/docs/docs/concepts/models/column-migrations.mdx b/website/src/content/docs/docs/concepts/models/column-migrations.mdx index 7cc03397f1..f9c8c8c628 100644 --- a/website/src/content/docs/docs/concepts/models/column-migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/column-migrations.mdx @@ -9,6 +9,8 @@ Column migrations apply to incremental models, including microbatch models, and ## Declaring a rename +[`sqb rename column:.`](/docs/cli/rename) renames the column in the model and every query that reads it, and adds the `migrate_from` declaration below for you. + Add `migrate_from` to the renamed column in the model header, naming the old column: ```sql diff --git a/website/src/content/docs/docs/concepts/models/migrations.mdx b/website/src/content/docs/docs/concepts/models/migrations.mdx index 31388aec32..d42a7addd4 100644 --- a/website/src/content/docs/docs/concepts/models/migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/migrations.mdx @@ -9,7 +9,9 @@ Table and view models can declare a migration too. They hold no history, so they ## Declaring a migration -Add `migrate_from` to the renamed model's header, naming the old model: +[`sqb rename`](/docs/cli/rename) renames a model and every reference to it, and adds `migrate_from` only when [automatic discovery](#automatic-discovery) would not find the old relation. + +To declare a migration by hand, add `migrate_from` to the renamed model's header, naming the old model: ```sql MODEL ( diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 487d278a8b..98bdf37bb5 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -188,6 +188,7 @@ "docs/cli/format", "docs/cli/contract", "docs/cli/scope", + "docs/cli/rename", "docs/cli/rules", "docs/cli/plan", "docs/cli/build", From 0c9e5d7ef4d00a71d05ef28d65d009c019bfd21a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:23:56 +0100 Subject: [PATCH 06/16] refactor(lint): share the quoted header value scanner with refactoring --- .../compiler/refactoring/_helpers/header_edits.py | 13 ++----------- src/sqlbuild/compiler/refactoring/constants.py | 1 - src/sqlbuild/lint/_helpers/native.py | 8 +++++--- src/sqlbuild/lint/main/quoted_value_end.py | 11 +++++++++++ 4 files changed, 18 insertions(+), 15 deletions(-) create mode 100644 src/sqlbuild/lint/main/quoted_value_end.py diff --git a/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py index 14d7f40aa0..f3a0154e1b 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py @@ -12,7 +12,6 @@ CURSOR_INPUTS_KEY, HEADER_CLOSERS, HEADER_DESCRIPTION_KEY, - HEADER_ESCAPE, HEADER_INDENT, HEADER_KEY_AND_VALUE_TOKENS, HEADER_OPEN_PAREN, @@ -27,6 +26,7 @@ ) from sqlbuild.compiler.refactoring.models import HeaderEntry, HeaderToken, TextEdit from sqlbuild.compiler.refactoring.types import EditKind, HeaderTokenKind +from sqlbuild.lint.main.quoted_value_end import quoted_value_end _VALUE_KINDS: frozenset[HeaderTokenKind] = frozenset({HeaderTokenKind.WORD, HeaderTokenKind.STRING}) @@ -373,16 +373,7 @@ def _is_separator(token: HeaderToken) -> bool: def _token_end(*, contents: str, kind: HeaderTokenKind, value: str, start: int) -> int: if kind != HeaderTokenKind.STRING: return start + len(value) - quote: str = contents[start] - index: int = start + 1 - while index < len(contents): - if contents[index] == HEADER_ESCAPE: - index += 2 - continue - if contents[index] == quote: - return index + 1 - index += 1 - return len(contents) + return quoted_value_end(text=contents, start=start) def _line_indent(*, contents: str, offset: int) -> str: diff --git a/src/sqlbuild/compiler/refactoring/constants.py b/src/sqlbuild/compiler/refactoring/constants.py index d16844106a..7d57ecefc9 100644 --- a/src/sqlbuild/compiler/refactoring/constants.py +++ b/src/sqlbuild/compiler/refactoring/constants.py @@ -75,7 +75,6 @@ HEADER_CLOSERS: frozenset[str] = frozenset({")", "]", "}"}) HEADER_SEPARATOR: str = "," HEADER_OPEN_PAREN: str = "(" -HEADER_ESCAPE: str = "\\" HEADER_DESCRIPTION_KEY: str = "description" HEADER_INDENT: str = " " PARENTHESIZED_EMPTY_TOKENS: int = 2 diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index ed30e0bd9d..a2b7a2684e 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -405,7 +405,7 @@ def _top_level_description_value_span(*, header_text: str) -> tuple[int, int] | while index < len(header_text): character: str = header_text[index] if character in _QUOTE_CHARACTERS: - index = _quoted_value_end(text=header_text, start=index) + index = quoted_value_end(text=header_text, start=index) continue if character in closing_delimiters: delimiters.append(closing_delimiters[character]) @@ -429,7 +429,7 @@ def _top_level_description_value_span(*, header_text: str) -> tuple[int, int] | while quote_start < len(header_text) and header_text[quote_start].isspace(): quote_start += 1 if quote_start < len(header_text) and header_text[quote_start] in _QUOTE_CHARACTERS: - quote_end: int = _quoted_value_end(text=header_text, start=quote_start) + quote_end: int = quoted_value_end(text=header_text, start=quote_start) return quote_start + 1, quote_end - 1 index = word_end continue @@ -437,7 +437,9 @@ def _top_level_description_value_span(*, header_text: str) -> tuple[int, int] | return None -def _quoted_value_end(*, text: str, start: int) -> int: +def quoted_value_end(*, text: str, start: int) -> int: + """Return the offset just past the quoted value that opens at start.""" + quote: str = text[start] index: int = start + 1 while index < len(text): diff --git a/src/sqlbuild/lint/main/quoted_value_end.py b/src/sqlbuild/lint/main/quoted_value_end.py new file mode 100644 index 0000000000..f397b2de2a --- /dev/null +++ b/src/sqlbuild/lint/main/quoted_value_end.py @@ -0,0 +1,11 @@ +"""Public end-offset finder for quoted MODEL header values.""" + +from __future__ import annotations + +from sqlbuild.lint._helpers.native import quoted_value_end as _quoted_value_end + + +def quoted_value_end(*, text: str, start: int) -> int: + """Return the offset just past the quoted value that opens at start.""" + + return _quoted_value_end(text=text, start=start) From 45d1c58281faf9123c4bbf984eb2c70eae94983a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:10:21 +0100 Subject: [PATCH 07/16] feat(plan): add the missing_migration_origin target policy, allowing missing origins by default --- .../missing_migration_origin.py | 68 +++++++++++++++++++ .../_helpers/build_planning/plan_policies.py | 14 +++- .../cli/commands/_helpers/entry/dispatch.py | 1 + .../_helpers/entry/parser_arguments.py | 1 + .../cli/commands/classes/cli_namespace.py | 2 + src/sqlbuild/cli/commands/models.py | 1 + .../discovery/_helpers/yml/project.py | 26 +++++++ .../planner/_helpers/migrations/columns.py | 14 ++-- .../planner/_helpers/migrations/display.py | 17 +++-- .../planner/_helpers/migrations/planning.py | 37 ++++++++-- src/sqlbuild/compiler/planner/constants.py | 16 +++++ src/sqlbuild/compiler/planner/models.py | 19 ++++++ .../spec/contracts/_helpers/targets.py | 5 ++ src/sqlbuild/spec/contracts/models.py | 3 + src/sqlbuild/spec/contracts/types.py | 8 +++ .../test_column_rename_safety.py | 9 --- .../commands/main/model_migrations/helpers.py | 17 +++++ .../test_manual_migrations.py | 14 ++-- .../discovery/_helpers/_test_types.py | 9 ++- .../discovery/_helpers/test_yml_project.py | 45 +++++++++++- .../pre_build/test_pre_build_work_display.py | 4 +- .../spec/contracts/main/_test_types.py | 10 +++ .../spec/contracts/main/test_targets.py | 54 +++++++++++++++ 23 files changed, 357 insertions(+), 37 deletions(-) create mode 100644 src/sqlbuild/cli/commands/_helpers/build_planning/missing_migration_origin.py diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/missing_migration_origin.py b/src/sqlbuild/cli/commands/_helpers/build_planning/missing_migration_origin.py new file mode 100644 index 0000000000..42ea26bd22 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/missing_migration_origin.py @@ -0,0 +1,68 @@ +"""Missing migration origin confirmation for build commands.""" + +from __future__ import annotations + +from typing import TextIO + +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy + + +def enforce_missing_migration_origin_policy( + *, + plan: PlanOutput, + allow_missing_migration_origin: bool, + input_stream: TextIO, + output_stream: TextIO, +) -> None: + """Confirm before building past declared migrations whose origin is missing.""" + + names: tuple[str, ...] = tuple( + sorted( + { + *( + entry.model_name + for entry in plan.migration_entries + if entry.origin_missing + and entry.missing_origin_policy + == MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION + ), + *( + entry.model_name + for entry in plan.column_migration_entries + if entry.origin_missing + and entry.missing_origin_policy + == MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION + ), + } + ) + ) + if not names or allow_missing_migration_origin: + return + if not input_stream.isatty(): + raise CliUserError( + f"migrate_from origin is missing for {_names(names)} and " + "missing_migration_origin requires confirmation", + code="M102", + help="Pass --allow-missing-migration-origin to build without migrating them.", + ) + expected: str = _confirmation_text(names) + _ = output_stream.write( + f"The migrate_from origin of {_names(names)} does not exist in this target; nothing " + "will be migrated.\n\n" + ) + _ = output_stream.write(f"Type `{expected}` to continue: ") + output_stream.flush() + if input_stream.readline().strip() != expected: + raise CliUserError("build without missing migration origins cancelled", code="M102") + + +def _confirmation_text(names: tuple[str, ...]) -> str: + if len(names) == 1: + return f"build {names[0]} without its migration" + return f"build {len(names)} models without their migrations" + + +def _names(names: tuple[str, ...]) -> str: + return ", ".join(f"'{name}'" for name in names) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py index 5c8db4d7ee..3f9591e84d 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -11,6 +11,9 @@ from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( enforce_snapshot_full_refresh_policy, ) +from sqlbuild.cli.commands._helpers.build_planning.missing_migration_origin import ( + enforce_missing_migration_origin_policy, +) from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( enforce_retention_decrease_policy, ) @@ -44,6 +47,12 @@ def enforce_build_plan_policies( _enforce_model_migration_policy(plan=plan) _enforce_column_migration_policy(plan=plan) + enforce_missing_migration_origin_policy( + plan=plan, + allow_missing_migration_origin=request.allow_missing_migration_origin, + input_stream=sys.stdin, + output_stream=sys.stdout, + ) _enforce_old_name_policy(plan=plan) enforce_model_execution_limit( model_count=executable_model_count(plan=plan), @@ -90,8 +99,9 @@ def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: f"{entry.origin.qualified_name or entry.origin.name})" for entry in missing ) - + " and no recorded migration into it was found; if the migration already happened " - "elsewhere or is no longer needed, remove migrate_from from the model header", + + " and no recorded migration into it was found, and this target sets " + "missing_migration_origin = deny; if the migration already happened elsewhere or is " + "no longer needed, remove migrate_from from the model header", code="M102", help="Run sqb plan to see every model migration decision.", ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 6ccfd72b83..b62457f553 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -164,6 +164,7 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) allow_snapshot_full_refresh=args.allow_snapshot_full_refresh, allow_table_type_downgrade=args.allow_table_type_downgrade, allow_retention_decrease=args.allow_retention_decrease, + allow_missing_migration_origin=args.allow_missing_migration_origin, allow_snapshot_schema_change=args.allow_snapshot_schema_change, concurrency=args.concurrency, select=select, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser_arguments.py b/src/sqlbuild/cli/commands/_helpers/entry/parser_arguments.py index 73036ec8e2..bbf8b91063 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser_arguments.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser_arguments.py @@ -63,6 +63,7 @@ def add_execution_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--allow-snapshot-full-refresh", action="store_true", default=False) parser.add_argument("--allow-table-type-downgrade", action="store_true", default=False) parser.add_argument("--allow-retention-decrease", action="store_true", default=False) + parser.add_argument("--allow-missing-migration-origin", action="store_true", default=False) parser.add_argument("--allow-snapshot-schema-change", action="store_true", default=False) parser.add_argument("--concurrency", type=int, default=None) parser.add_argument("--verbose", "-v", action="store_true", default=False) diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index 4c1fd9afb8..dd52ca05bf 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -56,6 +56,7 @@ "allow_snapshot_full_refresh": False, "allow_table_type_downgrade": False, "allow_retention_decrease": False, + "allow_missing_migration_origin": False, "allow_snapshot_schema_change": False, "concurrency": None, "verbose": False, @@ -221,6 +222,7 @@ class CliNamespace: allow_snapshot_full_refresh: bool allow_table_type_downgrade: bool allow_retention_decrease: bool + allow_missing_migration_origin: bool allow_snapshot_schema_change: bool concurrency: int | None verbose: bool diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index ab622e8694..cb5d042038 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -163,6 +163,7 @@ class BuildCommandRequest: allow_snapshot_full_refresh: bool = False allow_table_type_downgrade: bool = False allow_retention_decrease: bool = False + allow_missing_migration_origin: bool = False allow_snapshot_schema_change: bool = False concurrency: int | None = None select: tuple[str, ...] = () diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index b36a06510f..bc24612b2f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -82,6 +82,7 @@ EventExportSeverity, FutureCursorAction, MicrobatchLimitAction, + MissingMigrationOriginPolicy, RetentionDecreasePolicy, TableType, TableTypeDowngradePolicy, @@ -1127,6 +1128,21 @@ def _target_retention_by_materialization( return loaded +def _optional_missing_migration_origin_policy( + *, mapping: dict[str, object], target_name: str, file_path: Path +) -> MissingMigrationOriginPolicy | None: + value: object | None = mapping.get("missing_migration_origin") + if value is None: + return None + try: + return MissingMigrationOriginPolicy(value) + except (TypeError, ValueError) as exc: + raise ProjectConfigError( + f"{file_path} targets.{target_name}.missing_migration_origin must be 'deny', " + "'require_confirmation', or 'allow'" + ) from exc + + def _optional_retention_decrease_policy( *, mapping: dict[str, object], target_name: str, file_path: Path ) -> RetentionDecreasePolicy | None: @@ -1216,6 +1232,12 @@ def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig ) or RetentionDecreasePolicy.DENY ), + missing_migration_origin=( + _optional_missing_migration_origin_policy( + mapping=target_mapping, target_name=target_name, file_path=file_path + ) + or MissingMigrationOriginPolicy.ALLOW + ), execution_limits=execution_limits, clone=ClonePolicy( allow_as_clone_origin=_optional_bool( @@ -1304,6 +1326,9 @@ def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalT time_travel_retention_decrease=_optional_retention_decrease_policy( mapping=target_mapping, target_name=target_name, file_path=file_path ), + missing_migration_origin=_optional_missing_migration_origin_policy( + mapping=target_mapping, target_name=target_name, file_path=file_path + ), execution_limits=execution_limits, clone=LocalClonePolicy( allow_as_clone_origin=_optional_nullable_bool( @@ -1374,6 +1399,7 @@ def _validate_target_keys( "default_table_type", "table_type_downgrade", "time_travel_retention_decrease", + "missing_migration_origin", "clone", "execution_limits", } diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index 2db0141960..9500d43388 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -21,6 +21,7 @@ rename_hints, ) from sqlbuild.compiler.planner._helpers.migrations.planning import ( + missing_origin_policy, planning_database, redirected_cursor_snapshots, ) @@ -32,6 +33,7 @@ from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) +from sqlbuild.compiler.planner.constants import MISSING_ORIGIN_OUTCOMES from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanEntry, ColumnMigrationPlanning, @@ -47,6 +49,7 @@ ) from sqlbuild.compiler.planner.types import InputColumns, MaterializationType, WarningSeverity from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} @@ -141,6 +144,8 @@ def plan_column_migrations( entries.extend(model_entries) warnings.extend(model_warnings) hints.extend(_hints(candidate=candidate, entries=model_entries)) + origin_policy: MissingMigrationOriginPolicy = missing_origin_policy(runtime=runtime) + entries = [replace(entry, missing_origin_policy=origin_policy) for entry in entries] warnings.extend(warning for entry in entries if (warning := _entry_warning(entry))) overlaid: WarehouseSnapshot = _overlay_snapshot( snapshot=snapshot, @@ -399,7 +404,9 @@ def _overlay_snapshot( if not entries: return snapshot - blocked: frozenset[str] = frozenset(entry.model_name for entry in entries if entry.blocks_build) + blocked: frozenset[str] = frozenset( + entry.model_name for entry in entries if entry.blocks_build or entry.origin_missing + ) columns: dict[str, tuple[ColumnInfo, ...]] = dict(snapshot.existing_columns) fingerprints: dict[str, Fingerprint] = dict(snapshot.fingerprints.models) candidate: _ModelColumns @@ -521,8 +528,7 @@ def _entry_warning(entry: ColumnMigrationPlanEntry) -> PlanWarning | None: "M109", f"model '{entry.model_name}': {declaration}: column {entry.origin_column} does not " f"exist in {relation} and no recorded rename into {entry.destination_column} was " - "found; if the rename already happened elsewhere or is no longer needed, remove " - "migrate_from from the column", + f"found; {MISSING_ORIGIN_OUTCOMES[entry.missing_origin_policy]}", ), ColumnMigrationDecision.CONFLICT: ( "M110", @@ -546,7 +552,7 @@ def _entry_warning(entry: ColumnMigrationPlanEntry) -> PlanWarning | None: return None return PlanWarning( model_name=entry.model_name, - severity=WarningSeverity.ERROR, + severity=WarningSeverity.ERROR if entry.blocks_build else WarningSeverity.WARNING, message=coded[1], code=coded[0], ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py index 95b8c93128..e7dbb735c3 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py @@ -173,8 +173,10 @@ def _format_column_migrations( def _column_migration_text(*, entry: ColumnMigrationPlanEntry, style: CliStyle) -> str: rename: str = f"{entry.origin_column} -> {entry.destination_column}" decision: str - if entry.decision.blocks_build: + if entry.blocks_build: decision = style.error_strong(entry.decision.label) + elif entry.origin_missing: + decision = style.warning_strong(entry.decision.label) elif entry.decision == ColumnMigrationDecision.DONE: decision = style.muted(entry.decision.label) else: @@ -199,7 +201,7 @@ def _migration_lines( destination: str = entry.destination.qualified_name or entry.destination.name relation_move: str = f"{style.muted(f'{origin} ->')} {destination}" name: str = style.object_name(entry.model_name) - decision: str = _decision_text(decision=entry.decision, style=style) + decision: str = _decision_text(entry=entry, style=style) if entry.decision == MigrationDecision.RENAMED: decision = ( style.muted(MigrationDecision.DONE.label) @@ -253,8 +255,8 @@ def _migration_lines( ) if old_name is not None: rows.extend(old_name_rows(entry=old_name, style=style)) - blocking: bool = ( - entry.decision.blocks_build or entry.compatibility == MigrationCompatibility.INCOMPATIBLE + blocking: bool = entry.blocks_build or ( + entry.compatibility == MigrationCompatibility.INCOMPATIBLE ) finding_style: Callable[[str], str] = style.error if blocking else style.warning rows.extend(f" {finding_style(f'! {finding}')}" for finding in entry.compatibility_findings) @@ -273,9 +275,12 @@ def _property_row(*, label: str, value: str, style: CliStyle) -> str: return f" {style.muted(label)} {value}" -def _decision_text(*, decision: MigrationDecision, style: CliStyle) -> str: - if decision.blocks_build: +def _decision_text(*, entry: ModelMigrationPlanEntry, style: CliStyle) -> str: + decision: MigrationDecision = entry.decision + if entry.blocks_build and decision.blocks_build: return style.error_strong(decision.label) + if entry.origin_missing: + return style.warning_strong(decision.label) if decision in (MigrationDecision.SUPERSEDED_REPLACE, MigrationDecision.FORCED_REPLACE): return style.warning_strong(decision.label) if decision == MigrationDecision.DONE: diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index e759fa032b..7ebbaeb7fa 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -44,6 +44,7 @@ ) from sqlbuild.compiler.planner.constants import ( MIGRATION_MODEL_NAME_METADATA_KEY, + MISSING_ORIGIN_OUTCOMES, QUALIFIED_RELATION_MAX_PARTS, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError @@ -65,8 +66,9 @@ ) from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import SnapshotsConfig -from sqlbuild.spec.contracts.types import TableType +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, TableType _IDENTITY_MATERIALIZATIONS: frozenset[str] = frozenset( {MaterializationType.TABLE, MaterializationType.VIEW} @@ -177,6 +179,7 @@ def plan_model_migrations( if runtime.project_config is not None else SnapshotsConfig() ) + origin_policy: MissingMigrationOriginPolicy = missing_origin_policy(runtime=runtime) entries: list[ModelMigrationPlanEntry] = [] handovers: dict[str, Fingerprint | None] = {} request: ModelMigrationRequest @@ -195,7 +198,7 @@ def plan_model_migrations( state=state, snapshots_config=snapshots_config, ) - entries.append(entry) + entries.append(replace(entry, missing_origin_policy=origin_policy)) if applies: handovers[request.model.name] = _equivalent_definition( model=request.model, @@ -290,6 +293,22 @@ def planning_database(*, runtime: PlannerRuntime) -> str | None: ) +def missing_origin_policy(*, runtime: PlannerRuntime) -> MissingMigrationOriginPolicy: + """Return how the effective target treats a declared migration origin it cannot find.""" + + if ( + runtime.project_config is None + or runtime.local_config is None + or runtime.project.effective_target_name is None + ): + return MissingMigrationOriginPolicy.ALLOW + return resolve_target_config( + project_config=runtime.project_config, + local_config=runtime.local_config, + target_name=runtime.project.effective_target_name, + ).missing_migration_origin + + def project_schemas(*, runtime: PlannerRuntime) -> set[str]: """Return every destination schema owned by project models.""" @@ -755,14 +774,18 @@ def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: ), code="M101", ) - if entry.decision == MigrationDecision.ORIGIN_MISSING: + if entry.origin_missing: return PlanWarning( model_name=entry.model_name, - severity=WarningSeverity.ERROR, + severity=( + WarningSeverity.ERROR + if entry.missing_origin_policy == MissingMigrationOriginPolicy.DENY + else WarningSeverity.WARNING + ), message=( - f"model '{entry.model_name}': migrate_from origin {origin} does not exist and " - "no recorded migration into it was found; if the migration already happened " - "elsewhere or is no longer needed, remove migrate_from from the model header" + f"model '{entry.model_name}': migrate_from origin {origin} does not exist in " + "this target and no recorded migration into it was found; " + + MISSING_ORIGIN_OUTCOMES[entry.missing_origin_policy] ), code="M102", ) diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 008dbee04a..3adbfd48db 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -4,6 +4,7 @@ from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.cursor_algebra.types import BoundSentinel +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy PATH_SELECTOR_EXPLICIT_ROOT_ERROR: str = ( "path selectors require an explicit root: use 'models/' or 'python/'" @@ -87,3 +88,18 @@ "matched by unchanged definition; sqb rename model: also rewrites references " "and adds migrate_from when discovery cannot match" ) +MISSING_ORIGIN_OUTCOMES: dict[MissingMigrationOriginPolicy, str] = { + MissingMigrationOriginPolicy.ALLOW: ( + "nothing is migrated and the build continues as if migrate_from were absent; remove " + "migrate_from once no target still needs it" + ), + MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION: ( + "building without it needs confirmation (missing_migration_origin = " + "require_confirmation); pass --allow-missing-migration-origin, or remove migrate_from if " + "it is no longer needed" + ), + MissingMigrationOriginPolicy.DENY: ( + "the build stops (missing_migration_origin = deny); if the migration already happened " + "elsewhere or is no longer needed, remove migrate_from" + ), +} diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 9a715e90db..0868773451 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -91,6 +91,7 @@ from sqlbuild.spec.contracts.types import ( FutureCursorAction, MicrobatchLimitAction, + MissingMigrationOriginPolicy, SourceWriteStrategy, ) from sqlbuild.sql_values.models import SqlValue @@ -931,6 +932,13 @@ class ModelMigrationPlanEntry: compatibility_findings: tuple[str, ...] = () message: str | None = None origin_tracked: bool = False + missing_origin_policy: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW + + @property + def origin_missing(self) -> bool: + """Return whether the declared origin was not found in this target.""" + + return self.decision == MigrationDecision.ORIGIN_MISSING @property def storage_transition(self) -> str | None: @@ -951,6 +959,8 @@ def target_label(self) -> str: def blocks_build(self) -> bool: """Return whether this migration must stop a build before any execution.""" + if self.origin_missing: + return self.missing_origin_policy == MissingMigrationOriginPolicy.DENY return self.decision.blocks_build or ( self.decision.moves_data and self.compatibility == MigrationCompatibility.INCOMPATIBLE ) @@ -1010,11 +1020,20 @@ class ColumnMigrationPlanEntry: target_name: str | None completed_at: datetime | None = None message: str | None = None + missing_origin_policy: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW + + @property + def origin_missing(self) -> bool: + """Return whether the declared origin column was not found in this target.""" + + return self.decision == ColumnMigrationDecision.SOURCE_MISSING @property def blocks_build(self) -> bool: """Return whether this column migration must stop a build before any execution.""" + if self.origin_missing: + return self.missing_origin_policy == MissingMigrationOriginPolicy.DENY return self.decision.blocks_build diff --git a/src/sqlbuild/spec/contracts/_helpers/targets.py b/src/sqlbuild/spec/contracts/_helpers/targets.py index 7be6cdf3fc..aa62d4c54c 100644 --- a/src/sqlbuild/spec/contracts/_helpers/targets.py +++ b/src/sqlbuild/spec/contracts/_helpers/targets.py @@ -112,6 +112,11 @@ def resolve_target_config( if local_target.time_travel_retention_decrease is not None else project_target.time_travel_retention_decrease ), + missing_migration_origin=( + local_target.missing_migration_origin + if local_target.missing_migration_origin is not None + else project_target.missing_migration_origin + ), execution_limits=_merge_execution_limits( project_limits=project_target.execution_limits, local_limits=local_target.execution_limits, diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index 9559712e85..1643d02f19 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -14,6 +14,7 @@ EventExportSeverity, FutureCursorAction, MicrobatchLimitAction, + MissingMigrationOriginPolicy, RetentionDecreasePolicy, SourceFreshnessStrategy, SourceFreshnessValueKind, @@ -123,6 +124,7 @@ class TargetConfig: default_table_type: TableType | None = None table_type_downgrade: TableTypeDowngradePolicy = TableTypeDowngradePolicy.REQUIRE_CONFIRMATION time_travel_retention_decrease: RetentionDecreasePolicy = RetentionDecreasePolicy.DENY + missing_migration_origin: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) @@ -146,6 +148,7 @@ class LocalTargetConfig: default_table_type: TableType | None = None table_type_downgrade: TableTypeDowngradePolicy | None = None time_travel_retention_decrease: RetentionDecreasePolicy | None = None + missing_migration_origin: MissingMigrationOriginPolicy | None = None execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) diff --git a/src/sqlbuild/spec/contracts/types.py b/src/sqlbuild/spec/contracts/types.py index 7f822ee5a0..58c0ea6d92 100644 --- a/src/sqlbuild/spec/contracts/types.py +++ b/src/sqlbuild/spec/contracts/types.py @@ -101,6 +101,14 @@ class TableTypeDowngradePolicy(StrEnum): ALLOW = "allow" +class MissingMigrationOriginPolicy(StrEnum): + """Policy for a declared migrate_from whose origin does not exist in a target.""" + + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + ALLOW = "allow" + + class RetentionDecreasePolicy(StrEnum): """Policy for lowering live time-travel retention.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py index bdf8cafd20..d047d909fb 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py @@ -98,15 +98,6 @@ def test_given_interrupted_rename_when_retrying_then_column_is_renamed_once_and_ expected_decision="conflict", expected_error="error[M110]", ), - BlockedColumnRenameTestCase( - description="source column never existed", - renamed_sql=orders_sql( - columns="amount AS revenue", - extra_config=migrate_columns("revenue (migrate_from gross_amount)"), - ), - expected_decision="source_missing", - expected_error="error[M109]", - ), BlockedColumnRenameTestCase( description="source column is still produced", renamed_sql=orders_sql( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 4469ccbccc..c0e7c0a8c9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -33,6 +33,23 @@ old_name_views = false """ ).lstrip() +DENY_MISSING_ORIGIN_PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + default_target = "main" + + [connection] + database = "{DATABASE_FILE}" + + [targets.main] + schema = "main" + missing_migration_origin = "deny" + + [migrations] + old_name_views = false + """ +).lstrip() TARGETS_PROJECT_TOML: str = dedent( f""" name = "orders_project" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index e58feff3fe..f0eaac192f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -16,6 +16,7 @@ MigrationOutcomeTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + DENY_MISSING_ORIGIN_PROJECT_TOML, DESTINATION_MODEL, ORIGIN_MODEL, PROJECT_TOML, @@ -330,12 +331,13 @@ def test_given_destination_with_build_history_when_building_then_conflict_until_ "test_case", [ MigrationOutcomeTestCase( - description="missing origin without a recorded migration fails the build", + description="missing origin fails the build when the target denies it", expected_decisions=("origin_missing",), expected_output_fragment=( "'stg_customer_orders' (migrate_from main.retired_orders) and no recorded " - "migration into it was found; if the migration already happened elsewhere or " - "is no longer needed, remove migrate_from from the model header" + "migration into it was found, and this target sets missing_migration_origin = " + "deny; if the migration already happened elsewhere or is no longer needed, " + "remove migrate_from from the model header" ), ) ], @@ -347,6 +349,7 @@ def test_given_missing_origin_without_event_when_building_then_fails_before_buil write_project( project_dir=tmp_path, models={DESTINATION_MODEL: incremental_orders_sql(migrate_from="retired_orders")}, + project_toml=DENY_MISSING_ORIGIN_PROJECT_TOML, ) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) @@ -358,8 +361,9 @@ def test_given_missing_origin_without_event_when_building_then_fails_before_buil assert tuple(warning["severity"] for warning in plan["warnings"]) == ("error",) assert plan["warnings"][0]["message"] == ( "model 'stg_customer_orders': migrate_from origin main.retired_orders does not exist " - "and no recorded migration into it was found; if the migration already happened " - "elsewhere or is no longer needed, remove migrate_from from the model header" + "in this target and no recorded migration into it was found; the build stops " + "(missing_migration_origin = deny); if the migration already happened elsewhere or is " + "no longer needed, remove migrate_from" ) assert "origin missing main.retired_orders -> main.stg_customer_orders" in text.output assert result.exit_code == 1 diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 83ad6e4e03..7e9608c845 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -3,7 +3,7 @@ from pathlib import Path from sqlbuild.spec.contracts.models import AuthoredTimeTravelRetention -from sqlbuild.spec.contracts.types import RetentionDecreasePolicy +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy from sqlbuild.sql_values.types import CollectionRendering @@ -896,3 +896,10 @@ class ParseSeedAuditsYamlTestCase: contents: str expected_table_audits: tuple[str, ...] expected_column_audits: tuple[tuple[str, ...], ...] + + +@dataclass(frozen=True) +class MissingMigrationOriginConfigTestCase: + description: str + target_lines: tuple[str, ...] + expected_policy: MissingMigrationOriginPolicy diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 9ab93dc182..d4e94e1f67 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -18,7 +18,7 @@ ProjectConfig, TargetConfig, ) -from sqlbuild.spec.contracts.types import RetentionDecreasePolicy +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy from sqlbuild.sql_values.types import CollectionRendering from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ColumnContractModeConfigErrorTestCase, @@ -42,6 +42,7 @@ LoadTargetRetentionPoliciesTestCase, MicrobatchLimitConfigErrorTestCase, MicrobatchLimitConfigTestCase, + MissingMigrationOriginConfigTestCase, StartCursorConfigTestCase, TargetExecutionLimitsConfigTestCase, ) @@ -559,6 +560,43 @@ def test_given_target_retention_policies_when_loading_project_then_they_are_type assert target.time_travel_retention_decrease is test_case.expected_decrease_policy +@pytest.mark.parametrize( + "test_case", + [ + MissingMigrationOriginConfigTestCase( + description="missing migration origins are allowed by default", + target_lines=(), + expected_policy=MissingMigrationOriginPolicy.ALLOW, + ), + MissingMigrationOriginConfigTestCase( + description="deny is typed", + target_lines=('missing_migration_origin = "deny"',), + expected_policy=MissingMigrationOriginPolicy.DENY, + ), + MissingMigrationOriginConfigTestCase( + description="require_confirmation is typed", + target_lines=('missing_migration_origin = "require_confirmation"',), + expected_policy=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION, + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_migration_origin_policy_when_loading_project_then_it_is_typed( + test_case: MissingMigrationOriginConfigTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + "\n".join( + ('name = "demo"', 'adapter = "duckdb"', "[targets.prod]", *test_case.target_lines) + ), + encoding="utf-8", + ) + + target: TargetConfig = load_project_config(project_dir=tmp_path).targets["prod"] + + assert target.missing_migration_origin is test_case.expected_policy + + @pytest.mark.parametrize( "test_case", [ @@ -577,6 +615,11 @@ def test_given_target_retention_policies_when_loading_project_then_they_are_type target_lines=('time_travel_retention_decrease = "sometimes"',), expected_error_fragment="'deny', 'require_confirmation', or 'allow'", ), + LoadTargetRetentionPoliciesErrorTestCase( + description="missing migration origin policy must be a known value", + target_lines=('missing_migration_origin = "sometimes"',), + expected_error_fragment="targets.prod.missing_migration_origin must be", + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index 9c3588affa..9a1b404ce4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -92,11 +92,11 @@ ), ), MigrationDecisionStyleTestCase( - description="origin missing is bold red", + description="origin missing allowed by the default policy is bold yellow", decision=MigrationDecision.ORIGIN_MISSING, compatibility=MigrationCompatibility.NOT_CHECKED, findings=(), - expected_fragments=(f"{_BOLD_RED}origin missing{_RESET}",), + expected_fragments=(f"{_BOLD_YELLOW}origin missing{_RESET}",), ), MigrationDecisionStyleTestCase( description="renamed table is listed as a bold blue migration that rebuilds it", diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py index ea16b93e94..bd195c7ca4 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py @@ -9,6 +9,7 @@ LocalConfig, ProjectConfig, ) +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy from sqlbuild.sql_values.types import CollectionRendering @@ -55,3 +56,12 @@ class InvalidLoaderDestinationTestCase: description: str destination: str expected_error_fragment: str + + +@dataclass(frozen=True) +class MissingOriginPolicyResolutionTestCase: + description: str + project_config: ProjectConfig + local_config: LocalConfig + target_name: str + expected_policy: MissingMigrationOriginPolicy diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index 79bd7b2ac5..a01f23747d 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -11,8 +11,10 @@ ProjectConfig, TargetConfig, ) +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( ExecutionLimitsResolutionTestCase, + MissingOriginPolicyResolutionTestCase, TargetRetentionResolutionTestCase, ) @@ -134,3 +136,55 @@ def test_given_local_execution_limit_fields_when_resolving_then_each_field_overr ) assert target_config.execution_limits == test_case.expected_limits + + +@pytest.mark.parametrize( + "test_case", + [ + MissingOriginPolicyResolutionTestCase( + description="local target policy overrides the project target", + project_config=ProjectConfig( + name="shop", + adapter="duckdb", + targets={ + "dev": TargetConfig(missing_migration_origin=MissingMigrationOriginPolicy.DENY) + }, + ), + local_config=LocalConfig( + targets={ + "dev": LocalTargetConfig( + missing_migration_origin=MissingMigrationOriginPolicy.ALLOW + ) + } + ), + target_name="dev", + expected_policy=MissingMigrationOriginPolicy.ALLOW, + ), + MissingOriginPolicyResolutionTestCase( + description="project target policy applies without a local override", + project_config=ProjectConfig( + name="shop", + adapter="duckdb", + targets={ + "dev": TargetConfig( + missing_migration_origin=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION + ) + }, + ), + local_config=LocalConfig(targets={"dev": LocalTargetConfig()}), + target_name="dev", + expected_policy=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION, + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_policy_when_resolving_then_local_overrides_project( + test_case: MissingOriginPolicyResolutionTestCase, +) -> None: + target_config: TargetConfig = resolve_target_config( + project_config=test_case.project_config, + local_config=test_case.local_config, + target_name=test_case.target_name, + ) + + assert target_config.missing_migration_origin is test_case.expected_policy From e02a6154ad59b4bab147a90001ff45bb4376b8f8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:16:17 +0100 Subject: [PATCH 08/16] feat(cli): always declare migrate_from, refuse instead of partial edits, and rewrite declaration references --- .../cli/commands/_helpers/entry/dispatch.py | 1 - .../cli/commands/_helpers/entry/parser.py | 7 - .../cli/commands/_helpers/refactor/command.py | 33 ++- .../cli/commands/_helpers/refactor/output.py | 5 +- .../cli/commands/classes/cli_namespace.py | 2 - src/sqlbuild/cli/commands/models.py | 1 - .../refactoring/_helpers/columns/__init__.py | 1 + .../_helpers/{ => columns}/column_planning.py | 66 ++++- .../{ => columns}/column_references.py | 2 +- .../refactoring/_helpers/model_migration.py | 131 ---------- .../refactoring/_helpers/project/__init__.py | 1 + .../_helpers/{ => project}/project_files.py | 19 +- .../_helpers/{ => project}/workspace.py | 2 +- .../refactoring/_helpers/renames/__init__.py | 1 + .../_helpers/{ => renames}/model_planning.py | 114 ++++++++- .../{ => renames}/model_references.py | 14 +- .../refactoring/_helpers/text/__init__.py | 1 + .../_helpers/{ => text}/header_edits.py | 119 +++++++-- .../refactoring/_helpers/text/schema_edits.py | 55 +++++ .../_helpers/{ => text}/sql_sites.py | 11 + .../_helpers/{ => text}/text_edits.py | 0 .../refactoring/_helpers/text/yaml_edits.py | 171 +++++++++++++ .../compiler/refactoring/constants.py | 18 +- .../refactoring/main/commit_refactor_plan.py | 2 +- .../refactoring/main/plan_column_rename.py | 4 +- .../refactoring/main/plan_model_refactor.py | 4 +- .../refactoring/main/stage_refactor_plan.py | 2 +- .../refactoring/main/with_model_migration.py | 14 +- src/sqlbuild/compiler/refactoring/models.py | 21 ++ src/sqlbuild/compiler/refactoring/types.py | 2 + .../cli/commands/main/refactor/_test_types.py | 24 +- .../cli/commands/main/refactor/helpers.py | 123 +++++++++- .../refactor/test_missing_migration_origin.py | 135 ++++++++++ .../main/refactor/test_rename_columns.py | 101 ++++++-- .../main/refactor/test_rename_models.py | 72 ++++-- .../refactoring/_helpers/_test_types.py | 7 + .../compiler/refactoring/_helpers/helpers.py | 19 +- .../_helpers/test_column_references.py | 2 +- .../_helpers/test_declaration_edits.py | 232 ++++++++++++++++++ .../refactoring/_helpers/test_header_edits.py | 4 +- .../refactoring/_helpers/test_text_edits.py | 5 +- .../refactoring/_helpers/test_workspace.py | 2 +- 42 files changed, 1270 insertions(+), 280 deletions(-) create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py rename src/sqlbuild/compiler/refactoring/_helpers/{ => columns}/column_planning.py (88%) rename src/sqlbuild/compiler/refactoring/_helpers/{ => columns}/column_references.py (99%) delete mode 100644 src/sqlbuild/compiler/refactoring/_helpers/model_migration.py create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py rename src/sqlbuild/compiler/refactoring/_helpers/{ => project}/project_files.py (90%) rename src/sqlbuild/compiler/refactoring/_helpers/{ => project}/workspace.py (98%) create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/renames/__init__.py rename src/sqlbuild/compiler/refactoring/_helpers/{ => renames}/model_planning.py (65%) rename src/sqlbuild/compiler/refactoring/_helpers/{ => renames}/model_references.py (90%) create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py rename src/sqlbuild/compiler/refactoring/_helpers/{ => text}/header_edits.py (77%) create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py rename src/sqlbuild/compiler/refactoring/_helpers/{ => text}/sql_sites.py (93%) rename src/sqlbuild/compiler/refactoring/_helpers/{ => text}/text_edits.py (100%) create mode 100644 src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py create mode 100644 tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index b62457f553..895d4d6c6d 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -436,7 +436,6 @@ def _dispatch_compile_or_contract( destination=args.refactor_destination, project_dir=project_dir, cascade=args.refactor_cascade, - allow_manual=args.refactor_allow_manual, dry_run=args.dry_run, json_output=args.json, no_color=args.no_color, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index 58998b2623..d6c353f58c 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -133,13 +133,6 @@ def _add_refactor_parsers( ) parser: argparse.ArgumentParser for parser in (rename_parser, move_parser): - parser.add_argument( - "--allow-manual", - dest="refactor_allow_manual", - action="store_true", - default=False, - help="apply the safe edits and leave the listed locations to edit by hand", - ) parser.add_argument( "--dry-run", dest="dry_run", diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/command.py b/src/sqlbuild/cli/commands/_helpers/refactor/command.py index c163468b02..bb37351b9a 100644 --- a/src/sqlbuild/cli/commands/_helpers/refactor/command.py +++ b/src/sqlbuild/cli/commands/_helpers/refactor/command.py @@ -76,7 +76,7 @@ def _run( else plan_model_refactor(project=before.project, request=refactor_request) ) status.complete(message=f"Planned edits to {len(plan.changes)} files.") - if plan.blocking or (plan.manual and not request.allow_manual): + if plan.blocking or plan.manual: status.error("Refused: some references cannot be rewritten safely.") return _finish( request=request, @@ -92,15 +92,11 @@ def _run( project_dir=project_dir, staging_dir=staging_dir, plan=plan ) after: RefactorCompile = compile_for_refactor(project_dir=staging_dir, no_cache=True) - if ( - after.project is not None - and not after.errors - and refactor_request.operation != (RefactorOperation.RENAME_COLUMN) - ): + if refactor_request.operation != RefactorOperation.RENAME_COLUMN: migrated: RefactorPlan = with_model_migration( plan=plan, before=before.project.graph.project, - after=after.project.graph.project, + after=after.project.graph.project if after.project is not None else None, originals=originals, ) if migrated.blocking: @@ -118,16 +114,19 @@ def _run( project_dir=project_dir, staging_dir=staging_dir, plan=plan, copy_inputs=False ) after = compile_for_refactor(project_dir=staging_dir, no_cache=True) - if after.errors: - status.error("Verification failed: the edited project does not compile.") - return _finish( - request=request, - plan=plan, - status=RefactorStatus.COMPILE_FAILED, - diagnostics=_relative(errors=after.errors, staging_dir=staging_dir), - use_color=use_color, - ) - status.complete(message="Verified: the edited project compiles.") + diagnostics: tuple[CompilerDiagnostic, ...] = _relative( + errors=after.errors, staging_dir=staging_dir + ) + if diagnostics: + status.error("Verification failed: the edited project does not compile.") + return _finish( + request=request, + plan=plan, + status=RefactorStatus.COMPILE_FAILED, + diagnostics=diagnostics, + use_color=use_color, + ) + status.complete(message="Verified: the edited project compiles.") if request.dry_run: return _finish( request=request, diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/output.py b/src/sqlbuild/cli/commands/_helpers/refactor/output.py index fc7d7becdb..7c3a5317c7 100644 --- a/src/sqlbuild/cli/commands/_helpers/refactor/output.py +++ b/src/sqlbuild/cli/commands/_helpers/refactor/output.py @@ -44,11 +44,8 @@ def render_refactor_text( if blocked: lines.append(style.error_strong(f"Cannot {_verb(plan)} {_target(plan)} automatically")) lines.extend(_location_lines(style=style, locations=(*plan.blocking, *plan.manual))) - if plan.help and plan.manual and not plan.blocking: + if plan.help and plan.manual: lines.append(plan.help) - elif plan.manual: - lines.append(style.warning_strong("Edit by hand")) - lines.extend(_location_lines(style=style, locations=plan.manual)) if diagnostics: lines.append(style.error_strong("Compile errors in the edited project")) lines.extend(_diagnostic_lines(style=style, diagnostics=diagnostics)) diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index dd52ca05bf..13d2eb55bf 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -165,7 +165,6 @@ "refactor_new_name": None, "refactor_destination": None, "refactor_cascade": False, - "refactor_allow_manual": False, } @@ -249,7 +248,6 @@ class CliNamespace: refactor_new_name: str | None refactor_destination: str | None refactor_cascade: bool - refactor_allow_manual: bool cost_limit: int | None cost_no_limit: bool cost_sort: str | None diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index cb5d042038..be2c086de4 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -1065,7 +1065,6 @@ class RefactorCommandRequest: destination: str | None = None project_dir: Path | None = None cascade: bool = False - allow_manual: bool = False dry_run: bool = False json_output: bool = False no_color: bool = False diff --git a/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py new file mode 100644 index 0000000000..e0399ee685 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py @@ -0,0 +1 @@ +"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/column_planning.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py similarity index 88% rename from src/sqlbuild/compiler/refactoring/_helpers/column_planning.py rename to src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py index 97b94d5e3c..bcd5e6e449 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/column_planning.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py @@ -6,13 +6,24 @@ from sqlbuild.compiler.compile.models import CompiledModel from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.refactoring._helpers.column_references import ( +from sqlbuild.compiler.refactoring._helpers.columns.column_references import ( analyze_column, consumer_edits, output_edits, resource_columns, ) -from sqlbuild.compiler.refactoring._helpers.header_edits import ( +from sqlbuild.compiler.refactoring._helpers.project.project_files import ( + authored_bodies, + project_sql_files, + python_string_locations, + yaml_files, +) +from sqlbuild.compiler.refactoring._helpers.renames.model_planning import ( + find_model, + needs_column_migration, + validate_identifier, +) +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( add_column_entry_edit, column_config_edits, column_entry_edits, @@ -20,20 +31,19 @@ header_tokens, unhandled_word_offsets, ) -from sqlbuild.compiler.refactoring._helpers.model_migration import needs_column_migration -from sqlbuild.compiler.refactoring._helpers.model_planning import find_model, validate_identifier -from sqlbuild.compiler.refactoring._helpers.project_files import ( - authored_bodies, - project_sql_files, - python_string_locations, -) -from sqlbuild.compiler.refactoring._helpers.sql_sites import ( +from sqlbuild.compiler.refactoring._helpers.text.schema_edits import schema_column_edits +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import ( analysis_sql, authored_offset, authored_span, model_body, ) -from sqlbuild.compiler.refactoring._helpers.text_edits import manual_at, merge_parts, path_edits +from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( + manual_at, + merge_parts, + path_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.yaml_edits import yaml_column_edits from sqlbuild.compiler.refactoring.constants import ( EXPECTED_FIXTURE_PREFIX, FIXTURE_ROLES, @@ -62,6 +72,7 @@ RefactorRequest, TextEdit, ) +from sqlbuild.compiler.refactoring.types import SqlFileRole from sqlbuild.spec.contracts.models import SchemaColumn type _ModelContext = tuple[BodyContext | None, str, tuple[ManualLocation, ...]] @@ -84,6 +95,8 @@ def column_rename_context( columns=resource_columns(project=project.graph.project), contents={item.relative_path: item.contents for item in files}, bodies=authored_bodies(discovered=project.discovered), + yaml_files=yaml_files(discovered=project.discovered), + schema_files=tuple(item for item in files if item.role == SqlFileRole.SCHEMA), old=old, new=new, cascade=request.cascade, @@ -119,6 +132,17 @@ def column_rename_parts(*, context: ColumnRenameContext) -> RefactorParts: _declarations(context=context, model=model), consumers, _authored_body_parts(context=context, model=model), + RefactorParts( + edits=( + *yaml_column_edits( + files=context.yaml_files, + model=model.name, + old=context.old, + new=context.new, + ), + *_schema_edits(context=context, model=model), + ) + ), ) ) queue.extend(cascaded) @@ -126,6 +150,26 @@ def column_rename_parts(*, context: ColumnRenameContext) -> RefactorParts: return merge_parts(parts=(merged, _python_locations(context=context, merged=merged))) +def _schema_edits( + *, context: ColumnRenameContext, model: CompiledModel +) -> tuple[tuple[str, TextEdit], ...]: + edits: list[tuple[str, TextEdit]] = [] + item: ProjectSqlFile + for item in context.schema_files: + edits.extend( + path_edits( + path=item.relative_path, + edits=schema_column_edits( + contents=item.contents, + upstream=model.name, + old=context.old, + new=context.new, + ), + ) + ) + return tuple(edits) + + def _output_names(*, context: ColumnRenameContext) -> frozenset[str]: return frozenset( name.lower() for name in context.columns.get((REF_KIND, context.owner.name), ()) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/column_references.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py similarity index 99% rename from src/sqlbuild/compiler/refactoring/_helpers/column_references.py rename to src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py index 46b691d32d..ca6d6026a9 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/column_references.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py @@ -7,7 +7,7 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.refactoring._helpers.text_edits import manual_at, text_edit +from sqlbuild.compiler.refactoring._helpers.text.text_edits import manual_at, text_edit from sqlbuild.compiler.refactoring.constants import ( CTE_SCOPE_PREFIX, QUOTE_CHARACTERS, diff --git a/src/sqlbuild/compiler/refactoring/_helpers/model_migration.py b/src/sqlbuild/compiler/refactoring/_helpers/model_migration.py deleted file mode 100644 index d0dbb0accd..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/model_migration.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Decide whether a renamed or moved model needs an explicit migrate_from.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject -from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( - MigrationFingerprintCache, -) -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities -from sqlbuild.compiler.refactoring.constants import ( - HISTORY_MATERIALIZATIONS, - MATERIALIZED_KEY, - MIGRATABLE_MATERIALIZATIONS, - MIGRATE_FROM_KEY, -) -from sqlbuild.compiler.refactoring.models import ModelMigrationDecision -from sqlbuild.spec.contracts.main.get_config_str import get_config_str - - -def decide_model_migration( - *, before: CompiledProject, after: CompiledProject, old: str, new: str -) -> ModelMigrationDecision: - """Compare the old and new compile the way automatic rename discovery would.""" - - old_model: CompiledModel | None = _model(project=before, name=old) - new_model: CompiledModel | None = _model(project=after, name=new) - if old_model is None or new_model is None: - return ModelMigrationDecision(needed=False, reason="model not compiled") - materialized: str | None = get_config_str(values=new_model.config.values, key=MATERIALIZED_KEY) - if materialized not in MIGRATABLE_MATERIALIZATIONS: - return ModelMigrationDecision( - needed=False, reason=f"'{materialized}' models have no migration" - ) - if _location_key(old_model) == _location_key(new_model): - return ModelMigrationDecision(needed=False, reason="relation name is unchanged") - old_database: str = (old_model.destination.database or "").lower() - new_database: str = (new_model.destination.database or "").lower() - if old_database and new_database and old_database != new_database: - return ModelMigrationDecision( - needed=False, - blocked=True, - reason=( - f"moves from database {old_model.destination.database} to " - f"{new_model.destination.database}; migrations cannot cross databases" - ), - ) - old_schema: str = (old_model.destination.schema or "").lower() - project_schemas: frozenset[str] = frozenset( - (model.destination.schema or "").lower() for model in after.models - ) - if old_schema not in project_schemas: - return ModelMigrationDecision( - needed=False, - blocked=True, - reason=( - f"no model is left in schema {old_model.destination.schema}, so neither " - f"automatic discovery nor migrate_from {old} can find the old relation; add a " - "schema-qualified migrate_from for each target" - ), - ) - old_materialized: str | None = get_config_str( - values=old_model.config.values, key=MATERIALIZED_KEY - ) - if (old_materialized in HISTORY_MATERIALIZATIONS) != (materialized in HISTORY_MATERIALIZATIONS): - return ModelMigrationDecision( - needed=True, reason=f"materialization changes from {old_materialized}" - ) - before_fingerprint: str | None = _fingerprint(project=before, model=old_model) - after_fingerprint: str | None = _fingerprint(project=after, model=new_model) - if before_fingerprint is None: - return ModelMigrationDecision( - needed=True, - reason="automatic discovery cannot fingerprint its SQL, so it would not find the " - "old relation", - ) - if before_fingerprint != after_fingerprint: - return ModelMigrationDecision( - needed=True, - reason="its definition changes at the new location, so automatic discovery " - "cannot match it", - ) - return ModelMigrationDecision( - needed=False, reason="automatic rename discovery matches the unchanged definition" - ) - - -def needs_column_migration(*, model: CompiledModel) -> bool: - """Return whether renaming a column of this model must declare migrate_from.""" - - materialized: str | None = get_config_str(values=model.config.values, key=MATERIALIZED_KEY) - if materialized in HISTORY_MATERIALIZATIONS: - return True - return ( - materialized in MIGRATABLE_MATERIALIZATIONS - and model.config.values.get(MIGRATE_FROM_KEY) is not None - ) - - -def _fingerprint(*, project: CompiledProject, model: CompiledModel) -> str | None: - metadata_json: str = build_model_version_identity_metadata_json( - model=model, - function_local_hashes=build_function_local_hashes(functions=project.functions), - hook_version_hashes={ - name: identity.version_hash - for name, identity in build_hook_identities(project.hook_functions).items() - }, - ) - return MigrationFingerprintCache().fingerprint( - query_sql=model.query_sql, - metadata_json=metadata_json, - ref_identities={}, - dialect=project.sql_analysis_dialect, - ) - - -def _model(*, project: CompiledProject, name: str) -> CompiledModel | None: - return next((model for model in project.models if model.name == name), None) - - -def _location_key(model: CompiledModel) -> tuple[str, str, str]: - return ( - (model.destination.database or "").lower(), - (model.destination.schema or "").lower(), - model.destination.name.lower(), - ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py new file mode 100644 index 0000000000..e0399ee685 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py @@ -0,0 +1 @@ +"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project_files.py b/src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py similarity index 90% rename from src/sqlbuild/compiler/refactoring/_helpers/project_files.py rename to src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py index 1e87b63dfd..38f93ceca6 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/project_files.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py @@ -10,7 +10,7 @@ DiscoveredProjectInputs, DiscoveredSqlTestBlock, ) -from sqlbuild.compiler.refactoring._helpers.text_edits import whole_word_offsets +from sqlbuild.compiler.refactoring._helpers.text.text_edits import whole_word_offsets from sqlbuild.compiler.refactoring.constants import SQL_LIKE_PATTERN from sqlbuild.compiler.refactoring.models import AuthoredBody, ManualLocation, ProjectSqlFile from sqlbuild.compiler.refactoring.types import SqlFileRole @@ -36,6 +36,19 @@ def project_sql_files(*, discovered: DiscoveredProjectInputs) -> tuple[ProjectSq return tuple(files[path] for path in sorted(files)) +def yaml_files(*, discovered: DiscoveredProjectInputs) -> tuple[ProjectSqlFile, ...]: + """Return every source and seed YAML declaration file.""" + + files: dict[str, str] = { + item.relative_path.as_posix(): item.contents + for item in (*discovered.source_files, *discovered.schema_files) + } + return tuple( + ProjectSqlFile(relative_path=path, contents=files[path], role=SqlFileRole.YAML) + for path in sorted(files) + ) + + def authored_bodies(*, discovered: DiscoveredProjectInputs) -> tuple[AuthoredBody, ...]: """Locate every non-model SQL body in its file, in file order.""" @@ -118,6 +131,10 @@ def _authored_files(*, discovered: DiscoveredProjectInputs) -> tuple[_AuthoredFi (SqlFileRole.FUNCTION, item.relative_path, item.contents, (item.body_sql,)) for item in discovered.sql_function_files ) + files.extend( + (SqlFileRole.SCHEMA, item.relative_path, item.contents, ()) + for item in discovered.model_schema_files + ) return tuple(files) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/workspace.py b/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py similarity index 98% rename from src/sqlbuild/compiler/refactoring/_helpers/workspace.py rename to src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py index ee629c72ac..ddc5af27d0 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/workspace.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py @@ -6,7 +6,7 @@ import shutil from pathlib import Path, PurePosixPath -from sqlbuild.compiler.refactoring._helpers.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits from sqlbuild.compiler.refactoring.constants import ( COPIED_PROJECT_SUFFIXES, HIDDEN_PREFIX, diff --git a/src/sqlbuild/compiler/refactoring/_helpers/renames/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/renames/__init__.py new file mode 100644 index 0000000000..e0399ee685 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/renames/__init__.py @@ -0,0 +1 @@ +"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/model_planning.py b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py similarity index 65% rename from src/sqlbuild/compiler/refactoring/_helpers/model_planning.py rename to src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py index 1d2c35cece..2b80e35226 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/model_planning.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py @@ -1,4 +1,4 @@ -"""Resolve a model rename or move and find everything that blocks it.""" +"""Resolve a model rename or move, find what blocks it, and decide its migrations.""" from __future__ import annotations @@ -6,19 +6,24 @@ from pathlib import Path, PurePosixPath from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject -from sqlbuild.compiler.refactoring._helpers.header_edits import header_tokens, is_value -from sqlbuild.compiler.refactoring._helpers.model_references import ( - macro_reference_locations, - model_reference_edits, -) -from sqlbuild.compiler.refactoring._helpers.project_files import ( +from sqlbuild.compiler.refactoring._helpers.project.project_files import ( project_sql_files, python_string_locations, + yaml_files, ) -from sqlbuild.compiler.refactoring._helpers.text_edits import manual_at +from sqlbuild.compiler.refactoring._helpers.renames.model_references import ( + macro_reference_locations, + model_reference_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.header_edits import header_tokens, is_value +from sqlbuild.compiler.refactoring._helpers.text.text_edits import manual_at +from sqlbuild.compiler.refactoring._helpers.text.yaml_edits import yaml_model_edits from sqlbuild.compiler.refactoring.constants import ( GENERIC_DIALECT, + HISTORY_MATERIALIZATIONS, IDENTIFIER_PATTERN, + MATERIALIZED_KEY, + MIGRATABLE_MATERIALIZATIONS, MIGRATE_FROM_KEY, MODEL_KIND_PREFIX, PATH_SEPARATOR, @@ -28,6 +33,7 @@ from sqlbuild.compiler.refactoring.models import ( HeaderToken, ManualLocation, + ModelMigrationDecision, ModelTarget, ProjectSqlFile, RefactorParts, @@ -38,6 +44,7 @@ from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move from sqlbuild.compiler.scopes.models import DeclarationReport, MovePreview, ScopeDiagnostic +from sqlbuild.spec.contracts.main.get_config_str import get_config_str def find_model(*, project: CompiledProject, name: str) -> CompiledModel: @@ -103,7 +110,10 @@ def model_parts(*, project: RefactorProject, target: ModelTarget) -> RefactorPar old: str = target.model.name new: str = target.request.new_name return RefactorParts( - edits=model_reference_edits(files=files, old=old, new=new, dialect=dialect) + edits=( + *model_reference_edits(files=files, old=old, new=new, dialect=dialect), + *yaml_model_edits(files=yaml_files(discovered=project.discovered), old=old, new=new), + ) if new != old else (), manual=( @@ -237,3 +247,89 @@ def _pending_migration( ), ), ) + + +def decide_model_migration( + *, before: CompiledProject, after: CompiledProject | None, old: str, new: str +) -> ModelMigrationDecision: + """Declare migrate_from whenever the relation moves, unless migrations cannot follow it.""" + + old_model: CompiledModel | None = _model(project=before, name=old) + new_model: CompiledModel | None = None if after is None else _model(project=after, name=new) + if old_model is None: + return ModelMigrationDecision(needed=False, reason="model not compiled") + materialized: str | None = get_config_str( + values=(new_model or old_model).config.values, key=MATERIALIZED_KEY + ) + if materialized not in MIGRATABLE_MATERIALIZATIONS: + return ModelMigrationDecision( + needed=False, reason=f"'{materialized}' models keep no warehouse data" + ) + if after is None or new_model is None: + return ModelMigrationDecision( + needed=True, + reason="keeps the relation's history; the destination could not be checked because " + "the edited project does not compile", + ) + if _location_key(old_model) == _location_key(new_model): + return ModelMigrationDecision(needed=False, reason="relation name is unchanged") + return _blocked_move(after=after, old_model=old_model, new_model=new_model) or ( + ModelMigrationDecision( + needed=True, reason="keeps the relation's history and its old name working" + ) + ) + + +def _blocked_move( + *, after: CompiledProject, old_model: CompiledModel, new_model: CompiledModel +) -> ModelMigrationDecision | None: + old_database: str = (old_model.destination.database or "").lower() + new_database: str = (new_model.destination.database or "").lower() + if old_database and new_database and old_database != new_database: + return ModelMigrationDecision( + needed=False, + blocked=True, + reason=( + f"moves from database {old_model.destination.database} to " + f"{new_model.destination.database}; migrations cannot cross databases" + ), + ) + old_schema: str = (old_model.destination.schema or "").lower() + project_schemas: frozenset[str] = frozenset( + (model.destination.schema or "").lower() for model in after.models + ) + if old_schema in project_schemas: + return None + return ModelMigrationDecision( + needed=False, + blocked=True, + reason=( + f"no model is left in schema {old_model.destination.schema}, so migrate_from " + f"{old_model.name} cannot find the old relation; add a schema-qualified " + "migrate_from for each target" + ), + ) + + +def needs_column_migration(*, model: CompiledModel) -> bool: + """Return whether renaming a column of this model must declare migrate_from.""" + + materialized: str | None = get_config_str(values=model.config.values, key=MATERIALIZED_KEY) + if materialized in HISTORY_MATERIALIZATIONS: + return True + return ( + materialized in MIGRATABLE_MATERIALIZATIONS + and model.config.values.get(MIGRATE_FROM_KEY) is not None + ) + + +def _model(*, project: CompiledProject, name: str) -> CompiledModel | None: + return next((model for model in project.models if model.name == name), None) + + +def _location_key(model: CompiledModel) -> tuple[str, str, str]: + return ( + (model.destination.database or "").lower(), + (model.destination.schema or "").lower(), + model.destination.name.lower(), + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/model_references.py b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py similarity index 90% rename from src/sqlbuild/compiler/refactoring/_helpers/model_references.py rename to src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py index 40632f051a..9064c7bd39 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/model_references.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py @@ -4,13 +4,14 @@ from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.refactoring._helpers.header_edits import model_name_header_edits -from sqlbuild.compiler.refactoring._helpers.sql_sites import ( +from sqlbuild.compiler.refactoring._helpers.text.header_edits import model_name_header_edits +from sqlbuild.compiler.refactoring._helpers.text.schema_edits import schema_model_name_edits +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import ( authored_offset, model_body, resource_sites, ) -from sqlbuild.compiler.refactoring._helpers.text_edits import ( +from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( identifier_sites, manual_at, path_edits, @@ -60,6 +61,13 @@ def model_reference_edits( edits=model_name_header_edits(contents=item.contents, old=old, new=new), ) ) + if item.role == SqlFileRole.SCHEMA: + edits.extend( + path_edits( + path=item.relative_path, + edits=schema_model_name_edits(contents=item.contents, old=old, new=new), + ) + ) return tuple(edits) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py new file mode 100644 index 0000000000..e0399ee685 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py @@ -0,0 +1 @@ +"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py similarity index 77% rename from src/sqlbuild/compiler/refactoring/_helpers/header_edits.py rename to src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py index f3a0154e1b..c7e3fbc272 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/header_edits.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py @@ -5,7 +5,8 @@ import sqlbuild._native as _native from sqlbuild.compiler.discovery.main._model_header_spans import get_model_header_spans from sqlbuild.compiler.discovery.models import ModelHeaderSpans -from sqlbuild.compiler.refactoring._helpers.text_edits import text_edit, whole_word_offsets +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import embedded_ref_spans +from sqlbuild.compiler.refactoring._helpers.text.text_edits import text_edit, whole_word_offsets from sqlbuild.compiler.refactoring.constants import ( COLUMN_VALUED_CONFIG_KEYS, COLUMNS_KEY, @@ -20,11 +21,17 @@ IDENTIFIER_PATTERN, NATIVE_HEADER_TOKEN_KINDS, PARENTHESIZED_EMPTY_TOKENS, + REF_FUNCTION, RELATIONSHIPS_AUDIT, RELATIONSHIPS_FIELD_KEY, RELATIONSHIPS_TO_KEY, ) -from sqlbuild.compiler.refactoring.models import HeaderEntry, HeaderToken, TextEdit +from sqlbuild.compiler.refactoring.models import ( + HeaderEntry, + HeaderToken, + RelationshipTokens, + TextEdit, +) from sqlbuild.compiler.refactoring.types import EditKind, HeaderTokenKind from sqlbuild.lint.main.quoted_value_end import quoted_value_end @@ -37,8 +44,14 @@ def header_tokens(*, contents: str) -> tuple[HeaderToken, ...] | None: spans: ModelHeaderSpans = get_model_header_spans(contents=contents) if spans.body is None: return None - body_start: int = spans.body[0] - body: str = contents[spans.body[0] : spans.body[1]] + return span_tokens(contents=contents, span=spans.body) + + +def span_tokens(*, contents: str, span: tuple[int, int]) -> tuple[HeaderToken, ...]: + """Tokenize one declaration header body in file offsets.""" + + body_start: int = span[0] + body: str = contents[span[0] : span[1]] tokens: list[HeaderToken] = [] depth: int = 0 raw_kind: int @@ -106,12 +119,10 @@ def cursor_input_tokens( return () -def relationship_tokens( - *, tokens: tuple[HeaderToken, ...] -) -> tuple[tuple[HeaderToken | None, HeaderToken | None], ...]: - """Return the (to, field) value tokens of every relationships audit.""" +def relationship_tokens(*, tokens: tuple[HeaderToken, ...]) -> tuple[RelationshipTokens, ...]: + """Return the target and field value tokens of every relationships audit.""" - found: list[tuple[HeaderToken | None, HeaderToken | None]] = [] + found: list[RelationshipTokens] = [] index: int token: HeaderToken for index, token in enumerate(tokens): @@ -123,7 +134,7 @@ def relationship_tokens( ): continue depth: int = tokens[index + 1].depth + 1 - values: dict[str, HeaderToken] = {} + values: dict[str, int] = {} cursor: int = index + 2 while cursor < len(tokens) and tokens[cursor].depth >= depth: candidate: HeaderToken = tokens[cursor] @@ -133,9 +144,15 @@ def relationship_tokens( and candidate.value in {RELATIONSHIPS_TO_KEY, RELATIONSHIPS_FIELD_KEY} and cursor + 1 < len(tokens) ): - values[candidate.value] = tokens[cursor + 1] + values[candidate.value] = cursor + 1 cursor += 1 - found.append((values.get(RELATIONSHIPS_TO_KEY), values.get(RELATIONSHIPS_FIELD_KEY))) + found.append( + _relationship( + tokens=tokens, + to_index=values.get(RELATIONSHIPS_TO_KEY), + field_index=values.get(RELATIONSHIPS_FIELD_KEY), + ) + ) return tuple(found) @@ -265,23 +282,49 @@ def unhandled_word_offsets( def model_name_header_edits(*, contents: str, old: str, new: str) -> tuple[TextEdit, ...]: - """Rename a model in cursor_inputs keys and relationships `to` values.""" + """Rename a model in cursor_inputs keys, bare relationships targets, and quoted SQL.""" tokens: tuple[HeaderToken, ...] | None = header_tokens(contents=contents) if tokens is None: return () + return model_name_token_edits(contents=contents, tokens=tokens, old=old, new=new) + + +def model_name_token_edits( + *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename a model in the given header tokens.""" + names: list[HeaderToken] = [ name for name, _ in cursor_input_tokens(tokens=tokens) if is_value(token=name, value=old) ] names.extend( - to_token - for to_token, _ in relationship_tokens(tokens=tokens) - if to_token is not None and is_value(token=to_token, value=old) + relationship.target + for relationship in relationship_tokens(tokens=tokens) + if relationship.target is not None + and not relationship.called + and is_value(token=relationship.target, value=old) ) - return tuple( + edits: list[TextEdit] = [ rename_value_token(contents=contents, token=name, new_value=new, kind=EditKind.HEADER) for name in names - ) + ] + token: HeaderToken + for token in tokens: + if token.kind == HeaderTokenKind.STRING: + edits.extend( + text_edit( + text=contents, + start=token.start + start, + end=token.start + end, + replacement=new, + kind=EditKind.REFERENCE, + ) + for start, end in embedded_ref_spans( + text=contents[token.start : token.end], name=old + ) + ) + return tuple(edits) def consumer_column_header_edits( @@ -292,22 +335,29 @@ def consumer_column_header_edits( tokens: tuple[HeaderToken, ...] | None = header_tokens(contents=contents) if tokens is None: return () + return column_token_edits(contents=contents, tokens=tokens, upstream=upstream, old=old, new=new) + + +def column_token_edits( + *, contents: str, tokens: tuple[HeaderToken, ...], upstream: str, old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename an upstream column in the given header tokens.""" + values: list[HeaderToken] = [] name: HeaderToken inputs: tuple[HeaderToken, ...] for name, inputs in cursor_input_tokens(tokens=tokens): if is_value(token=name, value=upstream): values.extend(value for value in inputs if is_value(token=value, value=old)) - to_token: HeaderToken | None - field_token: HeaderToken | None - for to_token, field_token in relationship_tokens(tokens=tokens): + relationship: RelationshipTokens + for relationship in relationship_tokens(tokens=tokens): if ( - to_token is not None - and field_token is not None - and is_value(token=to_token, value=upstream) - and is_value(token=field_token, value=old) + relationship.target is not None + and relationship.field is not None + and is_value(token=relationship.target, value=upstream) + and is_value(token=relationship.field, value=old) ): - values.append(field_token) + values.append(relationship.field) return tuple( rename_value_token(contents=contents, token=value, new_value=new, kind=EditKind.HEADER) for value in values @@ -347,6 +397,23 @@ def _column_migration_edit( ) +def _relationship( + *, tokens: tuple[HeaderToken, ...], to_index: int | None, field_index: int | None +) -> RelationshipTokens: + field: HeaderToken | None = tokens[field_index] if field_index is not None else None + if to_index is None: + return RelationshipTokens(target=None, field=field, called=False) + called: bool = ( + tokens[to_index].value == REF_FUNCTION + and to_index + 2 < len(tokens) + and tokens[to_index + 1].value == HEADER_OPEN_PAREN + and tokens[to_index + 2].kind == HeaderTokenKind.STRING + ) + return RelationshipTokens( + target=tokens[to_index + 2] if called else tokens[to_index], field=field, called=called + ) + + def _nested_entries( *, tokens: tuple[HeaderToken, ...], depth: int ) -> tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...]: diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py new file mode 100644 index 0000000000..c6c9e58088 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py @@ -0,0 +1,55 @@ +"""Model and column references in reusable SCHEMA declaration headers.""" + +from __future__ import annotations + +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( + column_token_edits, + model_name_token_edits, + span_tokens, +) +from sqlbuild.compiler.refactoring.constants import SCHEMA_STATEMENT_PATTERN +from sqlbuild.compiler.refactoring.models import TextEdit + + +def schema_model_name_edits(*, contents: str, old: str, new: str) -> tuple[TextEdit, ...]: + """Rename a model in every SCHEMA header of a declaration file.""" + + edits: list[TextEdit] = [] + span: tuple[int, int] + for span in _schema_spans(contents): + edits.extend( + model_name_token_edits( + contents=contents, + tokens=span_tokens(contents=contents, span=span), + old=old, + new=new, + ) + ) + return tuple(edits) + + +def schema_column_edits( + *, contents: str, upstream: str, old: str, new: str +) -> tuple[TextEdit, ...]: + """Rename relationships `field` values pointing at a renamed column of a model.""" + + edits: list[TextEdit] = [] + span: tuple[int, int] + for span in _schema_spans(contents): + edits.extend( + column_token_edits( + contents=contents, + tokens=span_tokens(contents=contents, span=span), + upstream=upstream, + old=old, + new=new, + ) + ) + return tuple(edits) + + +def _schema_spans(contents: str) -> tuple[tuple[int, int], ...]: + return tuple( + (match.start("header"), match.end("header")) + for match in SCHEMA_STATEMENT_PATTERN.finditer(contents) + ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/sql_sites.py b/src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py similarity index 93% rename from src/sqlbuild/compiler/refactoring/_helpers/sql_sites.py rename to src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py index 876158c1ff..4f209e46a3 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/sql_sites.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py @@ -14,6 +14,7 @@ MappedOffset, ) from sqlbuild.compiler.refactoring.constants import ( + EMBEDDED_REF_PATTERN, GENERIC_PLACEHOLDER_KIND, PLACEHOLDER_BASE, PLACEHOLDER_PAD, @@ -46,6 +47,16 @@ def resource_sites(*, text: str, dialect: str) -> tuple[ResourceSite, ...]: return tuple(sites) +def embedded_ref_spans(*, text: str, name: str) -> tuple[tuple[int, int], ...]: + """Return the name spans of `__ref` calls to a model inside a quoted string's raw text.""" + + return tuple( + (match.start("name"), match.end("name")) + for match in EMBEDDED_REF_PATTERN.finditer(text) + if match.group("name") == name + ) + + def analysis_sql(*, text: str, dialect: str) -> AnalysisSql: """Replace every interpolation site with an identifier of the same length.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/text_edits.py similarity index 100% rename from src/sqlbuild/compiler/refactoring/_helpers/text_edits.py rename to src/sqlbuild/compiler/refactoring/_helpers/text/text_edits.py diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py new file mode 100644 index 0000000000..aec4de9fa9 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py @@ -0,0 +1,171 @@ +"""Model and column references in source and seed YAML declarations.""" + +from __future__ import annotations + +import yaml +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode + +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import embedded_ref_spans +from sqlbuild.compiler.refactoring._helpers.text.text_edits import path_edits, text_edit +from sqlbuild.compiler.refactoring.constants import ( + RELATIONSHIPS_AUDIT, + RELATIONSHIPS_FIELD_KEY, + RELATIONSHIPS_TO_KEY, +) +from sqlbuild.compiler.refactoring.models import ProjectSqlFile, TextEdit, YamlRelationship +from sqlbuild.compiler.refactoring.types import EditKind + + +def yaml_model_edits( + *, files: tuple[ProjectSqlFile, ...], old: str, new: str +) -> tuple[tuple[str, TextEdit], ...]: + """Rename `__ref` calls in YAML strings and bare relationships targets naming a model.""" + + edits: list[tuple[str, TextEdit]] = [] + item: ProjectSqlFile + for item in files: + root: Node | None = _compose(item.contents) + if root is None: + continue + found: list[TextEdit] = [] + scalar: ScalarNode + for scalar in _scalars(root): + found.extend( + text_edit( + text=item.contents, + start=_start(scalar) + start, + end=_start(scalar) + end, + replacement=new, + kind=EditKind.REFERENCE, + ) + for start, end in embedded_ref_spans( + text=_raw(contents=item.contents, scalar=scalar), name=old + ) + ) + found.extend( + _rename_scalar( + contents=item.contents, + scalar=relationship.target, + new=new, + kind=EditKind.REFERENCE, + ) + for relationship in _relationships(root) + if relationship.target is not None and relationship.target.value == old + ) + edits.extend(path_edits(path=item.relative_path, edits=tuple(found))) + return tuple(edits) + + +def yaml_column_edits( + *, files: tuple[ProjectSqlFile, ...], model: str, old: str, new: str +) -> tuple[tuple[str, TextEdit], ...]: + """Rename relationships `field` values that point at a renamed column of a model.""" + + edits: list[tuple[str, TextEdit]] = [] + item: ProjectSqlFile + for item in files: + root: Node | None = _compose(item.contents) + if root is None: + continue + edits.extend( + path_edits( + path=item.relative_path, + edits=tuple( + _rename_scalar( + contents=item.contents, + scalar=relationship.field, + new=new, + kind=EditKind.COLUMN, + ) + for relationship in _relationships(root) + if relationship.field is not None + and relationship.field.value.lower() == old.lower() + and _targets(contents=item.contents, relationship=relationship, model=model) + ), + ) + ) + return tuple(edits) + + +def _compose(contents: str) -> Node | None: + try: + return yaml.compose(contents, Loader=yaml.SafeLoader) + except yaml.YAMLError: + return None + + +def _scalars(node: Node) -> tuple[ScalarNode, ...]: + if isinstance(node, ScalarNode): + return (node,) + children: list[Node] = [] + if isinstance(node, SequenceNode): + children.extend(node.value) + if isinstance(node, MappingNode): + children.extend(value for _, value in node.value) + found: list[ScalarNode] = [] + child: Node + for child in children: + found.extend(_scalars(child)) + return tuple(found) + + +def _relationships(node: Node) -> tuple[YamlRelationship, ...]: + found: list[YamlRelationship] = [] + if isinstance(node, MappingNode): + key: Node + value: Node + for key, value in node.value: + if ( + isinstance(key, ScalarNode) + and key.value == RELATIONSHIPS_AUDIT + and isinstance(value, MappingNode) + ): + found.append(_relationship(value)) + found.extend(_relationships(value)) + if isinstance(node, SequenceNode): + child: Node + for child in node.value: + found.extend(_relationships(child)) + return tuple(found) + + +def _relationship(node: MappingNode) -> YamlRelationship: + values: dict[str, ScalarNode] = { + key.value: value + for key, value in node.value + if isinstance(key, ScalarNode) and isinstance(value, ScalarNode) + } + return YamlRelationship( + target=values.get(RELATIONSHIPS_TO_KEY), field=values.get(RELATIONSHIPS_FIELD_KEY) + ) + + +def _targets(*, contents: str, relationship: YamlRelationship, model: str) -> bool: + target: ScalarNode | None = relationship.target + if target is None: + return False + return target.value == model or bool( + embedded_ref_spans(text=_raw(contents=contents, scalar=target), name=model) + ) + + +def _rename_scalar(*, contents: str, scalar: ScalarNode, new: str, kind: EditKind) -> TextEdit: + raw: str = _raw(contents=contents, scalar=scalar) + offset: int = max(raw.lower().find(scalar.value.lower()), 0) + start: int = _start(scalar) + offset + return text_edit( + text=contents, + start=start, + end=start + len(scalar.value), + replacement=new, + kind=kind, + ) + + +def _raw(*, contents: str, scalar: ScalarNode) -> str: + end: int = scalar.end_mark.index if scalar.end_mark is not None else _start(scalar) + return contents[_start(scalar) : end] + + +def _start(scalar: ScalarNode) -> int: + return scalar.start_mark.index if scalar.start_mark is not None else 0 diff --git a/src/sqlbuild/compiler/refactoring/constants.py b/src/sqlbuild/compiler/refactoring/constants.py index 7d57ecefc9..7cee4805a0 100644 --- a/src/sqlbuild/compiler/refactoring/constants.py +++ b/src/sqlbuild/compiler/refactoring/constants.py @@ -4,7 +4,12 @@ import re -from sqlbuild.compiler.refactoring.types import HeaderTokenKind, RefactorOperation, SqlFileRole +from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN +from sqlbuild.compiler.refactoring.types import ( + HeaderTokenKind, + RefactorOperation, + SqlFileRole, +) MODEL_KIND_PREFIX: str = "model:" COLUMN_KIND_PREFIX: str = "column:" @@ -18,6 +23,9 @@ RESOURCE_CALL_PATTERN: re.Pattern[str] = re.compile( r"^__(?Pref|source|seed)\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" ) +EMBEDDED_REF_PATTERN: re.Pattern[str] = re.compile( + r"""__ref\s*\(\s*\\?['"](?P[^'"\\]+)\\?['"]\s*\)""" +) REF_FIXTURE_PREFIX: str = "__ref__" EXPECTED_FIXTURE_PREFIX: str = "__expected__" ASSERT_FIXTURE_PREFIX: str = "__assert__" @@ -96,9 +104,9 @@ UNKNOWN_COLUMN_TYPE: str = "UNKNOWN" MODEL_MANUAL_HELP: str = ( "Pass the model into the macro as an argument, or edit the listed locations, then run the " - "command again; --allow-manual applies the safe edits and leaves them to you." + "command again." ) -COLUMN_MANUAL_HELP: str = ( - "Edit the listed locations, or rerun with --allow-manual to apply the safe edits and finish " - "them by hand." +COLUMN_MANUAL_HELP: str = "Edit the listed locations, then run the command again." +SCHEMA_STATEMENT_PATTERN: re.Pattern[str] = re.compile( + r"\bSCHEMA\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;" ) diff --git a/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py b/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py index 88cf306a77..8b8ec24ce7 100644 --- a/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py +++ b/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.refactoring._helpers.workspace import commit_changes +from sqlbuild.compiler.refactoring._helpers.project.workspace import commit_changes from sqlbuild.compiler.refactoring.models import RefactorPlan diff --git a/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py b/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py index 67080f0b0d..40c92ac8ba 100644 --- a/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py +++ b/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py @@ -2,11 +2,11 @@ from __future__ import annotations -from sqlbuild.compiler.refactoring._helpers.column_planning import ( +from sqlbuild.compiler.refactoring._helpers.columns.column_planning import ( column_rename_context, column_rename_parts, ) -from sqlbuild.compiler.refactoring._helpers.text_edits import build_plan +from sqlbuild.compiler.refactoring._helpers.text.text_edits import build_plan from sqlbuild.compiler.refactoring.constants import COLUMN_MANUAL_HELP from sqlbuild.compiler.refactoring.models import ( ColumnRenameContext, diff --git a/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py index 6a244337c8..7548135426 100644 --- a/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py +++ b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py @@ -2,8 +2,8 @@ from __future__ import annotations -from sqlbuild.compiler.refactoring._helpers.model_planning import model_parts, model_target -from sqlbuild.compiler.refactoring._helpers.text_edits import build_plan +from sqlbuild.compiler.refactoring._helpers.renames.model_planning import model_parts, model_target +from sqlbuild.compiler.refactoring._helpers.text.text_edits import build_plan from sqlbuild.compiler.refactoring.constants import MODEL_MANUAL_HELP from sqlbuild.compiler.refactoring.models import ( ModelTarget, diff --git a/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py b/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py index b1c499a716..c0aa7be537 100644 --- a/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py +++ b/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.refactoring._helpers.workspace import ( +from sqlbuild.compiler.refactoring._helpers.project.workspace import ( copy_project_inputs, read_originals, write_staged_changes, diff --git a/src/sqlbuild/compiler/refactoring/main/with_model_migration.py b/src/sqlbuild/compiler/refactoring/main/with_model_migration.py index 09f146fc7c..2b9feb50f5 100644 --- a/src/sqlbuild/compiler/refactoring/main/with_model_migration.py +++ b/src/sqlbuild/compiler/refactoring/main/with_model_migration.py @@ -1,12 +1,12 @@ -"""Add model migrate_from to a staged rename exactly where discovery would miss it.""" +"""Add model migrate_from to a staged rename whenever its relation moves.""" from __future__ import annotations from dataclasses import replace from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.refactoring._helpers.header_edits import insert_header_entry_edit -from sqlbuild.compiler.refactoring._helpers.model_migration import decide_model_migration +from sqlbuild.compiler.refactoring._helpers.renames.model_planning import decide_model_migration +from sqlbuild.compiler.refactoring._helpers.text.header_edits import insert_header_entry_edit from sqlbuild.compiler.refactoring.constants import MIGRATE_FROM_KEY from sqlbuild.compiler.refactoring.models import ( FileChange, @@ -22,10 +22,10 @@ def with_model_migration( *, plan: RefactorPlan, before: CompiledProject, - after: CompiledProject, + after: CompiledProject | None, originals: dict[str, str], ) -> RefactorPlan: - """Return the plan with `migrate_from ` on the model when history would be lost.""" + """Return the plan with `migrate_from ` on the model whenever its relation moves.""" old: str = plan.request.model_name new: str = plan.request.new_name @@ -57,8 +57,8 @@ def with_model_migration( if edit is None: return replace( plan, - manual=( - *plan.manual, + blocking=( + *plan.blocking, ManualLocation( path=change.path, line=1, diff --git a/src/sqlbuild/compiler/refactoring/models.py b/src/sqlbuild/compiler/refactoring/models.py index e79594b157..61d7eee641 100644 --- a/src/sqlbuild/compiler/refactoring/models.py +++ b/src/sqlbuild/compiler/refactoring/models.py @@ -5,6 +5,8 @@ from dataclasses import dataclass, field from pathlib import Path +from yaml.nodes import ScalarNode + from sqlbuild.compiler.compile.models import CompiledModel, ExpansionSpan from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -185,6 +187,23 @@ class HeaderToken: depth: int +@dataclass(frozen=True) +class RelationshipTokens: + """The target and field value tokens of one relationships audit in a MODEL header.""" + + target: HeaderToken | None + field: HeaderToken | None + called: bool + + +@dataclass(frozen=True) +class YamlRelationship: + """The `to` and `field` scalars of one relationships audit in a YAML declaration.""" + + target: ScalarNode | None + field: ScalarNode | None + + @dataclass(frozen=True) class HeaderEntry: """One top-level `key value` entry of a MODEL header.""" @@ -281,6 +300,8 @@ class ColumnRenameContext: columns: ResourceColumns contents: dict[str, str] bodies: tuple[AuthoredBody, ...] + yaml_files: tuple[ProjectSqlFile, ...] + schema_files: tuple[ProjectSqlFile, ...] old: str new: str cascade: bool diff --git a/src/sqlbuild/compiler/refactoring/types.py b/src/sqlbuild/compiler/refactoring/types.py index ccf89da446..9377e1f1ed 100644 --- a/src/sqlbuild/compiler/refactoring/types.py +++ b/src/sqlbuild/compiler/refactoring/types.py @@ -43,6 +43,8 @@ class SqlFileRole(StrEnum): AUDIT = "audit" HOOK = "hook" FUNCTION = "function" + SCHEMA = "schema" + YAML = "yaml" class HeaderTokenKind(StrEnum): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py index 6d6cf0e07c..111e864dcf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -22,9 +22,11 @@ class ModelMigrationE2ETestCase: materialized: str udf: bool expected_declaration: str - expected_migrate_from_count: int expected_old_name_type: str expected_order_ids: tuple[int, ...] + extra_files: dict[str, str] = field(default_factory=dict) + removed_files: tuple[str, ...] = () + stripped_fingerprints: tuple[str, ...] = () @dataclass(frozen=True) @@ -44,6 +46,7 @@ class RefusedRefactorE2ETestCase: extra_files: dict[str, str] expected_status: str expected_reason: str + expected_paths: tuple[str, ...] @dataclass(frozen=True) @@ -59,3 +62,22 @@ class DryRunE2ETestCase: description: str command: tuple[str, ...] expected_paths: frozenset[str] + + +@dataclass(frozen=True) +class CombinedRenameE2ETestCase: + description: str + expected_declarations: tuple[str, ...] + expected_new_columns: tuple[str, ...] + expected_old_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class MissingOriginE2ETestCase: + description: str + target_settings: str + build_args: tuple[str, ...] + expected_exit: int + expected_output: str + expected_relation_type: str | None + expected_columns: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index a17df157c0..83c4280f7c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import json import subprocess from collections.abc import Mapping @@ -17,12 +18,20 @@ DATABASE_FILE: str = "orders.duckdb" SCHEMA: str = "analytics" -_PROJECT_TOML: str = ( - 'name = "orders_project"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' - f'[connection]\ndatabase = "{DATABASE_FILE}"\n\n' - f'[targets.dev]\nschema = "{SCHEMA}"\n\n' - "[janitor]\nenabled = true\n" -) + + +def project_toml(*, target_settings: str = "") -> str: + """Return the project config, with extra settings for the dev target.""" + + return ( + 'name = "orders_project"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + f'[connection]\ndatabase = "{DATABASE_FILE}"\n\n' + f'[targets.dev]\nschema = "{SCHEMA}"\n{target_settings}\n' + "[janitor]\nenabled = true\n" + ) + + +_PROJECT_TOML: str = project_toml() _SOURCES_YML: str = ( "sources:\n" " - name: raw_orders\n" @@ -88,6 +97,50 @@ 'MODEL (\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' ), } +CUSTOMER_DECLARATIONS: dict[str, str] = { + "seeds/customers.csv": "customer_id,customer_name\n10,Ada\n11,Bo\n", + "seeds/customers.yml": ( + "seeds:\n" + " - name: customers\n" + " columns:\n" + " - name: customer_id\n" + " type: INTEGER\n" + " audits:\n" + " - relationships:\n" + ' to: __ref("stg_orders")\n' + " field: customer_id\n" + " - name: customer_name\n" + " type: VARCHAR\n" + ), + "sources/customers.yml": ( + "sources:\n" + " - name: raw_customers\n" + ' expression: "SELECT 10 AS customer_id UNION ALL SELECT 11"\n' + " columns:\n" + " - name: customer_id\n" + " audits:\n" + " - relationships: {to: '__ref(\"stg_orders\")', field: customer_id}\n" + ), +} +ORDER_SHAPE: dict[str, str] = { + "models/marts/_sqlbuild/_schemas/order_shape.sql": ( + "SCHEMA (\n name order_shape,\n columns (\n customer_id (audits [relationships " + '(to __ref("stg_orders"), field customer_id)]),\n ),\n);\n' + ), + "models/marts/fact_orders.sql": FACT_ORDERS.replace( + "materialized table,", "materialized table,\n model_schema order_shape," + ), +} +ORDERS_MACRO: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_TOML + "\n[references]\nenforce_explicit = false\n", + "models/staging/_sqlbuild/_macros/orders.py": ( + "def staged_orders() -> str:\n return '__ref(\"stg_orders\")'\n" + ), + "models/staging/stg_order_count.sql": ( + "MODEL (\n materialized view,\n);\n\n" + "SELECT COUNT(*) AS order_count\nFROM @staged_orders()\n" + ), +} _INCREMENTAL_HEADER: str = ( " materialized incremental,\n" " incremental_strategy delete_insert,\n" @@ -137,6 +190,63 @@ def write_orders_project( return project_dir +def order_history_copy() -> dict[str, str]: + """Return a second model with exactly the order_history definition.""" + + return { + "models/marts/order_history_copy.sql": order_history_files( + materialized="incremental", udf=False + )["models/marts/order_history.sql"] + } + + +def strip_migration_fingerprints(*, project_dir: Path, models: tuple[str, ...]) -> None: + """Make built models look like ones built before rename matching recorded fingerprints.""" + + import duckdb + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / DATABASE_FILE)) + try: + rows: list[tuple[Any, ...]] = connection.execute( + f"SELECT node_name, metadata_json_b64 FROM {SCHEMA}._sqlbuild_fingerprints " + "WHERE list_contains(?, node_name)", + [list(models)], + ).fetchall() + model: str + encoded: str + for model, encoded in rows: + metadata: dict[str, Any] = json.loads(base64.b64decode(encoded)) + _ = metadata.pop("migration_fingerprint", None) + _ = connection.execute( + f"UPDATE {SCHEMA}._sqlbuild_fingerprints SET metadata_json_b64 = ? " + "WHERE node_name = ? AND metadata_json_b64 = ?", + [base64.b64encode(json.dumps(metadata).encode()).decode(), model, encoded], + ) + finally: + connection.close() + + +def declare_missing_column_origin(*, project_dir: Path) -> None: + """Add a revenue column that declares migrate_from a column the table never had.""" + + path: Path = project_dir / "models/marts/order_history.sql" + contents: str = path.read_text(encoding="utf-8") + _ = path.write_text( + contents.replace( + "MODEL (\n", "MODEL (\n columns (revenue (migrate_from gross_amount)),\n", 1 + ).replace(" order_date\n", " order_date,\n amount AS revenue\n", 1), + encoding="utf-8", + ) + + +def remove_files(*, project_dir: Path, paths: tuple[str, ...]) -> None: + """Delete project files by hand, as an author removing a model would.""" + + path: str + for path in paths: + (project_dir / path).unlink() + + def load_raw_orders(*, project_dir: Path, order_ids: tuple[int, ...]) -> None: """Replace the raw orders with one order per id, dated January of that day.""" @@ -172,6 +282,7 @@ def project_text(project_dir: Path) -> dict[str, str]: paths: tuple[Path, ...] = ( project_dir / "sqlbuild_project.toml", *project_dir.glob("sources/*.yml"), + *project_dir.glob("seeds/*.yml"), *project_dir.glob("models/**/*.sql"), *project_dir.glob("models/**/*.py"), *project_dir.glob("functions/**/*.sql"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py new file mode 100644 index 0000000000..2cebb92ade --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py @@ -0,0 +1,135 @@ +"""CLI e2e coverage for the missing_migration_origin target policy.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + MissingOriginE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + declare_missing_column_origin, + order_history_files, + project_toml, + relation_columns, + relation_type, + sqb, + write_orders_project, +) + +_DENY: str = 'missing_migration_origin = "deny"\n' +_CONFIRM: str = 'missing_migration_origin = "require_confirmation"\n' + + +@pytest.mark.parametrize( + "test_case", + [ + MissingOriginE2ETestCase( + description="fresh target builds with a warning by default", + target_settings="", + build_args=(), + expected_exit=0, + expected_output="migrate_from origin analytics.stg_orders does not exist", + expected_relation_type="VIEW", + ), + MissingOriginE2ETestCase( + description="deny stops the build", + target_settings=_DENY, + build_args=(), + expected_exit=1, + expected_output="error[M102]", + expected_relation_type=None, + ), + MissingOriginE2ETestCase( + description="require_confirmation stops an unconfirmed build", + target_settings=_CONFIRM, + build_args=(), + expected_exit=1, + expected_output="--allow-missing-migration-origin", + expected_relation_type=None, + ), + MissingOriginE2ETestCase( + description="require_confirmation builds once confirmed", + target_settings=_CONFIRM, + build_args=("--allow-missing-migration-origin",), + expected_exit=0, + expected_output="does not exist in this target", + expected_relation_type="VIEW", + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_model_on_fresh_target_when_building_then_policy_decides( + tmp_path: Path, test_case: MissingOriginE2ETestCase +) -> None: + """A rename's migrate_from finds no old relation on a target that never built it.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, + files={"sqlbuild_project.toml": project_toml(target_settings=test_case.target_settings)}, + ) + + renamed: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:stg_orders", "stg_order_lines" + ) + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build", *test_case.build_args) + + assert renamed.returncode == 0, renamed.stdout + renamed.stderr + assert built.returncode == test_case.expected_exit, built.stdout + built.stderr + assert test_case.expected_output in built.stdout + built.stderr + assert ( + relation_type(project_dir=project_dir, name="stg_order_lines") + == test_case.expected_relation_type + ) + + +@pytest.mark.parametrize( + "test_case", + [ + MissingOriginE2ETestCase( + description="missing origin column is allowed by default", + target_settings="", + build_args=(), + expected_exit=0, + expected_output="column gross_amount does not exist", + expected_relation_type="BASE TABLE", + expected_columns=("order_id", "amount", "order_date", "revenue"), + ), + MissingOriginE2ETestCase( + description="missing origin column stops the build under deny", + target_settings=_DENY, + build_args=(), + expected_exit=1, + expected_output="error[M109]", + expected_relation_type="BASE TABLE", + expected_columns=("order_id", "amount", "order_date"), + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_column_when_building_then_policy_decides( + tmp_path: Path, test_case: MissingOriginE2ETestCase +) -> None: + """A column migrate_from naming a column the table never had follows the same policy.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, + files={ + **order_history_files(materialized="incremental", udf=False), + "sqlbuild_project.toml": project_toml(target_settings=test_case.target_settings), + }, + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + declare_missing_column_origin(project_dir=project_dir) + + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert first.returncode == 0, first.stdout + first.stderr + assert built.returncode == test_case.expected_exit, built.stdout + built.stderr + assert test_case.expected_output in built.stdout + built.stderr + assert relation_columns(project_dir=project_dir, name="order_history") == ( + test_case.expected_columns + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py index 6de1d682bb..685c90cd92 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py @@ -12,11 +12,15 @@ from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( ColumnMigrationE2ETestCase, ColumnRenameE2ETestCase, + CombinedRenameE2ETestCase, RefusedRefactorE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( CENTS_MACRO, + CUSTOMER_DECLARATIONS, ORDER_EXPORT, + ORDER_SHAPE, + ORDERS_MACRO, fragments_present, load_raw_orders, order_history_files, @@ -58,24 +62,38 @@ }, ), ColumnRenameE2ETestCase( - description="cascade follows SELECT star consumers", - command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), - extra_files=ORDER_EXPORT, - expected_fragments={"models/marts/customer_totals.sql": ("SUM(revenue)",)}, + description="seed and source relationships follow the column", + command=("rename", "column:stg_orders.customer_id", "buyer_id"), + extra_files=CUSTOMER_DECLARATIONS, + expected_fragments={ + "seeds/customers.yml": ("field: buyer_id",), + "sources/customers.yml": ("field: buyer_id}",), + "models/marts/fact_orders.sql": (" o.buyer_id AS customer_id,",), + }, expected_columns={ - "fact_orders": ("order_id", "customer_id", "revenue", "order_date"), - "order_export": ("order_id", "customer_id", "revenue", "order_date"), + "stg_orders": ("order_id", "buyer_id", "amount", "order_date"), }, ), ColumnRenameE2ETestCase( - description="allow manual applies safe edits and lists the rest", - command=("rename", "column:fact_orders.amount", "revenue", "--allow-manual"), + description="reusable schema relationships follow the column", + command=("rename", "column:stg_orders.customer_id", "buyer_id"), + extra_files=ORDER_SHAPE, + expected_fragments={ + "models/marts/_sqlbuild/_schemas/order_shape.sql": ("field buyer_id)",), + }, + expected_columns={ + "fact_orders": ("order_id", "customer_id", "amount", "order_date"), + }, + ), + ColumnRenameE2ETestCase( + description="cascade follows SELECT star consumers", + command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), extra_files=ORDER_EXPORT, - expected_fragments={"models/marts/fact_orders.sql": (" o.amount AS revenue,",)}, + expected_fragments={"models/marts/customer_totals.sql": ("SUM(revenue)",)}, expected_columns={ + "fact_orders": ("order_id", "customer_id", "revenue", "order_date"), "order_export": ("order_id", "customer_id", "revenue", "order_date"), }, - expected_manual=("models/marts/order_export.sql",), ), ], ids=lambda case: case.description, @@ -115,6 +133,7 @@ def test_given_column_rename_when_applied_then_project_tests_and_builds( extra_files=CENTS_MACRO, expected_status="refused", expected_reason="macro:to_cents used by model:stg_order_cents is not visible", + expected_paths=("models/staging/_sqlbuild/_macros/cents.py",), ), RefusedRefactorE2ETestCase( description="rename onto an existing model", @@ -122,6 +141,7 @@ def test_given_column_rename_when_applied_then_project_tests_and_builds( extra_files={}, expected_status="refused", expected_reason="model:fact_orders already exists", + expected_paths=("models/marts/fact_orders.sql",), ), RefusedRefactorE2ETestCase( description="column used inside macro-generated SQL", @@ -129,6 +149,7 @@ def test_given_column_rename_when_applied_then_project_tests_and_builds( extra_files=CENTS_MACRO, expected_status="refused", expected_reason="referenced in SQL a macro generates", + expected_paths=("models/staging/stg_order_cents.sql",), ), RefusedRefactorE2ETestCase( description="SELECT star consumer without cascade", @@ -136,13 +157,15 @@ def test_given_column_rename_when_applied_then_project_tests_and_builds( extra_files=ORDER_EXPORT, expected_status="refused", expected_reason="rerun with --cascade", + expected_paths=("models/marts/order_export.sql",), ), RefusedRefactorE2ETestCase( - description="allow manual that leaves the project broken", - command=("rename", "column:stg_orders.amount", "revenue", "--allow-manual"), - extra_files=CENTS_MACRO, - expected_status="compile_failed", - expected_reason="Unknown column 'amount'", + description="model reference produced by a macro", + command=("rename", "model:stg_orders", "stg_order_lines"), + extra_files=ORDERS_MACRO, + expected_status="refused", + expected_reason="reference to stg_orders produced by a macro", + expected_paths=("models/staging/stg_order_count.sql",), ), ], ids=lambda case: case.description, @@ -163,6 +186,10 @@ def test_given_unsafe_refactor_when_running_then_no_file_changes( assert code == 1 assert payload["status"] == test_case.expected_status assert test_case.expected_reason in json.dumps(payload) + assert ( + tuple(sorted({item["path"] for item in (*payload["manual"], *payload["blocking"])})) + == test_case.expected_paths + ) assert text.returncode == 1 assert "no files changed" in text.stdout assert project_text(project_dir) == before @@ -207,3 +234,47 @@ def test_given_built_incremental_when_renaming_column_then_build_keeps_history( assert order_ids(project_dir=project_dir, name="order_history") == ( test_case.expected_order_ids ) + + +@pytest.mark.parametrize( + "test_case", + [ + CombinedRenameE2ETestCase( + description="table renamed, then its column, before one build", + expected_declarations=("migrate_from order_history", "revenue (migrate_from amount)"), + expected_new_columns=("order_id", "revenue", "order_date"), + expected_old_columns=("order_id", "amount", "order_date"), + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_table_when_renaming_its_column_then_build_keeps_both( + tmp_path: Path, test_case: CombinedRenameE2ETestCase +) -> None: + """A renamed table declares migrate_from, so its column rename is declared too.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, files=order_history_files(materialized="table", udf=False) + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + model: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:order_history", "order_ledger" + ) + column: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "column:order_ledger.amount", "revenue" + ) + header: str = (project_dir / "models/marts/order_ledger.sql").read_text(encoding="utf-8") + second: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert first.returncode == 0, first.stdout + first.stderr + assert model.returncode == 0, model.stdout + model.stderr + assert column.returncode == 0, column.stdout + column.stderr + assert all(declaration in header for declaration in test_case.expected_declarations), header + assert second.returncode == 0, second.stdout + second.stderr + assert relation_columns(project_dir=project_dir, name="order_ledger") == ( + test_case.expected_new_columns + ) + assert relation_columns(project_dir=project_dir, name="order_history") == ( + test_case.expected_old_columns + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py index 2d69fd9418..3e82dbbd6c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py @@ -14,14 +14,19 @@ ModelRefactorE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + CUSTOMER_DECLARATIONS, + ORDER_SHAPE, fragments_present, load_raw_orders, + order_history_copy, order_history_files, order_ids, project_text, relation_type, + remove_files, sqb, sqb_json, + strip_migration_fingerprints, write_orders_project, ) @@ -40,6 +45,32 @@ }, expected_relation="stg_order_lines", ), + ModelRefactorE2ETestCase( + description="rename rewrites seed and source relationships audits", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={ + "seeds/customers.yml": ('to: __ref("stg_order_lines")',), + "sources/customers.yml": ("to: '__ref(\"stg_order_lines\")'",), + "models/staging/stg_order_lines.sql": ("migrate_from stg_orders",), + }, + expected_relation="stg_order_lines", + extra_files=CUSTOMER_DECLARATIONS, + ), + ModelRefactorE2ETestCase( + description="rename rewrites reusable schema relationships audits", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={ + "models/marts/_sqlbuild/_schemas/order_shape.sql": ( + '(to __ref("stg_order_lines"), field customer_id)', + ), + }, + expected_relation="stg_order_lines", + extra_files=ORDER_SHAPE, + ), ModelRefactorE2ETestCase( description="rename rewrites cursor_inputs keys of incremental consumers", command=("rename", "model:stg_orders", "stg_order_lines"), @@ -142,40 +173,47 @@ def test_given_dry_run_json_when_refactoring_then_nothing_is_written( "test_case", [ ModelMigrationE2ETestCase( - description="unfingerprintable table gains migrate_from", + description="table", materialized="table", - udf=True, + udf=False, expected_declaration="migrate_from order_history", - expected_migrate_from_count=1, expected_old_name_type="VIEW", expected_order_ids=(3, 4), ), ModelMigrationE2ETestCase( - description="unfingerprintable view gains migrate_from", + description="view calling a UDF", materialized="view", udf=True, expected_declaration="migrate_from order_history", - expected_migrate_from_count=1, expected_old_name_type="VIEW", expected_order_ids=(3, 4), ), ModelMigrationE2ETestCase( - description="unfingerprintable incremental keeps its history", + description="incremental with an unchanged definition", materialized="incremental", - udf=True, + udf=False, + expected_declaration="migrate_from order_history", + expected_old_name_type="VIEW", + expected_order_ids=(1, 2, 3, 4), + ), + ModelMigrationE2ETestCase( + description="incremental that discovery would find ambiguous", + materialized="incremental", + udf=False, expected_declaration="migrate_from order_history", - expected_migrate_from_count=1, expected_old_name_type="VIEW", expected_order_ids=(1, 2, 3, 4), + extra_files=order_history_copy(), + removed_files=("models/marts/order_history_copy.sql",), ), ModelMigrationE2ETestCase( - description="unchanged definition is left to automatic discovery", + description="incremental built before rename matching", materialized="incremental", udf=False, - expected_declaration="", - expected_migrate_from_count=0, + expected_declaration="migrate_from order_history", expected_old_name_type="VIEW", expected_order_ids=(1, 2, 3, 4), + stripped_fingerprints=("order_history",), ), ], ids=lambda case: case.description, @@ -183,25 +221,29 @@ def test_given_dry_run_json_when_refactoring_then_nothing_is_written( def test_given_built_model_when_renamed_then_build_migrates_its_relation( tmp_path: Path, test_case: ModelMigrationE2ETestCase ) -> None: - """The rename declares migrate_from exactly when discovery would miss the old relation.""" + """Every rename declares migrate_from, so history survives whatever discovery would do.""" project_dir: Path = write_orders_project( tmp_path=tmp_path, - files=order_history_files(materialized=test_case.materialized, udf=test_case.udf), + files={ + **order_history_files(materialized=test_case.materialized, udf=test_case.udf), + **test_case.extra_files, + }, ) first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + strip_migration_fingerprints(project_dir=project_dir, models=test_case.stripped_fingerprints) result: subprocess.CompletedProcess[str] = sqb( project_dir, "rename", "model:order_history", "order_ledger" ) + remove_files(project_dir=project_dir, paths=test_case.removed_files) header: str = (project_dir / "models/marts/order_ledger.sql").read_text(encoding="utf-8") load_raw_orders(project_dir=project_dir, order_ids=(3, 4)) second: subprocess.CompletedProcess[str] = sqb(project_dir, "build") assert first.returncode == 0, first.stdout + first.stderr assert result.returncode == 0, result.stdout + result.stderr - assert test_case.expected_declaration in header - assert header.count("migrate_from") == test_case.expected_migrate_from_count, header + assert header.count(test_case.expected_declaration) == 1, header assert second.returncode == 0, second.stdout + second.stderr assert relation_type(project_dir=project_dir, name="order_history") == ( test_case.expected_old_name_type diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py index 8a1b6f9020..196c084397 100644 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py @@ -53,3 +53,10 @@ class CommitChangesTestCase: moved_to: str expected_error: type[Exception] expected_files: dict[str, str] + + +@dataclass(frozen=True) +class DeclarationEditTestCase: + description: str + contents: str + expected_contents: str diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py index 93d55e84ac..cfe5233aff 100644 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py @@ -4,18 +4,18 @@ from pathlib import Path -from sqlbuild.compiler.refactoring._helpers.column_references import ( +from sqlbuild.compiler.refactoring._helpers.columns.column_references import ( analyze_column, consumer_edits, ) -from sqlbuild.compiler.refactoring._helpers.header_edits import ( +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( add_column_entry_edit, column_config_edits, column_entry_edits, header_tokens, ) -from sqlbuild.compiler.refactoring._helpers.sql_sites import analysis_sql -from sqlbuild.compiler.refactoring._helpers.text_edits import text_edit +from sqlbuild.compiler.refactoring._helpers.text.sql_sites import analysis_sql +from sqlbuild.compiler.refactoring._helpers.text.text_edits import text_edit from sqlbuild.compiler.refactoring.models import ( AnalysisSql, BodyContext, @@ -23,9 +23,10 @@ ColumnFacts, ColumnQuery, HeaderToken, + ProjectSqlFile, TextEdit, ) -from sqlbuild.compiler.refactoring.types import EditKind +from sqlbuild.compiler.refactoring.types import EditKind, SqlFileRole DIALECT: str = "duckdb" COLUMNS: dict[tuple[str, str], tuple[str, ...]] = { @@ -106,3 +107,11 @@ def read_tree(*, root: Path) -> dict[str, str]: path.relative_to(root).as_posix(): path.read_text(encoding="utf-8") for path in sorted(root.rglob("*.sql")) } + + +def yaml_file(contents: str) -> ProjectSqlFile: + """Wrap YAML text as one project declaration file.""" + + return ProjectSqlFile( + relative_path="seeds/customers.yml", contents=contents, role=SqlFileRole.YAML + ) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py index d8dc0fc580..6baa3aebc1 100644 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.refactoring._helpers.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits from sqlbuild.compiler.refactoring.models import BodyEdits from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ColumnEditTestCase from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import plan_consumer diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py new file mode 100644 index 0000000000..5f922889e0 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py @@ -0,0 +1,232 @@ +"""Tests for model and column reference rewriting in YAML and declaration headers.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( + consumer_column_header_edits, + model_name_header_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.schema_edits import ( + schema_column_edits, + schema_model_name_edits, +) +from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring._helpers.text.yaml_edits import ( + yaml_column_edits, + yaml_model_edits, +) +from sqlbuild.compiler.refactoring.models import TextEdit +from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( + DeclarationEditTestCase, +) +from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import yaml_file + +_BODY: str = "SELECT order_id FROM orders\n" + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="relationships target written as a reference", + contents=( + "seeds:\n - name: customers\n columns:\n - name: customer_id\n" + " audits:\n - relationships:\n" + ' to: __ref("fact_orders")\n field: customer_id\n' + ), + expected_contents=( + "seeds:\n - name: customers\n columns:\n - name: customer_id\n" + " audits:\n - relationships:\n" + ' to: __ref("order_facts")\n field: customer_id\n' + ), + ), + DeclarationEditTestCase( + description="bare relationships target in a flow mapping", + contents="audits:\n - relationships: {to: fact_orders, field: customer_id}\n", + expected_contents="audits:\n - relationships: {to: order_facts, field: customer_id}\n", + ), + DeclarationEditTestCase( + description="reference inside an escaped double-quoted expression", + contents='audits:\n - expression_is_true: {expression: "id IN (SELECT id FROM ' + '__ref(\\"fact_orders\\"))"}\n', + expected_contents='audits:\n - expression_is_true: {expression: "id IN (SELECT id ' + 'FROM __ref(\\"order_facts\\"))"}\n', + ), + DeclarationEditTestCase( + description="similar names and descriptions are left alone", + contents=( + "sources:\n - name: raw\n description: fact_orders feed\n" + " audits:\n - relationships: {to: fact_orders_daily, field: id}\n" + ), + expected_contents=( + "sources:\n - name: raw\n description: fact_orders feed\n" + " audits:\n - relationships: {to: fact_orders_daily, field: id}\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_yaml_when_renaming_model_then_references_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[tuple[str, TextEdit], ...] = yaml_model_edits( + files=(yaml_file(test_case.contents),), old="fact_orders", new="order_facts" + ) + + assert ( + apply_text_edits(text=test_case.contents, edits=tuple(edit for _, edit in edits)) + == test_case.expected_contents + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="field of a relationship to the model", + contents="audits:\n - relationships: {to: '__ref(\"fact_orders\")', field: amount}\n", + expected_contents=( + "audits:\n - relationships: {to: '__ref(\"fact_orders\")', field: revenue}\n" + ), + ), + DeclarationEditTestCase( + description="field of a relationship to another model", + contents="audits:\n - relationships: {to: '__ref(\"customers\")', field: amount}\n", + expected_contents=( + "audits:\n - relationships: {to: '__ref(\"customers\")', field: amount}\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_yaml_when_renaming_column_then_relationship_fields_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[tuple[str, TextEdit], ...] = yaml_column_edits( + files=(yaml_file(test_case.contents),), model="fact_orders", old="amount", new="revenue" + ) + + assert ( + apply_text_edits(text=test_case.contents, edits=tuple(edit for _, edit in edits)) + == test_case.expected_contents + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="MODEL header quoted SQL and bare relationships target", + contents=( + 'MODEL (\n audits [expression_is_true (expression "id IN (SELECT id FROM ' + "__ref('fact_orders'))\")],\n columns (\n id (audits [relationships " + f"(to fact_orders, field id)]),\n ),\n);\n{_BODY}" + ), + expected_contents=( + 'MODEL (\n audits [expression_is_true (expression "id IN (SELECT id FROM ' + "__ref('order_facts'))\")],\n columns (\n id (audits [relationships " + f"(to order_facts, field id)]),\n ),\n);\n{_BODY}" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_model_header_when_renaming_model_then_quoted_and_bare_references_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = model_name_header_edits( + contents=test_case.contents, old="fact_orders", new="order_facts" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="field of a referenced relationship target", + contents=( + "MODEL (\n columns (\n id (audits [relationships " + f'(to __ref("fact_orders"), field amount)]),\n ),\n);\n{_BODY}' + ), + expected_contents=( + "MODEL (\n columns (\n id (audits [relationships " + f'(to __ref("fact_orders"), field revenue)]),\n ),\n);\n{_BODY}' + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_model_header_when_renaming_column_then_relationship_fields_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = consumer_column_header_edits( + contents=test_case.contents, upstream="fact_orders", old="amount", new="revenue" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="bare target and quoted SQL in every SCHEMA of a file", + contents=( + "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " + "(to fact_orders, field id)]),\n ),\n);\n\n" + "SCHEMA (\n name order_totals,\n audits [expression_is_true (expression " + "\"total > (SELECT 0 FROM __ref('fact_orders') LIMIT 1)\")],\n);\n" + ), + expected_contents=( + "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " + "(to order_facts, field id)]),\n ),\n);\n\n" + "SCHEMA (\n name order_totals,\n audits [expression_is_true (expression " + "\"total > (SELECT 0 FROM __ref('order_facts') LIMIT 1)\")],\n);\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_file_when_renaming_model_then_references_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = schema_model_name_edits( + contents=test_case.contents, old="fact_orders", new="order_facts" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationEditTestCase( + description="field of a relationship to the model", + contents=( + "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " + '(to __ref("fact_orders"), field amount)]),\n ),\n);\n' + ), + expected_contents=( + "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " + '(to __ref("fact_orders"), field revenue)]),\n ),\n);\n' + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_file_when_renaming_column_then_relationship_fields_follow( + test_case: DeclarationEditTestCase, +) -> None: + edits: tuple[TextEdit, ...] = schema_column_edits( + contents=test_case.contents, upstream="fact_orders", old="amount", new="revenue" + ) + + assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py index 397a8827e2..78159601c5 100644 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py @@ -4,11 +4,11 @@ import pytest -from sqlbuild.compiler.refactoring._helpers.header_edits import ( +from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( insert_header_entry_edit, model_name_header_edits, ) -from sqlbuild.compiler.refactoring._helpers.text_edits import apply_text_edits +from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits from sqlbuild.compiler.refactoring.models import TextEdit from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import HeaderEditTestCase from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import migrated_column_edits diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py index a9448ea4b5..d68fad0735 100644 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py @@ -4,7 +4,10 @@ import pytest -from sqlbuild.compiler.refactoring._helpers.text_edits import apply_text_edits, identifier_sites +from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( + apply_text_edits, + identifier_sites, +) from sqlbuild.compiler.refactoring.exceptions import RefactorEditError from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( ApplyEditsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py index cb29945bb3..c21491c1f0 100644 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py +++ b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.refactoring._helpers.workspace import commit_changes +from sqlbuild.compiler.refactoring._helpers.project.workspace import commit_changes from sqlbuild.compiler.refactoring.exceptions import RefactorWriteError from sqlbuild.compiler.refactoring.models import FileChange from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( From 75d662c36791fa7333f684e056c191bff79f0180 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:30:01 +0100 Subject: [PATCH 09/16] feat(cli): move scoped declarations with sqb mv to where placement requires --- .../refactoring/_helpers/project/workspace.py | 17 ++- .../_helpers/renames/model_planning.py | 126 ++++++++++++++---- .../refactoring/main/plan_model_refactor.py | 11 +- src/sqlbuild/compiler/refactoring/models.py | 1 + .../compiler/scopes/_helpers/placement.py | 113 ++++++++++++++-- .../main/relocate_declarations_for_move.py | 14 ++ .../cli/commands/main/refactor/_test_types.py | 9 ++ .../cli/commands/main/refactor/helpers.py | 25 ++++ .../main/refactor/test_move_declarations.py | 88 ++++++++++++ .../main/refactor/test_rename_columns.py | 7 +- .../sqlbuild/compiler/scopes/_test_types.py | 8 ++ .../src/sqlbuild/compiler/scopes/helpers.py | 46 +++++++ .../compiler/scopes/test_relocation.py | 62 +++++++++ 13 files changed, 482 insertions(+), 45 deletions(-) create mode 100644 src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_move_declarations.py create mode 100644 tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py b/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py index ddc5af27d0..824d494760 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py @@ -85,6 +85,7 @@ def write_staged_changes( target.parent.mkdir(parents=True, exist_ok=True) _ = target.write_text(contents[change.path], encoding=TEXT_ENCODING) written.append(target) + prune_emptied_directories(root=staging_dir, changes=changes) return tuple(written) @@ -120,7 +121,6 @@ def commit_changes( for change in changes: if change.moved: (project_dir / PurePosixPath(change.original_path)).unlink() - return tuple(written) except BaseException: _rollback( project_dir=project_dir, @@ -129,6 +129,21 @@ def commit_changes( created_directories=created_directories, ) raise + prune_emptied_directories(root=project_dir, changes=changes) + return tuple(written) + + +def prune_emptied_directories(*, root: Path, changes: tuple[FileChange, ...]) -> None: + """Remove folders a move left empty, walking up to the project root.""" + + change: FileChange + for change in changes: + if not change.moved: + continue + directory: Path = (root / PurePosixPath(change.original_path)).parent + while directory != root and directory.is_dir() and not any(directory.iterdir()): + directory.rmdir() + directory = directory.parent def _rollback( diff --git a/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py index 2b80e35226..b566d7490d 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_planning.py @@ -43,7 +43,18 @@ from sqlbuild.compiler.refactoring.types import RefactorOperation from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move -from sqlbuild.compiler.scopes.models import DeclarationReport, MovePreview, ScopeDiagnostic +from sqlbuild.compiler.scopes.main.relocate_declarations_for_move import ( + relocate_declarations_for_move, +) +from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + DeclarationRecord, + MovePreview, + ResourceIdentity, + ScopeDiagnostic, + ScopeIndex, +) +from sqlbuild.compiler.scopes.types import ResourceKind from sqlbuild.spec.contracts.main.get_config_str import get_config_str @@ -109,6 +120,9 @@ def model_parts(*, project: RefactorProject, target: ModelTarget) -> RefactorPar dialect: str = project.graph.project.sql_analysis_dialect or GENERIC_DIALECT old: str = target.model.name new: str = target.request.new_name + moves: tuple[tuple[str, str], ...] + move_blockers: tuple[ManualLocation, ...] + moves, move_blockers = _declaration_moves(project=project, target=target) return RefactorParts( edits=( *model_reference_edits(files=files, old=old, new=new, dialect=dialect), @@ -127,9 +141,10 @@ def model_parts(*, project: RefactorProject, target: ModelTarget) -> RefactorPar ), blocking=( *_collisions(project=project, target=target), - *_scope_losses(project=project, target=target), + *move_blockers, *_pending_migration(files=files, target=target), ), + moves=moves, ) @@ -175,18 +190,32 @@ def _collisions(*, project: RefactorProject, target: ModelTarget) -> tuple[Manua return tuple(found) -def _scope_losses(*, project: RefactorProject, target: ModelTarget) -> tuple[ManualLocation, ...]: +def _declaration_moves( + *, project: RefactorProject, target: ModelTarget +) -> tuple[tuple[tuple[str, str], ...], tuple[ManualLocation, ...]]: + """Move the declarations the model uses to where placement requires them afterwards.""" + if PurePosixPath(target.destination).parent == PurePosixPath(target.source_path).parent: - return () + return (), () + index: ScopeIndex = project.graph.project.scope_index move: MovePreview | None diagnostics: tuple[ScopeDiagnostic, ...] move, diagnostics = preview_scope_move( - lookup=build_scope_lookup(index=project.graph.project.scope_index), + lookup=build_scope_lookup(index=index), resource=f"{MODEL_KIND_PREFIX}{target.model.name}", destination=target.destination, ) - if move is None: - return tuple( + relocated: tuple[DeclarationRecord, ...] | None = ( + None + if move is None + else relocate_declarations_for_move( + index=index, + resource=ResourceIdentity(kind=ResourceKind.MODEL, name=target.model.name), + destination=target.destination, + ) + ) + if move is None or relocated is None: + return (), tuple( ManualLocation(path=target.destination, line=None, column=None, reason=item.message) for item in diagnostics ) or ( @@ -194,30 +223,73 @@ def _scope_losses(*, project: RefactorProject, target: ModelTarget) -> tuple[Man path=target.destination, line=None, column=None, - reason="declaration scopes at the destination could not be checked", + reason="declaration placement at the destination could not be worked out", ), ) - lost: dict[str, DeclarationReport] = {report.identity: report for report in move.lost} - return tuple( - _scope_loss(identity=identity, report=lost.get(identity), target=target) - for identity in move.invalidated_usages - ) + return _file_moves(project=project, index=index, relocated=relocated) + + +def _file_moves( + *, project: RefactorProject, index: ScopeIndex, relocated: tuple[DeclarationRecord, ...] +) -> tuple[tuple[tuple[str, str], ...], tuple[ManualLocation, ...]]: + destinations: dict[str, str] = {} + blocking: list[ManualLocation] = [] + record: DeclarationRecord + for record in sorted(relocated, key=lambda item: item.identity): + original: DeclarationRecord = next( + item for item in index.declarations if item.identity == record.identity + ) + label: str = _label(record.identity) + if not (project.project_dir / PurePosixPath(original.path)).is_file(): + blocking.append( + _declaration_blocker( + record=original, + reason=f"{label} must move to {record.path}, but it is not an authored " + "project file", + ) + ) + continue + planned: str | None = destinations.setdefault(original.path, record.path) + if planned != record.path: + blocking.append( + _declaration_blocker( + record=original, + reason=f"{original.path} holds declarations that must move to different " + f"folders ({planned}, {record.path}); split the file first", + ) + ) + moving: frozenset[DeclarationIdentity] = frozenset(item.identity for item in relocated) + staying: DeclarationRecord + for staying in index.declarations: + if staying.path in destinations and staying.identity not in moving: + blocking.append( + _declaration_blocker( + record=staying, + reason=f"{destinations[staying.path]} would take " + f"{_label(staying.identity)} along, which must " + f"stay in {staying.path}; split the file first", + ) + ) + new_path: str + for new_path in sorted(set(destinations.values())): + if (project.project_dir / PurePosixPath(new_path)).exists(): + blocking.append( + ManualLocation( + path=new_path, + line=None, + column=None, + reason="a declaration must move here, but the file already exists", + ) + ) + return tuple(sorted(destinations.items())), tuple(blocking) -def _scope_loss( - *, identity: str, report: DeclarationReport | None, target: ModelTarget -) -> ManualLocation: - reason: str = ( - f"{identity} used by model:{target.model.name} is not visible from {target.destination}" - ) - if report is None: - return ManualLocation(path=target.source_path, line=None, column=None, reason=reason) - return ManualLocation( - path=report.definition.path, - line=report.definition.line, - column=report.definition.column, - reason=reason, - ) +def _label(identity: DeclarationIdentity) -> str: + return f"{identity.kind.value}:{identity.name}" + + +def _declaration_blocker(*, record: DeclarationRecord, reason: str) -> ManualLocation: + return ManualLocation(path=record.path, line=record.line, column=record.column, reason=reason) def _pending_migration( diff --git a/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py index 7548135426..9770d28eb5 100644 --- a/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py +++ b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py @@ -22,8 +22,13 @@ def plan_model_refactor(*, project: RefactorProject, request: RefactorRequest) - return build_plan( request=target.request, parts=parts, - moves={target.source_path: target.destination} - if target.destination != target.source_path - else {}, + moves={ + **dict(parts.moves), + **( + {target.source_path: target.destination} + if target.destination != target.source_path + else {} + ), + }, help=MODEL_MANUAL_HELP, ) diff --git a/src/sqlbuild/compiler/refactoring/models.py b/src/sqlbuild/compiler/refactoring/models.py index 61d7eee641..02c1c82f27 100644 --- a/src/sqlbuild/compiler/refactoring/models.py +++ b/src/sqlbuild/compiler/refactoring/models.py @@ -123,6 +123,7 @@ class RefactorParts: blocking: tuple[ManualLocation, ...] = field(default_factory=tuple) migrations: tuple[MigrationDeclaration, ...] = field(default_factory=tuple) cascaded: tuple[str, ...] = field(default_factory=tuple) + moves: tuple[tuple[str, str], ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/scopes/_helpers/placement.py b/src/sqlbuild/compiler/scopes/_helpers/placement.py index 749865dd88..d2d1296274 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/placement.py +++ b/src/sqlbuild/compiler/scopes/_helpers/placement.py @@ -172,6 +172,102 @@ def resolve_required_placement( ) +def declaration_directory(*, kind: DeclarationKind, scope: ScopeKind, path: str | None) -> str: + """Return the project-relative folder where a declaration with this placement lives.""" + + role_parts: tuple[str, ...] = DECLARATION_ROLE_PARTS[kind] + if scope is ScopeKind.GLOBAL or path is None: + return "/".join(role_parts) + prefix: str = "_" if scope is ScopeKind.LOCAL else "" + role: str = "/".join((f"{prefix}{role_parts[0]}", *role_parts[1:])) + if path in _AUTHORED_ROOT_PATHS: + return f"{path}/{role}" + return f"{path}/{DECLARATION_GROUP_DIRECTORY}/{role}" + + +def relocate_for_resource_move( + *, index: ScopeIndex, resource: ResourceIdentity, destination: str +) -> tuple[DeclarationRecord, ...] | None: + """Return declarations whose placement changes when a resource moves, or None if unsettled.""" + + if not index.completeness.runtime_usage: + return None + misplaced: frozenset[DeclarationIdentity] = frozenset( + item.declaration + for item in build_placement_validated_index(index=index).diagnostics + if item.code in _PLACEMENT_CODES and item.declaration is not None + ) + current: ScopeIndex = replace( + index, + resources=tuple( + replace(item, path=destination) if item.identity == resource else item + for item in index.resources + ), + ) + usages_by_declaration: _UsagesByDeclaration = _usages_by_declaration(index=current) + for _ in range(len(index.declarations) + 1): + anchor_sets: _AnchorSets = _anchor_sets( + index=current, usages_by_declaration=usages_by_declaration + ) + relocated: dict[DeclarationIdentity, DeclarationRecord] = {} + record: DeclarationRecord + for record in current.declarations: + if record.scope is ScopeKind.PRIVATE or record.identity in misplaced: + continue + moved: DeclarationRecord | None = _relocated_record( + record=record, + usages_by_declaration=usages_by_declaration, + anchor_sets=anchor_sets, + ) + if moved is not None: + relocated[record.identity] = moved + if not relocated: + original: dict[DeclarationIdentity, DeclarationRecord] = { + item.identity: item for item in index.declarations + } + return tuple( + item for item in current.declarations if item.path != original[item.identity].path + ) + current = replace( + current, + declarations=tuple(relocated.get(item.identity, item) for item in current.declarations), + ) + return None + + +def _relocated_record( + *, + record: DeclarationRecord, + usages_by_declaration: _UsagesByDeclaration, + anchor_sets: _AnchorSets, +) -> DeclarationRecord | None: + required: tuple[ScopeKind, str | None, tuple[str, ...]] | None = _required_placement_for_record( + record=record, usages_by_declaration=usages_by_declaration, anchor_sets=anchor_sets + ) + if required is None: + return None + scope: ScopeKind = required[0] + path: str | None = required[1] + if record.scope is scope and record.owning_path == path: + return None + directory: str = declaration_directory(kind=record.identity.kind, scope=scope, path=path) + roots: set[OwnershipRoot] = { + root for root, _path in anchor_sets.get(record.identity, frozenset()) + } + ownership_root: OwnershipRoot = ( + OwnershipRoot(directory, OwnershipRootKind.GLOBAL) + if scope is ScopeKind.GLOBAL or len(roots) != 1 + else next(iter(roots)) + ) + return replace( + record, + path=f"{directory}/{PurePosixPath(record.path).name}", + scope=scope, + ownership_root=ownership_root, + owning_path=None if scope is ScopeKind.GLOBAL else path, + ) + + def _usages_by_declaration(*, index: ScopeIndex) -> _UsagesByDeclaration: usage_lists: dict[DeclarationIdentity, list[UsageRecord]] = {} for usage in index.usages: @@ -319,17 +415,12 @@ def _message( consumers: str, ) -> str: current_path: str = declaration.owning_path or declaration.ownership_root.path - role_parts: tuple[str, ...] = DECLARATION_ROLE_PARTS[declaration.identity.kind] - if required_scope is ScopeKind.GLOBAL: - target: str = f"top-level {'/'.join(role_parts)}/" - else: - prefix: str = "_" if required_scope is ScopeKind.LOCAL else "" - role: str = "/".join((f"{prefix}{role_parts[0]}", *role_parts[1:])) + "/" - target = ( - f"{required_path}/{role}" - if required_path in _AUTHORED_ROOT_PATHS - else f"{required_path}/{DECLARATION_GROUP_DIRECTORY}/{role}" - ) + directory: str = declaration_directory( + kind=declaration.identity.kind, scope=required_scope, path=required_path + ) + target: str = ( + f"top-level {directory}/" if required_scope is ScopeKind.GLOBAL else f"{directory}/" + ) consumer_label: str = ( "References" if declaration.identity.kind is DeclarationKind.SINGULAR_AUDIT else "Consumers" ) diff --git a/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py b/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py new file mode 100644 index 0000000000..9e4e436ddf --- /dev/null +++ b/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py @@ -0,0 +1,14 @@ +"""Public declaration relocation for a resource move.""" + +from __future__ import annotations + +from sqlbuild.compiler.scopes._helpers.placement import relocate_for_resource_move +from sqlbuild.compiler.scopes.models import DeclarationRecord, ResourceIdentity, ScopeIndex + + +def relocate_declarations_for_move( + *, index: ScopeIndex, resource: ResourceIdentity, destination: str +) -> tuple[DeclarationRecord, ...] | None: + """Return declarations that must move, at their new paths, when a resource moves.""" + + return relocate_for_resource_move(index=index, resource=resource, destination=destination) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py index 111e864dcf..18ecafd1bf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -81,3 +81,12 @@ class MissingOriginE2ETestCase: expected_output: str expected_relation_type: str | None expected_columns: tuple[str, ...] = () + + +@dataclass(frozen=True) +class DeclarationMoveE2ETestCase: + description: str + model: str + extra_files: dict[str, str] + destination: str + expected_moved: dict[str, str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index 83c4280f7c..5e4668aad8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -92,6 +92,22 @@ def project_toml(*, target_settings: str = "") -> str: 'SELECT\n order_id,\n @to_cents("amount") AS amount_cents\nFROM __ref("stg_orders")\n' ), } +ORDER_REFUNDS: dict[str, str] = { + "models/staging/stg_order_refunds.sql": ( + "MODEL (\n materialized view,\n);\n\n" + 'SELECT\n order_id,\n @to_cents("amount") AS refund_cents\nFROM __ref("stg_orders")\n' + ), +} +SPLIT_CENTS_MACROS: dict[str, str] = { + **CENTS_MACRO, + "models/staging/_sqlbuild/_macros/cents.py": ( + CENTS_MACRO["models/staging/_sqlbuild/_macros/cents.py"] + + '\n\ndef to_dollars(expression: str) -> str:\n return f"({expression}) / 100"\n' + ), + "models/staging/stg_order_refunds.sql": ORDER_REFUNDS[ + "models/staging/stg_order_refunds.sql" + ].replace("to_cents", "to_dollars"), +} ORDER_EXPORT: dict[str, str] = { "models/marts/order_export.sql": ( 'MODEL (\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' @@ -239,6 +255,15 @@ def declare_missing_column_origin(*, project_dir: Path) -> None: ) +def existing_files(*, project_dir: Path, paths: tuple[str, ...]) -> tuple[str, ...]: + """Return the given project-relative paths that exist, sorted.""" + + present: frozenset[str] = frozenset( + path.relative_to(project_dir).as_posix() for path in project_dir.rglob("*.*") + ) + return tuple(sorted(frozenset(paths) & present)) + + def remove_files(*, project_dir: Path, paths: tuple[str, ...]) -> None: """Delete project files by hand, as an author removing a model would.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_move_declarations.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_move_declarations.py new file mode 100644 index 0000000000..1305866406 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_move_declarations.py @@ -0,0 +1,88 @@ +"""CLI e2e coverage for `sqb mv` carrying scoped declarations to where placement requires.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + DeclarationMoveE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + CENTS_MACRO, + ORDER_REFUNDS, + ORDER_SHAPE, + existing_files, + relation_type, + sqb, + write_orders_project, +) + +_CENTS: str = "models/staging/_sqlbuild/_macros/cents.py" +_PRIORITY_ENUM: dict[str, str] = { + "models/marts/_sqlbuild/_enums/priority.sql": ( + "ENUM (\n name priority,\n members (LOW 1, HIGH 3),\n);\n" + ), + "models/marts/fact_orders.sql": ORDER_SHAPE["models/marts/fact_orders.sql"].replace( + "SELECT", 'SELECT\n @enum("priority").LOW AS priority,', 1 + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + DeclarationMoveE2ETestCase( + description="declarations only the moved model uses move with it", + model="fact_orders", + extra_files={**ORDER_SHAPE, **_PRIORITY_ENUM}, + destination="models/reporting/", + expected_moved={ + "models/marts/_sqlbuild/_schemas/order_shape.sql": ( + "models/reporting/_sqlbuild/_schemas/order_shape.sql" + ), + "models/marts/_sqlbuild/_enums/priority.sql": ( + "models/reporting/_sqlbuild/_enums/priority.sql" + ), + }, + ), + DeclarationMoveE2ETestCase( + description="a macro still used at the old folder moves up to the shared owner", + model="stg_order_cents", + extra_files={**CENTS_MACRO, **ORDER_REFUNDS}, + destination="models/staging/eu/", + expected_moved={_CENTS: "models/staging/_sqlbuild/macros/cents.py"}, + ), + DeclarationMoveE2ETestCase( + description="a macro shared across the models root moves to the top-level role", + model="stg_order_cents", + extra_files={**CENTS_MACRO, **ORDER_REFUNDS}, + destination="models/marts/", + expected_moved={_CENTS: "macros/cents.py"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_scoped_declarations_when_moving_model_then_they_follow_placement_rules( + tmp_path: Path, test_case: DeclarationMoveE2ETestCase +) -> None: + project_dir: Path = write_orders_project(tmp_path=tmp_path, files=test_case.extra_files) + model: str = test_case.model + + moved: subprocess.CompletedProcess[str] = sqb( + project_dir, "mv", f"model:{model}", test_case.destination + ) + compiled: subprocess.CompletedProcess[str] = sqb(project_dir, "compile") + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + + assert moved.returncode == 0, moved.stdout + moved.stderr + assert existing_files( + project_dir=project_dir, + paths=(*test_case.expected_moved, *test_case.expected_moved.values()), + ) == tuple(sorted(test_case.expected_moved.values())), moved.stdout + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + assert "error[S0" not in compiled.stdout + compiled.stderr + assert built.returncode == 0, built.stdout + built.stderr + assert relation_type(project_dir=project_dir, name=model) is not None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py index 685c90cd92..2cc4e39a1a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py @@ -21,6 +21,7 @@ ORDER_EXPORT, ORDER_SHAPE, ORDERS_MACRO, + SPLIT_CENTS_MACROS, fragments_present, load_raw_orders, order_history_files, @@ -128,11 +129,11 @@ def test_given_column_rename_when_applied_then_project_tests_and_builds( "test_case", [ RefusedRefactorE2ETestCase( - description="move out of a macro scope", + description="move needing to split a declaration file", command=("mv", "model:stg_order_cents", "models/marts/"), - extra_files=CENTS_MACRO, + extra_files=SPLIT_CENTS_MACROS, expected_status="refused", - expected_reason="macro:to_cents used by model:stg_order_cents is not visible", + expected_reason="would take macro:to_dollars along, which must stay in", expected_paths=("models/staging/_sqlbuild/_macros/cents.py",), ), RefusedRefactorE2ETestCase( diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py index 97b6caedeb..785f725133 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py @@ -221,3 +221,11 @@ class TolerantCategoryCase: files: dict[str, str] expected_kind: ResourceKind | DeclarationKind expected_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class RelocationCase: + description: str + consumer_paths: tuple[str, ...] + destination: str + expected_paths: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index 0ef42d0d8b..5e62b7decd 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -336,3 +336,49 @@ def report_scope_lookup(*, extra_globals: int = 0) -> ScopeLookup: completeness=ScopeCompleteness(), ) return build_scope_lookup(index=index) + + +_RELOCATION_ROOT: OwnershipRoot = OwnershipRoot("models", resource_kind=ResourceKind.MODEL) +_CENTS_MACRO: DeclarationIdentity = DeclarationIdentity(DeclarationKind.MACRO, "to_cents") +_ORDER_SHAPE: DeclarationIdentity = DeclarationIdentity(DeclarationKind.SCHEMA, "order_shape") + + +def relocation_index(consumer_paths: tuple[str, ...]) -> ScopeIndex: + """Return models sharing a staging schema that uses a staging macro.""" + + resources: tuple[ResourceRecord, ...] = tuple( + ResourceRecord( + identity=ResourceIdentity(ResourceKind.MODEL, f"orders_{position}"), + path=path, + ownership_root=_RELOCATION_ROOT, + ) + for position, path in enumerate(consumer_paths) + ) + return ScopeIndex( + ownership_roots=(_RELOCATION_ROOT,), + resources=resources, + declarations=( + DeclarationRecord( + identity=_ORDER_SHAPE, + path="models/staging/_sqlbuild/_schemas/order_shape.sql", + line=1, + column=1, + scope=ScopeKind.LOCAL, + ownership_root=_RELOCATION_ROOT, + owning_path="models/staging", + ), + DeclarationRecord( + identity=_CENTS_MACRO, + path="models/staging/_sqlbuild/_macros/cents.py", + line=1, + column=1, + scope=ScopeKind.LOCAL, + ownership_root=_RELOCATION_ROOT, + owning_path="models/staging", + ), + ), + usages=( + *(UsageRecord(consumer=item.identity, declaration=_ORDER_SHAPE) for item in resources), + UsageRecord(consumer=_ORDER_SHAPE, declaration=_CENTS_MACRO), + ), + ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py new file mode 100644 index 0000000000..8f76f0ce39 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py @@ -0,0 +1,62 @@ +"""Behavior tests for relocating declarations when a resource moves.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.scopes.main.relocate_declarations_for_move import ( + relocate_declarations_for_move, +) +from sqlbuild.compiler.scopes.models import ( + DeclarationRecord, + ResourceIdentity, +) +from sqlbuild.compiler.scopes.types import ResourceKind +from tests.unit.src.sqlbuild.compiler.scopes._test_types import RelocationCase +from tests.unit.src.sqlbuild.compiler.scopes.helpers import relocation_index + + +@pytest.mark.parametrize( + "test_case", + ( + RelocationCase( + description="sole consumer takes the schema and the macro it uses along", + consumer_paths=("models/staging/orders.sql",), + destination="models/marts/orders.sql", + expected_paths=( + "models/marts/_sqlbuild/_schemas/order_shape.sql", + "models/marts/_sqlbuild/_macros/cents.py", + ), + ), + RelocationCase( + description="remaining consumers lift the schema while its macro stays beside it", + consumer_paths=("models/staging/orders.sql", "models/staging/refunds.sql"), + destination="models/staging/eu/orders.sql", + expected_paths=("models/staging/_sqlbuild/schemas/order_shape.sql",), + ), + RelocationCase( + description="consumers spanning the models root use top-level roles", + consumer_paths=("models/staging/orders.sql", "models/staging/refunds.sql"), + destination="models/marts/orders.sql", + expected_paths=("schemas/order_shape.sql", "macros/cents.py"), + ), + RelocationCase( + description="move within the owner changes nothing", + consumer_paths=("models/staging/orders.sql",), + destination="models/staging/orders_v2.sql", + expected_paths=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_resource_move_when_relocating_then_declarations_follow_placement( + test_case: RelocationCase, +) -> None: + relocated: tuple[DeclarationRecord, ...] | None = relocate_declarations_for_move( + index=relocation_index(test_case.consumer_paths), + resource=ResourceIdentity(ResourceKind.MODEL, "orders_0"), + destination=test_case.destination, + ) + + assert relocated is not None + assert tuple(item.path for item in relocated) == test_case.expected_paths From c7f4c9e14db5bc0433d7256e8f07fa7753bdc01e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:32:02 +0100 Subject: [PATCH 10/16] docs: document declaration moves, always-on migrate_from, and missing_migration_origin --- src/sqlbuild/.agents/skills/sqlbuild/SKILL.md | 16 +++++---- .../sqlbuild/references/docs/cli/rename.md | 35 +++++++++++++++---- .../docs/concepts/models/column-migrations.md | 2 +- .../docs/concepts/models/migrations.md | 4 +-- .../docs/concepts/project-configuration.md | 18 ++++++++++ src/sqlbuild/compiler/planner/constants.py | 4 +-- .../test_identity_handover.py | 2 +- .../pre_build/test_pre_build_work_display.py | 2 +- website/src/content/docs/docs/cli/rename.mdx | 24 +++++++++---- .../concepts/models/column-migrations.mdx | 2 +- .../docs/docs/concepts/models/migrations.mdx | 4 +-- .../docs/concepts/project-configuration.mdx | 18 ++++++++++ 12 files changed, 100 insertions(+), 31 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index dc4bffac36..5894fca18c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -145,13 +145,15 @@ functions, but not in ordinary `MODEL()` config fields. Placement decides visibi **Renaming and moving.** Never rename or move a model or column by editing files by hand. `sqb rename` and `sqb mv` rewrite every `__ref`, fixture CTE, header reference and column use from -compiler facts, add `migrate_from` exactly where the warehouse history would otherwise be lost, -verify the edited project compiles in a scratch copy, and write nothing if any step fails. Run -with `--dry-run` first. A column rename is one step by default (downstream models keep their output -names via `AS `); `--cascade` renames pass-through outputs downstream too. When the command -refuses, fix the listed locations (usually a macro call or `SELECT *`) and rerun; use -`--allow-manual` only when the user agrees to finish those locations by hand. Then `sqb plan` to -see the migration. Read [references/docs/cli/rename.md](references/docs/cli/rename.md). +compiler facts (including YAML and `SCHEMA` relationships), add `migrate_from` whenever a +data-holding relation moves, carry scoped macros, enums, constants and schemas to the folder +placement requires, verify the edited project compiles in a scratch copy, and write nothing if any +step fails. Run with `--dry-run` first. A column rename is one step by default (downstream models +keep their output names via `AS `); `--cascade` renames pass-through outputs downstream too. +When the command refuses, fix the listed locations (usually a macro call, `SELECT *`, or a +declaration file to split) and rerun. Then `sqb plan` to see the migration; on a target that never +built the old name it warns about the missing origin and builds fresh unless the target sets +`missing_migration_origin`. Read [references/docs/cli/rename.md](references/docs/cli/rename.md). **Rules** are compile-time project checks (`sqb rules list`, `sqb rules show `). `sqb format` rewrites sources deterministically; `sqb format --check` only reports. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md index 5fe4bb8ff8..132044ab26 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -6,6 +6,17 @@ Online: https://sqlbuild.com/docs/cli/rename/ +## Contents + +- Usage +- What changes +- History +- Refusals +- Flags +- Output +- Exit codes +- Limitations + `sqb rename` and `sqb mv` change a name everywhere the compiler knows it is used, instead of leaving you to find every `__ref`, fixture, and header by hand. Every edit comes from compiled facts, not text search: a column rename follows the columns each query actually binds, so a same-named column of another table is left alone. Nothing is written until the edited project compiles. The command applies its edits to a scratch copy of the project, compiles it the way `sqb compile` does, and only then writes the files. If anything cannot be rewritten safely, the command lists those locations and writes nothing. @@ -34,7 +45,16 @@ Renaming or moving a model rewrites: - every `__ref("")` in models, tests, scenarios, audits, hooks, and SQL functions; - `__ref__` and `__expected__` fixture CTEs in tests and scenarios; -- `cursor_inputs` keys and `relationships (to )` audits in `MODEL` headers. +- `cursor_inputs` keys, `relationships (to ...)` audits, and `__ref("")` inside quoted SQL in `MODEL` headers and reusable `SCHEMA` declarations; +- `relationships` audits (`to:` and `field:`) and `__ref("")` in source and seed YAML. + +Moving a model to another folder also moves the macros, enums, constants, reusable schemas, generic audits, and named hooks it uses when their [placement](../concepts/declaration-scopes/placement.md) must change, following the same rules `sqb compile` enforces: + +- a declaration only the moved model uses moves with it, into the destination folder's matching role; +- a declaration other resources still use moves to the lowest folder they share, for example `models/staging/_sqlbuild/macros/`; +- when that folder would be an authored root such as `models/`, or the users span resource trees such as models and tests, it moves to the top-level role (`macros/`, `enums/`, `constants/`, `schemas/`, ...). + +Declarations are moved, never copied. Folders the move leaves empty are removed. Renaming a column rewrites the model that produces it and every query that reads it: @@ -47,7 +67,9 @@ By default a column rename is one step: downstream models keep their output name ## History -A renamed model keeps its warehouse history through a [model migration](../concepts/models/migrations.md). When SQLBuild's automatic rename discovery would match the renamed model by its unchanged definition, nothing is added. When it would not, for example because the model calls a UDF or its materialization family changes, the command adds `migrate_from ` to the model header. The next `sqb build` migrates the relation and leaves a compatibility view at the old name. +A renamed model keeps its warehouse history through a [model migration](../concepts/models/migrations.md). Whenever the model's relation moves and its materialization keeps warehouse data (table, view, incremental, or snapshot), the command adds `migrate_from ` to the model header, so the move does not depend on automatic discovery finding exactly one match. The next `sqb build` migrates the relation and leaves a compatibility view at the old name. A move that keeps the relation, such as a folder change in the same schema, adds nothing. + +On a target that never built the old name, the origin is missing. By default the plan warns and the model builds fresh; set [`missing_migration_origin`](../concepts/project-configuration.md#missing-migration-origins) on a target to require confirmation or stop the build instead. A renamed column of an incremental or snapshot model gets ` (migrate_from )` in its `columns` block, so the next build renames it in place as a [column migration](../concepts/models/column-migrations.md). Table and view models are rebuilt with the new column anyway; they get the declaration only when the model itself declares `migrate_from`. @@ -58,23 +80,22 @@ The command refuses a rename while the model, or the column, still declares `mig The command refuses, and writes nothing, when: - the new name already belongs to another model or column, or the destination file exists; -- a move would take the model out of the scope of a macro, enum, or constant it uses (see [`sqb scope`](scope.md)); +- a declaration a moved model uses must change folder, but its file also holds declarations that must stay, or it is not an authored project file, or a file with its name already exists at the new folder; - a move would leave no model in the old schema, so the old relation could not be found; - a reference is produced by a macro, or a column is read inside SQL that a macro generates; - a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; - a downstream model's `SELECT *` would silently rename its output, without `--cascade`; - Python code or Python node SQL strings name the model or column. -Each location is listed with its file, line, and column. Edit those locations, or pass the model or column to the macro as an argument, and run the command again. +Each location is listed with its file, line, and column, with the fix. Edit those locations, or pass the model or column to the macro as an argument, and run the command again. The command never applies part of a rename. -`--allow-manual` applies the safe edits and lists the remaining locations for you to finish. The edited project must still compile; if the remaining locations break it, the command writes nothing and prints the compile errors. +References produced by a macro are only possible when a project turns off explicit references (`[references] enforce_explicit = false`); such projects get the refusal list for them. ## Flags | Flag | Description | |------|-------------| | `--cascade` | For column renames, rename the output of every downstream model that passes the column through. | -| `--allow-manual` | Apply the safe edits even when some locations must be edited by hand. | | `--dry-run` | Print every edit and verify the edited project compiles, without writing files. | | `--json` | Print the result as JSON on stdout. Progress goes to stderr. | @@ -101,5 +122,5 @@ With `--json`, the result has `operation`, `target`, `new_name`, `destination`, ## Limitations -- `relationships` audits declared in source or seed YAML files, descriptions, and comments are not rewritten. +- Descriptions and comments are not rewritten. - Queries outside the project that read the old relation keep working through the old-name compatibility view until it expires. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md index 184babe8c6..eef9085a54 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md @@ -92,7 +92,7 @@ Automatically detected renames are marked `(automatic)`. The decision is based o | `already renamed, record` | The new column exists and the old one doesn't, but no rename is recorded, which is what an interrupted rename leaves behind. Record it without renaming again. | | `done` | The rename is recorded and the new column exists. Nothing to do. | | `conflict` | Both columns exist. The build stops (`M110`); drop one of them, or remove `migrate_from` and let `on_schema_change` handle the columns. | -| `source missing` | Neither column exists and no rename is recorded. The build stops (`M109`); remove `migrate_from` if the rename happened elsewhere or is no longer needed. | +| `source missing` | Neither column exists and no rename is recorded. The plan warns (`M109`) and the columns follow `on_schema_change` as if `migrate_from` were absent. A target can require confirmation or stop the build instead with [`missing_migration_origin`](../project-configuration.md#missing-migration-origins). | | `source still produced` | The query still selects the old column. The build stops (`M111`). | | `unsupported` | The warehouse can't rename this table's columns in place (see [Adapter support](#adapter-support)). The build stops (`M112`). | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index c6b898e9bc..e952d50b4a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -23,7 +23,7 @@ Table and view models can declare a migration too. They hold no history, so they ## Declaring a migration -[`sqb rename`](../../cli/rename.md) renames a model and every reference to it, and adds `migrate_from` only when [automatic discovery](#automatic-discovery) would not find the old relation. +[`sqb rename`](../../cli/rename.md) and `sqb mv` rename or move a model, update every reference to it, and add `migrate_from` whenever the model's relation moves and its materialization keeps warehouse data (table, view, incremental, or snapshot). To declare a migration by hand, add `migrate_from` to the renamed model's header, naming the old model: @@ -54,7 +54,7 @@ After the move is recorded, `migrate_from` has no further effect, and SQLBuild t | `superseded replace` | The new relation was itself the origin of the latest recorded move, for example after renaming a model back. Replace it with the newer data. | | `forced replace` | The new relation has its own build history and `migrate_force true` is set. Replace it. | | `conflict` | The new relation has its own build history. The build stops (`M103`) until you set `migrate_force true` or remove `migrate_from`. | -| `origin missing` | The old relation doesn't exist and no move was recorded. The build stops (`M102`). | +| `origin missing` | The old relation doesn't exist and no move was recorded, for example on a target that never built the old name. The plan warns (`M102`) and the model builds fresh. A target can require confirmation or stop the build instead with [`missing_migration_origin`](../project-configuration.md#missing-migration-origins). | The origin and destination must be compatible under the model's normal `on_schema_change` rules; otherwise the build stops (`M104`). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 0d23f38517..fe0e748869 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -152,6 +152,7 @@ A target is a named build context - the database and schema you build into, plus | `vars` | Target-specific project variables | | `defer_sources_to` | Target name to read managed source data from (see [Loaders](python-nodes/loaders.md#source-deferral)) | | `clone` | Clone policy (see below) | +| `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny` (see below) | ```toml [connections.warehouse] @@ -217,6 +218,23 @@ password = "${ENV:SNOWFLAKE_PASSWORD}" Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` still takes precedence for that invocation. +### Missing migration origins + +A `migrate_from` declaration, on a model or a column, names a relation or column that exists in the targets where the rename is pending. A target that never built the old name, such as a fresh development schema, has nothing to migrate. `missing_migration_origin` decides what happens there: + +| Value | Behavior | +|-------|----------| +| `allow` | Default. The plan warns (`M102` for models, `M109` for columns), nothing is migrated, and the model builds as if `migrate_from` were absent. | +| `require_confirmation` | The build asks for confirmation on a terminal, and stops otherwise; pass `--allow-missing-migration-origin` to confirm in non-interactive runs. | +| `deny` | The build stops with `M102` or `M109`. Use it for targets where a missing origin means the wrong target or a lost relation. | + +```toml +[targets.prod] +missing_migration_origin = "deny" +``` + +`sqlbuild_local.toml` can override the value for a target. + ### Clone policies Targets can declare whether they allow cloning to or from: diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 3adbfd48db..7e3d957eb2 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -85,8 +85,8 @@ SOURCE_INPUT_FUNCTION: str = "__source" REF_INPUT_FUNCTION: str = "__ref" MANUAL_RENAME_HINT: str = ( - "matched by unchanged definition; sqb rename model: also rewrites references " - "and adds migrate_from when discovery cannot match" + "matched by unchanged definition; sqb rename model: rewrites references and " + "declares migrate_from so the move does not depend on discovery" ) MISSING_ORIGIN_OUTCOMES: dict[MissingMigrationOriginPolicy, str] = { MissingMigrationOriginPolicy.ALLOW: ( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py index e20b441280..46b51b2328 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py @@ -154,7 +154,7 @@ def test_given_renamed_chain_with_full_replay_when_building_then_migrated_histor "main.customer_orders_enriched\n ├── transfer recreate (view)\n" " ├── discovery automatic\n" " ├── hint matched by unchanged definition; sqb rename model: " - "also rewrites references and adds migrate_from when discovery cannot match\n" + "rewrites references and declares migrate_from so the move does not depend on discovery\n" " └── old name main.orders_enriched\n" " └── left for janitor old_name_view false" ), diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index 9a1b404ce4..cfec0adb75 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -140,7 +140,7 @@ def test_given_migration_decision_when_formatting_then_applies_colour_roles( " ├── transfer physical copy, promote by transactional rename\n" " ├── discovery automatic\n" " └── hint matched by unchanged definition; sqb rename model: " - "also rewrites references and adds migrate_from when discovery cannot match" + "rewrites references and declares migrate_from so the move does not depend on discovery" ), ), ), diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx index e51828e58f..9230251a2a 100644 --- a/website/src/content/docs/docs/cli/rename.mdx +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -31,7 +31,16 @@ Renaming or moving a model rewrites: - every `__ref("")` in models, tests, scenarios, audits, hooks, and SQL functions; - `__ref__` and `__expected__` fixture CTEs in tests and scenarios; -- `cursor_inputs` keys and `relationships (to )` audits in `MODEL` headers. +- `cursor_inputs` keys, `relationships (to ...)` audits, and `__ref("")` inside quoted SQL in `MODEL` headers and reusable `SCHEMA` declarations; +- `relationships` audits (`to:` and `field:`) and `__ref("")` in source and seed YAML. + +Moving a model to another folder also moves the macros, enums, constants, reusable schemas, generic audits, and named hooks it uses when their [placement](/docs/concepts/declaration-scopes/placement) must change, following the same rules `sqb compile` enforces: + +- a declaration only the moved model uses moves with it, into the destination folder's matching role; +- a declaration other resources still use moves to the lowest folder they share, for example `models/staging/_sqlbuild/macros/`; +- when that folder would be an authored root such as `models/`, or the users span resource trees such as models and tests, it moves to the top-level role (`macros/`, `enums/`, `constants/`, `schemas/`, ...). + +Declarations are moved, never copied. Folders the move leaves empty are removed. Renaming a column rewrites the model that produces it and every query that reads it: @@ -44,7 +53,9 @@ By default a column rename is one step: downstream models keep their output name ## History -A renamed model keeps its warehouse history through a [model migration](/docs/concepts/models/migrations). When SQLBuild's automatic rename discovery would match the renamed model by its unchanged definition, nothing is added. When it would not, for example because the model calls a UDF or its materialization family changes, the command adds `migrate_from ` to the model header. The next `sqb build` migrates the relation and leaves a compatibility view at the old name. +A renamed model keeps its warehouse history through a [model migration](/docs/concepts/models/migrations). Whenever the model's relation moves and its materialization keeps warehouse data (table, view, incremental, or snapshot), the command adds `migrate_from ` to the model header, so the move does not depend on automatic discovery finding exactly one match. The next `sqb build` migrates the relation and leaves a compatibility view at the old name. A move that keeps the relation, such as a folder change in the same schema, adds nothing. + +On a target that never built the old name, the origin is missing. By default the plan warns and the model builds fresh; set [`missing_migration_origin`](/docs/concepts/project-configuration#missing-migration-origins) on a target to require confirmation or stop the build instead. A renamed column of an incremental or snapshot model gets ` (migrate_from )` in its `columns` block, so the next build renames it in place as a [column migration](/docs/concepts/models/column-migrations). Table and view models are rebuilt with the new column anyway; they get the declaration only when the model itself declares `migrate_from`. @@ -55,23 +66,22 @@ The command refuses a rename while the model, or the column, still declares `mig The command refuses, and writes nothing, when: - the new name already belongs to another model or column, or the destination file exists; -- a move would take the model out of the scope of a macro, enum, or constant it uses (see [`sqb scope`](/docs/cli/scope)); +- a declaration a moved model uses must change folder, but its file also holds declarations that must stay, or it is not an authored project file, or a file with its name already exists at the new folder; - a move would leave no model in the old schema, so the old relation could not be found; - a reference is produced by a macro, or a column is read inside SQL that a macro generates; - a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; - a downstream model's `SELECT *` would silently rename its output, without `--cascade`; - Python code or Python node SQL strings name the model or column. -Each location is listed with its file, line, and column. Edit those locations, or pass the model or column to the macro as an argument, and run the command again. +Each location is listed with its file, line, and column, with the fix. Edit those locations, or pass the model or column to the macro as an argument, and run the command again. The command never applies part of a rename. -`--allow-manual` applies the safe edits and lists the remaining locations for you to finish. The edited project must still compile; if the remaining locations break it, the command writes nothing and prints the compile errors. +References produced by a macro are only possible when a project turns off explicit references (`[references] enforce_explicit = false`); such projects get the refusal list for them. ## Flags | Flag | Description | |------|-------------| | `--cascade` | For column renames, rename the output of every downstream model that passes the column through. | -| `--allow-manual` | Apply the safe edits even when some locations must be edited by hand. | | `--dry-run` | Print every edit and verify the edited project compiles, without writing files. | | `--json` | Print the result as JSON on stdout. Progress goes to stderr. | @@ -98,5 +108,5 @@ With `--json`, the result has `operation`, `target`, `new_name`, `destination`, ## Limitations -- `relationships` audits declared in source or seed YAML files, descriptions, and comments are not rewritten. +- Descriptions and comments are not rewritten. - Queries outside the project that read the old relation keep working through the old-name compatibility view until it expires. diff --git a/website/src/content/docs/docs/concepts/models/column-migrations.mdx b/website/src/content/docs/docs/concepts/models/column-migrations.mdx index f9c8c8c628..4fb598e3a2 100644 --- a/website/src/content/docs/docs/concepts/models/column-migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/column-migrations.mdx @@ -80,7 +80,7 @@ Automatically detected renames are marked `(automatic)`. The decision is based o | `already renamed, record` | The new column exists and the old one doesn't, but no rename is recorded, which is what an interrupted rename leaves behind. Record it without renaming again. | | `done` | The rename is recorded and the new column exists. Nothing to do. | | `conflict` | Both columns exist. The build stops (`M110`); drop one of them, or remove `migrate_from` and let `on_schema_change` handle the columns. | -| `source missing` | Neither column exists and no rename is recorded. The build stops (`M109`); remove `migrate_from` if the rename happened elsewhere or is no longer needed. | +| `source missing` | Neither column exists and no rename is recorded. The plan warns (`M109`) and the columns follow `on_schema_change` as if `migrate_from` were absent. A target can require confirmation or stop the build instead with [`missing_migration_origin`](/docs/concepts/project-configuration#missing-migration-origins). | | `source still produced` | The query still selects the old column. The build stops (`M111`). | | `unsupported` | The warehouse can't rename this table's columns in place (see [Adapter support](#adapter-support)). The build stops (`M112`). | diff --git a/website/src/content/docs/docs/concepts/models/migrations.mdx b/website/src/content/docs/docs/concepts/models/migrations.mdx index d42a7addd4..e78483b251 100644 --- a/website/src/content/docs/docs/concepts/models/migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/migrations.mdx @@ -9,7 +9,7 @@ Table and view models can declare a migration too. They hold no history, so they ## Declaring a migration -[`sqb rename`](/docs/cli/rename) renames a model and every reference to it, and adds `migrate_from` only when [automatic discovery](#automatic-discovery) would not find the old relation. +[`sqb rename`](/docs/cli/rename) and `sqb mv` rename or move a model, update every reference to it, and add `migrate_from` whenever the model's relation moves and its materialization keeps warehouse data (table, view, incremental, or snapshot). To declare a migration by hand, add `migrate_from` to the renamed model's header, naming the old model: @@ -40,7 +40,7 @@ After the move is recorded, `migrate_from` has no further effect, and SQLBuild t | `superseded replace` | The new relation was itself the origin of the latest recorded move, for example after renaming a model back. Replace it with the newer data. | | `forced replace` | The new relation has its own build history and `migrate_force true` is set. Replace it. | | `conflict` | The new relation has its own build history. The build stops (`M103`) until you set `migrate_force true` or remove `migrate_from`. | -| `origin missing` | The old relation doesn't exist and no move was recorded. The build stops (`M102`). | +| `origin missing` | The old relation doesn't exist and no move was recorded, for example on a target that never built the old name. The plan warns (`M102`) and the model builds fresh. A target can require confirmation or stop the build instead with [`missing_migration_origin`](/docs/concepts/project-configuration#missing-migration-origins). | The origin and destination must be compatible under the model's normal `on_schema_change` rules; otherwise the build stops (`M104`). diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 4af4b22801..c308be37bd 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -129,6 +129,7 @@ A target is a named build context - the database and schema you build into, plus | `vars` | Target-specific project variables | | `defer_sources_to` | Target name to read managed source data from (see [Loaders](/docs/concepts/python-nodes/loaders#source-deferral)) | | `clone` | Clone policy (see below) | +| `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny` (see below) | ```toml [connections.warehouse] @@ -194,6 +195,23 @@ password = "${ENV:SNOWFLAKE_PASSWORD}" Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` still takes precedence for that invocation. +### Missing migration origins + +A `migrate_from` declaration, on a model or a column, names a relation or column that exists in the targets where the rename is pending. A target that never built the old name, such as a fresh development schema, has nothing to migrate. `missing_migration_origin` decides what happens there: + +| Value | Behavior | +|-------|----------| +| `allow` | Default. The plan warns (`M102` for models, `M109` for columns), nothing is migrated, and the model builds as if `migrate_from` were absent. | +| `require_confirmation` | The build asks for confirmation on a terminal, and stops otherwise; pass `--allow-missing-migration-origin` to confirm in non-interactive runs. | +| `deny` | The build stops with `M102` or `M109`. Use it for targets where a missing origin means the wrong target or a lost relation. | + +```toml +[targets.prod] +missing_migration_origin = "deny" +``` + +`sqlbuild_local.toml` can override the value for a target. + ### Clone policies Targets can declare whether they allow cloning to or from: From e1661f7a297dc539118d7be273d1fbd3aa72eb09 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:38:08 +0100 Subject: [PATCH 11/16] refactor: share target policy parsing and missing-origin confirmation --- .../_helpers/build_planning/confirmation.py | 32 +++ .../missing_migration_origin.py | 68 ------ .../_helpers/build_planning/plan_policies.py | 44 +++- .../discovery/_helpers/yml/project.py | 194 ++++++------------ .../planner/_helpers/migrations/planning.py | 14 +- .../planner/_helpers/planning/retention.py | 12 +- .../refactoring/_helpers/text/schema_edits.py | 51 ++--- 7 files changed, 161 insertions(+), 254 deletions(-) create mode 100644 src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/build_planning/missing_migration_origin.py diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py new file mode 100644 index 0000000000..8b0e1509a6 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py @@ -0,0 +1,32 @@ +"""Typed confirmation shared by the build safety policies.""" + +from __future__ import annotations + +from typing import TextIO + +from sqlbuild.cli.commands.exceptions import CliUserError + + +def confirm_typed_action( + *, + action: str, + flag: str, + warning: str, + expected: str, + input_stream: TextIO, + output_stream: TextIO, + code: str | None = None, +) -> None: + """Ask the user to type the expected text, or fail when the run is not interactive.""" + + if not input_stream.isatty(): + raise CliUserError( + f"{action} requires confirmation", + code=code, + help=f"Pass {flag} to confirm in non-interactive runs.", + ) + _ = output_stream.write(f"{warning}\n\n") + _ = output_stream.write(f"Type `{expected}` to continue: ") + output_stream.flush() + if input_stream.readline().strip() != expected: + raise CliUserError(f"{action} cancelled", code=code) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/missing_migration_origin.py b/src/sqlbuild/cli/commands/_helpers/build_planning/missing_migration_origin.py deleted file mode 100644 index 42ea26bd22..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/missing_migration_origin.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Missing migration origin confirmation for build commands.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy - - -def enforce_missing_migration_origin_policy( - *, - plan: PlanOutput, - allow_missing_migration_origin: bool, - input_stream: TextIO, - output_stream: TextIO, -) -> None: - """Confirm before building past declared migrations whose origin is missing.""" - - names: tuple[str, ...] = tuple( - sorted( - { - *( - entry.model_name - for entry in plan.migration_entries - if entry.origin_missing - and entry.missing_origin_policy - == MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION - ), - *( - entry.model_name - for entry in plan.column_migration_entries - if entry.origin_missing - and entry.missing_origin_policy - == MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION - ), - } - ) - ) - if not names or allow_missing_migration_origin: - return - if not input_stream.isatty(): - raise CliUserError( - f"migrate_from origin is missing for {_names(names)} and " - "missing_migration_origin requires confirmation", - code="M102", - help="Pass --allow-missing-migration-origin to build without migrating them.", - ) - expected: str = _confirmation_text(names) - _ = output_stream.write( - f"The migrate_from origin of {_names(names)} does not exist in this target; nothing " - "will be migrated.\n\n" - ) - _ = output_stream.write(f"Type `{expected}` to continue: ") - output_stream.flush() - if input_stream.readline().strip() != expected: - raise CliUserError("build without missing migration origins cancelled", code="M102") - - -def _confirmation_text(names: tuple[str, ...]) -> str: - if len(names) == 1: - return f"build {names[0]} without its migration" - return f"build {len(names)} models without their migrations" - - -def _names(names: tuple[str, ...]) -> str: - return ", ".join(f"'{name}'" for name in names) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py index 3f9591e84d..c2a2567429 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -4,6 +4,7 @@ import sys +from sqlbuild.cli.commands._helpers.build_planning.confirmation import confirm_typed_action from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( enforce_model_execution_limit, executable_model_count, @@ -11,9 +12,6 @@ from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( enforce_snapshot_full_refresh_policy, ) -from sqlbuild.cli.commands._helpers.build_planning.missing_migration_origin import ( - enforce_missing_migration_origin_policy, -) from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( enforce_retention_decrease_policy, ) @@ -30,6 +28,7 @@ PlanWarning, ) from sqlbuild.compiler.planner.types import WarningSeverity +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy _OLD_NAME_CODES: frozenset[str] = frozenset({"M114", "P008"}) _COLUMN_BLOCK_CODES: dict[ColumnMigrationDecision, str] = { @@ -47,11 +46,8 @@ def enforce_build_plan_policies( _enforce_model_migration_policy(plan=plan) _enforce_column_migration_policy(plan=plan) - enforce_missing_migration_origin_policy( - plan=plan, - allow_missing_migration_origin=request.allow_missing_migration_origin, - input_stream=sys.stdin, - output_stream=sys.stdout, + _confirm_missing_origins( + plan=plan, allow_missing_migration_origin=request.allow_missing_migration_origin ) _enforce_old_name_policy(plan=plan) enforce_model_execution_limit( @@ -146,6 +142,38 @@ def _enforce_column_migration_policy(*, plan: PlanOutput) -> None: ) +def _confirm_missing_origins(*, plan: PlanOutput, allow_missing_migration_origin: bool) -> None: + """Confirm before building past declared migrations whose origin is missing.""" + + names: tuple[str, ...] = tuple( + sorted( + { + entry.model_name + for entry in (*plan.migration_entries, *plan.column_migration_entries) + if entry.origin_missing + and entry.missing_origin_policy == MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION + } + ) + ) + if not names or allow_missing_migration_origin: + return + listed: str = ", ".join(f"'{name}'" for name in names) + confirm_typed_action( + action=f"building {listed} without its missing migrate_from origin", + flag="--allow-missing-migration-origin", + warning=f"The migrate_from origin of {listed} does not exist in this target; nothing " + "will be migrated.", + expected=( + f"build {names[0]} without its migration" + if len(names) == 1 + else f"build {len(names)} models without their migrations" + ), + input_stream=sys.stdin, + output_stream=sys.stdout, + code="M102", + ) + + def _enforce_old_name_policy(*, plan: PlanOutput) -> None: """Refuse to build over a compatibility view or while code reads an old name.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index bc24612b2f..415ce91d6e 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -6,6 +6,7 @@ from dataclasses import fields from datetime import date, datetime from decimal import Decimal +from enum import StrEnum from pathlib import Path from typing import cast @@ -1072,17 +1073,23 @@ def _optional_table_type( ) from exc -def _optional_table_type_downgrade_policy( - *, mapping: dict[str, object], key: str, label: str, file_path: Path -) -> TableTypeDowngradePolicy | None: +def _optional_target_policy[PolicyT: StrEnum]( + *, + mapping: dict[str, object], + key: str, + target_name: str, + file_path: Path, + policy: type[PolicyT], +) -> PolicyT | None: value: object | None = mapping.get(key) if value is None: return None try: - return TableTypeDowngradePolicy(value) + return policy(value) except (TypeError, ValueError) as exc: raise ProjectConfigError( - f"{file_path} {label} must be 'deny', 'require_confirmation', or 'allow'" + f"{file_path} targets.{target_name}.{key} must be 'deny', " + "'require_confirmation', or 'allow'" ) from exc @@ -1128,131 +1135,43 @@ def _target_retention_by_materialization( return loaded -def _optional_missing_migration_origin_policy( - *, mapping: dict[str, object], target_name: str, file_path: Path -) -> MissingMigrationOriginPolicy | None: - value: object | None = mapping.get("missing_migration_origin") - if value is None: - return None - try: - return MissingMigrationOriginPolicy(value) - except (TypeError, ValueError) as exc: - raise ProjectConfigError( - f"{file_path} targets.{target_name}.missing_migration_origin must be 'deny', " - "'require_confirmation', or 'allow'" - ) from exc - - -def _optional_retention_decrease_policy( - *, mapping: dict[str, object], target_name: str, file_path: Path -) -> RetentionDecreasePolicy | None: - value: object | None = mapping.get("time_travel_retention_decrease") - if value is None: - return None - try: - return RetentionDecreasePolicy(value) - except (TypeError, ValueError) as exc: - raise ProjectConfigError( - f"{file_path} targets.{target_name}.time_travel_retention_decrease must be " - "'deny', 'require_confirmation', or 'allow'" - ) from exc +def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig]: + return { + name: _project_target(local) + for name, local in _load_local_targets(payload=payload, file_path=file_path).items() + } -def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig]: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="targets", file_path=file_path +def _project_target(local: LocalTargetConfig) -> TargetConfig: + defaults: TargetConfig = TargetConfig() + return TargetConfig( + connection=local.connection, + connection_name=local.connection_name, + vars=local.vars, + database=local.database, + schema=local.schema, + loader_schema=local.loader_schema, + defer_sources_to=local.defer_sources_to, + defer_clone_from=local.defer_clone_from, + clone=ClonePolicy( + allow_as_clone_origin=local.clone.allow_as_clone_origin is True, + allow_as_clone_destination=local.clone.allow_as_clone_destination is True, + ), + compile_cache=local.compile_cache, + time_travel_retention=local.time_travel_retention, + time_travel_retention_by_materialization=( + local.time_travel_retention_by_materialization or {} + ), + owns_time_travel_retention_namespace=local.owns_time_travel_retention_namespace is True, + default_table_type=local.default_table_type, + table_type_downgrade=local.table_type_downgrade or defaults.table_type_downgrade, + time_travel_retention_decrease=( + local.time_travel_retention_decrease or defaults.time_travel_retention_decrease + ), + missing_migration_origin=local.missing_migration_origin + or defaults.missing_migration_origin, + execution_limits=local.execution_limits, ) - targets: dict[str, TargetConfig] = {} - target_name: str - target_payload: object - for target_name, target_payload in mapping.items(): - target_mapping: dict[str, object] = _coerce_mapping( - payload=target_payload, - label=f"targets.{target_name}", - file_path=file_path, - ) - _validate_target_keys( - target_mapping=target_mapping, target_name=target_name, file_path=file_path - ) - clone_mapping: dict[str, object] = _coerce_mapping( - payload=target_mapping.get("clone"), - label=f"targets.{target_name}.clone", - file_path=file_path, - ) - _validate_clone_keys( - clone_mapping=clone_mapping, target_name=target_name, file_path=file_path - ) - execution_limits: ExecutionLimitsConfig = _load_execution_limits( - payload=target_mapping.get("execution_limits"), - label=f"targets.{target_name}.execution_limits", - file_path=file_path, - ) - connection, connection_name = _load_target_connection( - target_mapping=target_mapping, target_name=target_name, file_path=file_path - ) - targets[target_name] = TargetConfig( - connection=connection, - connection_name=connection_name, - vars=_load_string_mapping(payload=target_mapping.get("vars"), file_path=file_path), - database=_optional_str(payload=target_mapping, key="database"), - schema=_optional_str(payload=target_mapping, key="schema"), - loader_schema=_optional_str(payload=target_mapping, key="loader_schema"), - defer_sources_to=_optional_str(payload=target_mapping, key="defer_sources_to"), - defer_clone_from=_optional_str(payload=target_mapping, key="defer_clone_from"), - compile_cache=_optional_nullable_bool(mapping=target_mapping, key="compile_cache"), - time_travel_retention=_optional_target_retention_default( - mapping=target_mapping, target_name=target_name, file_path=file_path - ), - time_travel_retention_by_materialization=_target_retention_by_materialization( - mapping=target_mapping, target_name=target_name, file_path=file_path - ), - owns_time_travel_retention_namespace=_optional_bool( - mapping=target_mapping, - key="owns_time_travel_retention_namespace", - default=False, - ), - default_table_type=_optional_table_type( - mapping=target_mapping, - key="default_table_type", - label=f"targets.{target_name}.default_table_type", - file_path=file_path, - ), - table_type_downgrade=( - _optional_table_type_downgrade_policy( - mapping=target_mapping, - key="table_type_downgrade", - label=f"targets.{target_name}.table_type_downgrade", - file_path=file_path, - ) - or TableTypeDowngradePolicy.REQUIRE_CONFIRMATION - ), - time_travel_retention_decrease=( - _optional_retention_decrease_policy( - mapping=target_mapping, target_name=target_name, file_path=file_path - ) - or RetentionDecreasePolicy.DENY - ), - missing_migration_origin=( - _optional_missing_migration_origin_policy( - mapping=target_mapping, target_name=target_name, file_path=file_path - ) - or MissingMigrationOriginPolicy.ALLOW - ), - execution_limits=execution_limits, - clone=ClonePolicy( - allow_as_clone_origin=_optional_bool( - mapping=clone_mapping, - key="allow_as_clone_origin", - default=False, - ), - allow_as_clone_destination=_optional_bool( - mapping=clone_mapping, - key="allow_as_clone_destination", - default=False, - ), - ), - ) - return targets def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalTargetConfig]: @@ -1317,17 +1236,26 @@ def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalT label=f"targets.{target_name}.default_table_type", file_path=file_path, ), - table_type_downgrade=_optional_table_type_downgrade_policy( + table_type_downgrade=_optional_target_policy( mapping=target_mapping, key="table_type_downgrade", - label=f"targets.{target_name}.table_type_downgrade", + target_name=target_name, file_path=file_path, + policy=TableTypeDowngradePolicy, ), - time_travel_retention_decrease=_optional_retention_decrease_policy( - mapping=target_mapping, target_name=target_name, file_path=file_path + time_travel_retention_decrease=_optional_target_policy( + mapping=target_mapping, + key="time_travel_retention_decrease", + target_name=target_name, + file_path=file_path, + policy=RetentionDecreasePolicy, ), - missing_migration_origin=_optional_missing_migration_origin_policy( - mapping=target_mapping, target_name=target_name, file_path=file_path + missing_migration_origin=_optional_target_policy( + mapping=target_mapping, + key="missing_migration_origin", + target_name=target_name, + file_path=file_path, + policy=MissingMigrationOriginPolicy, ), execution_limits=execution_limits, clone=LocalClonePolicy( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 7ebbaeb7fa..244c4b494c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -33,6 +33,7 @@ from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) +from sqlbuild.compiler.planner._helpers.planning.retention import effective_target_config from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( gather_redirected_cursor_snapshots, ) @@ -66,7 +67,6 @@ ) from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity from sqlbuild.spec.contracts.main.get_config_str import get_config_str -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import SnapshotsConfig from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, TableType @@ -296,17 +296,7 @@ def planning_database(*, runtime: PlannerRuntime) -> str | None: def missing_origin_policy(*, runtime: PlannerRuntime) -> MissingMigrationOriginPolicy: """Return how the effective target treats a declared migration origin it cannot find.""" - if ( - runtime.project_config is None - or runtime.local_config is None - or runtime.project.effective_target_name is None - ): - return MissingMigrationOriginPolicy.ALLOW - return resolve_target_config( - project_config=runtime.project_config, - local_config=runtime.local_config, - target_name=runtime.project.effective_target_name, - ).missing_migration_origin + return effective_target_config(runtime=runtime).missing_migration_origin def project_schemas(*, runtime: PlannerRuntime) -> set[str]: diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py index 1b63c8c8c1..9842ce637e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py @@ -59,7 +59,7 @@ def plan_table_types( ) if runtime.adapter.adapter_name != BuiltinAdapter.SNOWFLAKE: return () - target: TargetConfig = _effective_target(runtime=runtime) + target: TargetConfig = effective_target_config(runtime=runtime) entries: list[TableTypePlanEntry] = [] for model in selected_models: if not MaterializationType.is_table_backed( @@ -140,7 +140,9 @@ def plan_retention( entries.extend( _plan_relation_retention(runtime=runtime, warehouse=warehouse, model=model) ) - decrease_policy: str = _effective_target(runtime=runtime).time_travel_retention_decrease.value + decrease_policy: str = effective_target_config( + runtime=runtime + ).time_travel_retention_decrease.value return tuple(replace(entry, decrease_policy=decrease_policy) for entry in entries) @@ -262,7 +264,7 @@ def _plan_relation_retention( def _plan_bigquery_retention( *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] ) -> tuple[RetentionPlanEntry, ...]: - target: TargetConfig = _effective_target(runtime=runtime) + target: TargetConfig = effective_target_config(runtime=runtime) if not target.owns_time_travel_retention_namespace: raise PlannerInputError( "BigQuery managed time_travel_retention requires target " @@ -362,7 +364,9 @@ def _plan_bigquery_retention( return tuple(entries) -def _effective_target(*, runtime: PlannerRuntime) -> TargetConfig: +def effective_target_config(*, runtime: PlannerRuntime) -> TargetConfig: + """Return the resolved config of the target being planned, or defaults without one.""" + if ( runtime.project_config is None or runtime.local_config is None diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py index c6c9e58088..98cda55188 100644 --- a/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py +++ b/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py @@ -2,30 +2,26 @@ from __future__ import annotations +from collections.abc import Callable + from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( column_token_edits, model_name_token_edits, span_tokens, ) from sqlbuild.compiler.refactoring.constants import SCHEMA_STATEMENT_PATTERN -from sqlbuild.compiler.refactoring.models import TextEdit +from sqlbuild.compiler.refactoring.models import HeaderToken, TextEdit def schema_model_name_edits(*, contents: str, old: str, new: str) -> tuple[TextEdit, ...]: """Rename a model in every SCHEMA header of a declaration file.""" - edits: list[TextEdit] = [] - span: tuple[int, int] - for span in _schema_spans(contents): - edits.extend( - model_name_token_edits( - contents=contents, - tokens=span_tokens(contents=contents, span=span), - old=old, - new=new, - ) - ) - return tuple(edits) + return _each_schema( + contents=contents, + edit=lambda tokens: model_name_token_edits( + contents=contents, tokens=tokens, old=old, new=new + ), + ) def schema_column_edits( @@ -33,23 +29,20 @@ def schema_column_edits( ) -> tuple[TextEdit, ...]: """Rename relationships `field` values pointing at a renamed column of a model.""" + return _each_schema( + contents=contents, + edit=lambda tokens: column_token_edits( + contents=contents, tokens=tokens, upstream=upstream, old=old, new=new + ), + ) + + +def _each_schema( + *, contents: str, edit: Callable[[tuple[HeaderToken, ...]], tuple[TextEdit, ...]] +) -> tuple[TextEdit, ...]: edits: list[TextEdit] = [] - span: tuple[int, int] - for span in _schema_spans(contents): + for match in SCHEMA_STATEMENT_PATTERN.finditer(contents): edits.extend( - column_token_edits( - contents=contents, - tokens=span_tokens(contents=contents, span=span), - upstream=upstream, - old=old, - new=new, - ) + edit(span_tokens(contents=contents, span=(match.start("header"), match.end("header")))) ) return tuple(edits) - - -def _schema_spans(contents: str) -> tuple[tuple[int, int], ...]: - return tuple( - (match.start("header"), match.end("header")) - for match in SCHEMA_STATEMENT_PATTERN.finditer(contents) - ) From 347cbc30bde917d92fe76af32bf2611c163a647d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:59:20 +0100 Subject: [PATCH 12/16] fix(cli): let only the first set-operation branch name outputs when cascading column renames --- .../column_references/_helpers/references.rs | 58 ++++++++++--------- .../tests/test_column_references.rs | 44 ++++++++++++++ .../cli/commands/main/refactor/helpers.py | 17 ++++++ .../main/refactor/test_rename_columns.py | 30 ++++++++++ 4 files changed, 121 insertions(+), 28 deletions(-) diff --git a/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs b/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs index 3a2a7fcf23..392ffeb864 100644 --- a/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs +++ b/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs @@ -93,6 +93,7 @@ struct Analysis<'a> { output_ctes: HashSet, response: ReferenceResponse, seen: HashSet<(usize, usize)>, + later_branch: bool, } fn default_dialect() -> String { @@ -131,9 +132,10 @@ pub(crate) fn analyze_json(request_json: &str) -> Result { ..ReferenceResponse::default() }, seen: HashSet::new(), + later_branch: false, }; for scope in &scopes { - analysis.visit(scope, ROOT_SCOPE.to_string(), &[]); + analysis.visit(scope, ROOT_SCOPE.to_string(), &[], false); } serde_json::to_string(&analysis.response).map_err(|error| error.to_string()) } @@ -183,14 +185,14 @@ fn star_derived_ctes( } } +/// Only the first branch of a set operation names its output columns. fn selects_star_over_targets( body: &Expression, target_tables: &HashSet, target_ctes: &HashSet, ) -> bool { - branch_selects(body) - .iter() - .any(|select| select_stars_over_targets(select, target_tables, target_ctes)) + first_branch_select(body) + .is_some_and(|select| select_stars_over_targets(select, target_tables, target_ctes)) } fn select_stars_over_targets( @@ -224,23 +226,15 @@ fn is_target_name( target_tables.contains(name) || target_ctes.contains(name) } -fn branch_selects(body: &Expression) -> Vec<&polyglot_sql::expressions::Select> { +fn first_branch_select(body: &Expression) -> Option<&polyglot_sql::expressions::Select> { match body { - Expression::Select(select) => vec![select], - Expression::Union(union) => { - [branch_selects(&union.left), branch_selects(&union.right)].concat() - } - Expression::Intersect(intersect) => [ - branch_selects(&intersect.left), - branch_selects(&intersect.right), - ] - .concat(), - Expression::Except(except) => { - [branch_selects(&except.left), branch_selects(&except.right)].concat() - } - Expression::Subquery(subquery) => branch_selects(&subquery.this), - Expression::Paren(paren) => branch_selects(&paren.this), - _ => Vec::new(), + Expression::Select(select) => Some(select), + Expression::Union(union) => first_branch_select(&union.left), + Expression::Intersect(intersect) => first_branch_select(&intersect.left), + Expression::Except(except) => first_branch_select(&except.left), + Expression::Subquery(subquery) => first_branch_select(&subquery.this), + Expression::Paren(paren) => first_branch_select(&paren.this), + _ => None, } } @@ -308,11 +302,13 @@ fn select_body(expression: &Expression) -> Option<&Expression> { } impl Analysis<'_> { - fn visit(&mut self, scope: &Scope, role: String, outer: &[&Scope]) { + /// Visit one scope; `later_branch` marks a set-operation branch that names no outputs. + fn visit(&mut self, scope: &Scope, role: String, outer: &[&Scope], later_branch: bool) { if let Some(select) = scope_select(&scope.expression) { let local: Scope = local_scope(scope, select); let mut chain: Vec<&Scope> = vec![&local]; chain.extend(outer.iter().copied()); + self.later_branch = later_branch; self.visit_select(select, &scope.expression, &chain, &role); } for child in &scope.cte_scopes { @@ -320,18 +316,18 @@ impl Analysis<'_> { Expression::Cte(cte) => cte.alias.name.clone(), _ => String::new(), }; - self.visit(child, format!("{CTE_SCOPE_PREFIX}{name}"), &[]); + self.visit(child, format!("{CTE_SCOPE_PREFIX}{name}"), &[], false); } for child in &scope.derived_table_scopes { - self.visit(child, DERIVED_SCOPE.to_string(), &[]); + self.visit(child, DERIVED_SCOPE.to_string(), &[], false); } let mut nested: Vec<&Scope> = vec![scope]; nested.extend(outer.iter().copied()); for child in scope.subquery_scopes.iter().chain(scope.udtf_scopes.iter()) { - self.visit(child, SUBQUERY_SCOPE.to_string(), &nested); + self.visit(child, SUBQUERY_SCOPE.to_string(), &nested, false); } - for child in &scope.union_scopes { - self.visit(child, role.clone(), outer); + for (position, child) in scope.union_scopes.iter().enumerate() { + self.visit(child, role.clone(), outer, later_branch || position > 0); } } @@ -343,7 +339,9 @@ impl Analysis<'_> { role: &str, ) { self.record_outputs(select, role); - self.record_stars(select, chain[0], role); + if !self.later_branch { + self.record_stars(select, chain[0], role); + } self.record_joins(select, chain[0], role); let Some(body) = select_body(expression) else { return; @@ -379,7 +377,11 @@ impl Analysis<'_> { name_start: name_span.start, name_end: name_span.end, scope: role.to_string(), - projection: projection_site(select, span.start), + projection: if self.later_branch { + None + } else { + projection_site(select, span.start) + }, }), Resolution::Other => {} Resolution::Unknown => { diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs b/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs index eb358a8b03..ec7ab77d68 100644 --- a/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs +++ b/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs @@ -91,6 +91,50 @@ fn given_column_uses_when_analyzing_then_reports_rename_sites() -> Result<(), St expected_output_aliases: &["amount", "amount"], expected_outputs: 3, }, + ColumnReferenceTestCase { + description: "a later set-operation branch does not name the output", + sql: "SELECT amount FROM customers UNION ALL SELECT amount FROM orders_fact", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount"], + expected_scopes: &["root"], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "the first branch of a nested set operation names the output", + sql: "(SELECT amount FROM orders_fact EXCEPT SELECT amount FROM orders_fact) \ + UNION ALL SELECT * FROM orders_fact", + output_ctes: &[], + expected_parsed: true, + expected_names: &["amount", "amount"], + expected_scopes: &["root", "root"], + expected_bare_projections: 1, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, + ColumnReferenceTestCase { + description: "a star in a later branch does not pass the column through", + sql: "WITH x AS (SELECT 1 AS order_id, 2 AS amount UNION ALL SELECT * FROM orders_fact) \ + SELECT * FROM customers UNION ALL SELECT * FROM orders_fact", + output_ctes: &[], + expected_parsed: true, + expected_names: &[], + expected_scopes: &[], + expected_bare_projections: 0, + expected_aliases: &[], + expected_star_scopes: &[], + expected_joins: 0, + expected_output_aliases: &[], + expected_outputs: 0, + }, ColumnReferenceTestCase { description: "unparseable SQL reports that it was not analysed", sql: "SELECT FROM WHERE (", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index 5e4668aad8..60388c6653 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -108,6 +108,23 @@ def project_toml(*, target_settings: str = "") -> str: "models/staging/stg_order_refunds.sql" ].replace("to_cents", "to_dollars"), } + + +def order_amount_union(*, first: str, second: str) -> dict[str, str]: + """Return a view unioning amount from two models, and a view reading its output.""" + + return { + "models/marts/order_amounts.sql": ( + "MODEL (\n materialized view,\n);\n\n" + f'SELECT amount FROM __ref("{first}")\nUNION ALL\n' + f'SELECT amount FROM __ref("{second}")\n' + ), + "models/marts/order_amount_reads.sql": ( + 'MODEL (\n materialized view,\n);\n\nSELECT amount FROM __ref("order_amounts")\n' + ), + } + + ORDER_EXPORT: dict[str, str] = { "models/marts/order_export.sql": ( 'MODEL (\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py index 2cc4e39a1a..da5f81b6ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py @@ -24,6 +24,7 @@ SPLIT_CENTS_MACROS, fragments_present, load_raw_orders, + order_amount_union, order_history_files, order_ids, project_text, @@ -86,6 +87,35 @@ "fact_orders": ("order_id", "customer_id", "amount", "order_date"), }, ), + ColumnRenameE2ETestCase( + description="cascade stops at a later set-operation branch", + command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), + extra_files=order_amount_union(first="stg_orders", second="fact_orders"), + expected_fragments={ + "models/marts/order_amounts.sql": ('SELECT revenue FROM __ref("fact_orders")',), + "models/marts/order_amount_reads.sql": ("SELECT amount FROM",), + }, + expected_columns={ + "order_amounts": ("amount",), + "order_amount_reads": ("amount",), + }, + ), + ColumnRenameE2ETestCase( + description="cascade follows the first set-operation branch", + command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), + extra_files=order_amount_union(first="fact_orders", second="stg_orders"), + expected_fragments={ + "models/marts/order_amounts.sql": ( + 'SELECT revenue FROM __ref("fact_orders")', + 'SELECT amount FROM __ref("stg_orders")', + ), + "models/marts/order_amount_reads.sql": ("SELECT revenue FROM",), + }, + expected_columns={ + "order_amounts": ("revenue",), + "order_amount_reads": ("revenue",), + }, + ), ColumnRenameE2ETestCase( description="cascade follows SELECT star consumers", command=("rename", "column:fact_orders.amount", "revenue", "--cascade"), From 494c22c389149a0e16e7a45b991a7bc87a8cb975 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:05:21 +0100 Subject: [PATCH 13/16] fix(dbt): apply migration policies and missing-origin confirmation to dbt run and build --- .../_helpers/build_planning/plan_policies.py | 41 +++++++++++++++---- .../commands/_helpers/dbt/sqlbuild_work.py | 9 ++++ .../dbt/_helpers/pipeline/plan_output.py | 2 + .../cli/commands/main/dbt/_test_types.py | 7 ++++ .../sqlbuild/cli/commands/main/dbt/helpers.py | 20 +++++++++ .../cli/commands/main/dbt/test_execute.py | 35 ++++++++++++++++ 6 files changed, 107 insertions(+), 7 deletions(-) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py index c2a2567429..29d7236a8c 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -3,6 +3,7 @@ from __future__ import annotations import sys +from typing import TextIO from sqlbuild.cli.commands._helpers.build_planning.confirmation import confirm_typed_action from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( @@ -44,10 +45,11 @@ def enforce_build_plan_policies( ) -> None: """Apply migration, execution-limit, and storage safety gates in their required order.""" - _enforce_model_migration_policy(plan=plan) - _enforce_column_migration_policy(plan=plan) - _confirm_missing_origins( - plan=plan, allow_missing_migration_origin=request.allow_missing_migration_origin + enforce_migration_plan_policies( + plan=plan, + allow_missing_migration_origin=request.allow_missing_migration_origin, + input_stream=sys.stdin, + output_stream=sys.stdout, ) _enforce_old_name_policy(plan=plan) enforce_model_execution_limit( @@ -76,6 +78,25 @@ def enforce_build_plan_policies( ) +def enforce_migration_plan_policies( + *, + plan: PlanOutput, + allow_missing_migration_origin: bool, + input_stream: TextIO, + output_stream: TextIO, +) -> None: + """Refuse blocked model and column migrations, then confirm building past missing origins.""" + + _enforce_model_migration_policy(plan=plan) + _enforce_column_migration_policy(plan=plan) + _confirm_missing_origins( + plan=plan, + allow_missing_migration_origin=allow_missing_migration_origin, + input_stream=input_stream, + output_stream=output_stream, + ) + + def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: """Refuse to build while any planned migration conflicts or is incompatible.""" @@ -142,7 +163,13 @@ def _enforce_column_migration_policy(*, plan: PlanOutput) -> None: ) -def _confirm_missing_origins(*, plan: PlanOutput, allow_missing_migration_origin: bool) -> None: +def _confirm_missing_origins( + *, + plan: PlanOutput, + allow_missing_migration_origin: bool, + input_stream: TextIO, + output_stream: TextIO, +) -> None: """Confirm before building past declared migrations whose origin is missing.""" names: tuple[str, ...] = tuple( @@ -168,8 +195,8 @@ def _confirm_missing_origins(*, plan: PlanOutput, allow_missing_migration_origin if len(names) == 1 else f"build {len(names)} models without their migrations" ), - input_stream=sys.stdin, - output_stream=sys.stdout, + input_stream=input_stream, + output_stream=output_stream, code="M102", ) diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index f38709f558..b9c36f2390 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -7,6 +7,9 @@ from typing import TextIO from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.cli.commands._helpers.build_planning.plan_policies import ( + enforce_migration_plan_policies, +) from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( enforce_retention_decrease_policy, ) @@ -50,6 +53,12 @@ def execute_sqlbuild_build_work( adapter_name: str = context.adapter_name output_stream: TextIO = context.output_stream use_color: bool = context.use_color + enforce_migration_plan_policies( + plan=plan_output, + allow_missing_migration_origin=False, + input_stream=sys.stdin, + output_stream=output_stream, + ) enforce_retention_decrease_policy( plan=plan_output, allow_retention_decrease=False, diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py index 6c2fab0341..5ddec9b493 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py @@ -90,6 +90,8 @@ def build_sqlbuild_plan_output( deferral=deferral, policies=PlannerPolicies(), on_progress=hooks.on_progress, + project_config=environment.discovered_inputs.project_config, + local_config=environment.discovered_inputs.local_config, ) except PlannerInputError: return build_display_only_sqlbuild_plan( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py index b2fa50709f..a6c9f07cff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py @@ -107,3 +107,10 @@ class DbtAutoInitE2ETestCase: class DbtCliFlagAmbiguityE2ETestCase: description: str expected_stdout_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DbtMissingOriginE2ETestCase: + description: str + command: tuple[str, ...] + expected_output: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py index b3f7d14d57..3cb72f0257 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -129,6 +129,26 @@ def write_sqlbuild_defer_target_models(*, project_dir: Path) -> None: ) +def declare_missing_origin_requiring_confirmation(*, project_dir: Path) -> None: + """Give local_only a migrate_from origin that was never built, under require_confirmation.""" + + config_path: Path = project_dir / "sqlbuild_project.toml" + config_path.write_text( + config_path.read_text(encoding="utf-8").replace( + 'adapter = "duckdb"\n', 'adapter = "duckdb"\ndefault_target = "main"\n' + ) + + '\n[targets.main]\nschema = "main"\nmissing_migration_origin = "require_confirmation"\n', + encoding="utf-8", + ) + model_path: Path = project_dir / "models" / "local_only.sql" + model_path.write_text( + model_path.read_text(encoding="utf-8").replace( + "MODEL (\n", "MODEL (\n materialized table,\n migrate_from retired_local_orders,\n", 1 + ), + encoding="utf-8", + ) + + def load_json_stdout(stdout: str) -> dict[str, object]: """Load JSON command output.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py index f9e3c8b924..6f387d5ddc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py @@ -11,9 +11,11 @@ DbtExecutionCliTestCase, DbtExecutionFailureCliTestCase, DbtExecutionQueryAssertion, + DbtMissingOriginE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.dbt.helpers import ( break_dbt_interop_fact_orders_model, + declare_missing_origin_requiring_confirmation, load_json_stdout, prepare_dbt_interop_project, skip_unless_dbt_is_runnable, @@ -541,3 +543,36 @@ def test_given_failing_dbt_model_when_running_command_then_dependent_sqlbuild_is relation_name: str for relation_name in test_case.expected_absent_relations: assert not table_exists(db_path=db_path, table_name=relation_name), relation_name + + +@pytest.mark.parametrize( + "test_case", + [ + DbtMissingOriginE2ETestCase( + description="dbt build refuses a missing origin that needs confirmation", + command=("dbt", "build", "--select", "tag:sqb_only"), + expected_output="--allow-missing-migration-origin", + ), + DbtMissingOriginE2ETestCase( + description="dbt run refuses a missing origin that needs confirmation", + command=("dbt", "run", "--select", "local_only"), + expected_output="--allow-missing-migration-origin", + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_requiring_confirmation_when_running_dbt_then_refuses( + test_case: DbtMissingOriginE2ETestCase, + tmp_path: Path, +) -> None: + skip_unless_dbt_is_runnable() + project_dir: Path = prepare_dbt_interop_project(tmp_path=tmp_path) + declare_missing_origin_requiring_confirmation(project_dir=project_dir) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir, input_text="" + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert test_case.expected_output in result.stdout + result.stderr + assert not table_exists(db_path=project_dir / "dbt_interop.duckdb", table_name="local_only") From 6f52c969acb0136b4165bedb9ce78e48fbac4b3c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:11:59 +0100 Subject: [PATCH 14/16] fix(plan): stop on a missing migration origin that was built in this target, whatever the policy --- .../docs/concepts/models/migrations.md | 2 +- .../docs/concepts/project-configuration.md | 2 + .../_helpers/build_planning/plan_policies.py | 26 ++++++-- .../planner/_helpers/migrations/planning.py | 11 ++++ src/sqlbuild/compiler/planner/constants.py | 5 ++ src/sqlbuild/compiler/planner/models.py | 11 +++- .../cli/commands/main/refactor/_test_types.py | 7 +++ .../cli/commands/main/refactor/helpers.py | 12 ++++ .../refactor/test_missing_migration_origin.py | 44 +++++++++++++ .../sqlbuild/compiler/planner/_test_types.py | 13 ++++ .../sqlbuild/compiler/planner/test_models.py | 61 +++++++++++++++++++ .../docs/docs/concepts/models/migrations.mdx | 2 +- .../docs/concepts/project-configuration.mdx | 2 + 13 files changed, 190 insertions(+), 8 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index e952d50b4a..6bce224583 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -54,7 +54,7 @@ After the move is recorded, `migrate_from` has no further effect, and SQLBuild t | `superseded replace` | The new relation was itself the origin of the latest recorded move, for example after renaming a model back. Replace it with the newer data. | | `forced replace` | The new relation has its own build history and `migrate_force true` is set. Replace it. | | `conflict` | The new relation has its own build history. The build stops (`M103`) until you set `migrate_force true` or remove `migrate_from`. | -| `origin missing` | The old relation doesn't exist and no move was recorded, for example on a target that never built the old name. The plan warns (`M102`) and the model builds fresh. A target can require confirmation or stop the build instead with [`missing_migration_origin`](../project-configuration.md#missing-migration-origins). | +| `origin missing` | The old relation doesn't exist and no move was recorded, for example on a target that never built the old name. The plan warns (`M102`) and the model builds fresh. A target can require confirmation or stop the build instead with [`missing_migration_origin`](../project-configuration.md#missing-migration-origins). If SQLBuild has build history for the old relation in this target, it should exist, so the build stops (`M102`) whatever the policy: the connection may lack permission to see it, or it was dropped outside SQLBuild. Restore access, or remove `migrate_from` to build fresh deliberately. | The origin and destination must be compatible under the model's normal `on_schema_change` rules; otherwise the build stops (`M104`). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index fe0e748869..68f9a3e789 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -235,6 +235,8 @@ missing_migration_origin = "deny" `sqlbuild_local.toml` can override the value for a target. +The policy covers only origins with no build history in the target. A model origin that SQLBuild built in this target but can no longer see, because of permissions or because it was dropped outside SQLBuild, always stops the build with `M102`. Column origins have no such history, so `M109` always follows the policy. `sqb dbt run` and `sqb dbt build` apply the policy too, without a confirmation flag: they ask on a terminal and stop otherwise. + ### Clone policies Targets can declare whether they allow cloning to or from: diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py index 29d7236a8c..86304f51d3 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -22,6 +22,7 @@ from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import BuildCommandRequest, BuildInvocation from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision +from sqlbuild.compiler.planner.constants import HIDDEN_ORIGIN_REMEDY from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanEntry, ModelMigrationPlanEntry, @@ -105,17 +106,24 @@ def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: ) if not blocked: return + hidden: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in blocked if entry.origin_hidden + ) + if hidden: + raise CliUserError( + "model migration origin was built in this target but is not visible for " + + _origins(hidden) + + f"; {HIDDEN_ORIGIN_REMEDY}", + code="M102", + help="Run sqb plan to see every model migration decision.", + ) missing: tuple[ModelMigrationPlanEntry, ...] = tuple( entry for entry in blocked if entry.decision == MigrationDecision.ORIGIN_MISSING ) if missing: raise CliUserError( "model migration origin does not exist for " - + ", ".join( - f"'{entry.model_name}' (migrate_from " - f"{entry.origin.qualified_name or entry.origin.name})" - for entry in missing - ) + + _origins(missing) + " and no recorded migration into it was found, and this target sets " "missing_migration_origin = deny; if the migration already happened elsewhere or is " "no longer needed, remove migrate_from from the model header", @@ -143,6 +151,13 @@ def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: ) +def _origins(entries: tuple[ModelMigrationPlanEntry, ...]) -> str: + return ", ".join( + f"'{entry.model_name}' (migrate_from {entry.origin.qualified_name or entry.origin.name})" + for entry in entries + ) + + def _enforce_column_migration_policy(*, plan: PlanOutput) -> None: """Refuse to build while any declared column rename cannot be applied safely.""" @@ -178,6 +193,7 @@ def _confirm_missing_origins( entry.model_name for entry in (*plan.migration_entries, *plan.column_migration_entries) if entry.origin_missing + and not entry.blocks_build and entry.missing_origin_policy == MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION } ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 244c4b494c..b6af40b215 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -44,6 +44,7 @@ MigrationStateInspection, ) from sqlbuild.compiler.planner.constants import ( + HIDDEN_ORIGIN_REMEDY, MIGRATION_MODEL_NAME_METADATA_KEY, MISSING_ORIGIN_OUTCOMES, QUALIFIED_RELATION_MAX_PARTS, @@ -764,6 +765,16 @@ def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: ), code="M101", ) + if entry.origin_hidden: + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.ERROR, + message=( + f"model '{entry.model_name}': migrate_from origin {origin} was built in this " + f"target but is not visible; {HIDDEN_ORIGIN_REMEDY}" + ), + code="M102", + ) if entry.origin_missing: return PlanWarning( model_name=entry.model_name, diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 7e3d957eb2..59b474cb02 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -88,6 +88,11 @@ "matched by unchanged definition; sqb rename model: rewrites references and " "declares migrate_from so the move does not depend on discovery" ) +HIDDEN_ORIGIN_REMEDY: str = ( + "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore " + "access to it, or remove migrate_from to build the model fresh deliberately " + "(missing_migration_origin does not apply)" +) MISSING_ORIGIN_OUTCOMES: dict[MissingMigrationOriginPolicy, str] = { MissingMigrationOriginPolicy.ALLOW: ( "nothing is migrated and the build continues as if migrate_from were absent; remove " diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 0868773451..8afa5db8ca 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -940,6 +940,12 @@ def origin_missing(self) -> bool: return self.decision == MigrationDecision.ORIGIN_MISSING + @property + def origin_hidden(self) -> bool: + """Return whether the missing origin has build history in this target.""" + + return self.origin_missing and self.origin_tracked + @property def storage_transition(self) -> str | None: """Return the Snowflake table-type transition label, when both types are known.""" @@ -960,7 +966,10 @@ def blocks_build(self) -> bool: """Return whether this migration must stop a build before any execution.""" if self.origin_missing: - return self.missing_origin_policy == MissingMigrationOriginPolicy.DENY + return ( + self.origin_tracked + or self.missing_origin_policy == MissingMigrationOriginPolicy.DENY + ) return self.decision.blocks_build or ( self.decision.moves_data and self.compatibility == MigrationCompatibility.INCOMPATIBLE ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py index 18ecafd1bf..54bd686e72 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -90,3 +90,10 @@ class DeclarationMoveE2ETestCase: extra_files: dict[str, str] destination: str expected_moved: dict[str, str] + + +@dataclass(frozen=True) +class HiddenOriginE2ETestCase: + description: str + target_settings: str + expected_output: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index 60388c6653..78b1284e7a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -259,6 +259,18 @@ def strip_migration_fingerprints(*, project_dir: Path, models: tuple[str, ...]) connection.close() +def drop_view_outside_sqlbuild(*, project_dir: Path, name: str) -> None: + """Drop a built view directly in the warehouse, leaving SQLBuild's history in place.""" + + import duckdb + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / DATABASE_FILE)) + try: + _ = connection.execute(f"DROP VIEW {SCHEMA}.{name}") + finally: + connection.close() + + def declare_missing_column_origin(*, project_dir: Path) -> None: """Add a revenue column that declares migrate_from a column the table never had.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py index 2cebb92ade..9bedf27127 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_missing_migration_origin.py @@ -8,10 +8,12 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + HiddenOriginE2ETestCase, MissingOriginE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( declare_missing_column_origin, + drop_view_outside_sqlbuild, order_history_files, project_toml, relation_columns, @@ -133,3 +135,45 @@ def test_given_missing_origin_column_when_building_then_policy_decides( assert relation_columns(project_dir=project_dir, name="order_history") == ( test_case.expected_columns ) + + +@pytest.mark.parametrize( + "test_case", + [ + HiddenOriginE2ETestCase( + description="an origin built here but dropped outside SQLBuild stops under allow", + target_settings="", + expected_output="was built in this target but is not visible", + ), + HiddenOriginE2ETestCase( + description="require_confirmation cannot confirm past a hidden origin", + target_settings=_CONFIRM, + expected_output="error[M102]", + ), + ], + ids=lambda case: case.description, +) +def test_given_built_origin_not_visible_when_building_then_stops_regardless_of_policy( + tmp_path: Path, test_case: HiddenOriginE2ETestCase +) -> None: + """Recorded build history means the origin should exist, so its absence is an error.""" + + project_dir: Path = write_orders_project( + tmp_path=tmp_path, + files={"sqlbuild_project.toml": project_toml(target_settings=test_case.target_settings)}, + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + drop_view_outside_sqlbuild(project_dir=project_dir, name="stg_orders") + renamed: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:stg_orders", "stg_order_lines" + ) + + built: subprocess.CompletedProcess[str] = sqb( + project_dir, "build", "--allow-missing-migration-origin" + ) + + assert first.returncode == 0, first.stdout + first.stderr + assert renamed.returncode == 0, renamed.stdout + renamed.stderr + assert built.returncode == 1, built.stdout + built.stderr + assert test_case.expected_output in built.stdout + built.stderr + assert relation_type(project_dir=project_dir, name="stg_order_lines") is None diff --git a/tests/unit/src/sqlbuild/compiler/planner/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_test_types.py index e69de29bb2..5563adc22d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_test_types.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy + + +@dataclass(frozen=True) +class MissingOriginBlockTestCase: + description: str + origin_tracked: bool + policy: MissingMigrationOriginPolicy + expected_blocks: bool diff --git a/tests/unit/src/sqlbuild/compiler/planner/test_models.py b/tests/unit/src/sqlbuild/compiler/planner/test_models.py index 8b13789179..847e0b6039 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/planner/test_models.py @@ -1 +1,62 @@ +"""Tests for planner model decisions.""" +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from sqlbuild.compiler.migrations.types import MigrationDecision +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy +from tests.unit.src.sqlbuild.compiler.planner._test_types import MissingOriginBlockTestCase +from tests.unit.src.sqlbuild.compiler.planner.main.pre_build.helpers import migration_entry + + +@pytest.mark.parametrize( + "test_case", + [ + MissingOriginBlockTestCase( + description="never-built origin follows allow", + origin_tracked=False, + policy=MissingMigrationOriginPolicy.ALLOW, + expected_blocks=False, + ), + MissingOriginBlockTestCase( + description="never-built origin follows require_confirmation", + origin_tracked=False, + policy=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION, + expected_blocks=False, + ), + MissingOriginBlockTestCase( + description="never-built origin follows deny", + origin_tracked=False, + policy=MissingMigrationOriginPolicy.DENY, + expected_blocks=True, + ), + MissingOriginBlockTestCase( + description="origin built in this target blocks even under allow", + origin_tracked=True, + policy=MissingMigrationOriginPolicy.ALLOW, + expected_blocks=True, + ), + MissingOriginBlockTestCase( + description="origin built in this target blocks under require_confirmation", + origin_tracked=True, + policy=MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION, + expected_blocks=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_when_deciding_then_history_overrides_policy( + test_case: MissingOriginBlockTestCase, +) -> None: + entry: ModelMigrationPlanEntry = replace( + migration_entry(decision=MigrationDecision.ORIGIN_MISSING), + origin_tracked=test_case.origin_tracked, + missing_origin_policy=test_case.policy, + ) + + assert entry.blocks_build is test_case.expected_blocks + assert entry.origin_hidden is test_case.origin_tracked diff --git a/website/src/content/docs/docs/concepts/models/migrations.mdx b/website/src/content/docs/docs/concepts/models/migrations.mdx index e78483b251..fde311af7c 100644 --- a/website/src/content/docs/docs/concepts/models/migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/migrations.mdx @@ -40,7 +40,7 @@ After the move is recorded, `migrate_from` has no further effect, and SQLBuild t | `superseded replace` | The new relation was itself the origin of the latest recorded move, for example after renaming a model back. Replace it with the newer data. | | `forced replace` | The new relation has its own build history and `migrate_force true` is set. Replace it. | | `conflict` | The new relation has its own build history. The build stops (`M103`) until you set `migrate_force true` or remove `migrate_from`. | -| `origin missing` | The old relation doesn't exist and no move was recorded, for example on a target that never built the old name. The plan warns (`M102`) and the model builds fresh. A target can require confirmation or stop the build instead with [`missing_migration_origin`](/docs/concepts/project-configuration#missing-migration-origins). | +| `origin missing` | The old relation doesn't exist and no move was recorded, for example on a target that never built the old name. The plan warns (`M102`) and the model builds fresh. A target can require confirmation or stop the build instead with [`missing_migration_origin`](/docs/concepts/project-configuration#missing-migration-origins). If SQLBuild has build history for the old relation in this target, it should exist, so the build stops (`M102`) whatever the policy: the connection may lack permission to see it, or it was dropped outside SQLBuild. Restore access, or remove `migrate_from` to build fresh deliberately. | The origin and destination must be compatible under the model's normal `on_schema_change` rules; otherwise the build stops (`M104`). diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index c308be37bd..34add19ee6 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -212,6 +212,8 @@ missing_migration_origin = "deny" `sqlbuild_local.toml` can override the value for a target. +The policy covers only origins with no build history in the target. A model origin that SQLBuild built in this target but can no longer see, because of permissions or because it was dropped outside SQLBuild, always stops the build with `M102`. Column origins have no such history, so `M109` always follows the policy. `sqb dbt run` and `sqb dbt build` apply the policy too, without a confirmation flag: they ask on a terminal and stop otherwise. + ### Clone policies Targets can declare whether they allow cloning to or from: From 27481cfa8e9511dd0f06025104aee0fbdfbe8d28 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:45:35 +0100 Subject: [PATCH 15/16] fix(janitor): keep the origin of a pending model migration --- .../sqlbuild/references/docs/cli/janitor.md | 5 ++ .../executor/janitor/_helpers/plan.py | 55 ++++++++++++++ src/sqlbuild/executor/janitor/constants.py | 1 + src/sqlbuild/executor/janitor/main/plan.py | 66 +++++++++++++++-- .../cli/commands/main/refactor/_test_types.py | 7 ++ .../refactor/test_janitor_pending_origin.py | 72 +++++++++++++++++++ .../executor/janitor/main/_test_types.py | 1 + .../sqlbuild/executor/janitor/main/helpers.py | 3 +- .../executor/janitor/main/test_plan.py | 27 ++++++- website/src/content/docs/docs/cli/janitor.mdx | 4 ++ 10 files changed, 235 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 253b81aa58..6d929df51e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/cli/janitor/ - Flags - Lifecycle - Compatibility views +- Pending migration origins - Relation age - Audit record - Restoring an archive @@ -74,6 +75,10 @@ A compatibility view is never archived as a stale relation. Once it expires, or Run the same SQLBuild version for `sqb janitor` as for builds. An older janitor does not know compatibility views and archives them as stale relations. +## Pending migration origins + +A relation that a project model still has to migrate from is never archived. While a model declares `migrate_from` and its own relation does not exist yet in the target, the janitor skips the origin and lists it as `pending migration origin for `. A bare `migrate_from` name is matched in each schema the project builds into; a qualified name only at that location. Once the model has built, the origin is handled as usual: after a migration, the old name holds a compatibility view, which expires as described above. + ## Relation age `retention_days` is measured from the last time a relation was written, as reported by the warehouse. For a relation the project no longer builds, that is roughly when it stopped being built. diff --git a/src/sqlbuild/executor/janitor/_helpers/plan.py b/src/sqlbuild/executor/janitor/_helpers/plan.py index 3e8e4c466f..883c44cb42 100644 --- a/src/sqlbuild/executor/janitor/_helpers/plan.py +++ b/src/sqlbuild/executor/janitor/_helpers/plan.py @@ -8,12 +8,14 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup +from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledProject, CompiledSeed, CompiledSource, ) +from sqlbuild.executor.janitor.constants import QUALIFIED_ORIGIN_PARTS from sqlbuild.executor.janitor.models import JanitorRelationKey @@ -42,6 +44,59 @@ def collect_desired_keys(project: CompiledProject) -> set[JanitorRelationKey]: return keys +def pending_migration_origins( + *, + project: CompiledProject, + relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], +) -> dict[JanitorRelationKey, str]: + """Return listed origins of migrations whose destination relation does not exist yet.""" + + listed: dict[tuple[str | None, str | None, str], JanitorRelationKey] = {} + relations: tuple[RelationInfo, ...] + for relations in relations_by_schema.values(): + listed.update({_folded(relation_key(item)): relation_key(item) for item in relations}) + schemas: set[tuple[str | None, str | None]] = collect_target_schemas(project) + pending: dict[JanitorRelationKey, str] = {} + model: CompiledModel + for model in project.models: + raw: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) + destination: JanitorRelationKey = JanitorRelationKey( + database=model.destination.database, + schema=model.destination.schema, + name=model.destination.name, + ) + if not isinstance(raw, str) or _folded(destination) in listed: + continue + origin: JanitorRelationKey + for origin in _origin_keys(raw=raw, destination=destination, schemas=schemas): + found: JanitorRelationKey | None = listed.get(_folded(origin)) + if found is not None: + pending.setdefault(found, f"pending migration origin for {model.name}") + return pending + + +def _origin_keys( + *, raw: str, destination: JanitorRelationKey, schemas: set[tuple[str | None, str | None]] +) -> tuple[JanitorRelationKey, ...]: + parts: list[str] = [part.strip().strip('"`') for part in raw.split(".")] + if len(parts) == 1: + return tuple( + JanitorRelationKey(database=database, schema=schema, name=parts[0]) + for database, schema in sorted(schemas, key=_schema_sort_key) + ) + return ( + JanitorRelationKey( + database=parts[0] if len(parts) == QUALIFIED_ORIGIN_PARTS else destination.database, + schema=parts[-2], + name=parts[-1], + ), + ) + + +def _folded(key: JanitorRelationKey) -> tuple[str | None, str | None, str]: + return RelationLookup.key(database=key.database, schema=key.schema, name=key.name) + + def collect_target_schemas(project: CompiledProject) -> set[tuple[str | None, str | None]]: """Collect schemas where the compiled project writes relations, one spelling per schema.""" diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 2d9db78951..ce81109fec 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -43,3 +43,4 @@ MALFORMED_ARCHIVE_REASON: str = ( "name resembles a janitor archive but does not match the strict archive grammar" ) +QUALIFIED_ORIGIN_PARTS: int = 3 diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py index 0550f41374..e681589a3a 100644 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ b/src/sqlbuild/executor/janitor/main/plan.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import replace from datetime import UTC, datetime from typing import Any @@ -14,7 +15,11 @@ collect_query_diff_artifact_candidates, gather_janitor_warehouse_facts, ) -from sqlbuild.executor.janitor._helpers.plan import collect_scan_schemas, collect_target_schemas +from sqlbuild.executor.janitor._helpers.plan import ( + collect_scan_schemas, + collect_target_schemas, + pending_migration_origins, +) from sqlbuild.executor.janitor._helpers.schema_planning import classify_target_schemas from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader from sqlbuild.executor.janitor.models import ( @@ -23,6 +28,7 @@ JanitorDirectStatePruneCandidate, JanitorOldNameViewPlanning, JanitorPlan, + JanitorRelationKey, JanitorRelationScope, JanitorSchemaClassification, JanitorWarehouseFacts, @@ -107,12 +113,11 @@ def build_janitor_plan( ) inspection.completed(metadata={"item_count": len(target_schemas)}) old_names: JanitorOldNameViewPlanning - old_names, scope = plan_old_name_view_cleanup( + old_names, scope = _plan_protected_names( adapter=adapter, connection=connection, managed_target_schemas=managed_target_schemas, - relations_by_schema=facts.relations_by_schema, - target_name=project.effective_target_name, + facts=facts, early_drops=early_old_name_view_drops, project=project, scope=scope, @@ -173,3 +178,56 @@ def build_janitor_plan( age_metadata_supported=age_supported, planned_at=now, ) + + +def _plan_protected_names( + *, + adapter: BaseAdapter, + connection: Any, + managed_target_schemas: set[tuple[str | None, str | None]], + facts: JanitorWarehouseFacts, + early_drops: tuple[str, ...], + project: CompiledProject, + scope: JanitorRelationScope, + now: datetime, +) -> tuple[JanitorOldNameViewPlanning, JanitorRelationScope]: + """Plan old-name views, then keep general cleanup away from pending migration origins.""" + + old_names: JanitorOldNameViewPlanning + old_names, scope = plan_old_name_view_cleanup( + adapter=adapter, + connection=connection, + managed_target_schemas=managed_target_schemas, + relations_by_schema=facts.relations_by_schema, + target_name=project.effective_target_name, + early_drops=early_drops, + project=project, + scope=scope, + now=now, + ) + return old_names, _protect_pending_origins( + project=project, facts=facts, scope=scope, old_names=old_names + ) + + +def _protect_pending_origins( + *, + project: CompiledProject, + facts: JanitorWarehouseFacts, + scope: JanitorRelationScope, + old_names: JanitorOldNameViewPlanning, +) -> JanitorRelationScope: + pending: dict[JanitorRelationKey, str] = { + key: reason + for key, reason in pending_migration_origins( + project=project, relations_by_schema=facts.relations_by_schema + ).items() + if key not in old_names.keys + } + if not pending: + return scope + return replace( + scope, + protected_relation_keys=scope.protected_relation_keys | frozenset(pending), + protected_relation_reasons={**(scope.protected_relation_reasons or {}), **pending}, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py index 54bd686e72..910f4adea4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -97,3 +97,10 @@ class HiddenOriginE2ETestCase: description: str target_settings: str expected_output: str + + +@dataclass(frozen=True) +class PendingOriginJanitorE2ETestCase: + description: str + janitor_args: tuple[str, ...] + expected_order_ids: tuple[int, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py new file mode 100644 index 0000000000..f890b505b4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py @@ -0,0 +1,72 @@ +"""CLI e2e coverage for the janitor keeping the origin of a pending model migration.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + PendingOriginJanitorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + load_raw_orders, + order_history_files, + order_ids, + relation_type, + sqb, + write_orders_project, +) + +_PENDING_REASON: str = "analytics.order_history pending migration origin for order_ledger" + + +@pytest.mark.parametrize( + "test_case", + [ + PendingOriginJanitorE2ETestCase( + description="janitor keeps the origin until the rename is migrated", + janitor_args=("janitor", "--retention-days", "0", "--auto-approve"), + expected_order_ids=(1, 2, 3, 4), + ), + ], + ids=lambda case: case.description, +) +def test_given_pending_rename_when_janitor_runs_then_origin_survives_until_migrated( + tmp_path: Path, test_case: PendingOriginJanitorE2ETestCase +) -> None: + project_dir: Path = write_orders_project( + tmp_path=tmp_path, files=order_history_files(materialized="incremental", udf=False) + ) + first: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + renamed: subprocess.CompletedProcess[str] = sqb( + project_dir, "rename", "model:order_history", "order_ledger" + ) + + pending: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.janitor_args) + origin_after_pending: str | None = relation_type(project_dir=project_dir, name="order_history") + load_raw_orders(project_dir=project_dir, order_ids=(3, 4)) + built: subprocess.CompletedProcess[str] = sqb(project_dir, "build") + migrated: tuple[int, ...] = order_ids(project_dir=project_dir, name="order_ledger") + settled: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.janitor_args) + old_name_after_settled: str | None = relation_type( + project_dir=project_dir, name="order_history" + ) + dropped: subprocess.CompletedProcess[str] = sqb( + project_dir, *test_case.janitor_args, "--drop-old-name-view", "order_history" + ) + + assert first.returncode == 0, first.stdout + first.stderr + assert renamed.returncode == 0, renamed.stdout + renamed.stderr + assert pending.returncode == 0, pending.stdout + pending.stderr + assert _PENDING_REASON in pending.stdout + assert origin_after_pending == "BASE TABLE" + assert built.returncode == 0, built.stdout + built.stderr + assert migrated == test_case.expected_order_ids + assert settled.returncode == 0, settled.stdout + settled.stderr + assert "pending migration origin" not in settled.stdout + assert "analytics.order_history -> model:order_ledger" in settled.stdout + assert old_name_after_settled == "VIEW" + assert dropped.returncode == 0, dropped.stdout + dropped.stderr + assert relation_type(project_dir=project_dir, name="order_history") is None diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 5a4971c5c2..8561369fad 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -22,6 +22,7 @@ class JanitorPlanTestCase: tracked_relations: tuple[tuple[str | None, str | None, str], ...] = field(default_factory=tuple) exclude_patterns: tuple[str, ...] = field(default_factory=tuple) protected_relation_keys: frozenset[JanitorRelationKey] = frozenset() + migrate_from: str | None = None expected_candidate_names: tuple[str, ...] = field(default_factory=tuple) expected_direct_state_table_names: tuple[str, ...] = field(default_factory=tuple) expected_skipped_relation_reasons: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index aa0eb7cfbf..ae6494a16c 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -540,6 +540,7 @@ def build_project( source_database: str | None = None, destination_database: str | None = None, destination_schema: str = "analytics", + migrate_from: str | None = None, ) -> CompiledProject: source: CompiledSource = CompiledSource( key=CompiledObjectKey( @@ -569,7 +570,7 @@ def build_project( name="orders", relative_path=Path("models/orders.sql"), query_sql="select 1", - config=CompileModelConfig(), + config=CompileModelConfig(values={"migrate_from": migrate_from}), destination=CompiledRelationLocation( database=destination_database, schema=destination_schema, diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index c50974b1d2..8deb2b0c05 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -164,6 +164,29 @@ expected_candidate_names=(), expected_skipped_relation_reasons=("relation is not tracked by SQLBuild",), ), + JanitorPlanTestCase( + description="origin of a pending migration is kept", + relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), + migrate_from="old_orders", + expected_candidate_names=(), + expected_skipped_relation_reasons=("pending migration origin for orders",), + ), + JanitorPlanTestCase( + description="qualified origin of a pending migration is kept", + relation_infos=(relation_info("OLD_ORDERS", created_at=OLD_TIME),), + migrate_from="analytics.old_orders", + expected_candidate_names=(), + expected_skipped_relation_reasons=("pending migration origin for orders",), + ), + JanitorPlanTestCase( + description="origin is cleaned normally once the destination exists", + relation_infos=( + relation_info("orders", created_at=OLD_TIME), + relation_info("old_orders", created_at=OLD_TIME), + ), + migrate_from="old_orders", + expected_candidate_names=("old_orders",), + ), JanitorPlanTestCase( description="existing direct state tables are eligible for history pruning", relation_infos=( @@ -204,7 +227,9 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp ) plan: JanitorPlan = build_janitor_plan( - project=build_project(source_schema=test_case.source_schema), + project=build_project( + source_schema=test_case.source_schema, migrate_from=test_case.migrate_from + ), adapter=adapter, connection=object(), retention_days=test_case.retention_days, diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 55bdb6d92f..0c0868e469 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -59,6 +59,10 @@ A compatibility view is never archived as a stale relation. Once it expires, or Run the same SQLBuild version for `sqb janitor` as for builds. An older janitor does not know compatibility views and archives them as stale relations. +## Pending migration origins + +A relation that a project model still has to migrate from is never archived. While a model declares `migrate_from` and its own relation does not exist yet in the target, the janitor skips the origin and lists it as `pending migration origin for `. A bare `migrate_from` name is matched in each schema the project builds into; a qualified name only at that location. Once the model has built, the origin is handled as usual: after a migration, the old name holds a compatibility view, which expires as described above. + ## Relation age `retention_days` is measured from the last time a relation was written, as reported by the warehouse. For a relation the project no longer builds, that is roughly when it stopped being built. From ddca09d2d748ef4ed80050ddab84d6f89f3cfb01 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:52:36 +0100 Subject: [PATCH 16/16] fix(dbt): give dbt-appropriate remediation when a missing origin needs confirmation --- .../references/docs/concepts/project-configuration.md | 2 +- .../cli/commands/_helpers/build_planning/confirmation.py | 4 ++-- .../cli/commands/_helpers/build_planning/plan_policies.py | 7 ++++++- src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py | 2 ++ src/sqlbuild/cli/commands/constants.py | 8 ++++++++ .../src/sqlbuild/cli/commands/main/dbt/test_execute.py | 5 +++-- .../content/docs/docs/concepts/project-configuration.mdx | 2 +- 7 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 68f9a3e789..cef8b9084c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -235,7 +235,7 @@ missing_migration_origin = "deny" `sqlbuild_local.toml` can override the value for a target. -The policy covers only origins with no build history in the target. A model origin that SQLBuild built in this target but can no longer see, because of permissions or because it was dropped outside SQLBuild, always stops the build with `M102`. Column origins have no such history, so `M109` always follows the policy. `sqb dbt run` and `sqb dbt build` apply the policy too, without a confirmation flag: they ask on a terminal and stop otherwise. +The policy covers only origins with no build history in the target. A model origin that SQLBuild built in this target but can no longer see, because of permissions or because it was dropped outside SQLBuild, always stops the build with `M102`. Column origins have no such history, so `M109` always follows the policy. `sqb dbt run` and `sqb dbt build` apply the policy too, but have no confirmation flag: they ask on a terminal and stop otherwise. In non-interactive dbt runs, build the affected models with `sqb build --allow-missing-migration-origin` and leave them out of the dbt selection, or set the target's `missing_migration_origin` to `allow`. ### Clone policies diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py index 8b0e1509a6..7080ea11cb 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py @@ -10,7 +10,7 @@ def confirm_typed_action( *, action: str, - flag: str, + non_interactive_help: str, warning: str, expected: str, input_stream: TextIO, @@ -23,7 +23,7 @@ def confirm_typed_action( raise CliUserError( f"{action} requires confirmation", code=code, - help=f"Pass {flag} to confirm in non-interactive runs.", + help=non_interactive_help, ) _ = output_stream.write(f"{warning}\n\n") _ = output_stream.write(f"Type `{expected}` to continue: ") diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py index 86304f51d3..152ea7bb45 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -19,6 +19,7 @@ from sqlbuild.cli.commands._helpers.build_planning.table_type import ( enforce_table_type_downgrade_policy, ) +from sqlbuild.cli.commands.constants import MISSING_ORIGIN_BUILD_HELP from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import BuildCommandRequest, BuildInvocation from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision @@ -49,6 +50,7 @@ def enforce_build_plan_policies( enforce_migration_plan_policies( plan=plan, allow_missing_migration_origin=request.allow_missing_migration_origin, + non_interactive_help=MISSING_ORIGIN_BUILD_HELP, input_stream=sys.stdin, output_stream=sys.stdout, ) @@ -83,6 +85,7 @@ def enforce_migration_plan_policies( *, plan: PlanOutput, allow_missing_migration_origin: bool, + non_interactive_help: str, input_stream: TextIO, output_stream: TextIO, ) -> None: @@ -93,6 +96,7 @@ def enforce_migration_plan_policies( _confirm_missing_origins( plan=plan, allow_missing_migration_origin=allow_missing_migration_origin, + non_interactive_help=non_interactive_help, input_stream=input_stream, output_stream=output_stream, ) @@ -182,6 +186,7 @@ def _confirm_missing_origins( *, plan: PlanOutput, allow_missing_migration_origin: bool, + non_interactive_help: str, input_stream: TextIO, output_stream: TextIO, ) -> None: @@ -203,7 +208,7 @@ def _confirm_missing_origins( listed: str = ", ".join(f"'{name}'" for name in names) confirm_typed_action( action=f"building {listed} without its missing migrate_from origin", - flag="--allow-missing-migration-origin", + non_interactive_help=non_interactive_help, warning=f"The migrate_from origin of {listed} does not exist in this target; nothing " "will be migrated.", expected=( diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index b9c36f2390..3e8c27f173 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -17,6 +17,7 @@ BuildProgressCallbacks, format_build_footer, ) +from sqlbuild.cli.commands.constants import MISSING_ORIGIN_DBT_HELP from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.cli.progress.main._execution_header import format_execution_header @@ -56,6 +57,7 @@ def execute_sqlbuild_build_work( enforce_migration_plan_policies( plan=plan_output, allow_missing_migration_origin=False, + non_interactive_help=MISSING_ORIGIN_DBT_HELP, input_stream=sys.stdin, output_stream=output_stream, ) diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index c2d025fcca..ad8a34121e 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -103,4 +103,12 @@ QUERY_DIFF_INCOMPLETE_PREPARATION_CODES: frozenset[str] = frozenset( {"C229", "C230", "C231", "C232", "C233", "C234", "C235", "C240", "C244"} ) +MISSING_ORIGIN_BUILD_HELP: str = ( + "Pass --allow-missing-migration-origin to confirm in non-interactive runs." +) +MISSING_ORIGIN_DBT_HELP: str = ( + "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " + "models with sqb build --allow-missing-migration-origin and leave them out of this dbt " + "selection, or set the target's missing_migration_origin to 'allow'." +) QUERY_DIFF_INCOMPLETE_EXECUTION_CODES: frozenset[str] = frozenset({"C237", "C238", "C239"}) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py index 6f387d5ddc..a2a29bb6a5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py @@ -551,12 +551,12 @@ def test_given_failing_dbt_model_when_running_command_then_dependent_sqlbuild_is DbtMissingOriginE2ETestCase( description="dbt build refuses a missing origin that needs confirmation", command=("dbt", "build", "--select", "tag:sqb_only"), - expected_output="--allow-missing-migration-origin", + expected_output="build the affected models with sqb build --allow-missing-migration-origin", ), DbtMissingOriginE2ETestCase( description="dbt run refuses a missing origin that needs confirmation", command=("dbt", "run", "--select", "local_only"), - expected_output="--allow-missing-migration-origin", + expected_output="build the affected models with sqb build --allow-missing-migration-origin", ), ], ids=lambda case: case.description, @@ -575,4 +575,5 @@ def test_given_missing_origin_requiring_confirmation_when_running_dbt_then_refus assert result.returncode == 1, result.stdout + result.stderr assert test_case.expected_output in result.stdout + result.stderr + assert "Pass --allow-missing-migration-origin" not in result.stdout + result.stderr assert not table_exists(db_path=project_dir / "dbt_interop.duckdb", table_name="local_only") diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 34add19ee6..1a97666da6 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -212,7 +212,7 @@ missing_migration_origin = "deny" `sqlbuild_local.toml` can override the value for a target. -The policy covers only origins with no build history in the target. A model origin that SQLBuild built in this target but can no longer see, because of permissions or because it was dropped outside SQLBuild, always stops the build with `M102`. Column origins have no such history, so `M109` always follows the policy. `sqb dbt run` and `sqb dbt build` apply the policy too, without a confirmation flag: they ask on a terminal and stop otherwise. +The policy covers only origins with no build history in the target. A model origin that SQLBuild built in this target but can no longer see, because of permissions or because it was dropped outside SQLBuild, always stops the build with `M102`. Column origins have no such history, so `M109` always follows the policy. `sqb dbt run` and `sqb dbt build` apply the policy too, but have no confirmation flag: they ask on a terminal and stop otherwise. In non-interactive dbt runs, build the affected models with `sqb build --allow-missing-migration-origin` and leave them out of the dbt selection, or set the target's `missing_migration_origin` to `allow`. ### Clone policies