diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/aspects.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/aspects.rs new file mode 100644 index 0000000000..99f896fc06 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/aspects.rs @@ -0,0 +1,57 @@ +//! Aspect comparison and the words reasons use for aspects and kinds. + +use crate::definition_changes::constants::{ASPECT_LABELS, KIND_LABELS}; +use crate::definition_changes::models::NodeDefinition; + +fn aspect_rank(aspect: &str) -> usize { + ASPECT_LABELS + .iter() + .position(|(name, _)| *name == aspect) + .unwrap_or(ASPECT_LABELS.len()) +} + +/// The aspects present on either side whose digests differ, in reason order. +pub(crate) fn changed_aspects(base: &NodeDefinition, current: &NodeDefinition) -> Vec { + let mut aspects: Vec = base + .aspects + .keys() + .chain(current.aspects.keys()) + .filter(|aspect| base.aspects.get(*aspect) != current.aspects.get(*aspect)) + .cloned() + .collect(); + aspects.sort_by(|left, right| { + aspect_rank(left) + .cmp(&aspect_rank(right)) + .then(left.cmp(right)) + }); + aspects.dedup(); + aspects +} + +/// How an aspect reads in a reason. +pub(crate) fn aspect_label(aspect: &str) -> &str { + ASPECT_LABELS + .iter() + .find(|(name, _)| *name == aspect) + .map_or(aspect, |(_, label)| label) +} + +/// The singular and plural words of a node kind. +pub(crate) fn kind_words(kind: &str) -> (&str, String) { + KIND_LABELS + .iter() + .find(|(name, _, _)| *name == kind) + .map_or_else( + || (kind, format!("{kind}s")), + |(_, singular, plural)| (*singular, (*plural).to_owned()), + ) +} + +/// `a`, `a and b`, `a, b and c`. +pub(crate) fn joined(words: &[String]) -> String { + match words { + [] => String::new(), + [only] => only.clone(), + [init @ .., last] => format!("{} and {last}", init.join(", ")), + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs new file mode 100644 index 0000000000..57a4708da1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod aspects; +pub(crate) mod reasons; diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs new file mode 100644 index 0000000000..6cc9143e0a --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs @@ -0,0 +1,131 @@ +//! The reason texts of selected tests and the declarations an executed item reaches. + +use std::collections::BTreeSet; + +use crate::definition_changes::_helpers::aspects::{aspect_label, joined, kind_words}; +use crate::definition_changes::constants::{ + COLUMNS_ASPECT, CONNECTION_KIND, CONSTANT_KIND, DEPENDENCIES_KIND, ENUM_KIND, MODEL_KIND, + SEED_KIND, SETTINGS_KIND, SOURCE_KIND, TEST_FILE_KIND, +}; + +/// Project-wide kinds whose reasons name the kind, not the single node. +const PROJECT_WIDE_KINDS: [&str; 3] = [SETTINGS_KIND, CONNECTION_KIND, DEPENDENCIES_KIND]; +use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem, NodeChange}; + +fn aspect_phrase(change: &NodeChange) -> String { + let labels: Vec = change + .aspects + .iter() + .map(|aspect| aspect_label(aspect).to_owned()) + .collect(); + joined(&labels) +} + +/// `test file changed`, `scenario file and snapshots changed`, `test file added`. +pub(crate) fn own_reason(change: &NodeChange) -> String { + let (singular, _) = kind_words(&change.kind); + if change.added { + return format!("{singular} added"); + } + if change.kind == TEST_FILE_KIND { + return format!("{singular} changed"); + } + format!("{singular} {} changed", aspect_phrase(change)) +} + +/// `added`, `model SQL changed`, `renamed from old_orders, config changed`, `changed`. +fn change_detail(change: &NodeChange) -> String { + if change.added { + return "added".to_owned(); + } + let detail: String = if change.kind == MODEL_KIND { + format!("model {} changed", aspect_phrase(change)) + } else { + "changed".to_owned() + }; + match &change.renamed_from { + Some(old_name) => format!("renamed from {old_name}, {detail}"), + None => detail, + } +} + +/// `executes stg_orders (model SQL changed)`, `reads seed countries (changed)`. +pub(crate) fn executed_reason(change: &NodeChange) -> String { + let subject: String = if change.kind == MODEL_KIND { + change.name.clone() + } else { + format!("{} {}", kind_words(&change.kind).0, change.name) + }; + let verb: &str = if change.kind == SEED_KIND || change.kind == SOURCE_KIND { + "reads" + } else { + "executes" + }; + format!("{verb} {subject} ({})", change_detail(change)) +} + +/// `executes macro to_utc (changed)`, `uses enum status (changed)`, `uses project settings (changed)`. +pub(crate) fn declaration_reason(change: &NodeChange) -> String { + if PROJECT_WIDE_KINDS.contains(&change.kind.as_str()) { + return format!( + "uses {} ({})", + kind_words(&change.kind).0, + change_detail(change) + ); + } + if change.kind == SOURCE_KIND { + return executed_reason(change); + } + let verb: &str = if change.kind == ENUM_KIND || change.kind == CONSTANT_KIND { + "uses" + } else { + "executes" + }; + format!( + "{verb} {} {} ({})", + kind_words(&change.kind).0, + change.name, + change_detail(change) + ) +} + +/// Declarations `item` reaches, transitively through its SQL and executed nodes, minus its mocks. +pub(crate) fn reached_declarations( + item: &ExecutedItem, + current: &DefinitionSnapshot, +) -> BTreeSet { + let mut pending: Vec = item.declarations.clone(); + for key in &item.executed { + if let Some(node) = current.nodes.get(key) { + pending.extend(node.uses.iter().cloned()); + } + } + let mut reached: BTreeSet = BTreeSet::new(); + while let Some(key) = pending.pop() { + if item.mocked.contains(&key) || !reached.insert(key.clone()) { + continue; + } + if let Some(node) = current.nodes.get(&key) { + pending.extend(node.uses.iter().cloned()); + } + } + reached +} + +/// `mocks stg_orders (model columns changed)` when a mocked relation's declared shape changed. +pub(crate) fn mocked_reason(change: &NodeChange) -> Option { + if !change.added && !change.aspects.iter().any(|aspect| aspect == COLUMNS_ASPECT) { + return None; + } + let subject: String = if change.kind == MODEL_KIND { + change.name.clone() + } else { + format!("{} {}", kind_words(&change.kind).0, change.name) + }; + let detail: String = if change.added { + "added".to_owned() + } else { + format!("{} columns changed", kind_words(&change.kind).0) + }; + Some(format!("mocks {subject} ({detail})")) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/constants.rs b/crates/sqlbuild-analysis/src/definition_changes/constants.rs new file mode 100644 index 0000000000..f17cdf9140 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/constants.rs @@ -0,0 +1,89 @@ +//! Node kinds, definition aspects and the snapshot format version. + +/// The version of the stored snapshot layout and fingerprint scheme; older snapshots are recomputed. +pub const DEFINITION_SNAPSHOT_FORMAT: u32 = 3; + +pub const MODEL_KIND: &str = "model"; +pub const SEED_KIND: &str = "seed"; +pub const UDF_KIND: &str = "udf"; +pub const TABLE_FUNCTION_KIND: &str = "table_fn"; +pub const MACRO_KIND: &str = "macro"; +pub const ENUM_KIND: &str = "enum"; +pub const CONSTANT_KIND: &str = "constant"; +pub const TEST_FILE_KIND: &str = "test_file"; +pub const SCENARIO_KIND: &str = "scenario"; +pub const SOURCE_KIND: &str = "source"; +pub const MATERIALIZATION_KIND: &str = "materialization"; +pub const SETTINGS_KIND: &str = "settings"; +/// Adapter, connection and target configuration, adapter modules and referenced environment. +pub const CONNECTION_KIND: &str = "connection"; +/// Python dependency manifests and lock files of the project and its repository. +pub const DEPENDENCIES_KIND: &str = "dependencies"; +/// The one node of each project-wide kind. +pub const PROJECT_NODE_NAME: &str = "project"; +/// Python nodes by decorator. +pub const PYTHON_NODE_KINDS: [&str; 4] = ["loader", "task", "asset", "check"]; + +/// Normalised SQL of a model, a function body or a scenario. +pub const SQL_ASPECT: &str = "sql"; +/// Header configuration, without descriptions. +pub const CONFIG_ASPECT: &str = "config"; +/// Declared columns and contract, without descriptions. +pub const COLUMNS_ASPECT: &str = "columns"; +/// Attached audits, without descriptions. +pub const AUDITS_ASPECT: &str = "audits"; +/// The whole definition of a seed, function, macro, enum or constant. +pub const DEFINITION_ASPECT: &str = "definition"; +/// The authored file of a test or scenario. +pub const FILE_ASPECT: &str = "file"; +/// Captured local snapshots of a scenario. +pub const SNAPSHOTS_ASPECT: &str = "snapshots"; +/// Values of the project variables and environment variables a definition reads. +pub const VARS_ASPECT: &str = "vars"; + +/// How each aspect reads in a reason, in the order reasons list them. +pub(crate) const ASPECT_LABELS: [(&str, &str); 8] = [ + (SQL_ASPECT, "SQL"), + (CONFIG_ASPECT, "config"), + (COLUMNS_ASPECT, "columns"), + (AUDITS_ASPECT, "audits"), + (DEFINITION_ASPECT, "definition"), + (FILE_ASPECT, "file"), + (SNAPSHOTS_ASPECT, "snapshots"), + (VARS_ASPECT, "vars"), +]; + +/// How each node kind reads in summaries: singular, plural. +pub(crate) const KIND_LABELS: [(&str, &str, &str); 23] = [ + (SETTINGS_KIND, "project settings", "project settings"), + ( + CONNECTION_KIND, + "connection and target configuration", + "connection and target configurations", + ), + ( + DEPENDENCIES_KIND, + "Python environment", + "Python environments", + ), + (MODEL_KIND, "model", "models"), + (SOURCE_KIND, "source", "sources"), + (SEED_KIND, "seed", "seeds"), + (UDF_KIND, "function", "functions"), + (TABLE_FUNCTION_KIND, "table function", "table functions"), + (MACRO_KIND, "macro", "macros"), + (ENUM_KIND, "enum", "enums"), + (CONSTANT_KIND, "constant", "constants"), + ("audit", "audit", "audits"), + ("singular_audit", "singular audit", "singular audits"), + ("schema", "schema", "schemas"), + ("sql_hook", "SQL hook", "SQL hooks"), + ("python_hook", "Python hook", "Python hooks"), + (MATERIALIZATION_KIND, "materialization", "materializations"), + ("loader", "loader", "loaders"), + ("task", "task", "tasks"), + ("asset", "asset", "assets"), + ("check", "check", "checks"), + (TEST_FILE_KIND, "test file", "test files"), + (SCENARIO_KIND, "scenario", "scenarios"), +]; diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs b/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs new file mode 100644 index 0000000000..cac6c5d918 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs @@ -0,0 +1,70 @@ +//! A one-line account of what changed since the base revision. + +use std::collections::BTreeMap; + +use crate::definition_changes::_helpers::aspects::kind_words; +use crate::definition_changes::constants::KIND_LABELS; +use crate::definition_changes::models::DefinitionChanges; + +fn kind_rank(kind: &str) -> usize { + KIND_LABELS + .iter() + .position(|(name, _, _)| *name == kind) + .unwrap_or(KIND_LABELS.len()) +} + +fn counted<'a>(kinds: impl Iterator) -> Option { + let mut counts: BTreeMap<(usize, &str), usize> = BTreeMap::new(); + for kind in kinds { + *counts.entry((kind_rank(kind), kind)).or_default() += 1; + } + if counts.is_empty() { + return None; + } + let parts: Vec = counts + .into_iter() + .map(|((_, kind), count)| { + let (singular, plural) = kind_words(kind); + if count == 1 { + format!("1 {singular}") + } else { + format!("{count} {plural}") + } + }) + .collect(); + Some(parts.join(", ")) +} + +/// `2 models, 1 macro changed; 1 seed added; deleted: model old_orders`, or `no definition changes`. +pub fn changes_summary(changes: &DefinitionChanges) -> String { + let mut parts: Vec = Vec::new(); + let modified = changes.changed.values().filter(|change| !change.added); + if let Some(text) = counted(modified.map(|change| change.kind.as_str())) { + parts.push(format!("{text} changed")); + } + let added = changes.changed.values().filter(|change| change.added); + if let Some(text) = counted(added.map(|change| change.kind.as_str())) { + parts.push(format!("{text} added")); + } + let mut renamed: Vec<&str> = Vec::new(); + for (_, key) in &changes.renamed { + if let Some((kind, _)) = key.split_once(':') { + renamed.push(kind); + } + } + if let Some(text) = counted(renamed.into_iter()) { + parts.push(format!("{text} renamed")); + } + if !changes.deleted.is_empty() { + let names: Vec = changes + .deleted + .iter() + .map(|node| format!("{} {}", kind_words(&node.kind).0, node.name)) + .collect(); + parts.push(format!("deleted: {}", names.join(", "))); + } + if parts.is_empty() { + return "no definition changes".to_owned(); + } + parts.join("; ") +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs b/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs new file mode 100644 index 0000000000..81fe3874ab --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs @@ -0,0 +1,88 @@ +//! The nodes that changed between a base and a current definition snapshot. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::definition_changes::_helpers::aspects::changed_aspects; +use crate::definition_changes::main::node_key::node_key; +use crate::definition_changes::models::{ + DefinitionChanges, DefinitionSnapshot, DeletedNode, NodeChange, NodeDefinition, +}; + +/// The base node a current node renamed through `migrate_from`, if one is unclaimed. +fn renamed_base<'a>( + current_key: &str, + node: &NodeDefinition, + (base, current): (&'a DefinitionSnapshot, &DefinitionSnapshot), + claimed: &BTreeSet, +) -> Option<(String, &'a NodeDefinition)> { + for old_name in &node.migrate_from { + let old_key: String = node_key(&node.kind, old_name); + if old_key == current_key + || claimed.contains(&old_key) + || current.nodes.contains_key(&old_key) + { + continue; + } + if let Some(definition) = base.nodes.get(&old_key) { + return Some((old_key, definition)); + } + } + None +} + +/// Added nodes are changed, deleted ones listed; a `migrate_from` rename changes only with its definition. +pub fn compare_definitions( + base: &DefinitionSnapshot, + current: &DefinitionSnapshot, +) -> DefinitionChanges { + let mut changed: BTreeMap = BTreeMap::new(); + let mut renamed: Vec<(String, String)> = Vec::new(); + let mut claimed: BTreeSet = BTreeSet::new(); + for (key, node) in ¤t.nodes { + let change = |added: bool, aspects: Vec, renamed_from: Option| NodeChange { + kind: node.kind.clone(), + name: node.name.clone(), + added, + aspects, + renamed_from, + }; + if let Some(previous) = base.nodes.get(key) { + let aspects: Vec = changed_aspects(previous, node); + if !aspects.is_empty() { + changed.insert(key.clone(), change(false, aspects, None)); + } + continue; + } + match renamed_base(key, node, (base, current), &claimed) { + Some((old_key, previous)) => { + let aspects: Vec = changed_aspects(previous, node); + if !aspects.is_empty() { + changed.insert( + key.clone(), + change(false, aspects, Some(previous.name.clone())), + ); + } + claimed.insert(old_key.clone()); + renamed.push((old_key, key.clone())); + } + None => { + changed.insert(key.clone(), change(true, Vec::new(), None)); + } + } + } + let mut deleted: Vec = base + .nodes + .iter() + .filter(|(key, _)| !current.nodes.contains_key(*key) && !claimed.contains(*key)) + .map(|(_, node)| DeletedNode { + kind: node.kind.clone(), + name: node.name.clone(), + }) + .collect(); + deleted.sort(); + DefinitionChanges { + changed, + deleted, + renamed, + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs b/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs new file mode 100644 index 0000000000..6c84c8b477 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs @@ -0,0 +1,14 @@ +//! The digest of definition parts. + +use sha2::{Digest, Sha256}; + +/// The SHA-256 hex digest of `parts`, each length-prefixed so boundaries cannot shift. +pub fn definition_digest(parts: &[&str]) -> String { + let mut digest = Sha256::new(); + for part in parts { + digest.update(part.len().to_string().as_bytes()); + digest.update(b":"); + digest.update(part.as_bytes()); + } + format!("{:x}", digest.finalize()) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs b/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs new file mode 100644 index 0000000000..9b283e84ef --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs @@ -0,0 +1,8 @@ +//! The stored JSON form of a definition snapshot. + +use crate::definition_changes::models::DefinitionSnapshot; + +/// `snapshot` as compact JSON. +pub fn definition_snapshot_json(snapshot: &DefinitionSnapshot) -> String { + serde_json::to_string(snapshot).unwrap_or_default() +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs new file mode 100644 index 0000000000..4faa4cb4ae --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs @@ -0,0 +1,10 @@ +pub mod changes_summary; +pub mod compare_definitions; +pub mod definition_digest; +pub mod definition_snapshot_json; +pub mod node_key; +pub mod parse_definition_snapshot; +pub mod selection_json; +pub mod selection_reasons; +pub mod selection_report; +pub mod sql_definition_digest; diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/node_key.rs b/crates/sqlbuild-analysis/src/definition_changes/main/node_key.rs new file mode 100644 index 0000000000..b2767a3e46 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/node_key.rs @@ -0,0 +1,6 @@ +//! The snapshot key of one node. + +/// `kind:identity`, where identity is a name, or a declaration identity for scoped declarations. +pub fn node_key(kind: &str, identity: &str) -> String { + format!("{kind}:{identity}") +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/parse_definition_snapshot.rs b/crates/sqlbuild-analysis/src/definition_changes/main/parse_definition_snapshot.rs new file mode 100644 index 0000000000..62e1cd972a --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/parse_definition_snapshot.rs @@ -0,0 +1,12 @@ +//! Read a stored definition snapshot. + +use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; +use crate::definition_changes::models::DefinitionSnapshot; + +/// The snapshot `text` stores, or `None` when it is unreadable or from another format version. +pub fn parse_definition_snapshot(text: &str) -> Option { + match serde_json::from_str::(text) { + Ok(snapshot) if snapshot.format == DEFINITION_SNAPSHOT_FORMAT => Some(snapshot), + Ok(_) | Err(_) => None, + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_json.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_json.rs new file mode 100644 index 0000000000..550624991f --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_json.rs @@ -0,0 +1,53 @@ +//! The `since` section of a `--since` run's JSON document. + +use serde_json::{Value, json}; + +use crate::definition_changes::models::{DefinitionChanges, SelectedItem}; + +/// The ref, merge base, changed, renamed and deleted nodes, and each selected item with its reasons. +pub fn selection_json( + (reference, merge_base): (&str, &str), + changes: &DefinitionChanges, + (total, selected): (usize, &[SelectedItem]), +) -> Value { + let changed: Vec = changes + .changed + .values() + .map(|change| { + json!({ + "kind": change.kind, + "name": change.name, + "added": change.added, + "aspects": change.aspects, + "renamed_from": change.renamed_from, + }) + }) + .collect(); + let renamed: Vec = changes + .renamed + .iter() + .map(|(from, to)| { + let (kind, old_name) = from.split_once(':').unwrap_or_default(); + let new_name: &str = to.split_once(':').map_or(to.as_str(), |(_, name)| name); + json!({ "kind": kind, "from": old_name, "to": new_name }) + }) + .collect(); + let deleted: Vec = changes + .deleted + .iter() + .map(|node| json!({ "kind": node.kind, "name": node.name })) + .collect(); + let items: Vec = selected + .iter() + .map(|item| json!({ "name": item.label, "reasons": item.reasons })) + .collect(); + json!({ + "ref": reference, + "merge_base": merge_base, + "changed": changed, + "renamed": renamed, + "deleted": deleted, + "candidate_count": total, + "selected": items, + }) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs new file mode 100644 index 0000000000..6ecafb8286 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs @@ -0,0 +1,41 @@ +//! Why a test or scenario is affected by the changes since the base revision. + +use crate::definition_changes::_helpers::reasons::{ + declaration_reason, executed_reason, mocked_reason, own_reason, reached_declarations, +}; +use crate::definition_changes::models::{DefinitionChanges, DefinitionSnapshot, ExecutedItem}; + +/// One reason per changed dependency of `item`, mocked relations by their declared columns only. +pub fn selection_reasons( + item: &ExecutedItem, + current: &DefinitionSnapshot, + changes: &DefinitionChanges, +) -> Vec { + let mut reasons: Vec = Vec::new(); + let mut push = |reason: String| { + if !reasons.contains(&reason) { + reasons.push(reason); + } + }; + for key in &item.own_keys { + if let Some(change) = changes.changed.get(key) { + push(own_reason(change)); + } + } + for key in &item.executed { + if let Some(change) = changes.changed.get(key) { + push(executed_reason(change)); + } + } + for key in &item.mocked_relations { + if let Some(reason) = changes.changed.get(key).and_then(mocked_reason) { + push(reason); + } + } + for key in reached_declarations(item, current) { + if let Some(change) = changes.changed.get(&key) { + push(declaration_reason(change)); + } + } + reasons +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs new file mode 100644 index 0000000000..617e67e589 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs @@ -0,0 +1,30 @@ +//! The lines a `--since` run prints about what changed and what it selected. + +use crate::definition_changes::models::SelectedItem; + +/// The change summary, then the selected count with reasons when `explain` is set. +pub fn selection_report( + (reference, summary): (&str, &str), + (singular, plural): (&str, &str), + (total, selected): (usize, &[SelectedItem]), + explain: bool, +) -> String { + let mut text: String = format!("Changed since {reference}: {summary}\n"); + if selected.is_empty() { + text.push_str(&format!( + "No {plural} are affected by the changes since {reference}\n" + )); + return text; + } + let noun: &str = if total == 1 { singular } else { plural }; + text.push_str(&format!("Selected {} of {total} {noun}\n", selected.len())); + if explain { + for item in selected { + text.push_str(&format!(" {}\n", item.label)); + for reason in &item.reasons { + text.push_str(&format!(" {reason}\n")); + } + } + } + text +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/sql_definition_digest.rs b/crates/sqlbuild-analysis/src/definition_changes/main/sql_definition_digest.rs new file mode 100644 index 0000000000..0e206f9ef8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/sql_definition_digest.rs @@ -0,0 +1,12 @@ +//! The digest of SQL with whitespace, comments and keyword case ignored. + +use crate::definition_changes::main::definition_digest::definition_digest; +use crate::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; + +/// The generic-tokenizer fingerprint of `sql`, or the digest of its text when it cannot tokenize. +pub fn sql_definition_digest(sql: &str) -> String { + match compiler_query_fingerprint(sql, None) { + Ok(fingerprint) => fingerprint, + Err(_) => definition_digest(&[sql]), + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/mod.rs new file mode 100644 index 0000000000..b4c56eaa3e --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/mod.rs @@ -0,0 +1,9 @@ +//! Definition fingerprints of project nodes, their changes, and why tests that run them are selected. + +mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/definition_changes/models.rs b/crates/sqlbuild-analysis/src/definition_changes/models.rs new file mode 100644 index 0000000000..7057e2eefa --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/models.rs @@ -0,0 +1,80 @@ +//! Definition snapshots, the changes between two of them, and what a test or scenario executes. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +/// The definition fingerprint of one project node. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeDefinition { + pub kind: String, + pub name: String, + /// The digest of each part of the definition, by aspect name. + pub aspects: BTreeMap, + /// Keys of the macros, enums and constants the node uses, resolved through scopes. + #[serde(default)] + pub uses: Vec, + /// Names the node was renamed from, as its `migrate_from` declares them. + #[serde(default)] + pub migrate_from: Vec, +} + +/// Every node definition of one compiled project revision, by node key. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DefinitionSnapshot { + pub format: u32, + pub nodes: BTreeMap, +} + +/// How one node differs from the base revision. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NodeChange { + pub kind: String, + pub name: String, + /// The node did not exist at the base revision. + pub added: bool, + /// The aspects whose digests differ, in aspect order; empty for an added node. + pub aspects: Vec, + /// The base name of a node its `migrate_from` renamed. + pub renamed_from: Option, +} + +/// A base node that no longer exists. +#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct DeletedNode { + pub kind: String, + pub name: String, +} + +/// The nodes that changed between a base and a current snapshot. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DefinitionChanges { + /// Added and modified nodes, by current node key. + pub changed: BTreeMap, + /// Base nodes absent from the current revision, which select nothing. + pub deleted: Vec, + /// `(base key, current key)` of renames that `migrate_from` declares. + pub renamed: Vec<(String, String)>, +} + +/// What one test or scenario depends on, honouring what it mocks. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExecutedItem { + /// Keys of the item's own definition, such as its test file. + pub own_keys: Vec, + /// Keys of the models, seeds and functions it really executes, in execution order. + pub executed: Vec, + /// Keys of the macros, enums and constants its own SQL uses. + pub declarations: Vec, + /// Keys of declarations the item mocks, whose changes never select it. + pub mocked: BTreeSet, + /// Keys of relations the item mocks, whose declared columns still shape the mock. + pub mocked_relations: Vec, +} + +/// One selected test or scenario and why it was selected. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SelectedItem { + pub label: String, + pub reasons: Vec, +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests.rs b/crates/sqlbuild-analysis/src/definition_changes/tests.rs new file mode 100644 index 0000000000..ded3fb082a --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_definition_changes.rs"] +mod definition_changes; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs new file mode 100644 index 0000000000..eee20b0773 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs @@ -0,0 +1,141 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; +use crate::definition_changes::main::node_key::node_key; +use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem, NodeDefinition}; + +/// One node with `aspects` as `(aspect, digest)` and `uses` as node keys. +pub(super) fn node( + kind: &str, + name: &str, + aspects: &[(&str, &str)], + uses: &[&str], +) -> (String, NodeDefinition) { + ( + node_key(kind, name), + NodeDefinition { + kind: kind.to_owned(), + name: name.to_owned(), + aspects: aspects + .iter() + .map(|(aspect, digest)| ((*aspect).to_owned(), (*digest).to_owned())) + .collect(), + uses: uses.iter().map(|key| (*key).to_owned()).collect(), + migrate_from: Vec::new(), + }, + ) +} + +/// `definition` renamed from `old_names`. +pub(super) fn renamed( + (key, mut definition): (String, NodeDefinition), + old_names: &[&str], +) -> (String, NodeDefinition) { + definition.migrate_from = old_names.iter().map(|name| (*name).to_owned()).collect(); + (key, definition) +} + +pub(super) fn snapshot(nodes: Vec<(String, NodeDefinition)>) -> DefinitionSnapshot { + DefinitionSnapshot { + format: DEFINITION_SNAPSHOT_FORMAT, + nodes: nodes.into_iter().collect::>(), + } +} + +/// The orders project at its base revision. +pub(super) fn base_orders() -> DefinitionSnapshot { + snapshot(vec![ + node( + "model", + "stg_orders", + &[("sql", "s1"), ("config", "c1"), ("columns", "k1")], + &["macro:to_utc"], + ), + node( + "model", + "orders", + &[("sql", "s2"), ("config", "c2")], + &["enum:order_status"], + ), + node("model", "old_customers", &[("sql", "s3")], &[]), + node("seed", "countries", &[("definition", "d1")], &[]), + node( + "macro", + "to_utc", + &[("definition", "m1")], + &["macro:to_timestamp"], + ), + node("macro", "to_timestamp", &[("definition", "m2")], &[]), + node("enum", "order_status", &[("definition", "e1")], &[]), + node("test_file", "tests/orders.sql", &[("file", "t1")], &[]), + ]) +} + +pub(super) fn keys(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// The base orders project with `stg_orders` SQL edited, `customers` renamed and `countries` gone. +pub(super) fn current_orders() -> DefinitionSnapshot { + snapshot(vec![ + node( + "model", + "stg_orders", + &[("sql", "s1-edited"), ("config", "c1"), ("columns", "k1")], + &["macro:to_utc"], + ), + node( + "model", + "orders", + &[("sql", "s2"), ("config", "c2")], + &["enum:order_status"], + ), + renamed( + node("model", "customers", &[("sql", "s3")], &[]), + &["old_customers"], + ), + node( + "macro", + "to_utc", + &[("definition", "m1")], + &["macro:to_timestamp"], + ), + node("macro", "to_timestamp", &[("definition", "m2")], &[]), + node("enum", "order_status", &[("definition", "e1")], &[]), + node("test_file", "tests/orders.sql", &[("file", "t1")], &[]), + ]) +} + +/// What an item executes, by node keys. +pub(super) fn item( + own: &[&str], + executed: &[&str], + declarations: &[&str], + mocked: &[&str], +) -> ExecutedItem { + ExecutedItem { + own_keys: keys(own), + executed: keys(executed), + declarations: keys(declarations), + mocked: mocked + .iter() + .map(|key| (*key).to_owned()) + .collect::>(), + mocked_relations: Vec::new(), + } +} + +/// `current` with `entry` added or replaced. +pub(super) fn with_node( + mut current: DefinitionSnapshot, + entry: (String, NodeDefinition), +) -> DefinitionSnapshot { + current.nodes.insert(entry.0, entry.1); + current +} + +/// `item` mocking the relations `keys`. +pub(super) fn mocking(mut item: ExecutedItem, keys: &[&str]) -> ExecutedItem { + item.mocked_relations = keys.iter().map(|key| (*key).to_owned()).collect(); + item +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs new file mode 100644 index 0000000000..bdb9583a5d --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs @@ -0,0 +1,426 @@ +use serde_json::json; + +use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; +use crate::definition_changes::main::changes_summary::changes_summary; +use crate::definition_changes::main::compare_definitions::compare_definitions; +use crate::definition_changes::main::definition_snapshot_json::definition_snapshot_json; +use crate::definition_changes::main::parse_definition_snapshot::parse_definition_snapshot; +use crate::definition_changes::main::selection_json::selection_json; +use crate::definition_changes::main::selection_reasons::selection_reasons; +use crate::definition_changes::main::selection_report::selection_report; +use crate::definition_changes::main::sql_definition_digest::sql_definition_digest; +use crate::definition_changes::models::{DefinitionChanges, SelectedItem}; +use crate::definition_changes::tests::helpers::{ + base_orders, current_orders, item, keys, mocking, node, renamed, snapshot, with_node, +}; +use crate::definition_changes::tests::test_types::{ + CompareTestCase, SelectionJsonTestCase, SelectionReasonTestCase, SelectionReportTestCase, + SnapshotJsonTestCase, SqlDigestTestCase, +}; + +#[test] +fn given_two_snapshots_when_comparing_then_reports_changed_renamed_and_deleted_nodes() { + let test_cases = [ + CompareTestCase { + description: "an edit, a pure rename and a deletion", + base: base_orders(), + current: current_orders(), + expected_changes: &[("model:stg_orders", &["sql"], false, None)], + expected_renamed: &[("model:old_customers", "model:customers")], + expected_deleted: &["seed countries"], + expected_summary: "1 model changed; 1 model renamed; deleted: seed countries", + }, + CompareTestCase { + description: "a rename whose definition also changed", + base: snapshot(vec![node("model", "old_customers", &[("sql", "s3")], &[])]), + current: snapshot(vec![renamed( + node( + "model", + "customers", + &[("sql", "s4"), ("config", "c4")], + &[], + ), + &["old_customers"], + )]), + expected_changes: &[( + "model:customers", + &["sql", "config"], + false, + Some("old_customers"), + )], + expected_renamed: &[("model:old_customers", "model:customers")], + expected_deleted: &[], + expected_summary: "1 model changed; 1 model renamed", + }, + CompareTestCase { + description: "added nodes count as changed and kinds are counted", + base: base_orders(), + current: with_node( + with_node( + current_orders(), + node("model", "returns", &[("sql", "s9")], &[]), + ), + node("macro", "to_timestamp", &[("definition", "m3")], &[]), + ), + expected_changes: &[ + ("macro:to_timestamp", &["definition"], false, None), + ("model:returns", &[], true, None), + ("model:stg_orders", &["sql"], false, None), + ], + expected_renamed: &[("model:old_customers", "model:customers")], + expected_deleted: &["seed countries"], + expected_summary: "1 model, 1 macro changed; 1 model added; 1 model renamed; \ + deleted: seed countries", + }, + CompareTestCase { + description: "identical snapshots", + base: base_orders(), + current: base_orders(), + expected_changes: &[], + expected_renamed: &[], + expected_deleted: &[], + expected_summary: "no definition changes", + }, + ]; + for test_case in test_cases { + let changes: DefinitionChanges = compare_definitions(&test_case.base, &test_case.current); + + let observed: Vec<(String, Vec, bool, Option)> = changes + .changed + .iter() + .map(|(key, change)| { + ( + key.clone(), + change.aspects.clone(), + change.added, + change.renamed_from.clone(), + ) + }) + .collect(); + let expected: Vec<(String, Vec, bool, Option)> = test_case + .expected_changes + .iter() + .map(|(key, aspects, added, renamed_from)| { + ( + (*key).to_owned(), + keys(aspects), + *added, + renamed_from.map(str::to_owned), + ) + }) + .collect(); + let deleted: Vec = changes + .deleted + .iter() + .map(|node| format!("{} {}", node.kind, node.name)) + .collect(); + let renamed: Vec<(String, String)> = test_case + .expected_renamed + .iter() + .map(|(old, new)| ((*old).to_owned(), (*new).to_owned())) + .collect(); + assert_eq!(observed, expected, "{}", test_case.description); + assert_eq!(changes.renamed, renamed, "{}", test_case.description); + assert_eq!( + deleted, + keys(test_case.expected_deleted), + "{}", + test_case.description + ); + assert_eq!( + changes_summary(&changes), + test_case.expected_summary, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_executed_item_when_explaining_selection_then_lists_each_changed_dependency() { + let test_cases = [ + SelectionReasonTestCase { + description: "an executed model whose SQL changed", + current: current_orders(), + item: item( + &["test_file:tests/orders.sql"], + &["model:stg_orders", "model:orders"], + &[], + &[], + ), + expected_reasons: &["executes stg_orders (model SQL changed)"], + }, + SelectionReasonTestCase { + description: "a mocked model is not executed, so its change selects nothing", + current: current_orders(), + item: item(&["test_file:tests/orders.sql"], &["model:orders"], &[], &[]), + expected_reasons: &[], + }, + SelectionReasonTestCase { + description: "a mocked model's SQL change does not select the test", + current: current_orders(), + item: mocking( + item(&["test_file:tests/orders.sql"], &["model:orders"], &[], &[]), + &["model:stg_orders"], + ), + expected_reasons: &[], + }, + SelectionReasonTestCase { + description: "a mocked model's declared columns shape the mock", + current: with_node( + base_orders(), + node( + "model", + "stg_orders", + &[("sql", "s1"), ("config", "c1"), ("columns", "k2")], + &["macro:to_utc"], + ), + ), + item: mocking( + item(&["test_file:tests/orders.sql"], &["model:orders"], &[], &[]), + &["model:stg_orders"], + ), + expected_reasons: &["mocks stg_orders (model columns changed)"], + }, + SelectionReasonTestCase { + description: "a macro changed through another macro the model calls", + current: with_node( + base_orders(), + node("macro", "to_timestamp", &[("definition", "m3")], &[]), + ), + item: item(&[], &["model:stg_orders"], &[], &[]), + expected_reasons: &["executes macro to_timestamp (changed)"], + }, + SelectionReasonTestCase { + description: "a mocked macro hides its own and its callees' changes", + current: with_node( + base_orders(), + node("macro", "to_timestamp", &[("definition", "m3")], &[]), + ), + item: item(&[], &["model:stg_orders"], &[], &["macro:to_utc"]), + expected_reasons: &[], + }, + SelectionReasonTestCase { + description: "an enum used by an executed model", + current: with_node( + base_orders(), + node("enum", "order_status", &[("definition", "e2")], &[]), + ), + item: item(&[], &["model:orders"], &[], &[]), + expected_reasons: &["uses enum order_status (changed)"], + }, + SelectionReasonTestCase { + description: "the test's own file and an added model", + current: with_node( + with_node( + base_orders(), + node("test_file", "tests/orders.sql", &[("file", "t2")], &[]), + ), + node("model", "returns", &[("sql", "s9")], &[]), + ), + item: item( + &["test_file:tests/orders.sql"], + &["model:returns"], + &[], + &[], + ), + expected_reasons: &["test file changed", "executes returns (added)"], + }, + SelectionReasonTestCase { + description: "a renamed model whose config changed", + current: snapshot(vec![renamed( + node( + "model", + "customers", + &[("sql", "s3"), ("config", "c9")], + &[], + ), + &["old_customers"], + )]), + item: item(&[], &["model:customers"], &[], &[]), + expected_reasons: &[ + "executes customers (renamed from old_customers, model config changed)", + ], + }, + SelectionReasonTestCase { + description: "a changed seed the test really reads", + current: with_node( + base_orders(), + node("seed", "countries", &[("definition", "d2")], &[]), + ), + item: item(&[], &["model:orders", "seed:countries"], &[], &[]), + expected_reasons: &["reads seed countries (changed)"], + }, + ]; + for test_case in test_cases { + let changes: DefinitionChanges = compare_definitions(&base_orders(), &test_case.current); + + let reasons: Vec = selection_reasons(&test_case.item, &test_case.current, &changes); + + assert_eq!( + reasons, + keys(test_case.expected_reasons), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selected_items_when_reporting_then_counts_and_explains_only_on_request() { + let selected = || { + vec![SelectedItem { + label: "orders_rollup".to_owned(), + reasons: keys(&["executes stg_orders (model SQL changed)"]), + }] + }; + let test_cases = [ + SelectionReportTestCase { + description: "explain lists each item's reasons", + summary: "1 model changed", + nouns: ("test", "tests"), + total: 9, + selected: selected(), + explain: true, + expected_text: "Changed since origin/main: 1 model changed\nSelected 1 of 9 tests\n \ + orders_rollup\n executes stg_orders (model SQL changed)\n", + }, + SelectionReportTestCase { + description: "without explain only counts are shown", + summary: "1 model changed", + nouns: ("test", "tests"), + total: 9, + selected: selected(), + explain: false, + expected_text: "Changed since origin/main: 1 model changed\nSelected 1 of 9 tests\n", + }, + SelectionReportTestCase { + description: "nothing affected says so", + summary: "no definition changes", + nouns: ("scenario", "scenarios"), + total: 3, + selected: Vec::new(), + explain: true, + expected_text: "Changed since origin/main: no definition changes\nNo scenarios are \ + affected by the changes since origin/main\n", + }, + ]; + for test_case in test_cases { + let text: String = selection_report( + ("origin/main", test_case.summary), + test_case.nouns, + (test_case.total, &test_case.selected), + test_case.explain, + ); + + assert_eq!(text, test_case.expected_text, "{}", test_case.description); + } +} + +#[test] +fn given_stored_snapshot_when_reading_then_accepts_only_the_same_format() { + let test_cases = [ + SnapshotJsonTestCase { + description: "a stored snapshot reads back", + stored: |text| text, + expected_snapshot: Some(base_orders()), + }, + SnapshotJsonTestCase { + description: "another format version is recomputed", + stored: |text| { + text.replacen( + &format!("\"format\":{DEFINITION_SNAPSHOT_FORMAT}"), + "\"format\":0", + 1, + ) + }, + expected_snapshot: None, + }, + SnapshotJsonTestCase { + description: "unreadable text is recomputed", + stored: |_| "not json".to_owned(), + expected_snapshot: None, + }, + ]; + for test_case in test_cases { + let text: String = (test_case.stored)(definition_snapshot_json(&base_orders())); + + assert_eq!( + parse_definition_snapshot(&text), + test_case.expected_snapshot, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_when_fingerprinting_definition_then_ignores_layout_and_comments_only() { + let original: String = sql_definition_digest("SELECT order_id FROM orders"); + let test_cases = [ + SqlDigestTestCase { + description: "layout, comments and keyword case are ignored", + sql: "select\n order_id -- the key\nfrom orders", + expected_same_as_original: true, + }, + SqlDigestTestCase { + description: "a different column is a change", + sql: "SELECT customer_id FROM orders", + expected_same_as_original: false, + }, + ]; + for test_case in test_cases { + let digest: String = sql_definition_digest(test_case.sql); + + assert_eq!( + digest == original, + test_case.expected_same_as_original, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_changes_and_selection_when_writing_json_then_lists_nodes_and_reasons() { + let test_cases = [SelectionJsonTestCase { + description: "changed, renamed and deleted nodes with each selected item's reasons", + selected: vec![SelectedItem { + label: "orders_rollup".to_owned(), + reasons: keys(&["executes stg_orders (model SQL changed)"]), + }], + expected_json: json!({ + "ref": "origin/main", + "merge_base": "abc123", + "changed": [{ + "kind": "model", + "name": "stg_orders", + "added": false, + "aspects": ["sql"], + "renamed_from": null, + }], + "renamed": [{ "kind": "model", "from": "old_customers", "to": "customers" }], + "deleted": [{ "kind": "seed", "name": "countries" }], + "candidate_count": 9, + "selected": [{ + "name": "orders_rollup", + "reasons": ["executes stg_orders (model SQL changed)"], + }], + }), + }]; + for test_case in test_cases { + let changes: DefinitionChanges = compare_definitions(&base_orders(), ¤t_orders()); + + let document: serde_json::Value = selection_json( + ("origin/main", "abc123"), + &changes, + (9, &test_case.selected), + ); + + assert_eq!( + document, test_case.expected_json, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs new file mode 100644 index 0000000000..a9e6045877 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs @@ -0,0 +1,57 @@ +use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem, SelectedItem}; + +/// One expected changed node: key, changed aspects, added, renamed-from name. +pub(crate) type ExpectedChange = ( + &'static str, + &'static [&'static str], + bool, + Option<&'static str>, +); + +pub(crate) struct CompareTestCase { + pub(crate) description: &'static str, + pub(crate) base: DefinitionSnapshot, + pub(crate) current: DefinitionSnapshot, + pub(crate) expected_changes: &'static [ExpectedChange], + /// `(base key, current key)`. + pub(crate) expected_renamed: &'static [(&'static str, &'static str)], + /// `kind name` of each deleted node. + pub(crate) expected_deleted: &'static [&'static str], + pub(crate) expected_summary: &'static str, +} + +pub(crate) struct SelectionReasonTestCase { + pub(crate) description: &'static str, + pub(crate) current: DefinitionSnapshot, + pub(crate) item: ExecutedItem, + pub(crate) expected_reasons: &'static [&'static str], +} + +pub(crate) struct SelectionReportTestCase { + pub(crate) description: &'static str, + pub(crate) summary: &'static str, + pub(crate) nouns: (&'static str, &'static str), + pub(crate) total: usize, + pub(crate) selected: Vec, + pub(crate) explain: bool, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct SnapshotJsonTestCase { + pub(crate) description: &'static str, + /// Turns the stored text of the base orders snapshot into the text to read back. + pub(crate) stored: fn(String) -> String, + pub(crate) expected_snapshot: Option, +} + +pub(crate) struct SqlDigestTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_same_as_original: bool, +} + +pub(crate) struct SelectionJsonTestCase { + pub(crate) description: &'static str, + pub(crate) selected: Vec, + pub(crate) expected_json: serde_json::Value, +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 3f2e538f9d..af65f6af67 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -11,6 +11,7 @@ pub mod compiler; mod constants; pub mod contracts; pub mod dag_artifact; +pub mod definition_changes; pub mod diff_selection; pub mod explicit_references; pub mod fingerprint_storage; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs index f446d3496b..f0b830b251 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs @@ -110,6 +110,9 @@ impl Run<'_> { "test --inspect cannot be combined with --json or --json-output", )); } + if let Some(message) = since_error(&namespace) { + return Err(self.error(parser, message)); + } if command == JANITOR_COMMAND && namespace.value("as_target") != Value::None && truthy(&namespace.value("auto_approve")) @@ -147,6 +150,17 @@ impl Run<'_> { } } +/// The first rule `--since` and `--explain` break, on the commands that accept them. +fn since_error(namespace: &Namespace) -> Option<&'static str> { + match namespace.value("since") { + Value::None if truthy(&namespace.value("explain")) => Some("--explain requires --since"), + Value::Str(reference) if reference.trim().is_empty() => { + Some("--since requires a git ref, such as origin/main or @base") + } + _ => None, + } +} + /// `_validate_scope_args`: the first rule the scope options break. fn scope_error(parser: &ParserSpec, namespace: &Namespace) -> Option<&'static str> { let present = |name: &str| namespace.value(name) != Value::None; diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 6ba7934b09..e47f29cad0 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1731,6 +1731,28 @@ "help": "Show the resolved test plan", "group": 1 }, + { + "kind": "store", + "options": [ + "--since" + ], + "dest": "since", + "metavar": "REF", + "help": "run only the tests affected by changes since the merge base of REF and HEAD; @base reads [git] base_ref", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--explain" + ], + "dest": "explain", + "nargs": 0, + "const": true, + "default": false, + "help": "with --since, show why each test was selected", + "group": 1 + }, { "kind": "store", "options": [ @@ -5625,6 +5647,28 @@ "help": "Per-scenario timeout for this run, such as '10m', or 'disabled'", "group": 1 }, + { + "kind": "store", + "options": [ + "--since" + ], + "dest": "since", + "metavar": "REF", + "help": "run only the scenarios affected by changes since the merge base of REF and HEAD; @base reads [git] base_ref", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--explain" + ], + "dest": "explain", + "nargs": 0, + "const": true, + "default": false, + "help": "with --since, show why each scenario was selected", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 050122eac5..5f8f0d494b 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -63,7 +63,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 177] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 179] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -83,6 +83,8 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 177] = [ ("warehouse", NamespaceDefault::None), ("case", NamespaceDefault::None), ("inspect", NamespaceDefault::Bool(false)), + ("since", NamespaceDefault::None), + ("explain", NamespaceDefault::Bool(false)), ("target_range", NamespaceDefault::None), ("from_target", NamespaceDefault::None), ("to_target", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 9070e10680..e7a5de0990 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -205,7 +205,7 @@ pub(crate) const FRESHNESS_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const TEST_FIELDS: [Field; 13] = [ +const TEST_FIELDS: [Field; 15] = [ PROJECT_DIR, NO_SQL_VALIDATION, NO_COLOR, @@ -219,6 +219,8 @@ const TEST_FIELDS: [Field; 13] = [ attr("case_name", "case"), same("inspect"), same("test_timeout"), + same("since"), + same("explain"), ]; const TEST_ARGUMENTS: [Field; 1] = [request(class("TestCommandRequest"), &TEST_FIELDS)]; pub(crate) const TEST_ROUTE: Route = routed( @@ -717,7 +719,7 @@ const SCENARIO_COMMON: [Field; 7] = [ EXCLUDE, attr("retain", "scenario_retain"), ]; -const SCENARIO_TEST_FIELDS: [Field; 16] = [ +const SCENARIO_TEST_FIELDS: [Field; 18] = [ SCENARIO_COMMON[0], SCENARIO_COMMON[1], SCENARIO_COMMON[2], @@ -734,6 +736,8 @@ const SCENARIO_TEST_FIELDS: [Field; 16] = [ JSON_OUTPUT, JSON_OUTPUT_PATH, same("scenario_timeout"), + same("since"), + same("explain"), ]; const SCENARIO_TEST_ARGUMENTS: [Field; 1] = [request( class("ScenarioTestCommandRequest"), diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 5690123c50..c5c121bc45 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -14,9 +14,9 @@ use crate::project::_helpers::help::{ }; use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ - load_constants, load_cost, load_cursors, load_dbt, load_diff, load_janitor, load_local_dbt, - load_microbatches, load_migrations, load_policy_flag, load_scenario, load_settings, - load_snapshots, setting_overrides, + load_constants, load_cost, load_cursors, load_dbt, load_diff, load_git, load_janitor, + load_local_dbt, load_microbatches, load_migrations, load_policy_flag, load_scenario, + load_settings, load_snapshots, setting_overrides, }; use crate::project::_helpers::sinks::load_sinks; use crate::project::_helpers::targets::{ @@ -276,6 +276,7 @@ fn project_fields(payload: &Entries) -> Loaded { recall_section(recall, RECALL_SECTION) .map_err(|message| ConfigFailure::at(PROJECT_FILE, message))?; } + load_git(get(payload, "git"), PROJECT_FILE)?; let mut fields = identity(payload)?; fields.extend(sections(payload)?); fields.push(("targets", targets(payload)?)); @@ -423,3 +424,11 @@ pub(crate) fn local_sql_analysis_failure( .with_help(help), ) } + +/// `[git] base_ref` of `sqlbuild_project.toml`, validated as project loading validates it. +pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { + let file = resolve(paths, false)?; + let table = load_toml_mapping(paths, file)?; + let payload = table.as_map().unwrap_or_default(); + Ok(load_git(get(payload, "git"), PROJECT_FILE)?) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 021b10dbc3..4dc70f4a5b 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -536,6 +536,18 @@ pub(crate) fn load_diff(payload: Option<&ConfigValue>, file: ConfigFile) -> Load )) } +/// `[git]`: the `base_ref` that `--since @base` compares against, when set. +pub(crate) fn load_git(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded> { + let mapping = coerce_mapping(payload, place(file, "git"))?; + validate_allowed_keys(mapping, &["base_ref"], place(file, "git"))?; + optional_str(mapping, "base_ref").map_err(|_| { + ConfigFailure::at( + file, + "git.base_ref must be a non-empty git ref such as 'origin/main'", + ) + }) +} + fn janitor_count(mapping: &Entries, key: &str, default: i64) -> Loaded { optional_int(mapping, key, default) } diff --git a/crates/sqlbuild-config/src/project/main/load_git_base_ref.rs b/crates/sqlbuild-config/src/project/main/load_git_base_ref.rs new file mode 100644 index 0000000000..63d4ff0c07 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/load_git_base_ref.rs @@ -0,0 +1,9 @@ +//! Read `[git] base_ref` from `sqlbuild_project.toml`. + +use crate::project::errors::LoadFailure; +use crate::project::models::ConfigPaths; + +/// The configured base ref that `--since @base` names, or `None` when `[git]` sets none. +pub fn load_git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { + crate::project::_helpers::documents::git_base_ref(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index a3e65d82e8..7198e4d7de 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -1,4 +1,5 @@ pub mod explicit_references_help; +pub mod load_git_base_ref; pub mod load_local_config; pub mod load_project_config; pub mod local_sql_analysis_failure; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 1f460c6232..1bcced4f42 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,6 +1,7 @@ +use crate::project::main::load_git_base_ref::load_git_base_ref; use crate::project::models::ConfigFile; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; -use crate::project::tests::test_types::ProjectFailureTestCase; +use crate::project::tests::test_types::{GitBaseRefTestCase, ProjectFailureTestCase}; const PROJECT: &str = "name = 'orders'\nadapter = 'duckdb'\n"; @@ -39,6 +40,32 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { ), ), }, + ProjectFailureTestCase { + description: "a git base ref loads", + project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n", + local: "", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "an empty git base ref is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = ''\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "git.base_ref must be a non-empty git ref such as 'origin/main'", + None, + ), + }, + ProjectFailureTestCase { + description: "a misspelt git key suggests base_ref", + project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_rf = 'origin/main'\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "git contains unknown key(s): base_rf. Allowed keys: base_ref", + Some("did you mean 'base_ref'?"), + ), + }, ProjectFailureTestCase { description: "a shutdown timeout over ten minutes is rejected", project: "name = 'orders'\nadapter = 'duckdb'\n[sinks.lifecycle]\nshutdown_timeout = '601s'\n", @@ -90,3 +117,31 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { ); } } + +#[test] +fn given_git_section_when_reading_base_ref_then_returns_configured_ref() { + let test_cases = [ + GitBaseRefTestCase { + description: "a configured base ref", + project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n", + expected_base_ref: Some("origin/main"), + }, + GitBaseRefTestCase { + description: "no git section", + project: PROJECT, + expected_base_ref: None, + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let base_ref = load_git_base_ref(&paths).expect(test_case.description); + + assert_eq!( + base_ref.as_deref(), + test_case.expected_base_ref, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index a3f1dd6441..09c6e090ef 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -13,3 +13,9 @@ pub(super) struct GuidanceTestCase { pub(super) expected_note: &'static str, pub(super) expected_help: &'static str, } + +pub(crate) struct GitBaseRefTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_base_ref: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs new file mode 100644 index 0000000000..7121192d36 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs @@ -0,0 +1,70 @@ +//! Which files record a project's Python dependencies, and where they are looked for. + +use std::collections::BTreeSet; +use std::path::Path; + +use crate::git_revisions::_helpers::failures::export_failed; +use crate::git_revisions::_helpers::git_command::git_output; +use crate::git_revisions::constants::{DEPENDENCY_FILE_NAMES, REQUIREMENTS_FILE_AFFIXES}; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +/// Whether `name` is a Python dependency manifest or lock file. +pub(crate) fn is_dependency_file(name: &str) -> bool { + let (prefix, suffix) = REQUIREMENTS_FILE_AFFIXES; + DEPENDENCY_FILE_NAMES.contains(&name) || (name.starts_with(prefix) && name.ends_with(suffix)) +} + +/// The repository root and the project directory, as repository-relative directory prefixes. +pub(crate) fn directories(base: &GitBase) -> BTreeSet { + let project: &str = base.project_prefix.trim_end_matches('/'); + let mut directories: BTreeSet = BTreeSet::from([String::new()]); + if !project.is_empty() { + directories.insert(format!("{project}/")); + } + directories +} + +/// `text` with CRLF line endings as LF and surrounding whitespace removed, as git stores it. +pub(crate) fn normalised(text: &str) -> String { + text.replace("\r\n", "\n").trim().to_owned() +} + +/// Repository paths of the dependency files git tracks directly in `directory`. +pub(crate) fn tracked_dependency_paths( + base: &GitBase, + directory: &str, +) -> Result, GitRevisionError> { + let listing: String = git_output( + &base.repository_root, + &[ + "ls-files", + "-z", + "--full-name", + "--", + &format!("{directory}*"), + ], + &[], + ) + .map_err(|failure| export_failed(base, failure.detail()))?; + Ok(listing + .split('\0') + .filter_map(|path| { + let name: &str = path.strip_prefix(directory)?; + (!name.contains('/') && is_dependency_file(name)).then(|| path.to_owned()) + }) + .collect()) +} + +/// The normalised checkout text of repository path `path`, or `None` when it was deleted. +pub(crate) fn checkout_text( + base: &GitBase, + path: &str, +) -> Result, GitRevisionError> { + let file: &Path = &base.repository_root.join(path); + match std::fs::read(file) { + Ok(bytes) => Ok(Some(normalised(&String::from_utf8_lossy(&bytes)))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(export_failed(base, &error.to_string())), + } +} diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/failures.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/failures.rs new file mode 100644 index 0000000000..13c328ad89 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/failures.rs @@ -0,0 +1,18 @@ +//! The error of a base revision that cannot be read. + +use crate::git_revisions::constants::EXPORT_FAILED_CODE; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +/// The project at the merge base cannot be read, with git's `detail`. +pub(crate) fn export_failed(base: &GitBase, detail: &str) -> GitRevisionError { + GitRevisionError { + message: format!( + "Cannot read the project at merge base {}: {detail}", + base.merge_base + ), + code: EXPORT_FAILED_CODE, + help: "Check that the repository is readable and complete, for example with `git fsck`" + .to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/git_command.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/git_command.rs new file mode 100644 index 0000000000..f8fab4f441 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/git_command.rs @@ -0,0 +1,50 @@ +//! Run one git command in a directory and read its trimmed standard output. + +use std::path::Path; +use std::process::{Command, Output, Stdio}; + +use crate::git_revisions::constants::GIT_PROGRAM; + +/// How one git command failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum GitFailure { + /// git could not be started at all. + Unavailable(String), + /// git ran and exited unsuccessfully, with its trimmed standard error. + Failed(String), +} + +impl GitFailure { + /// The failure text a message can quote. + pub(crate) fn detail(&self) -> &str { + match self { + Self::Unavailable(detail) | Self::Failed(detail) => detail, + } + } +} + +/// The trimmed stdout of `git ` run in `directory` with `environment` added. +pub(crate) fn git_output( + directory: &Path, + arguments: &[&str], + environment: &[(&str, &Path)], +) -> Result { + let mut command: Command = Command::new(GIT_PROGRAM); + command + .current_dir(directory) + .args(arguments) + .stdin(Stdio::null()) + .env("GIT_TERMINAL_PROMPT", "0"); + for (name, value) in environment { + command.env(name, value); + } + let output: Output = command + .output() + .map_err(|error| GitFailure::Unavailable(error.to_string()))?; + if !output.status.success() { + return Err(GitFailure::Failed( + String::from_utf8_lossy(&output.stderr).trim().to_owned(), + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs new file mode 100644 index 0000000000..050dde0f95 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod dependency_files; +pub(crate) mod failures; +pub(crate) mod git_command; diff --git a/crates/sqlbuild-core/src/git_revisions/constants.rs b/crates/sqlbuild-core/src/git_revisions/constants.rs new file mode 100644 index 0000000000..c5091e5ae8 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/constants.rs @@ -0,0 +1,30 @@ +//! The `--since` ref alias, git invocation details and error codes. + +/// The `--since` value that reads `[git] base_ref` from the project configuration. +pub const BASE_REF_ALIAS: &str = "@base"; +/// Hex digits of a commit hash shown in progress and summaries. +pub const SHORT_COMMIT_LENGTH: usize = 10; + +/// Python dependency manifests and lock files, by exact name. +pub(crate) const DEPENDENCY_FILE_NAMES: [&str; 5] = [ + "pyproject.toml", + "uv.lock", + "poetry.lock", + "pdm.lock", + "Pipfile.lock", +]; +/// `requirements*.txt`: the prefix and suffix of pip requirement files. +pub(crate) const REQUIREMENTS_FILE_AFFIXES: (&str, &str) = ("requirements", ".txt"); + +pub(crate) const GIT_PROGRAM: &str = "git"; +pub(crate) const INDEX_FILE_ENV: &str = "GIT_INDEX_FILE"; +pub(crate) const SHALLOW_TRUE: &str = "true"; + +pub(crate) const NOT_A_REPOSITORY_CODE: &str = "C480"; +pub(crate) const UNKNOWN_REF_CODE: &str = "C481"; +pub(crate) const NO_MERGE_BASE_CODE: &str = "C482"; +pub(crate) const EXPORT_FAILED_CODE: &str = "C483"; +pub(crate) const BASE_REF_UNSET_CODE: &str = "C485"; +pub(crate) const DEPTH_HELP: &str = "Fetch enough history to reach the merge base, for example \ + `git fetch --unshallow`, or in CI clone with a larger fetch \ + depth"; diff --git a/crates/sqlbuild-core/src/git_revisions/errors.rs b/crates/sqlbuild-core/src/git_revisions/errors.rs new file mode 100644 index 0000000000..130babf10e --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/errors.rs @@ -0,0 +1,9 @@ +//! A `--since` ref that cannot be resolved or read, with its code and remedy. + +/// The message, error code and help of one failed git step. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GitRevisionError { + pub message: String, + pub code: &'static str, + pub help: String, +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs new file mode 100644 index 0000000000..ea8c0c4b00 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs @@ -0,0 +1,34 @@ +//! The Python dependency manifests and lock files of a project at the merge base. + +use std::path::Path; + +use crate::git_revisions::_helpers::dependency_files::{ + directories, is_dependency_file, normalised, +}; +use crate::git_revisions::_helpers::failures::export_failed; +use crate::git_revisions::_helpers::git_command::{GitFailure, git_output}; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +/// `(repository path, text)` of each dependency file at the merge base, sorted. +pub fn base_dependency_files(base: &GitBase) -> Result, GitRevisionError> { + let root: &Path = &base.repository_root; + let mut files: Vec<(String, String)> = Vec::new(); + for directory in directories(base) { + let tree: String = format!("{}:{directory}", base.merge_base); + let listing: String = match git_output(root, &["ls-tree", "--name-only", &tree], &[]) { + Ok(listing) => listing, + Err(GitFailure::Failed(_)) => continue, + Err(failure) => return Err(export_failed(base, failure.detail())), + }; + for name in listing.lines().filter(|name| is_dependency_file(name)) { + let path: String = format!("{directory}{name}"); + let blob: String = format!("{}:{path}", base.merge_base); + let text: String = git_output(root, &["show", &blob], &[]) + .map_err(|failure| export_failed(base, failure.detail()))?; + files.push((path, normalised(&text))); + } + } + files.sort(); + Ok(files) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs new file mode 100644 index 0000000000..bef9d44aa7 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs @@ -0,0 +1,21 @@ +//! The git-tracked Python dependency manifests and lock files of a project in the checkout. + +use crate::git_revisions::_helpers::dependency_files::{ + checkout_text, directories, tracked_dependency_paths, +}; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +/// `(repository path, text)` of each tracked dependency file as the checkout has it, sorted. +pub fn current_dependency_files(base: &GitBase) -> Result, GitRevisionError> { + let mut files: Vec<(String, String)> = Vec::new(); + for directory in directories(base) { + for path in tracked_dependency_paths(base, &directory)? { + if let Some(text) = checkout_text(base, &path)? { + files.push((path, text)); + } + } + } + files.sort(); + Ok(files) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs b/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs new file mode 100644 index 0000000000..4d090694d1 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs @@ -0,0 +1,54 @@ +//! Write the project's files at the base commit into a directory, without touching the checkout. + +use std::path::{Path, PathBuf}; + +use crate::git_revisions::_helpers::failures::export_failed; +use crate::git_revisions::_helpers::git_command::git_output; +use crate::git_revisions::constants::INDEX_FILE_ENV; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +fn tree_of(base: &GitBase) -> String { + let prefix: &str = base.project_prefix.trim_end_matches('/'); + if prefix.is_empty() { + format!("{}^{{tree}}", base.merge_base) + } else { + format!("{}:{prefix}", base.merge_base) + } +} + +/// Write the base project tree to `destination` through a private index; `false` without a project. +pub fn export_git_tree(base: &GitBase, destination: &Path) -> Result { + let root: &Path = &base.repository_root; + let tree: String = tree_of(base); + if git_output(root, &["cat-file", "-e", &tree], &[]).is_err() { + return Ok(false); + } + std::fs::create_dir_all(destination) + .map_err(|error| export_failed(base, &error.to_string()))?; + let index: PathBuf = destination.with_extension("index"); + let environment: [(&str, &Path); 1] = [(INDEX_FILE_ENV, index.as_path())]; + let mut prefix: String = destination.to_string_lossy().replace('\\', "/"); + if !prefix.ends_with('/') { + prefix.push('/'); + } + let exported: Result<(), GitRevisionError> = + git_output(root, &["read-tree", &tree], &environment) + .and_then(|_| { + git_output( + root, + &[ + "checkout-index", + "--all", + "--force", + &format!("--prefix={prefix}"), + ], + &environment, + ) + }) + .map(|_| ()) + .map_err(|failure| export_failed(base, failure.detail())); + let _ = std::fs::remove_file(&index); + exported?; + Ok(true) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/mod.rs b/crates/sqlbuild-core/src/git_revisions/main/mod.rs new file mode 100644 index 0000000000..c841f13156 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/mod.rs @@ -0,0 +1,6 @@ +pub mod base_dependency_files; +pub mod current_dependency_files; +pub mod export_git_tree; +pub mod resolve_git_base; +pub mod resolve_since_reference; +pub mod short_commit; diff --git a/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs b/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs new file mode 100644 index 0000000000..f6d449d56d --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs @@ -0,0 +1,98 @@ +//! The merge base of a ref and `HEAD` for the repository containing a project. + +use std::path::{Path, PathBuf}; + +use crate::git_revisions::_helpers::git_command::{GitFailure, git_output}; +use crate::git_revisions::constants::{ + DEPTH_HELP, NO_MERGE_BASE_CODE, NOT_A_REPOSITORY_CODE, SHALLOW_TRUE, UNKNOWN_REF_CODE, +}; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +fn not_a_repository(project_dir: &Path, failure: &GitFailure) -> GitRevisionError { + let message: String = match failure { + GitFailure::Unavailable(detail) => { + format!("--since needs git, which could not be run: {detail}") + } + GitFailure::Failed(_) => format!( + "--since needs a git repository, but {} is not inside one", + project_dir.display() + ), + }; + GitRevisionError { + message, + code: NOT_A_REPOSITORY_CODE, + help: "Run the command inside a git checkout of the project, or drop --since".to_owned(), + } +} + +fn is_shallow(root: &Path) -> bool { + git_output(root, &["rev-parse", "--is-shallow-repository"], &[]) + .is_ok_and(|value| value == SHALLOW_TRUE) +} + +/// Resolve `reference` and `HEAD` in the repository of `project_dir` to their merge base. +pub fn resolve_git_base(project_dir: &Path, reference: &str) -> Result { + let root: PathBuf = git_output(project_dir, &["rev-parse", "--show-toplevel"], &[]) + .map(PathBuf::from) + .map_err(|failure| not_a_repository(project_dir, &failure))?; + let prefix: String = git_output(project_dir, &["rev-parse", "--show-prefix"], &[]) + .map_err(|failure| not_a_repository(project_dir, &failure))?; + let shallow: bool = is_shallow(&root); + let commit: String = git_output( + &root, + &[ + "rev-parse", + "--verify", + "--quiet", + "--end-of-options", + &format!("{reference}^{{commit}}"), + ], + &[], + ) + .map_err(|_| GitRevisionError { + message: format!("Git ref '{reference}' does not name a commit in this repository"), + code: UNKNOWN_REF_CODE, + help: if shallow { + format!( + "This is a shallow clone; fetch the ref first, for example `git fetch origin \ + `, with enough depth to reach the merge base. {DEPTH_HELP}" + ) + } else { + "Check the spelling, or fetch the ref first, for example `git fetch origin `" + .to_owned() + }, + })?; + let merge_base: String = + git_output(&root, &["merge-base", &commit, "HEAD"], &[]).map_err(|failure| { + if shallow { + GitRevisionError { + message: format!( + "Cannot find the merge base of '{reference}' and HEAD: this is a shallow \ + clone without the shared history" + ), + code: NO_MERGE_BASE_CODE, + help: DEPTH_HELP.to_owned(), + } + } else { + let detail: &str = failure.detail(); + GitRevisionError { + message: if detail.is_empty() { + format!("'{reference}' and HEAD share no history") + } else { + format!("Cannot find the merge base of '{reference}' and HEAD: {detail}") + }, + code: NO_MERGE_BASE_CODE, + help: "Pass a ref that shares history with HEAD, such as the branch this \ + work started from" + .to_owned(), + } + } + })?; + Ok(GitBase { + reference: reference.to_owned(), + merge_base, + repository_root: root, + project_prefix: prefix, + }) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/resolve_since_reference.rs b/crates/sqlbuild-core/src/git_revisions/main/resolve_since_reference.rs new file mode 100644 index 0000000000..b9621ea9d8 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/resolve_since_reference.rs @@ -0,0 +1,27 @@ +//! The git ref a `--since` value names, reading `@base` from `[git] base_ref`. + +use crate::git_revisions::constants::{BASE_REF_ALIAS, BASE_REF_UNSET_CODE}; +use crate::git_revisions::errors::GitRevisionError; + +/// `value` itself, or the configured base ref when `value` is `@base`. +pub fn resolve_since_reference( + value: &str, + configured_base_ref: Option<&str>, +) -> Result { + let value: &str = value.trim(); + if value != BASE_REF_ALIAS { + return Ok(value.to_owned()); + } + match configured_base_ref.map(str::trim) { + Some(base_ref) if !base_ref.is_empty() => Ok(base_ref.to_owned()), + _ => Err(GitRevisionError { + message: format!( + "--since {BASE_REF_ALIAS} needs [git] base_ref in sqlbuild_project.toml" + ), + code: BASE_REF_UNSET_CODE, + help: "Add [git] base_ref = \"origin/main\" (your default branch) to \ + sqlbuild_project.toml, or pass a ref such as --since origin/main" + .to_owned(), + }), + } +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/short_commit.rs b/crates/sqlbuild-core/src/git_revisions/main/short_commit.rs new file mode 100644 index 0000000000..a00c8d49e4 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/short_commit.rs @@ -0,0 +1,8 @@ +//! The abbreviated commit hash shown in progress and summaries. + +use crate::git_revisions::constants::SHORT_COMMIT_LENGTH; + +/// The first hex digits of `commit`. +pub fn short_commit(commit: &str) -> &str { + commit.get(..SHORT_COMMIT_LENGTH).unwrap_or(commit) +} diff --git a/crates/sqlbuild-core/src/git_revisions/mod.rs b/crates/sqlbuild-core/src/git_revisions/mod.rs new file mode 100644 index 0000000000..19b88f3412 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/mod.rs @@ -0,0 +1,10 @@ +//! Git revisions a `--since` comparison reads: the merge base of a ref and `HEAD`, and its tree. + +mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/git_revisions/models.rs b/crates/sqlbuild-core/src/git_revisions/models.rs new file mode 100644 index 0000000000..4372d5a2ee --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/models.rs @@ -0,0 +1,15 @@ +//! The resolved base revision of a `--since` comparison. + +use std::path::PathBuf; + +/// The merge base of a ref and `HEAD`, and where the project lives in the repository. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GitBase { + /// The ref as the user gave it, after `@base` resolution. + pub reference: String, + /// The full hash of the merge-base commit. + pub merge_base: String, + pub repository_root: PathBuf, + /// The project directory relative to the repository root, with `/` separators, empty at the root. + pub project_prefix: String, +} diff --git a/crates/sqlbuild-core/src/git_revisions/tests.rs b/crates/sqlbuild-core/src/git_revisions/tests.rs new file mode 100644 index 0000000000..4a6c249d67 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_git_revisions.rs"] +mod git_revisions; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs b/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs new file mode 100644 index 0000000000..de11e65211 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs @@ -0,0 +1,89 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT_DIRECTORY: AtomicUsize = AtomicUsize::new(0); + +/// A fresh empty directory under the system temporary directory. +pub(super) fn scratch_directory(label: &str) -> PathBuf { + let directory: PathBuf = std::env::temp_dir().join(format!( + "sqlbuild-git-revisions-{label}-{}-{}", + std::process::id(), + NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).expect("the scratch directory is created"); + directory +} + +/// Run git in `directory` with a fixed identity, returning trimmed stdout. +pub(super) fn git(directory: &Path, arguments: &[&str]) -> String { + let output = Command::new("git") + .current_dir(directory) + .args([ + "-c", + "user.name=Example", + "-c", + "user.email=dev@example.com", + ]) + .args([ + "-c", + "init.defaultBranch=main", + "-c", + "commit.gpgsign=false", + ]) + .args(arguments) + .output() + .expect("git runs"); + assert!( + output.status.success(), + "git {arguments:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_owned() +} + +/// Write `contents` to `relative` under `root`, creating parent directories. +pub(super) fn write(root: &Path, relative: &str, contents: &str) { + let path: PathBuf = root.join(relative); + std::fs::create_dir_all(path.parent().expect("a parent")).expect("parents are created"); + std::fs::write(path, contents).expect("the file is written"); +} + +/// A repository with a project in `warehouse/` and branches `main` and `feature`; root and `main`. +pub(super) fn project_repository() -> (PathBuf, String) { + let root: PathBuf = scratch_directory("repo"); + git(&root, &["init", "--quiet"]); + write( + &root, + "warehouse/sqlbuild_project.toml", + "name = \"orders\"\n", + ); + write( + &root, + "warehouse/models/orders.sql", + "SELECT 1 AS order_id\n", + ); + git(&root, &["add", "."]); + git(&root, &["commit", "--quiet", "-m", "orders"]); + let main: String = git(&root, &["rev-parse", "HEAD"]); + git(&root, &["checkout", "--quiet", "-b", "feature"]); + write( + &root, + "warehouse/models/orders.sql", + "SELECT 2 AS order_id\n", + ); + write( + &root, + "warehouse/models/customers.sql", + "SELECT 1 AS customer_id\n", + ); + git(&root, &["add", "."]); + git(&root, &["commit", "--quiet", "-m", "customers"]); + (root, main) +} + +/// The text of `path`, or `None` when it is absent. +pub(super) fn read(path: &Path) -> Option { + std::fs::read_to_string(path).ok() +} diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs new file mode 100644 index 0000000000..2ef025b90d --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs @@ -0,0 +1,366 @@ +use std::path::PathBuf; + +use crate::git_revisions::main::base_dependency_files::base_dependency_files; +use crate::git_revisions::main::current_dependency_files::current_dependency_files; +use crate::git_revisions::main::export_git_tree::export_git_tree; +use crate::git_revisions::main::resolve_git_base::resolve_git_base; +use crate::git_revisions::main::resolve_since_reference::resolve_since_reference; +use crate::git_revisions::models::GitBase; +use crate::git_revisions::tests::helpers::{ + git, project_repository, read, scratch_directory, write, +}; +use crate::git_revisions::tests::test_types::{ + AbsentProjectTestCase, DependencyFilesTestCase, ExportTreeTestCase, GitBaseFailureTestCase, + GitBaseTestCase, SinceReferenceTestCase, +}; + +#[test] +fn given_since_value_when_resolving_then_reads_base_alias_from_config() { + let test_cases = [ + SinceReferenceTestCase { + description: "a plain ref is used as given", + value: "origin/main", + configured: Some("origin/release"), + expected_reference: Ok("origin/main"), + }, + SinceReferenceTestCase { + description: "@base reads the configured base ref", + value: "@base", + configured: Some("origin/main"), + expected_reference: Ok("origin/main"), + }, + SinceReferenceTestCase { + description: "@base without a configured base ref fails", + value: "@base", + configured: None, + expected_reference: Err("C485"), + }, + SinceReferenceTestCase { + description: "@base with a blank base ref fails", + value: "@base", + configured: Some(" "), + expected_reference: Err("C485"), + }, + ]; + for test_case in test_cases { + let resolved = resolve_since_reference(test_case.value, test_case.configured); + + assert_eq!( + resolved.as_deref().map_err(|error| error.code), + test_case.expected_reference, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_project_directory_when_resolving_base_then_returns_merge_base_and_prefix() { + let (root, main) = project_repository(); + let test_cases = [ + GitBaseTestCase { + description: "a project in a subdirectory", + project_directory: "warehouse", + reference: "main", + expected_prefix: "warehouse/", + }, + GitBaseTestCase { + description: "a nested directory of the project", + project_directory: "warehouse/models", + reference: "main", + expected_prefix: "warehouse/models/", + }, + ]; + for test_case in test_cases { + let base: GitBase = + resolve_git_base(&root.join(test_case.project_directory), test_case.reference) + .expect(test_case.description); + + assert_eq!(base.merge_base, main, "{}", test_case.description); + assert_eq!( + base.project_prefix, test_case.expected_prefix, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unresolvable_ref_when_resolving_base_then_fails_with_code() { + let (root, _) = project_repository(); + git(&root, &["checkout", "--quiet", "--orphan", "unrelated"]); + git(&root, &["commit", "--quiet", "-m", "unrelated"]); + git(&root, &["checkout", "--quiet", "feature"]); + let test_cases = [ + GitBaseFailureTestCase { + description: "a missing ref", + reference: "origin/missing", + expected_code: "C481", + expected_message_fragment: "'origin/missing' does not name a commit", + expected_help_fragment: "git fetch origin ", + }, + GitBaseFailureTestCase { + description: "a ref without shared history", + reference: "unrelated", + expected_code: "C482", + expected_message_fragment: "'unrelated' and HEAD", + expected_help_fragment: "shares history with HEAD", + }, + ]; + for test_case in test_cases { + let error = resolve_git_base(&root.join("warehouse"), test_case.reference) + .expect_err(test_case.description); + + assert_eq!( + error.code, test_case.expected_code, + "{}", + test_case.description + ); + assert!( + error.message.contains(test_case.expected_message_fragment) + && error.help.contains(test_case.expected_help_fragment), + "{}: {} ({})", + test_case.description, + error.message, + error.help + ); + } +} + +#[test] +fn given_directory_outside_repository_when_resolving_base_then_fails_with_code() { + let directory: PathBuf = scratch_directory("plain"); + let test_cases = [GitBaseFailureTestCase { + description: "a plain directory", + reference: "main", + expected_code: "C480", + expected_message_fragment: "is not inside one", + expected_help_fragment: "drop --since", + }]; + for test_case in test_cases { + let error = + resolve_git_base(&directory, test_case.reference).expect_err(test_case.description); + + assert_eq!( + error.code, test_case.expected_code, + "{}", + test_case.description + ); + assert!( + error.message.contains(test_case.expected_message_fragment) + && error.help.contains(test_case.expected_help_fragment), + "{}: {}", + test_case.description, + error.message + ); + } +} + +#[test] +fn given_shallow_clone_without_shared_history_when_resolving_base_then_asks_for_depth() { + let (root, _) = project_repository(); + let clone: PathBuf = scratch_directory("shallow"); + let origin: String = format!("file://{}", root.display()); + git( + &clone, + &[ + "clone", + "--quiet", + "--depth", + "1", + "--branch", + "feature", + "--no-single-branch", + &origin, + "checkout", + ], + ); + let checkout: PathBuf = clone.join("checkout"); + git( + &checkout, + &["fetch", "--quiet", "--depth", "1", "origin", "main"], + ); + let test_cases = [GitBaseFailureTestCase { + description: "the merge base is outside the shallow history", + reference: "origin/main", + expected_code: "C482", + expected_message_fragment: "shallow clone", + expected_help_fragment: "--unshallow", + }]; + for test_case in test_cases { + let error = resolve_git_base(&checkout.join("warehouse"), test_case.reference) + .expect_err(test_case.description); + + assert_eq!( + error.code, test_case.expected_code, + "{}", + test_case.description + ); + assert!( + error.message.contains(test_case.expected_message_fragment) + && error.help.contains(test_case.expected_help_fragment), + "{}: {} ({})", + test_case.description, + error.message, + error.help + ); + } +} + +#[test] +fn given_merge_base_when_exporting_tree_then_writes_base_files_and_leaves_checkout_alone() { + let (root, _) = project_repository(); + write( + &root, + "warehouse/models/orders.sql", + "SELECT 3 AS order_id\n", + ); + let status_before: String = git(&root, &["status", "--porcelain"]); + let base: GitBase = resolve_git_base(&root.join("warehouse"), "main").expect("main resolves"); + let destination: PathBuf = scratch_directory("export").join("base"); + let exported: bool = export_git_tree(&base, &destination).expect("the tree exports"); + assert!(exported); + assert_eq!(git(&root, &["status", "--porcelain"]), status_before); + let test_cases = [ + ExportTreeTestCase { + description: "an edited model is exported as committed at the base", + relative_path: "models/orders.sql", + expected_exported: Some("SELECT 1 AS order_id\n"), + expected_working_tree: Some("SELECT 3 AS order_id\n"), + }, + ExportTreeTestCase { + description: "a model added after the base is not exported", + relative_path: "models/customers.sql", + expected_exported: None, + expected_working_tree: Some("SELECT 1 AS customer_id\n"), + }, + ExportTreeTestCase { + description: "the private index file is removed", + relative_path: "../base.index", + expected_exported: None, + expected_working_tree: None, + }, + ]; + for test_case in test_cases { + let exported_text: Option = read(&destination.join(test_case.relative_path)); + let working_text: Option = + read(&root.join("warehouse").join(test_case.relative_path)); + + assert_eq!( + exported_text.as_deref(), + test_case.expected_exported, + "{}", + test_case.description + ); + assert_eq!( + working_text.as_deref(), + test_case.expected_working_tree, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_project_absent_at_merge_base_when_exporting_tree_then_reports_no_project() { + let (root, main) = project_repository(); + write( + &root, + "billing/sqlbuild_project.toml", + "name = \"billing\"\n", + ); + git(&root, &["add", "."]); + git(&root, &["commit", "--quiet", "-m", "billing"]); + let test_cases = [ + AbsentProjectTestCase { + description: "a project added after the base", + project_prefix: "billing/", + expected_exported: false, + }, + AbsentProjectTestCase { + description: "a project present at the base", + project_prefix: "warehouse/", + expected_exported: true, + }, + ]; + for test_case in test_cases { + let base: GitBase = GitBase { + reference: "main".to_owned(), + merge_base: main.clone(), + repository_root: root.clone(), + project_prefix: test_case.project_prefix.to_owned(), + }; + + let exported: bool = export_git_tree(&base, &scratch_directory("absent").join("base")) + .expect(test_case.description); + + assert_eq!( + exported, test_case.expected_exported, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dependency_files_when_reading_both_sides_then_lists_root_and_project_files() { + let (root, _) = project_repository(); + write(&root, "uv.lock", "version = 1\n"); + write( + &root, + "warehouse/requirements-dev.txt", + "pytest\r\nruff\r\n", + ); + write(&root, "warehouse/requirements-local.txt", "untracked\n"); + write(&root, "warehouse/notes.txt", "not a dependency file\n"); + git(&root, &["add", "uv.lock"]); + git(&root, &["commit", "--quiet", "-m", "lock"]); + git(&root, &["add", "warehouse/requirements-dev.txt"]); + let current: Vec<(&str, &str)> = vec![ + ("uv.lock", "version = 1"), + ("warehouse/requirements-dev.txt", "pytest\nruff"), + ]; + let test_cases = [ + DependencyFilesTestCase { + description: "tracked files added after the base are only in the checkout, with LF \ + line endings; untracked ones are ignored", + reference: "main", + expected_base: vec![], + expected_current: current.clone(), + }, + DependencyFilesTestCase { + description: "committed files are read from the merge base", + reference: "feature", + expected_base: vec![("uv.lock", "version = 1")], + expected_current: current.clone(), + }, + ]; + for test_case in test_cases { + let base: GitBase = + resolve_git_base(&root.join("warehouse"), test_case.reference).expect("ref resolves"); + + let base_files: Vec<(String, String)> = + base_dependency_files(&base).expect("the base reads"); + let current_files: Vec<(String, String)> = + current_dependency_files(&base).expect("the checkout reads"); + + let owned = |files: &[(&str, &str)]| -> Vec<(String, String)> { + files + .iter() + .map(|(path, text)| ((*path).to_owned(), (*text).to_owned())) + .collect() + }; + assert_eq!( + base_files, + owned(&test_case.expected_base), + "{}", + test_case.description + ); + assert_eq!( + current_files, + owned(&test_case.expected_current), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs new file mode 100644 index 0000000000..3953a0c601 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs @@ -0,0 +1,42 @@ +pub(crate) struct SinceReferenceTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) configured: Option<&'static str>, + /// The resolved ref, or the error code. + pub(crate) expected_reference: Result<&'static str, &'static str>, +} + +pub(crate) struct GitBaseTestCase { + pub(crate) description: &'static str, + pub(crate) project_directory: &'static str, + pub(crate) reference: &'static str, + pub(crate) expected_prefix: &'static str, +} + +pub(crate) struct GitBaseFailureTestCase { + pub(crate) description: &'static str, + pub(crate) reference: &'static str, + pub(crate) expected_code: &'static str, + pub(crate) expected_message_fragment: &'static str, + pub(crate) expected_help_fragment: &'static str, +} + +pub(crate) struct ExportTreeTestCase { + pub(crate) description: &'static str, + pub(crate) relative_path: &'static str, + pub(crate) expected_exported: Option<&'static str>, + pub(crate) expected_working_tree: Option<&'static str>, +} + +pub(crate) struct AbsentProjectTestCase { + pub(crate) description: &'static str, + pub(crate) project_prefix: &'static str, + pub(crate) expected_exported: bool, +} + +pub(crate) struct DependencyFilesTestCase { + pub(crate) description: &'static str, + pub(crate) reference: &'static str, + pub(crate) expected_base: Vec<(&'static str, &'static str)>, + pub(crate) expected_current: Vec<(&'static str, &'static str)>, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 2569bf6dc1..29ad9c99e3 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -30,6 +30,7 @@ pub mod dlt_destination; pub mod event_export; pub mod format_command; pub mod freshness_text; +pub mod git_revisions; pub mod graph_walk; pub mod identifier_text; pub mod ingestr_command; diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs b/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs index c79d505b15..9787452733 100644 --- a/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs +++ b/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs @@ -37,6 +37,9 @@ pub fn render_test_inspection( " boundary: {mock} is replaced by __ref__{mock}\n" )); } + for reason in &entry.selection_reasons { + text.push_str(&format!(" selected: {reason}\n")); + } text.push('\n'); } let errors: usize = warnings diff --git a/crates/sqlbuild-core/src/test_command/models.rs b/crates/sqlbuild-core/src/test_command/models.rs index aa17cc236e..0c8b931617 100644 --- a/crates/sqlbuild-core/src/test_command/models.rs +++ b/crates/sqlbuild-core/src/test_command/models.rs @@ -21,6 +21,8 @@ pub struct TestInspectionEntry { pub mocked_table_functions: Vec, pub real_models: Vec, pub expected_models: Vec, + /// Why `--since` selected the test; empty without `--since`. + pub selection_reasons: Vec, } /// One plan warning: its severity value and message. diff --git a/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs b/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs index 46b2e84661..47e203b230 100644 --- a/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs +++ b/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs @@ -242,6 +242,18 @@ fn given_plan_when_inspecting_then_lists_boundaries_and_counts_errors() { expected_text: "\nResolved test plan\n\norder_status [open_case]\n mocked sources: raw_orders\n mocked refs: stg\n mocked seeds: none\n mocked dbt refs: none\n mocked table functions: none\n real models: orders\n expected models: orders\n boundary: stg is replaced by __ref__stg\n\n warning: careful\n error: broken\nTest plan inspection failed: 1 selected, 1 errors.\n", expected_exit_code: 1, }, + InspectionTestCase { + description: "a test selected by --since lists its reasons", + entries: vec![TestInspectionEntry { + label: "orders_rollup".to_owned(), + real_models: vec!["stg_orders".to_owned()], + selection_reasons: vec!["executes stg_orders (model SQL changed)".to_owned()], + ..TestInspectionEntry::default() + }], + warnings: Vec::new(), + expected_text: "\nResolved test plan\n\norders_rollup\n mocked sources: none\n mocked refs: none\n mocked seeds: none\n mocked dbt refs: none\n mocked table functions: none\n real models: stg_orders\n expected models: none\n selected: executes stg_orders (model SQL changed)\n\nTest plan inspection complete: 1 selected, 0 errors.\n", + expected_exit_code: 0, + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/closure.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/closure.rs new file mode 100644 index 0000000000..56fbc22e87 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/closure.rs @@ -0,0 +1,38 @@ +//! The local modules one Python module executes through its imports, transitively. + +use std::collections::BTreeSet; +use std::convert::Infallible; +use std::path::{Path, PathBuf}; + +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::rule_evidence::_helpers::imports::read_source; +use crate::rule_evidence::_helpers::paths::{local_import_paths, package_inits}; + +/// Every module and package initializer under `root` that `source_path` executes, sorted. +pub(crate) fn module_closure( + source_path: &Path, + (project, root): (&Path, &Path), + package: Option<&str>, + python_version: (u8, u8), +) -> Vec { + let mut helpers: BTreeSet = BTreeSet::new(); + let Ok(()) = breadth_first_reach([source_path.to_path_buf()], None, &mut helpers, |current| { + let mut candidates: BTreeSet = package_inits(current, root).into_iter().collect(); + let parsed: Option = + parse_cpython_module(&read_source(current), python_version); + if let Some(parsed) = &parsed { + let roots: Vec> = + parsed.module.body.iter().map(CpythonNode::Stmt).collect(); + for node in walk_breadth_first(roots) { + candidates.extend(local_import_paths(&node, current, (project, root), package)); + } + } + candidates.remove(source_path); + Ok::<_, Infallible>(candidates) + }); + helpers.into_iter().collect() +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs index af1a7d23cc..c4524381f9 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod cases; +pub(crate) mod closure; pub(crate) mod evidence; pub(crate) mod imports; pub(crate) mod owners; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs index c929921954..f22f797a7e 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs @@ -6,9 +6,7 @@ use std::path::{Component, Path, PathBuf}; use ruff_python_ast::Stmt; use crate::cpython_tree::models::CpythonNode; -use crate::rule_evidence::constants::{ - INIT_FILE_NAME, PYTHON_SUFFIX, RULES_DIRECTORY_NAME, STAR_IMPORT_NAME, -}; +use crate::rule_evidence::constants::{INIT_FILE_NAME, PYTHON_SUFFIX, STAR_IMPORT_NAME}; use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM as INIT_MODULE_NAME; /// `Path.resolve()`: symlinks resolved through the existing prefix, the rest kept lexically. @@ -67,22 +65,21 @@ pub(crate) fn package_inits(path: &Path, rules_root: &Path) -> Vec { inits } -/// Every repository-owned rules module one import statement can execute. +/// Modules under `rules_root` one import executes, from `project` and only `package` when set. pub(crate) fn local_import_paths( node: &CpythonNode<'_>, source_path: &Path, - rules_root: &Path, + (project, rules_root): (&Path, &Path), + package: Option<&str>, ) -> Vec { - let project: PathBuf = rules_root - .parent() - .map_or(PathBuf::new(), Path::to_path_buf); + let project: PathBuf = project.to_path_buf(); let mut targets: Vec<(PathBuf, Vec)> = Vec::new(); match node { CpythonNode::Stmt(Stmt::Import(import)) => { for alias in &import.names { let parts: Vec = alias.name.as_str().split('.').map(str::to_owned).collect(); - if parts[0] == RULES_DIRECTORY_NAME { + if package.is_none_or(|package| parts[0] == package) { targets.push((project.clone(), parts)); } } @@ -100,7 +97,9 @@ pub(crate) fn local_import_paths( for _ in 1..import.level { base = base.parent().map_or(base.clone(), Path::to_path_buf); } - } else if module_parts.first().map(String::as_str) != Some(RULES_DIRECTORY_NAME) { + } else if package + .is_some_and(|package| module_parts.first().map(String::as_str) != Some(package)) + { return Vec::new(); } targets.push((base.clone(), module_parts.clone())); diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs index 05ceadd2f0..1bdbd3a7f9 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs @@ -1,16 +1,9 @@ //! `import_closure`: the repository-owned helper modules one rule module imports. -use std::collections::BTreeSet; -use std::convert::Infallible; use std::path::{Path, PathBuf}; -use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; - -use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; -use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; -use crate::cpython_tree::models::{CpythonNode, ParsedSource}; -use crate::rule_evidence::_helpers::imports::read_source; -use crate::rule_evidence::_helpers::paths::{local_import_paths, package_inits, resolve}; +use crate::rule_evidence::_helpers::closure::module_closure; +use crate::rule_evidence::_helpers::paths::resolve; use crate::rule_evidence::constants::RULES_DIRECTORY_NAME; /// Every rules module and package initializer `source_path` executes, sorted; paths are resolved. @@ -20,21 +13,13 @@ pub fn import_closure( python_version: (u8, u8), ) -> Vec { let rules_root: PathBuf = resolve(&project_dir.join(RULES_DIRECTORY_NAME)); - let mut helpers: BTreeSet = BTreeSet::new(); - let Ok(()) = breadth_first_reach([source_path.to_path_buf()], None, &mut helpers, |current| { - let mut candidates: BTreeSet = - package_inits(current, &rules_root).into_iter().collect(); - let parsed: Option = - parse_cpython_module(&read_source(current), python_version); - if let Some(parsed) = &parsed { - let roots: Vec> = - parsed.module.body.iter().map(CpythonNode::Stmt).collect(); - for node in walk_breadth_first(roots) { - candidates.extend(local_import_paths(&node, current, &rules_root)); - } - } - candidates.remove(source_path); - Ok::<_, Infallible>(candidates) - }); - helpers.into_iter().collect() + let project: PathBuf = rules_root + .parent() + .map_or(PathBuf::new(), Path::to_path_buf); + module_closure( + source_path, + (&project, &rules_root), + Some(RULES_DIRECTORY_NAME), + python_version, + ) } diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs index 449b4c6551..db81695ac7 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs @@ -2,4 +2,5 @@ pub mod allowed_custom_rule_import; pub mod disallowed_import; pub mod implementation_fingerprint; pub mod import_closure; +pub mod project_import_closure; pub mod test_evidence; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs new file mode 100644 index 0000000000..6a74ff0773 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs @@ -0,0 +1,16 @@ +//! `project_import_closure`: the project modules one Python file imports, transitively. + +use std::path::{Path, PathBuf}; + +use crate::rule_evidence::_helpers::closure::module_closure; +use crate::rule_evidence::_helpers::paths::resolve; + +/// Project modules and package initializers `source_path` imports, resolved from `project_dir`. +pub fn project_import_closure( + source_path: &Path, + project_dir: &Path, + python_version: (u8, u8), +) -> Vec { + let root: PathBuf = resolve(project_dir); + module_closure(&resolve(source_path), (&root, &root), None, python_version) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs index b1599b4202..2111e866c7 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs @@ -29,3 +29,15 @@ fn write(path: &Path, contents: &str) { let _ = std::fs::create_dir_all(path.parent().unwrap_or(path)); let _ = std::fs::write(path, contents); } + +/// A fresh project directory named `name` holding `files`. +pub(crate) fn project_with(name: &str, files: &[(&str, &str)]) -> PathBuf { + let root: PathBuf = std::env::temp_dir() + .join(format!("sqlbuild-project-closure-{}", std::process::id())) + .join(name); + let _ = std::fs::remove_dir_all(&root); + for (path, contents) in files { + write(&root.join(path), contents); + } + std::fs::canonicalize(&root).unwrap_or(root) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs index 5035eb0930..32e2da73e9 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs @@ -1,9 +1,10 @@ use std::path::{Path, PathBuf}; use crate::rule_evidence::main::import_closure::import_closure; +use crate::rule_evidence::main::project_import_closure::project_import_closure; use crate::rule_evidence::main::test_evidence::test_evidence; -use crate::rule_evidence::tests::helpers::rule_project; -use crate::rule_evidence::tests::test_types::EvidenceTestCase; +use crate::rule_evidence::tests::helpers::{project_with, rule_project}; +use crate::rule_evidence::tests::test_types::{EvidenceTestCase, ProjectClosureTestCase}; const PYTHON: (u8, u8) = (3, 12); @@ -74,3 +75,54 @@ fn given_rule_project_when_reading_evidence_then_matches_python_closure_and_case ); } } + +#[test] +fn given_project_module_when_reading_project_closure_then_lists_imported_project_files() { + let test_cases = [ + ProjectClosureTestCase { + description: "helper modules across directories, transitively", + files: vec![ + ( + "macros/money.py", + "import os\nfrom macros._rates import RATE\n\ndef cents(x):\n return x\n", + ), + ( + "macros/_rates.py", + "from python import shared\nRATE = 100\n", + ), + ("python/__init__.py", ""), + ("python/shared.py", "VALUE = 1\n"), + ("python/unused.py", "VALUE = 2\n"), + ], + source: "macros/money.py", + expected_closure: vec!["macros/_rates.py", "python/__init__.py", "python/shared.py"], + }, + ProjectClosureTestCase { + description: "a module importing nothing local has an empty closure", + files: vec![("macros/plain.py", "import json\n")], + source: "macros/plain.py", + expected_closure: vec![], + }, + ]; + + for test_case in test_cases { + let root: PathBuf = project_with(test_case.description, &test_case.files); + let closure: Vec = + project_import_closure(&root.join(test_case.source), &root, PYTHON); + let relative: Vec<&Path> = closure + .iter() + .map(|path| path.strip_prefix(&root).unwrap_or(path)) + .collect(); + + assert_eq!( + relative, + test_case + .expected_closure + .iter() + .map(Path::new) + .collect::>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs index fe7802a5cc..3e8802056c 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs @@ -20,3 +20,10 @@ pub(crate) struct DisallowedImportTestCase { crate::rule_evidence::models::UnparsedSource, >, } + +pub(crate) struct ProjectClosureTestCase { + pub(crate) description: &'static str, + pub(crate) files: Vec<(&'static str, &'static str)>, + pub(crate) source: &'static str, + pub(crate) expected_closure: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs new file mode 100644 index 0000000000..2e88520c54 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs @@ -0,0 +1,146 @@ +//! Where the base snapshot is cached, what its cache key covers, and how old entries are pruned. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; +use sqlbuild_rules::constants::NATIVE_BUILD_IDENTITY; + +use crate::bindings::_helpers::change_selection::base_compile::NOTES_EXTENSION; +use crate::bindings::_helpers::change_selection::fingerprints::{ + environment_digest_with, process_environment, referenced_environment, +}; + +/// Cached base snapshots kept per project; older ones are removed after each compile. +const KEPT_SNAPSHOTS: usize = 8; +const SNAPSHOT_SUFFIX: &str = "json"; +const STAGED_SUFFIX: &str = "tmp"; +const WORK_PREFIX: &str = "work-"; +/// Work directories and staged files older than this belong to runs that did not clean up. +const STALE_AFTER: Duration = Duration::from_secs(24 * 60 * 60); +/// Authored file types that can read environment variables through SQL or configuration. +const AUTHORED_SUFFIXES: [&str; 5] = ["toml", "yml", "yaml", "sql", "py"]; +/// Directories of build output, tools and dependencies, never authored project files. +const SKIPPED_DIRECTORIES: [&str; 4] = ["target", "logs", "node_modules", "__pycache__"]; + +/// Authored files below `directory`, skipping hidden, build and dependency directories. +fn authored_files(directory: &Path) -> std::io::Result> { + let mut files: Vec = Vec::new(); + let mut pending: Vec = vec![directory.to_path_buf()]; + while let Some(next) = pending.pop() { + for entry in std::fs::read_dir(&next)? { + let path: PathBuf = entry?.path(); + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + if name.starts_with('.') { + continue; + } + if path.is_dir() { + if !SKIPPED_DIRECTORIES.contains(&name.as_str()) { + pending.push(path); + } + } else if path + .extension() + .is_some_and(|suffix| AUTHORED_SUFFIXES.contains(&&*suffix.to_string_lossy())) + { + files.push(path); + } + } + } + files.sort(); + Ok(files) +} + +/// The digest of every environment variable the project's files read, by its `lookup` value. +fn environment_inputs( + project_dir: &Path, + lookup: &dyn Fn(&str) -> Option, +) -> std::io::Result { + let mut names: BTreeSet = BTreeSet::new(); + for file in authored_files(project_dir)? { + let bytes: Vec = std::fs::read(&file)?; + names.extend(referenced_environment(&String::from_utf8_lossy(&bytes))); + } + Ok(environment_digest_with(&names, lookup)) +} + +/// The cache key of a base compile besides its commit: build, target, vars, local config, env. +pub(crate) fn cache_key( + project_dir: &Path, + inputs: (Option<&str>, &str, &str), +) -> std::io::Result { + cache_key_with(project_dir, inputs, &process_environment) +} + +/// `cache_key` with environment variables read through `lookup`. +pub(crate) fn cache_key_with( + project_dir: &Path, + (target, vars, local_config): (Option<&str>, &str, &str), + lookup: &dyn Fn(&str) -> Option, +) -> std::io::Result { + Ok(definition_digest(&[ + NATIVE_BUILD_IDENTITY, + target.unwrap_or_default(), + vars, + local_config, + &environment_inputs(project_dir, lookup)?, + ])) +} + +/// A work directory name no other run, even one reusing this process id, can choose. +pub(crate) fn work_directory_name(short: &str) -> String { + let nanos: u128 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos()); + format!("{WORK_PREFIX}{short}-{}-{nanos:x}", std::process::id()) +} + +fn removed(result: std::io::Result<()>) -> std::io::Result<()> { + match result { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Keep the newest snapshots and their notes; drop the rest and day-old staged files and work dirs. +pub(crate) fn prune_snapshots(cache_dir: &Path) -> std::io::Result<()> { + let now: SystemTime = SystemTime::now(); + let mut snapshots: Vec<(SystemTime, PathBuf)> = Vec::new(); + let mut leftovers: Vec = Vec::new(); + for entry in std::fs::read_dir(cache_dir)? { + let path: PathBuf = entry?.path(); + let modified: SystemTime = std::fs::metadata(&path)?.modified()?; + let stale: bool = now + .duration_since(modified) + .is_ok_and(|age| age > STALE_AFTER); + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let suffix: String = path + .extension() + .map(|suffix| suffix.to_string_lossy().into_owned()) + .unwrap_or_default(); + if path.is_dir() { + if stale && name.starts_with(WORK_PREFIX) { + removed(std::fs::remove_dir_all(&path))?; + } + } else if suffix == SNAPSHOT_SUFFIX { + snapshots.push((modified, path)); + } else if suffix == STAGED_SUFFIX && stale { + leftovers.push(path); + } + } + snapshots.sort_by(|left, right| right.cmp(left)); + for (_, path) in snapshots.into_iter().skip(KEPT_SNAPSHOTS) { + leftovers.push(path.with_extension(NOTES_EXTENSION)); + leftovers.push(path); + } + for path in leftovers { + removed(std::fs::remove_file(&path))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs new file mode 100644 index 0000000000..9dfb3aa2af --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs @@ -0,0 +1,132 @@ +//! The base-revision side of `--since`: compile an exported tree and write its snapshot. + +use std::path::PathBuf; + +use pyo3::prelude::{ + Bound, IntoPyObject, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::definition_snapshot_json; +use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; + +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, keywords}; +use crate::bindings::_helpers::change_selection::definition_snapshot::definition_snapshot; +use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::pipeline_entries::run_compile_only_pipeline; +use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; + +/// The extension of the file beside the snapshot that holds notes for the run's output. +pub(crate) const NOTES_EXTENSION: &str = "notes"; + +fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +/// Whether `target` is unset or defined by the project or its local configuration. +fn target_defined(discovered: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>) -> PyResult { + if target.is_none() { + return Ok(true); + } + for config in ["project_config", "local_config"] { + let targets: Bound<'_, PyAny> = + native_field(&native_field(discovered, config)?, "targets")?; + if let Ok(targets) = targets.cast::() + && targets.contains(target)? + { + return Ok(true); + } + } + Ok(false) +} + +/// Compile an exported base tree without SQL analysis and write its definition snapshot. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_base_definition_snapshot( + py: Python<'_>, + options: Option>, +) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "write_base_definition_snapshot", + options, + ( + &["project_dir", "output_path"], + &["selected_target", "cli_vars"], + ), + )?; + let none = || py.None().into_bound(py); + let project_dir: PathBuf = arguments.value("project_dir")?.extract()?; + let output_path: PathBuf = arguments.value("output_path")?.extract()?; + let directory: Bound<'_, PyAny> = project_dir.clone().into_pyobject(py)?.into_any(); + let target: Bound<'_, PyAny> = arguments.value("selected_target")?; + let vars: Bound<'_, PyAny> = arguments.value("cli_vars")?; + let discovered: Bound<'_, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", directory.clone())])?), + )?; + let target: Bound<'_, PyAny> = if target_defined(&discovered, &target)? { + target + } else { + std::fs::write( + output_path.with_extension(NOTES_EXTENSION), + format!( + "Target '{}' is not defined at the merge base, so the base compiles with its \ + default target and everything that depends on the target is selected\n", + target.str()? + ), + )?; + none() + }; + let context: Bound<'_, PyAny> = resolve_adapter_connection_context( + &discovered, + &directory, + (target.clone(), vars.clone()), + false, + )?; + let resolver: Bound<'_, PyAny> = + resolve_external_sql_reference_resolver(directory.clone(), discovered.clone())?; + let options: Bound<'_, PyAny> = new_record_with( + py, + RecordId::CompilePipelineOptions, + &[ + ("selected_target", target), + ("no_sql_validation", flag(py, true)), + ("source_deferral_enabled", flag(py, false)), + ( + "connection_config", + native_field(&context, "connection_config")?, + ), + ("cli_vars", vars), + ("external_sql_reference_resolver", resolver), + ], + )?; + let pipeline: Bound<'_, PyAny> = run_compile_only_pipeline( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", native_field(&context, "adapter")?), + ("options", options), + ("on_progress", none()), + ], + )?), + )?; + let snapshot: DefinitionSnapshot = definition_snapshot( + &native_field(&pipeline, "project")?, + &discovered, + &project_dir, + )?; + std::fs::write(&output_path, definition_snapshot_json(&snapshot))?; + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_base_definition_snapshot, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs new file mode 100644 index 0000000000..2b25114035 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -0,0 +1,343 @@ +//! The `--since` base snapshot: resolved from git, compiled in a child interpreter, cached per commit. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_analysis::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; +use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::definition_snapshot_json; +use sqlbuild_analysis::definition_changes::main::parse_definition_snapshot::parse_definition_snapshot; +use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; +use sqlbuild_core::git_revisions::constants::BASE_REF_ALIAS; +use sqlbuild_core::git_revisions::errors::GitRevisionError; +use sqlbuild_core::git_revisions::main::export_git_tree::export_git_tree; +use sqlbuild_core::git_revisions::main::resolve_git_base::resolve_git_base; +use sqlbuild_core::git_revisions::main::resolve_since_reference::resolve_since_reference; +use sqlbuild_core::git_revisions::main::short_commit::short_commit; +use sqlbuild_core::git_revisions::models::GitBase; +use sqlbuild_core::scenario_command::errors::ScenarioCommandError; + +use crate::bindings::_helpers::change_selection::base_cache::{ + cache_key, prune_snapshots, work_directory_name, +}; +use crate::bindings::_helpers::change_selection::base_compile::NOTES_EXTENSION; +use crate::bindings::_helpers::change_selection::environment_nodes::with_base_dependencies; +use crate::bindings::_helpers::compile_command::runtime_identity::interpreter_values; +use crate::bindings::_helpers::discovery::project_config::configured_git_base_ref; +use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, sorted_dumps}; +use crate::bindings::_helpers::scenario_command::setup::{cli_user_error, flush, write}; + +const CACHE_DIRECTORY: [&str; 2] = ["target", "since"]; +/// Where base compiles work when the project's cache directory cannot be created. +const TEMPORARY_WORK_ROOT: &str = "sqlbuild-since"; +const LOCAL_CONFIG_FILE: &str = "sqlbuild_local.toml"; +const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; +const BASE_COMPILE_FAILED_CODE: &str = "C484"; +const FAILURE_TAIL_LINES: usize = 20; +/// Compiles the exported base tree in a separate interpreter; a failure prints only its error. +const BASE_COMPILE_PROGRAM: &str = concat!( + "import json, sys\n", + "from sqlbuild._native import write_base_definition_snapshot\n", + "try:\n", + " write_base_definition_snapshot(project_dir=sys.argv[1], output_path=sys.argv[2],\n", + " selected_target=sys.argv[3] or None, cli_vars=json.loads(sys.argv[4]))\n", + "except Exception as error:\n", + " code = getattr(error, 'code', None)\n", + " message = f'error[{code}]: {error}' if code else f'{type(error).__name__}: {error}'\n", + " print(message, file=sys.stderr)\n", + " sys.exit(1)\n", +); + +/// The base revision and its definition snapshot. +pub(crate) struct BaseComparison { + pub(crate) base: GitBase, + pub(crate) snapshot: DefinitionSnapshot, +} + +/// The inputs that decide what the base compile produces besides its commit. +pub(crate) struct BaseVariant { + pub(crate) selected_target: Option, + /// `--vars` as sorted compact JSON. + pub(crate) vars_json: String, +} + +/// The base variant of a command's `--target` and `--vars`. +pub(crate) fn base_variant( + selected_target: Option, + cli_vars: &Bound<'_, PyAny>, +) -> PyResult { + Ok(BaseVariant { + selected_target, + vars_json: vars_json(cli_vars)?, + }) +} + +pub(crate) fn git_error(py: Python<'_>, error: GitRevisionError) -> PyErr { + cli_user_error( + py, + ScenarioCommandError { + code: error.code, + message: error.message, + help: Some(error.help), + }, + ) +} + +fn compile_failed(py: Python<'_>, base: &GitBase, detail: &str) -> PyErr { + cli_user_error( + py, + ScenarioCommandError { + code: BASE_COMPILE_FAILED_CODE, + message: format!( + "Cannot compile the project at merge base {} of '{}' and HEAD:\n{detail}", + short_commit(&base.merge_base), + base.reference + ), + help: Some( + "--since compares with the project as it was at the merge base, so that \ + revision must compile. Compare with a later ref whose project compiles, or run \ + without --since to run everything" + .to_owned(), + ), + }, + ) +} + +fn io_failed(py: Python<'_>, base: &GitBase, error: &std::io::Error) -> PyErr { + compile_failed(py, base, &error.to_string()) +} + +/// The last lines of a failed base compile's output. +fn failure_tail(output: &Output) -> String { + let text: String = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let lines: Vec<&str> = text + .lines() + .filter(|line| !line.trim().is_empty()) + .collect(); + lines[lines.len().saturating_sub(FAILURE_TAIL_LINES)..].join("\n") +} + +fn vars_json(cli_vars: &Bound<'_, PyAny>) -> PyResult { + if cli_vars.is_none() { + return Ok("{}".to_owned()); + } + sorted_dumps(cli_vars, COMPACT, true) +} + +/// Run the base compile of `tree` into `output` in a separate interpreter. +fn compile_base_tree( + py: Python<'_>, + base: &GitBase, + (tree, output): (&Path, &Path), + (selected_target, vars): (Option<&str>, &str), +) -> PyResult<()> { + let [_, executable, _, _] = interpreter_values(py)?; + let finished: std::io::Result = py.detach(|| { + Command::new(&executable) + .arg("-c") + .arg(BASE_COMPILE_PROGRAM) + .arg(tree) + .arg(output) + .arg(selected_target.unwrap_or_default()) + .arg(vars) + .current_dir(tree) + .stdin(Stdio::null()) + .output() + }); + let finished: Output = finished.map_err(|error| io_failed(py, base, &error))?; + if finished.status.success() && output.is_file() { + return Ok(()); + } + Err(compile_failed(py, base, &failure_tail(&finished))) +} + +/// Export the base tree, carry over the untracked local config, and compile it. +fn compiled_base_snapshot( + py: Python<'_>, + (base, project_dir, work): (&GitBase, &Path, &Path), + (selected_target, vars): (Option<&str>, &str), +) -> PyResult<(DefinitionSnapshot, String)> { + let tree: PathBuf = work.join("project"); + let exported: bool = export_git_tree(base, &tree).map_err(|error| git_error(py, error))?; + if !exported || !tree.join(PROJECT_CONFIG_FILE).is_file() { + let empty: DefinitionSnapshot = DefinitionSnapshot { + format: DEFINITION_SNAPSHOT_FORMAT, + ..DefinitionSnapshot::default() + }; + return Ok((empty, String::new())); + } + let local: PathBuf = project_dir.join(LOCAL_CONFIG_FILE); + if local.is_file() && !tree.join(LOCAL_CONFIG_FILE).exists() { + std::fs::copy(&local, tree.join(LOCAL_CONFIG_FILE)) + .map_err(|error| io_failed(py, base, &error))?; + } + let output: PathBuf = work.join("definitions.json"); + compile_base_tree(py, base, (&tree, &output), (selected_target, vars))?; + let text: String = + std::fs::read_to_string(&output).map_err(|error| io_failed(py, base, &error))?; + let snapshot: DefinitionSnapshot = parse_definition_snapshot(&text) + .ok_or_else(|| compile_failed(py, base, "the base compile wrote an unreadable snapshot"))?; + Ok((snapshot, read_notes(py, base, &output)?)) +} + +/// The notes stored beside snapshot `path`, empty when there are none. +fn read_notes(py: Python<'_>, base: &GitBase, path: &Path) -> PyResult { + match std::fs::read_to_string(path.with_extension(NOTES_EXTENSION)) { + Ok(notes) => Ok(notes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(String::new()), + Err(error) => Err(io_failed(py, base, &error)), + } +} + +/// The notes cached beside snapshot `path`; unreadable notes are skipped with a warning. +fn cached_notes(path: &Path) -> String { + let notes: PathBuf = path.with_extension(NOTES_EXTENSION); + match std::fs::read_to_string(¬es) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => { + warn(&format!( + "Could not read the cached notes at {}: {error}", + notes.display() + )); + String::new() + } + } +} + +/// Write one warning line to stderr, which machine-readable stdout never carries. +fn warn(message: &str) { + let mut stderr: std::io::StderrLock<'_> = std::io::stderr().lock(); + match writeln!(stderr, "warning: {message}") { + Ok(()) | Err(_) => {} + } +} + +/// The cache only saves time, so a failure to write it is a warning and the run continues. +fn warn_uncached(path: &Path, error: &std::io::Error) { + warn(&format!( + "Could not cache the base revision at {}: {error}; continuing without the cache", + path.display() + )); +} + +/// A fresh work directory under `cache_dir`, or the system temporary directory without one. +fn work_directory(cache_dir: &Path, short: &str) -> std::io::Result { + let name: String = work_directory_name(short); + let work: PathBuf = cache_dir.join(&name); + match std::fs::create_dir_all(&work) { + Ok(()) => Ok(work), + Err(_unavailable) => { + let fallback: PathBuf = std::env::temp_dir().join(TEMPORARY_WORK_ROOT).join(&name); + std::fs::create_dir_all(&fallback)?; + Ok(fallback) + } + } +} + +/// Cache `snapshot` at `cached` with its `notes`, staging the snapshot so readers never see half. +fn store_snapshot( + cached: &Path, + (snapshot, notes): (&DefinitionSnapshot, &str), +) -> std::io::Result<()> { + let notes_path: PathBuf = cached.with_extension(NOTES_EXTENSION); + if notes.is_empty() { + match std::fs::remove_file(¬es_path) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error), + _ => {} + } + } else { + std::fs::write(¬es_path, notes)?; + } + let staged: PathBuf = cached.with_extension(format!("{}.tmp", std::process::id())); + let stored: std::io::Result<()> = std::fs::write(&staged, definition_snapshot_json(snapshot)) + .and_then(|()| std::fs::rename(&staged, cached)); + if stored.is_err() { + match std::fs::remove_file(&staged) { + Ok(()) | Err(_) => {} + } + } + stored +} + +/// Resolve `since` to its merge base and load, or compile and cache, the base snapshot. +pub(crate) fn base_comparison<'py>( + (stream, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + since: &str, + variant: &BaseVariant, +) -> PyResult { + let py: Python<'py> = stream.py(); + let directory: PathBuf = project_dir.extract()?; + let configured: Option = if since.trim() == BASE_REF_ALIAS { + configured_git_base_ref(project_dir)? + } else { + None + }; + let reference: String = resolve_since_reference(since, configured.as_deref()) + .map_err(|error| git_error(py, error))?; + let base: GitBase = py + .detach(|| resolve_git_base(&directory, &reference)) + .map_err(|error| git_error(py, error))?; + let short: &str = short_commit(&base.merge_base); + write( + stream, + &format!("Comparing with {reference} (merge base {short})\n"), + )?; + let vars: &str = &variant.vars_json; + let target: Option<&str> = variant.selected_target.as_deref(); + let local_config: String = match std::fs::read_to_string(directory.join(LOCAL_CONFIG_FILE)) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(io_failed(py, &base, &error)), + }; + let key: String = cache_key(&directory, (target, vars, &local_config)) + .map_err(|error| io_failed(py, &base, &error))?; + let cache_dir: PathBuf = CACHE_DIRECTORY + .iter() + .fold(directory.clone(), |path, part| path.join(part)); + let cached: PathBuf = cache_dir.join(format!("{}-{}.json", base.merge_base, &key[..16])); + let stored: Option = match std::fs::read_to_string(&cached) { + Ok(text) => parse_definition_snapshot(&text), + Err(_) => None, + }; + if let Some(snapshot) = stored { + write(stream, &format!("Using cached base revision {short}\n"))?; + write(stream, &cached_notes(&cached))?; + flush(stream)?; + let snapshot: DefinitionSnapshot = with_base_dependencies(py, &base, &snapshot)?; + return Ok(BaseComparison { base, snapshot }); + } + write(stream, &format!("Compiling base revision {short}...\n"))?; + flush(stream)?; + let started: Instant = Instant::now(); + let work: PathBuf = + work_directory(&cache_dir, short).map_err(|error| io_failed(py, &base, &error))?; + let compiled: PyResult<(DefinitionSnapshot, String)> = + compiled_base_snapshot(py, (&base, &directory, &work), (target, vars)); + let _ = std::fs::remove_dir_all(&work); + let (snapshot, notes) = compiled?; + write(stream, ¬es)?; + match store_snapshot(&cached, (&snapshot, ¬es)) { + Ok(()) => match prune_snapshots(&cache_dir) { + Ok(()) => {} + Err(error) => warn_uncached(&cache_dir, &error), + }, + Err(error) => warn_uncached(&cached, &error), + } + write( + stream, + &format!( + "Compiled base revision {short} ({:.2}s)\n", + started.elapsed().as_secs_f64() + ), + )?; + flush(stream)?; + let snapshot: DefinitionSnapshot = with_base_dependencies(py, &base, &snapshot)?; + Ok(BaseComparison { base, snapshot }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs new file mode 100644 index 0000000000..61fecd18ef --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -0,0 +1,592 @@ +//! The definition snapshot of a compiled project: one fingerprint per node and its declarations. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use serde_json::{Map, Value}; +use sqlbuild_analysis::definition_changes::constants::{ + AUDITS_ASPECT, COLUMNS_ASPECT, CONFIG_ASPECT, CONNECTION_KIND, DEFINITION_ASPECT, + DEFINITION_SNAPSHOT_FORMAT, DEPENDENCIES_KIND, FILE_ASPECT, MACRO_KIND, MATERIALIZATION_KIND, + MODEL_KIND, PROJECT_NODE_NAME, PYTHON_NODE_KINDS, SCENARIO_KIND, SEED_KIND, SETTINGS_KIND, + SNAPSHOTS_ASPECT, SOURCE_KIND, SQL_ASPECT, TEST_FILE_KIND, VARS_ASPECT, +}; +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; +use sqlbuild_analysis::definition_changes::main::node_key::node_key; +use sqlbuild_analysis::definition_changes::main::sql_definition_digest::sql_definition_digest; +use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; + +use crate::bindings::_helpers::change_selection::fingerprints::{ + Fingerprinting, directory_digest, file_digest, text_of, +}; +use crate::bindings::_helpers::compiled_views::function_reads; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; +use crate::bindings::_helpers::compiled_views::seed_reads; +use crate::bindings::_helpers::compiled_views::source_reads; +use crate::bindings::_helpers::planner_identity::metadata::{function_local_hashes, seed_identity}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Header values that rename rather than redefine a model. +const RENAME_CONFIG_KEYS: [&str; 2] = ["migrate_from", "description"]; +const MIGRATE_FROM_KEY: &str = "migrate_from"; +const MATERIALIZED_KEY: &str = "materialized"; +/// Which header keys were authored, rather than inherited from defaults, does not change a model. +const AUTHORED_KEYS_FIELD: &str = "model_header_keys"; +const SOURCE_EXPRESSION_FIELD: &str = "expression"; +const SCENARIO_SNAPSHOT_ROOT: [&str; 2] = ["tests", "_scenario_snapshots"]; +const CONTRACT_KEY: &str = "contract"; +/// Source declaration fields that shape a source mock's columns. +const SOURCE_SHAPE_FIELDS: [&str; 3] = ["columns", "contract", "type_enforcement"]; +/// Project-local adapter modules. +const ADAPTERS_DIRECTORY: &str = "adapters"; +/// Project configuration fields that other nodes fingerprint, or that do not shape compiled SQL. +const PROJECT_CONFIG_ELSEWHERE: [&str; 11] = [ + "name", + "settings", + "vars", + "defaults", + "path_defaults", + "cost", + "diff", + "janitor", + "migrations", + "sinks", + "scenario", +]; +const LOCAL_CONFIG_ELSEWHERE: [&str; 4] = ["settings", "setting_overrides", "vars", "scenario"]; +/// Settings that tune how a run executes or validates, not what a node computes. +const RUNTIME_SETTINGS: [&str; 9] = [ + "sql_analysis", + "require_sql_analysis", + "concurrency", + "microbatch_concurrency", + "test_timeout", + "test_timeout_seconds", + "scenario_timeout", + "scenario_timeout_seconds", + "delta_compile", +]; +const PYTHON_SUFFIX: &str = ".py"; +/// Discovered Python node collections, in `PYTHON_NODE_KINDS` order. +const PYTHON_NODE_COLLECTIONS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; + +fn node(kind: &str, name: &str, aspects: Vec<(&str, String)>) -> NodeDefinition { + NodeDefinition { + kind: kind.to_owned(), + name: name.to_owned(), + aspects: aspects + .into_iter() + .map(|(aspect, digest)| (aspect.to_owned(), digest)) + .collect(), + uses: Vec::new(), + migrate_from: Vec::new(), + } +} + +/// The node key of a scope declaration identity: `kind:name`, with `@owner` for owned ones. +pub(crate) fn declaration_key(identity: &Bound<'_, PyAny>) -> PyResult { + let kind: String = text_of(&native_field(&native_field(identity, "kind")?, "value")?)?; + let name: String = native_field(identity, "name")?.extract()?; + let owner: Bound<'_, PyAny> = native_field(identity, "owner")?; + if owner.is_none() { + return Ok(node_key(&kind, &name)); + } + let owner_kind: String = text_of(&native_field(&native_field(&owner, "kind")?, "value")?)?; + let owner_name: String = native_field(&owner, "name")?.extract()?; + Ok(node_key( + &kind, + &format!("{name}@{}", node_key(&owner_kind, &owner_name)), + )) +} + +/// `kind:name` of a usage consumer. +fn consumer_key(consumer: &Bound<'_, PyAny>) -> PyResult { + Ok(node_key( + &text_of(&native_field(&native_field(consumer, "kind")?, "value")?)?, + &native_field(consumer, "name")?.extract::()?, + )) +} + +/// Declaration keys each consumer uses, by consumer key, from the project's scope index. +pub(crate) fn declaration_uses( + project: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut uses: BTreeMap> = BTreeMap::new(); + let index: Bound<'_, PyAny> = native_field(project, "scope_index")?; + if index.is_none() { + return Ok(uses); + } + for usage in native_field(&index, "usages")?.try_iter()? { + let usage: Bound<'_, PyAny> = usage?; + uses.entry(consumer_key(&native_field(&usage, "consumer")?)?) + .or_default() + .insert(declaration_key(&native_field(&usage, "declaration")?)?); + } + Ok(uses) +} + +/// The digest of a declaration file: Python with its imports, anything else as normalised SQL. +fn declaration_file_digest(fingerprints: &Fingerprinting, path: &str) -> String { + if path.ends_with(PYTHON_SUFFIX) { + return fingerprints.python_module_digest(Path::new(path)); + } + match std::fs::read_to_string(fingerprints.project_dir.join(path)) { + Ok(contents) => { + let mut parts: Vec = vec![sql_definition_digest(&contents)]; + parts.extend(fingerprints.referenced_vars_digest(&contents)); + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + definition_digest(&borrowed) + } + Err(_) => definition_digest(&["missing"]), + } +} + +/// Scope declarations by their file; a Python macro also by its facts and macro dependencies. +fn declaration_nodes( + project: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult> { + let mut nodes: BTreeMap = BTreeMap::new(); + let index: Bound<'_, PyAny> = native_field(project, "scope_index")?; + if index.is_none() { + return Ok(nodes); + } + for declaration in native_field(&index, "declarations")?.try_iter()? { + let declaration: Bound<'_, PyAny> = declaration?; + let identity: Bound<'_, PyAny> = native_field(&declaration, "identity")?; + let kind: String = text_of(&native_field(&native_field(&identity, "kind")?, "value")?)?; + let name: String = native_field(&identity, "name")?.extract()?; + let path: String = native_field(&declaration, "path")?.extract()?; + let macro_facts: Bound<'_, PyAny> = native_field(&declaration, "macro")?; + let (digest, uses): (String, Vec) = if kind == MACRO_KIND && !macro_facts.is_none() + { + let dependencies: Vec = native_field(¯o_facts, "dependencies")? + .try_iter()? + .map(|dependency| declaration_key(&dependency?)) + .collect::>()?; + ( + definition_digest(&[ + &fingerprints.canonical_digest(¯o_facts, (&[], &[]))?, + &declaration_file_digest(fingerprints, &path), + ]), + dependencies, + ) + } else { + (declaration_file_digest(fingerprints, &path), Vec::new()) + }; + let mut definition: NodeDefinition = node(&kind, &name, vec![(DEFINITION_ASPECT, digest)]); + definition.uses = uses; + nodes.insert(declaration_key(&identity)?, definition); + } + Ok(nodes) +} + +fn migrate_from(config_values: &Bound<'_, PyAny>) -> PyResult> { + let Ok(values) = config_values.cast::() else { + return Ok(Vec::new()); + }; + let Some(value) = values.get_item(MIGRATE_FROM_KEY)? else { + return Ok(Vec::new()); + }; + if value.is_instance_of::() { + return Ok(vec![value.extract()?]); + } + value.try_iter()?.map(|name| name?.extract()).collect() +} + +/// The resolved configuration of a model, defaults and path defaults included, without renames. +fn config_digest(fingerprints: &Fingerprinting, config: &Bound<'_, PyAny>) -> PyResult { + let mut encoded: Value = fingerprints.canonical(config, &[AUTHORED_KEYS_FIELD])?; + if let Value::Object(fields) = &mut encoded + && let Some(Value::Object(values)) = fields.get_mut("values") + { + for key in RENAME_CONFIG_KEYS { + values.remove(key); + } + } + Ok(definition_digest(&[&encoded.to_string()])) +} + +/// A model by its authored SQL, resolved configuration, columns, audits and variables it reads. +fn model_node( + model: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, + uses: &BTreeMap>, +) -> PyResult<(String, NodeDefinition)> { + let name: String = model_reads::model_name(model)?; + let config: Bound<'_, PyAny> = model_reads::config(model)?; + let config_values: Bound<'_, PyAny> = native_field(&config, "values")?; + let schema: Bound<'_, PyAny> = model_reads::schema_entry(model)?; + let contract: String = match config_values.cast::() { + Ok(values) => match values.get_item(CONTRACT_KEY)? { + Some(value) => fingerprints.canonical(&value, &[])?.to_string(), + None => String::new(), + }, + Err(_) => String::new(), + }; + let (columns, audits): (String, String) = if schema.is_none() { + (definition_digest(&[&contract]), definition_digest(&[])) + } else { + let mut column_audits: Vec = + vec![fingerprints.canonical(&native_field(&schema, "audits")?, &[])?]; + for column in native_field(&schema, "columns")?.try_iter()? { + column_audits.push(fingerprints.canonical(&native_field(&column?, "audits")?, &[])?); + } + ( + definition_digest(&[ + &fingerprints.canonical_digest(&schema, (&["name", "tags"], &["audits"]))?, + &contract, + ]), + definition_digest(&[&Value::Array(column_audits).to_string()]), + ) + }; + let query_sql: String = native_field(model, "authored_query_sql")?.extract()?; + let authored_sql: String = native_field(model, "authored_sql")?.extract()?; + let mut aspects: Vec<(&str, String)> = vec![ + (SQL_ASPECT, sql_definition_digest(&query_sql)), + (CONFIG_ASPECT, config_digest(fingerprints, &config)?), + (COLUMNS_ASPECT, columns), + (AUDITS_ASPECT, audits), + ]; + if let Some(digest) = fingerprints.referenced_vars_digest(&authored_sql) { + aspects.push((VARS_ASPECT, digest)); + } + let mut definition: NodeDefinition = node(MODEL_KIND, &name, aspects); + let key: String = node_key(MODEL_KIND, &name); + definition.uses = uses.get(&key).into_iter().flatten().cloned().collect(); + definition.uses.extend(project_wide_keys()); + if let Ok(values) = config_values.cast::() + && let Some(materialized) = values.get_item(MATERIALIZED_KEY)? + && let Ok(materialized) = materialized.cast::() + { + definition + .uses + .push(node_key(MATERIALIZATION_KIND, materialized.to_str()?)); + } + definition.migrate_from = migrate_from(&config_values)?; + Ok((key, definition)) +} + +/// A source by its declared columns, which shape mocks, and the rest of its declaration. +fn source_node( + source: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult<(String, NodeDefinition)> { + let name: String = source_reads::name(source)?.extract()?; + let mut encoded: Value = fingerprints.canonical(&source_reads::source_entry(source)?, &[])?; + let mut shape: Map = Map::new(); + let mut reads: Option = None; + if let Value::Object(fields) = &mut encoded { + for field in SOURCE_SHAPE_FIELDS { + if let Some(value) = fields.remove(field) { + shape.insert(field.to_owned(), value); + } + } + if let Some(Value::String(expression)) = fields.get(SOURCE_EXPRESSION_FIELD) { + reads = fingerprints.referenced_vars_digest(expression); + let normalised: String = sql_definition_digest(expression); + fields.insert(SOURCE_EXPRESSION_FIELD.to_owned(), Value::from(normalised)); + } + } + let mut aspects: Vec<(&str, String)> = vec![ + ( + DEFINITION_ASPECT, + definition_digest(&[&encoded.to_string()]), + ), + ( + COLUMNS_ASPECT, + definition_digest(&[&Value::Object(shape).to_string()]), + ), + ]; + aspects.extend(reads.map(|digest| (VARS_ASPECT, digest))); + let mut definition: NodeDefinition = node(SOURCE_KIND, &name, aspects); + definition.uses = project_wide_keys(); + Ok((node_key(SOURCE_KIND, &name), definition)) +} + +/// A seed by its data and settings, and by the declared columns that shape its mocks. +fn seed_node( + seed: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult<(String, NodeDefinition)> { + let name: String = seed_reads::name(seed)?.extract()?; + let (version_hash, _) = seed_identity(seed)?; + let schema: Bound<'_, PyAny> = seed_reads::schema_entry(seed)?; + let columns: String = if schema.is_none() { + definition_digest(&[]) + } else { + definition_digest(&[&fingerprints + .canonical(&native_field(&schema, "columns")?, &[])? + .to_string()]) + }; + Ok(( + node_key(SEED_KIND, &name), + node( + SEED_KIND, + &name, + vec![(DEFINITION_ASPECT, version_hash), (COLUMNS_ASPECT, columns)], + ), + )) +} + +/// Keys of the project-wide nodes that every compiled definition depends on. +pub(crate) fn project_wide_keys() -> Vec { + [SETTINGS_KIND, CONNECTION_KIND, DEPENDENCIES_KIND] + .iter() + .map(|kind| node_key(kind, PROJECT_NODE_NAME)) + .collect() +} + +/// The adapter, connections and targets as configured, the selected target, and adapter modules. +fn connection_node( + project: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult<(String, NodeDefinition)> { + let mut parts: Vec = vec![ + fingerprints.canonical_digest( + &native_field(discovered, "project_config")?, + (&PROJECT_CONFIG_ELSEWHERE, &[]), + )?, + fingerprints.canonical_digest( + &native_field(discovered, "local_config")?, + (&LOCAL_CONFIG_ELSEWHERE, &[]), + )?, + text_of(&native_field(project, "effective_target_name")?)?, + ]; + let adapter_file: Bound<'_, PyAny> = native_field(discovered, "adapter_file")?; + if !adapter_file.is_none() { + let path: PathBuf = native_field(&adapter_file, "file_path")?.extract()?; + parts.push(fingerprints.python_module_digest(&path)); + } + for module in python_files(&fingerprints.project_dir.join(ADAPTERS_DIRECTORY))? { + parts.push(fingerprints.python_module_digest(&module)); + } + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + Ok(( + node_key(CONNECTION_KIND, PROJECT_NODE_NAME), + node( + CONNECTION_KIND, + PROJECT_NODE_NAME, + vec![(DEFINITION_ASPECT, definition_digest(&borrowed))], + ), + )) +} + +/// The `.py` files directly in `directory`, sorted; none when it does not exist. +fn python_files(directory: &Path) -> PyResult> { + let entries: std::fs::ReadDir = match std::fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error.into()), + }; + let mut files: Vec = Vec::new(); + for entry in entries { + let path: PathBuf = entry?.path(); + if path.is_file() && path.to_string_lossy().ends_with(PYTHON_SUFFIX) { + files.push(path); + } + } + files.sort(); + Ok(files) +} + +/// Loaders, tasks, assets, checks and materializations by their Python modules and imports. +fn python_nodes( + discovered: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult> { + let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); + let collections = PYTHON_NODE_COLLECTIONS + .iter() + .zip(PYTHON_NODE_KINDS) + .map(|(collection, kind)| (*collection, kind)) + .chain([("materialization_files", MATERIALIZATION_KIND)]); + for (collection, kind) in collections { + for item in native_field(discovered, collection)?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let name: String = native_field(&item, "name")?.extract()?; + let path: PathBuf = native_field(&item, "file_path")?.extract()?; + nodes.push(( + node_key(kind, &name), + node( + kind, + &name, + vec![(DEFINITION_ASPECT, fingerprints.python_module_digest(&path))], + ), + )); + } + } + Ok(nodes) +} + +/// User-defined and table functions by their definitions, Python ones with their imports. +fn function_nodes( + project: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult> { + let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); + let functions: Bound<'_, PyAny> = project_reads::functions(project)?; + let hashes: Bound<'_, PyDict> = function_local_hashes(&functions, None)?; + for function in functions.try_iter()? { + let function: Bound<'_, PyAny> = function?; + let key: Bound<'_, PyAny> = function_reads::key(&function)?; + let kind: String = text_of(&native_field( + &native_field(&key, "resource_type")?, + "value", + )?)?; + let name: String = function_reads::name(&function)?.extract()?; + let Some(hash) = hashes.get_item(&name)? else { + continue; + }; + let mut digest: String = hash.extract()?; + let source_file: Bound<'_, PyAny> = function_reads::source_file_path(&function)?; + if !source_file.is_none() { + let path: String = text_of(&source_file)?; + if path.ends_with(PYTHON_SUFFIX) { + digest = definition_digest(&[ + &digest, + &fingerprints.python_module_digest(Path::new(&path)), + ]); + } + } + let mut definition: NodeDefinition = node(&kind, &name, vec![(DEFINITION_ASPECT, digest)]); + for dependency in function_reads::deps(&function)?.try_iter()? { + let dependency: Bound<'_, PyAny> = dependency?; + definition.uses.push(node_key( + &text_of(&native_field( + &native_field(&dependency, "resource_type")?, + "value", + )?)?, + &native_field(&dependency, "name")?.extract::()?, + )); + } + definition.uses.extend(project_wide_keys()); + nodes.push((node_key(&kind, &name), definition)); + } + Ok(nodes) +} + +/// Each unit test file by its text and the variables it reads. +fn test_file_nodes( + project: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult> { + let mut nodes: BTreeMap = BTreeMap::new(); + for test in project_reads::sql_tests(project)?.try_iter()? { + let file: Bound<'_, PyAny> = native_field(&test?, "test_file")?; + let path: String = text_of(&native_field(&file, "relative_path")?)?; + let key: String = node_key(TEST_FILE_KIND, &path); + if nodes.contains_key(&key) { + continue; + } + let contents: String = native_field(&file, "contents")?.extract()?; + let mut aspects: Vec<(&str, String)> = vec![( + FILE_ASPECT, + file_digest(&fingerprints.project_dir.join(&path)), + )]; + if let Some(digest) = fingerprints.referenced_vars_digest(&contents) { + aspects.push((VARS_ASPECT, digest)); + } + nodes.insert(key, node(TEST_FILE_KIND, &path, aspects)); + } + Ok(nodes) +} + +/// Each scenario by its file, captured snapshots and the variables it reads. +fn scenario_nodes( + project: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, + uses: &BTreeMap>, +) -> PyResult> { + let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); + let snapshot_root: PathBuf = SCENARIO_SNAPSHOT_ROOT + .iter() + .fold(fingerprints.project_dir.clone(), |path, part| { + path.join(part) + }); + for scenario in native_field(project, "sql_scenarios")?.try_iter()? { + let scenario: Bound<'_, PyAny> = scenario?; + let name: String = native_field(&scenario, "name")?.extract()?; + let path: String = text_of(&native_field( + &native_field(&scenario, "scenario_file")?, + "relative_path", + )?)?; + let file: PathBuf = fingerprints.project_dir.join(&path); + let mut aspects: Vec<(&str, String)> = vec![(FILE_ASPECT, file_digest(&file))]; + if let Some(digest) = directory_digest(&snapshot_root.join(&name)) { + aspects.push((SNAPSHOTS_ASPECT, digest)); + } + let contents: String = match std::fs::read_to_string(&file) { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(error.into()), + }; + if let Some(digest) = fingerprints.referenced_vars_digest(&contents) { + aspects.push((VARS_ASPECT, digest)); + } + let key: String = node_key(SCENARIO_KIND, &name); + let mut definition: NodeDefinition = node(SCENARIO_KIND, &name, aspects); + definition.uses = uses.get(&key).into_iter().flatten().cloned().collect(); + nodes.push((key, definition)); + } + Ok(nodes) +} + +/// The definition snapshot of `project`, discovered as `discovered` under `project_dir`. +pub(crate) fn definition_snapshot( + project: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, + project_dir: &Path, +) -> PyResult { + let fingerprints: Fingerprinting = Fingerprinting::new( + project, + project_dir, + &native_field(project, "effective_vars")?, + )?; + let uses: BTreeMap> = declaration_uses(project)?; + let mut nodes: BTreeMap = declaration_nodes(project, &fingerprints)?; + nodes.insert( + node_key(SETTINGS_KIND, PROJECT_NODE_NAME), + node( + SETTINGS_KIND, + PROJECT_NODE_NAME, + vec![( + DEFINITION_ASPECT, + fingerprints.canonical_digest( + &native_field(project, "settings")?, + (&RUNTIME_SETTINGS, &[]), + )?, + )], + ), + ); + let (connection_key, connection) = connection_node(project, discovered, &fingerprints)?; + nodes.insert(connection_key, connection); + for model in project_reads::models(project)?.try_iter()? { + let (key, definition) = model_node(&model?, &fingerprints, &uses)?; + nodes.insert(key, definition); + } + for source in project_reads::sources(project)?.try_iter()? { + let (key, definition) = source_node(&source?, &fingerprints)?; + nodes.insert(key, definition); + } + for seed in project_reads::seeds(project)?.try_iter()? { + let (key, definition) = seed_node(&seed?, &fingerprints)?; + nodes.insert(key, definition); + } + nodes.extend(function_nodes(project, &fingerprints)?); + nodes.extend(python_nodes(discovered, &fingerprints)?); + nodes.extend(test_file_nodes(project, &fingerprints)?); + nodes.extend(scenario_nodes(project, &fingerprints, &uses)?); + Ok(DefinitionSnapshot { + format: DEFINITION_SNAPSHOT_FORMAT, + nodes, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs new file mode 100644 index 0000000000..a675d91201 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs @@ -0,0 +1,59 @@ +//! Project-wide inputs compared outside the cached base snapshot: Python dependency files. + +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::definition_changes::constants::{ + DEFINITION_ASPECT, DEPENDENCIES_KIND, PROJECT_NODE_NAME, +}; +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; +use sqlbuild_analysis::definition_changes::main::node_key::node_key; +use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; +use sqlbuild_core::git_revisions::main::base_dependency_files::base_dependency_files; +use sqlbuild_core::git_revisions::main::current_dependency_files::current_dependency_files; +use sqlbuild_core::git_revisions::models::GitBase; + +use crate::bindings::_helpers::change_selection::base_snapshot::git_error; + +fn add_dependencies( + snapshot: &DefinitionSnapshot, + files: &[(String, String)], +) -> DefinitionSnapshot { + let parts: Vec<&str> = files + .iter() + .flat_map(|(path, text)| [path.as_str(), text.as_str()]) + .collect(); + let mut updated: DefinitionSnapshot = snapshot.clone(); + updated.nodes.insert( + node_key(DEPENDENCIES_KIND, PROJECT_NODE_NAME), + NodeDefinition { + kind: DEPENDENCIES_KIND.to_owned(), + name: PROJECT_NODE_NAME.to_owned(), + aspects: [(DEFINITION_ASPECT.to_owned(), definition_digest(&parts))].into(), + ..NodeDefinition::default() + }, + ); + updated +} + +/// `snapshot` of the base revision with its dependency files. +pub(crate) fn with_base_dependencies( + py: Python<'_>, + base: &GitBase, + snapshot: &DefinitionSnapshot, +) -> PyResult { + let files: Vec<(String, String)> = py + .detach(|| base_dependency_files(base)) + .map_err(|error| git_error(py, error))?; + Ok(add_dependencies(snapshot, &files)) +} + +/// `snapshot` of the working tree with its dependency files. +pub(crate) fn with_current_dependencies( + py: Python<'_>, + base: &GitBase, + snapshot: &DefinitionSnapshot, +) -> PyResult { + let files: Vec<(String, String)> = py + .detach(|| current_dependency_files(base)) + .map_err(|error| git_error(py, error))?; + Ok(add_dependencies(snapshot, &files)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs new file mode 100644 index 0000000000..8d983c663e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs @@ -0,0 +1,329 @@ +//! Stable digests of compiled records, authored files, project variables and Python modules. + +use std::cell::RefCell; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, PyTuple, + PyTypeMethods, +}; +use serde_json::{Map, Value}; +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; +use sqlbuild_pysource::rule_evidence::main::project_import_closure::project_import_closure; + +use crate::bindings::_helpers::records::models::NativeRecord; + +/// Fields that document a definition without changing what it does. +const DOCUMENTATION_FIELDS: [&str; 2] = ["description", "location"]; +/// Python source that mentions this can read project variables through `ctx.vars`. +const VARS_ATTRIBUTE: &str = "vars"; +const VARIABLE_SIGIL: &str = "@@"; +const ENVIRONMENT_VARIABLE: &str = "ENV"; +/// How SQL and configuration read an environment variable. +const ENVIRONMENT_MARKERS: [&str; 2] = ["@@ENV:", "${ENV:"]; + +/// What fingerprints of one project revision need: where it lives and its variables. +pub(crate) struct Fingerprinting { + pub(crate) project_dir: PathBuf, + /// Spellings of the project directory, stripped from values so both revisions compare. + roots: Vec, + /// Effective project variables as canonical JSON, by name. + vars: BTreeMap, + python_version: (u8, u8), + /// Python module digests already computed, by module path; many macros share one file. + module_digests: RefCell>, +} + +impl Fingerprinting { + pub(crate) fn new( + project: &Bound<'_, PyAny>, + project_dir: &Path, + effective_vars: &Bound<'_, PyAny>, + ) -> PyResult { + let py: Python<'_> = project.py(); + let mut roots: Vec = Vec::new(); + let canonical: Option = match std::fs::canonicalize(project_dir) { + Ok(resolved) => Some(resolved), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + for root in canonical + .into_iter() + .chain(std::iter::once(project_dir.to_path_buf())) + { + let text: String = format!("{}/", root.to_string_lossy().replace('\\', "/")); + if !roots.contains(&text) { + roots.push(text); + } + } + roots.sort_by_key(|root| std::cmp::Reverse(root.len())); + let mut fingerprinting: Self = Self { + project_dir: project_dir.to_path_buf(), + roots, + vars: BTreeMap::new(), + python_version: (py.version_info().major, py.version_info().minor), + module_digests: RefCell::new(BTreeMap::new()), + }; + if let Ok(values) = effective_vars.cast::() { + for (name, value) in values.iter() { + let encoded: String = fingerprinting.canonical(&value, &[])?.to_string(); + fingerprinting.vars.insert(name.str()?.to_string(), encoded); + } + } + Ok(fingerprinting) + } + + fn relative(&self, text: &str) -> String { + let mut text: String = text.replace('\\', "/"); + for root in &self.roots { + text = text.replace(root.as_str(), ""); + } + text + } + + /// A stable JSON form of a Python value, without documentation fields or `skipped` fields. + pub(crate) fn canonical(&self, value: &Bound<'_, PyAny>, skipped: &[&str]) -> PyResult { + if value.is_none() { + return Ok(Value::Null); + } + if let Ok(cell) = value.cast::() { + let record: &NativeRecord = cell.get(); + let mut fields: Map = Map::new(); + fields.insert("record".to_owned(), Value::from(record.spec.name)); + for (spec, item) in record.spec.fields.iter().zip(&record.values) { + if DOCUMENTATION_FIELDS.contains(&spec.name) || skipped.contains(&spec.name) { + continue; + } + fields.insert( + spec.name.to_owned(), + self.canonical(item.bind(value.py()), skipped)?, + ); + } + return Ok(Value::Object(fields)); + } + if value.is_instance_of::() { + return Ok(Value::from(value.extract::()?)); + } + if value.is_instance_of::() { + return Ok(Value::from(self.relative(&value.extract::()?))); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(Value::from(value.repr()?.to_string())); + } + if value.is_instance_of::() || value.is_instance_of::() { + let items: Vec = value + .try_iter()? + .map(|item| self.canonical(&item?, skipped)) + .collect::>()?; + return Ok(Value::Array(items)); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = value + .try_iter()? + .map(|item| Ok(self.canonical(&item?, skipped)?.to_string())) + .collect::>()?; + items.sort(); + return Ok(Value::from(items)); + } + if let Ok(mapping) = value.cast::() { + let mut entries: BTreeMap = BTreeMap::new(); + for (key, item) in mapping.iter() { + entries.insert(key.str()?.to_string(), self.canonical(&item, skipped)?); + } + return Ok(Value::Object(entries.into_iter().collect())); + } + let text: String = value.repr()?.to_string(); + if text.contains(" at 0x") { + return Ok(Value::from(value.get_type().name()?.to_string())); + } + Ok(Value::from(self.relative(&text))) + } + + /// The digest of `value`'s canonical form, without the top-level record fields `top_skipped`. + pub(crate) fn canonical_digest( + &self, + value: &Bound<'_, PyAny>, + (top_skipped, skipped): (&[&str], &[&str]), + ) -> PyResult { + let mut encoded: Value = self.canonical(value, skipped)?; + if let Value::Object(fields) = &mut encoded { + for field in top_skipped { + fields.remove(*field); + } + } + Ok(definition_digest(&[&encoded.to_string()])) + } + + /// The digest of the variables `names`, each by its effective value or its absence. + fn vars_digest<'a>(&self, names: impl Iterator) -> String { + let mut parts: Vec<&str> = Vec::new(); + for name in names { + parts.push(name); + parts.push(self.vars.get(name).map_or("missing", String::as_str)); + } + definition_digest(&parts) + } + + /// The digest of the project and environment variables `text` reads, or `None` without any. + pub(crate) fn referenced_vars_digest(&self, text: &str) -> Option { + let names: BTreeSet = referenced_vars(text); + let environment: BTreeSet = referenced_environment(text); + if names.is_empty() && environment.is_empty() { + return None; + } + Some(definition_digest(&[ + &self.vars_digest(names.iter()), + &environment_digest(&environment), + ])) + } + + /// A Python file and its imported project modules; code mentioning `vars` covers every variable. + pub(crate) fn python_module_digest(&self, path: &Path) -> String { + let source: PathBuf = if path.is_absolute() { + path.to_path_buf() + } else { + self.project_dir.join(path) + }; + if let Some(digest) = self.module_digests.borrow().get(&source) { + return digest.clone(); + } + let mut files: Vec = vec![source.clone()]; + files.extend(project_import_closure( + &source, + &self.project_dir, + self.python_version, + )); + let mut parts: Vec = Vec::new(); + let mut reads_vars: bool = false; + for file in files { + let contents: String = std::fs::read(&file) + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .unwrap_or_else(|_| "missing".to_owned()); + reads_vars |= contents.contains(VARS_ATTRIBUTE); + parts.push(self.relative(&file.to_string_lossy())); + parts.push(definition_digest(&[&contents])); + } + if reads_vars { + parts.push(self.vars_digest(self.vars.keys())); + } + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + let digest: String = definition_digest(&borrowed); + self.module_digests + .borrow_mut() + .insert(source, digest.clone()); + digest + } +} + +/// Names of the project variables `text` reads with `@@name`, without runtime `@@@` and `@@ENV:`. +fn referenced_vars(text: &str) -> BTreeSet { + let mut names: BTreeSet = BTreeSet::new(); + let mut rest: &str = text; + while let Some(index) = rest.find(VARIABLE_SIGIL) { + let preceded: bool = rest[..index].ends_with('@'); + let after: &str = &rest[index + VARIABLE_SIGIL.len()..]; + rest = after.trim_start_matches('@'); + if preceded || after.starts_with('@') { + continue; + } + let name: String = after + .chars() + .take_while(|character| character.is_ascii_alphanumeric() || *character == '_') + .collect(); + if name.is_empty() || (name == ENVIRONMENT_VARIABLE && after[name.len()..].starts_with(':')) + { + continue; + } + names.insert(name); + } + names +} + +/// Names of the environment variables `text` reads with `@@ENV:NAME` or `${ENV:NAME}`. +pub(crate) fn referenced_environment(text: &str) -> BTreeSet { + let mut names: BTreeSet = BTreeSet::new(); + for marker in ENVIRONMENT_MARKERS { + for (index, _) in text.match_indices(marker) { + let name: String = text[index + marker.len()..] + .chars() + .take_while(|character| character.is_ascii_alphanumeric() || *character == '_') + .collect(); + if !name.is_empty() { + names.insert(name); + } + } + } + names +} + +/// The value of environment variable `name` in this process. +pub(crate) fn process_environment(name: &str) -> Option { + std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) +} + +/// The digest of the environment variables `names` by their current values, which it never shows. +pub(crate) fn environment_digest(names: &BTreeSet) -> String { + environment_digest_with(names, &process_environment) +} + +/// The digest of the environment variables `names` by the values `lookup` gives them. +pub(crate) fn environment_digest_with( + names: &BTreeSet, + lookup: &dyn Fn(&str) -> Option, +) -> String { + let mut parts: Vec = Vec::new(); + for name in names { + parts.push(name.clone()); + parts.push(match lookup(name) { + Some(value) => definition_digest(&[&value]), + None => "unset".to_owned(), + }); + } + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + definition_digest(&borrowed) +} + +pub(crate) fn text_of(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string().replace('\\', "/")) +} + +/// The digest of a file's bytes, or of its absence. +pub(crate) fn file_digest(path: &Path) -> String { + match std::fs::read(path) { + Ok(bytes) => definition_digest(&[&String::from_utf8_lossy(&bytes)]), + Err(_) => definition_digest(&["missing"]), + } +} + +/// The digest of every file below `directory`, by relative path, or `None` when it is absent. +pub(crate) fn directory_digest(directory: &Path) -> Option { + let mut files: Vec<(String, String)> = Vec::new(); + let mut pending: Vec = vec![directory.to_path_buf()]; + while let Some(next) = pending.pop() { + let entries: std::fs::ReadDir = match std::fs::read_dir(&next) { + Ok(entries) => entries, + Err(_) => return None, + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + if path.is_dir() { + pending.push(path); + } else { + let relative: String = path + .strip_prefix(directory) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + files.push((relative, file_digest(&path))); + } + } + } + files.sort(); + let parts: Vec<&str> = files + .iter() + .flat_map(|(path, digest)| [path.as_str(), digest.as_str()]) + .collect(); + Some(definition_digest(&parts)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs new file mode 100644 index 0000000000..ab0bd562ee --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod base_cache; +pub(crate) mod base_compile; +pub(crate) mod base_snapshot; +pub(crate) mod definition_snapshot; +pub(crate) mod environment_nodes; +pub(crate) mod fingerprints; +pub(crate) mod registration; +pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/registration.rs new file mode 100644 index 0000000000..9c99d47c7e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/registration.rs @@ -0,0 +1,9 @@ +//! Register the base-revision compile entry that `--since` runs in a separate interpreter. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::change_selection::base_compile; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + base_compile::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs new file mode 100644 index 0000000000..33a5191efa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -0,0 +1,341 @@ +//! Which tests and scenarios a `--since` run selects, and why, from what each really executes. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::Path; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::Value; +use sqlbuild_analysis::definition_changes::constants::{ + MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, + TEST_FILE_KIND, +}; +use sqlbuild_analysis::definition_changes::main::changes_summary::changes_summary; +use sqlbuild_analysis::definition_changes::main::compare_definitions::compare_definitions; +use sqlbuild_analysis::definition_changes::main::node_key::node_key; +use sqlbuild_analysis::definition_changes::main::selection_json::selection_json; +use sqlbuild_analysis::definition_changes::main::selection_reasons::selection_reasons; +use sqlbuild_analysis::definition_changes::main::selection_report::selection_report; +use sqlbuild_analysis::definition_changes::models::{ + DefinitionChanges, DefinitionSnapshot, ExecutedItem, SelectedItem, +}; + +use crate::bindings::_helpers::change_selection::base_snapshot::BaseComparison; +use crate::bindings::_helpers::change_selection::definition_snapshot::{ + declaration_uses, definition_snapshot, project_wide_keys, +}; +use crate::bindings::_helpers::change_selection::environment_nodes::with_current_dependencies; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; +use crate::bindings::_helpers::graph::keys::native_key; +use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::scenario_command::setup::{flush, write}; + +const MACRO_TEST_MODE: &str = "macro"; +/// The JSON document key of the changes and selection of a `--since` run. +const SINCE_SECTION: &str = "since"; +/// The relation call a test or scenario file reads a source with. +const SOURCE_CALL: &str = "__source("; + +/// Compiled tests or models by name. +pub(crate) type ByName<'py> = HashMap>; +const TEST_CONSUMER_KIND: &str = "test"; + +/// The changes since the base revision and what the current revision uses. +pub(crate) struct ChangedProject { + pub(crate) reference: String, + merge_base: String, + pub(crate) current: DefinitionSnapshot, + pub(crate) changes: DefinitionChanges, + uses: BTreeMap>, +} + +/// Compare the compiled working tree `project`, discovered as `discovered`, with the base snapshot. +pub(crate) fn changed_project( + (project, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + project_dir: &Path, + base: BaseComparison, +) -> PyResult { + let current: DefinitionSnapshot = with_current_dependencies( + project.py(), + &base.base, + &definition_snapshot(project, discovered, project_dir)?, + )?; + let changes: DefinitionChanges = compare_definitions(&base.snapshot, ¤t); + Ok(ChangedProject { + reference: base.base.reference, + merge_base: base.base.merge_base, + current, + changes, + uses: declaration_uses(project)?, + }) +} + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() +} + +/// Names of the sources `sql` reads with `__source("name")`. +fn source_calls(sql: &str) -> Vec { + let mut names: Vec = Vec::new(); + for (index, _) in sql.match_indices(SOURCE_CALL) { + let rest: &str = sql[index + SOURCE_CALL.len()..].trim_start(); + let Some(quote) = rest + .chars() + .next() + .filter(|quote| *quote == '"' || *quote == '\'') + else { + continue; + }; + if let Some(end) = rest[1..].find(quote) { + names.push(rest[1..=end].to_owned()); + } + } + names +} + +/// Node keys of mocked models, sources and seeds. +fn relation_keys((refs, sources, seeds): (&[String], &[String], &[String])) -> Vec { + let models = refs.iter().map(|name| node_key(MODEL_KIND, name)); + let sources = sources.iter().map(|name| node_key(SOURCE_KIND, name)); + let seeds = seeds.iter().map(|name| node_key(SEED_KIND, name)); + models.chain(sources).chain(seeds).collect() +} + +/// The sources `model` reads, other than `mocked` ones, as node keys. +fn read_sources(model: &Bound<'_, PyAny>, mocked: &[String]) -> PyResult> { + let mut keys: Vec = Vec::new(); + for dependency in model_reads::deps(model)?.try_iter()? { + let (kind, name) = native_key(&dependency?)?; + if kind == SOURCE_KIND && !mocked.contains(&name) { + keys.push(node_key(SOURCE_KIND, &name)); + } + } + Ok(keys) +} + +impl ChangedProject { + /// Keys of current nodes of `kind` named in `names`. + fn keys_named(&self, kind: &str, names: &[String]) -> BTreeSet { + self.current + .nodes + .iter() + .filter(|(_, node)| node.kind == kind && names.contains(&node.name)) + .map(|(key, _)| key.clone()) + .collect() + } + + fn declarations_of(&self, consumer: &str) -> Vec { + self.uses + .get(consumer) + .into_iter() + .flatten() + .cloned() + .collect() + } + + /// Why `item` is affected; empty when nothing it runs changed. + pub(crate) fn reasons(&self, item: &ExecutedItem) -> Vec { + selection_reasons(item, &self.current, &self.changes) + } + + /// The `since` section of the run's JSON document. + pub(crate) fn json(&self, (total, selected): (usize, &[SelectedItem])) -> String { + selection_json( + (&self.reference, &self.merge_base), + &self.changes, + (total, selected), + ) + .to_string() + } + + /// Write the change summary and the selection to `stream`. + pub(crate) fn report( + &self, + stream: &Bound<'_, PyAny>, + nouns: (&str, &str), + (total, selected, explain): (usize, &[SelectedItem], bool), + ) -> PyResult<()> { + write( + stream, + &selection_report( + (&self.reference, &changes_summary(&self.changes)), + nouns, + (total, selected), + explain, + ), + )?; + flush(stream) + } + + /// What one planned unit test executes, honouring its mocks. + pub(crate) fn test_item( + &self, + entry: &Bound<'_, PyAny>, + (tests, models): (&ByName<'_>, &ByName<'_>), + ) -> PyResult { + let name: String = native_field(entry, "name")?.extract()?; + let parent: Option = native_field(entry, "parent_name")?.extract()?; + let mut item: ExecutedItem = ExecutedItem { + declarations: project_wide_keys(), + ..ExecutedItem::default() + }; + let mocked_seeds: Vec = strings(&native_field(entry, "mock_seed_names")?)?; + let mocked_sources: Vec = strings(&native_field(entry, "mock_source_names")?)?; + let mocked_refs: Vec = strings(&native_field(entry, "mock_ref_names")?)?; + let mocked_functions: Vec = + strings(&native_field(entry, "mock_table_function_names")?)?; + item.mocked + .extend(self.keys_named(TABLE_FUNCTION_KIND, &mocked_functions)); + item.mocked_relations = relation_keys((&mocked_refs, &mocked_sources, &mocked_seeds)); + let mut consumers: Vec = vec![name.clone()]; + consumers.extend(parent.clone()); + let compiled: Option<&Bound<'_, PyAny>> = tests + .get(&name) + .or_else(|| parent.as_ref().and_then(|parent| tests.get(parent))); + if let Some(test) = compiled { + let file: Bound<'_, PyAny> = native_field(test, "test_file")?; + let path: Bound<'_, PyAny> = native_field(&file, "relative_path")?; + item.own_keys.push(node_key( + TEST_FILE_KIND, + &path.str()?.to_string().replace('\\', "/"), + )); + consumers.push(native_field(&path, "stem")?.extract()?); + for block in native_field(&file, "blocks")?.try_iter()? { + let block_name: Option = native_field(&block?, "name")?.extract()?; + consumers.extend(block_name); + } + let contents: String = native_field(&file, "contents")?.extract()?; + for name in source_calls(&contents) { + if !mocked_sources.contains(&name) { + item.executed.push(node_key(SOURCE_KIND, &name)); + } + } + let payload: Bound<'_, PyAny> = native_field(test, "payload")?; + if let Ok(mocks) = native_field(&payload, "macro_mocks") + && let Ok(mocks) = mocks.cast::() + { + let names: Vec = mocks + .iter() + .map(|(key, _)| key.extract()) + .collect::>()?; + item.mocked.extend(self.keys_named(MACRO_KIND, &names)); + } + if let Ok(mode) = native_field(&payload, "mode") + && native_field(&mode, "value")?.extract::()? == MACRO_TEST_MODE + { + let tested: Vec = + strings(&native_field(&payload, "tested_resource_names")?)?; + item.declarations + .extend(self.keys_named(MACRO_KIND, &tested)); + } + } + for step in native_field(entry, "chain")?.try_iter()? { + let model_name: String = native_field(&step?, "model_name")?.extract()?; + item.executed.push(node_key(MODEL_KIND, &model_name)); + let Some(model) = models.get(&model_name) else { + continue; + }; + item.executed.extend(read_sources(model, &mocked_sources)?); + for dependency in model_reads::deps(model)?.try_iter()? { + let (kind, dependency_name) = native_key(&dependency?)?; + if kind == SEED_KIND && !mocked_seeds.contains(&dependency_name) { + item.executed.push(node_key(SEED_KIND, &dependency_name)); + } + } + } + for key in native_field(entry, "function_deps")?.try_iter()? { + let (kind, function_name) = native_key(&key?)?; + item.executed.push(node_key(&kind, &function_name)); + } + for consumer in consumers { + item.declarations + .extend(self.declarations_of(&node_key(TEST_CONSUMER_KIND, &consumer))); + } + Ok(item) + } + + /// What one compiled scenario builds and loads, honouring its fixtures. + pub(crate) fn scenario_item( + &self, + scenario: &Bound<'_, PyAny>, + (project, models): (&Bound<'_, PyAny>, &ByName<'_>), + ) -> PyResult { + let name: String = native_field(scenario, "name")?.extract()?; + let key: String = node_key(SCENARIO_KIND, &name); + let (graph, _) = plan_scenario_graph(scenario, project, true, None)?; + let mut declarations: Vec = project_wide_keys(); + declarations.extend(self.declarations_of(&key)); + let mut item: ExecutedItem = ExecutedItem { + own_keys: vec![key.clone()], + declarations, + ..ExecutedItem::default() + }; + let source_fixtures: Vec = strings(&native_field(&graph, "source_fixture_names")?)?; + item.mocked_relations = relation_keys(( + &strings(&native_field(&graph, "ref_fixture_names")?)?, + &source_fixtures, + &strings(&native_field(&graph, "seed_fixture_names")?)?, + )); + for name in source_calls(&native_field(scenario, "sql_body")?.extract::()?) { + if !source_fixtures.contains(&name) { + item.executed.push(node_key(SOURCE_KIND, &name)); + } + } + for model_name in strings(&native_field(&graph, "model_names")?)? { + item.executed.push(node_key(MODEL_KIND, &model_name)); + if let Some(model) = models.get(&model_name) { + item.executed.extend(read_sources(model, &source_fixtures)?); + } + } + let fixtures: Vec = strings(&native_field(&graph, "seed_fixture_names")?)?; + for seed_name in strings(&native_field(&graph, "seed_names")?)? { + if !fixtures.contains(&seed_name) { + item.executed.push(node_key(SEED_KIND, &seed_name)); + } + } + for function in native_field(&graph, "function_deps")?.try_iter()? { + let (kind, function_name) = native_key(&function?)?; + item.executed.push(node_key(&kind, &function_name)); + } + Ok(item) + } +} + +/// `payload`, a JSON document, with the `since` section added when the run used `--since`. +pub(crate) fn with_since_section(payload: String, since: Option<&str>) -> PyResult { + let Some(since) = since else { + return Ok(payload); + }; + let invalid = |error: serde_json::Error| PyValueError::new_err(error.to_string()); + let mut document: Value = serde_json::from_str(&payload).map_err(invalid)?; + if let Value::Object(fields) = &mut document { + fields.insert( + SINCE_SECTION.to_owned(), + serde_json::from_str(since).map_err(invalid)?, + ); + } + Ok(format!( + "{}\n", + serde_json::to_string_pretty(&document).map_err(invalid)? + )) +} + +/// The project's compiled tests and models by name. +pub(crate) fn tests_and_models<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult<(ByName<'py>, ByName<'py>)> { + let mut tests: ByName<'py> = HashMap::new(); + for test in project_reads::sql_tests(project)?.try_iter()? { + let test: Bound<'py, PyAny> = test?; + tests.insert(native_field(&test, "name")?.extract()?, test); + } + let mut models: ByName<'py> = HashMap::new(); + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + models.insert(model_reads::model_name(&model)?, model); + } + Ok((tests, models)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 84a52f27ab..f829d01f1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -5,12 +5,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{progress, progress_reporters, progress_text}; use crate::bindings::_helpers::cli::{argv, entry, launch_state, namespace, observability}; use crate::bindings::_helpers::{ - audit_command, build_command, build_progress, check, clone, command_runtime, compute_logs, - contract_command, cost_command, cost_report, cost_telemetry, dbt_command, dbt_events, - dbt_interop, debug_command, diff_report, event_exporting, execution_history, format_command, - freshness, init_command, janitor, load, output_capture, plan_command, presentation_text, - query_command, recall, result_output, scenario_command, seed_command, sql_test_command, - workspace_commands, + audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, + compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, + dbt_events, dbt_interop, debug_command, diff_report, event_exporting, execution_history, + format_command, freshness, init_command, janitor, load, output_capture, plan_command, + presentation_text, query_command, recall, result_output, scenario_command, seed_command, + sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -43,6 +43,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { workspace_commands::registration::register(module)?; check::registration::register(module)?; sql_test_command::registration::register(module)?; + change_selection::registration::register(module)?; seed_command::registration::register(module)?; audit_command::registration::register(module)?; build_command::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs index 6d105a3a13..e695e2ed5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{ use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_config::project::errors::{ConfigFailure, FailureClass, LoadFailure}; +use sqlbuild_config::project::main::load_git_base_ref::load_git_base_ref; use sqlbuild_config::project::main::load_local_config::load_local_config; use sqlbuild_config::project::main::load_project_config::load_project_config; use sqlbuild_config::project::main::local_sql_analysis_failure::local_sql_analysis_failure; @@ -173,6 +174,13 @@ pub(crate) fn load_project_config_file( } } +/// `[git] base_ref` of the project at `project_dir`, which `--since @base` names. +pub(crate) fn configured_git_base_ref(project_dir: &Bound<'_, PyAny>) -> PyResult> { + let files = files(project_dir)?; + load_git_base_ref(&files.native) + .or_else(|failure| Err(load_error(project_dir.py(), &failure, &files)?)) +} + /// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. #[pyfunction] pub(crate) fn load_local_config_file( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 3f78468b90..ae28284cad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod authored_values; pub(crate) mod build_command; pub(crate) mod build_progress; pub(crate) mod cache; +pub(crate) mod change_selection; pub(crate) mod check; pub(crate) mod cli; pub(crate) mod clone; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index 5296f7a525..3bacaebca1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -6,7 +6,7 @@ mod constants; mod entries; pub(crate) mod execution; mod fixtures; -mod graph; +pub(crate) mod graph; mod helper_scope; mod models; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 5ff12ba1d6..dad59be8f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -584,6 +584,8 @@ const SCENARIO_TEST_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("json_output", FieldDefault::False), FieldSpec::new("json_output_path", FieldDefault::None), FieldSpec::new("scenario_timeout", FieldDefault::None), + FieldSpec::new("since", FieldDefault::None), + FieldSpec::new("explain", FieldDefault::False), ], }; const SCENARIO_CAPTURE_COMMAND_REQUEST: RecordSpec = RecordSpec { @@ -648,6 +650,8 @@ const TEST_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("case_name", FieldDefault::None), FieldSpec::new("inspect", FieldDefault::False), FieldSpec::new("test_timeout", FieldDefault::None), + FieldSpec::new("since", FieldDefault::None), + FieldSpec::new("explain", FieldDefault::False), ], }; const REFACTOR_COMMAND_REQUEST: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 700f55b4fe..0f4c8df59c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -6,6 +6,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::definition_changes::models::{ExecutedItem, SelectedItem}; use sqlbuild_core::cli_text::main::format_summary_footer::format_summary_footer; use sqlbuild_core::cli_text::models::{CliStyle, SummaryCount}; use sqlbuild_core::scenario_command::main::render_scenario_failures::render_scenario_failures; @@ -17,6 +18,12 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::change_selection::base_snapshot::{ + BaseComparison, base_comparison, base_variant, +}; +use crate::bindings::_helpers::change_selection::selection::{ + ChangedProject, changed_project, tests_and_models, with_since_section, +}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -53,6 +60,8 @@ pub(crate) struct TestRun<'py> { pub(crate) stream: Bound<'py, PyAny>, pub(crate) style: CliStyle, pub(crate) concurrency: i64, + /// The `since` JSON section of a `--since` run. + pub(crate) since_json: Option, } fn flag(request: &Bound<'_, PyAny>, name: &str) -> PyResult { @@ -123,6 +132,7 @@ fn prepare<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = PyString::new( py, - &format_scenario_execution_json( - py, - Some(keywords( + &with_since_section( + format_scenario_execution_json( py, - &[ - ("results", results.clone().into_any()), - ("local", boolean(py, run.local)), - ( - "run_namespace", - optional_text(py, namespace.value.as_deref()), - ), - ("namespace_source", text(py, &namespace.source)), - ( - "duration_ms", - pyo3::types::PyFloat::new(py, elapsed_seconds * 1000.0).into_any(), - ), - ], - )?), + Some(keywords( + py, + &[ + ("results", results.clone().into_any()), + ("local", boolean(py, run.local)), + ( + "run_namespace", + optional_text(py, namespace.value.as_deref()), + ), + ("namespace_source", text(py, &namespace.source)), + ( + "duration_ms", + pyo3::types::PyFloat::new(py, elapsed_seconds * 1000.0).into_any(), + ), + ], + )?), + )?, + run.since_json.as_deref(), )?, ) .into_any(); @@ -335,6 +348,46 @@ fn run_scenarios( Ok(if ok { 0 } else { 1 }) } +/// Keep only the scenarios that execute something changed since the base revision. +fn select_changed<'py>( + run: &TestRun<'py>, + (pipeline, scenarios): (&Bound<'py, PyAny>, Bound<'py, PyTuple>), + base: BaseComparison, + explain: bool, +) -> PyResult<(Bound<'py, PyTuple>, String)> { + let py: Python<'py> = pipeline.py(); + let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; + let changed: ChangedProject = changed_project( + (&project, &run.project.discovered), + &run.project.project_dir.extract::()?, + base, + )?; + let (_, models) = tests_and_models(&project)?; + let mut kept: Vec> = Vec::new(); + let mut selected: Vec = Vec::new(); + for scenario in scenarios.iter() { + let item: ExecutedItem = changed.scenario_item(&scenario, (&project, &models))?; + let reasons: Vec = changed.reasons(&item); + if reasons.is_empty() { + continue; + } + selected.push(SelectedItem { + label: native_field(&scenario, "name")?.extract()?, + reasons, + }); + kept.push(scenario); + } + changed.report( + &run.stream, + ("scenario", "scenarios"), + (scenarios.len(), &selected, explain), + )?; + Ok(( + PyTuple::new(py, kept)?, + changed.json((scenarios.len(), &selected)), + )) +} + /// Run one `sqb scenario test` request. #[pyfunction] #[pyo3(signature = (*, request))] @@ -342,7 +395,7 @@ pub(crate) fn run_scenario_test_command( py: Python<'_>, request: &Bound<'_, PyAny>, ) -> PyResult { - let run: TestRun<'_> = prepare(py, request)?; + let mut run: TestRun<'_> = prepare(py, request)?; let planning: Bound<'_, PyAny> = write_headers( &run.stream, &run.project, @@ -356,6 +409,15 @@ pub(crate) fn run_scenario_test_command( run.style, ), )?; + let since: Option = native_field(request, "since")?.extract()?; + let base: Option = match since.as_deref() { + Some(since) => Some(base_comparison( + (&run.stream, &run.project.project_dir), + since, + &base_variant(None, &py.None().into_bound(py))?, + )?), + None => None, + }; let no_sql_validation: bool = run.local || flag(request, "no_sql_validation")?; let compiled: PyResult> = compile( &run.project, @@ -374,8 +436,18 @@ pub(crate) fn run_scenario_test_command( finished?; let selectors: Vec = run.project.selectors.extract()?; let exclude: Vec = run.project.exclude.extract()?; - let scenarios: Bound<'_, PyTuple> = + let mut scenarios: Bound<'_, PyTuple> = selected_scenarios(&pipeline, (&selectors, &exclude, &run.project.project_dir))?; + if let Some(base) = base { + let since_json: String; + (scenarios, since_json) = select_changed( + &run, + (&pipeline, scenarios), + base, + flag(request, "explain")?, + )?; + run.since_json = Some(since_json); + } if run.local && (flag(request, "sync_snapshots")? || flag(request, "refresh")?) { let synced: LocalSync<'_> = sync_local_snapshots(request, &run, (&pipeline, &scenarios))?; if synced.exit_code != 0 { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 5e34f9a244..aff77775db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -1,12 +1,14 @@ //! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; +use std::path::PathBuf; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_analysis::definition_changes::models::{ExecutedItem, SelectedItem}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::test_command::main::render_test_completion::render_test_completion; use sqlbuild_core::test_command::main::render_test_inspection::render_test_inspection; @@ -28,6 +30,12 @@ use crate::bindings::_helpers::build_progress::progress_text::{ }; use crate::bindings::_helpers::build_progress::result_text::test_label; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::change_selection::base_snapshot::{ + BaseComparison, base_comparison, base_variant, +}; +use crate::bindings::_helpers::change_selection::selection::{ + ChangedProject, changed_project, tests_and_models, with_since_section, +}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; @@ -39,6 +47,7 @@ use crate::bindings::_helpers::planner_entries::command_plans::build_test_comman use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::result_output::documents::format_test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; @@ -49,6 +58,11 @@ use crate::bindings::_helpers::sql_test_command::timeouts::{ }; const PASS_OUTCOME: &str = "pass"; + +/// Why `--since` selected each test, by inspection label. +type SelectionReasons = HashMap>; +/// What a `--since` selection kept: the narrowed pipeline, reasons by label, and its JSON section. +type ChangedSelection<'py> = (Bound<'py, PyAny>, SelectionReasons, String); const EXPECT_LABEL: &str = "expect"; const UNKNOWN_GROUP: &str = "(unknown)"; const EXECUTION_COMMAND: &str = "sqb test"; @@ -385,14 +399,23 @@ fn inspection_entry(entry: &Bound<'_, PyAny>) -> PyResult { mocked_table_functions: strings(entry.getattr("mock_table_function_names")?)?, real_models, expected_models, + selection_reasons: Vec::new(), }) } /// Write the resolved plan without executing it; returns the exit code. -fn inspect(stream: &Bound<'_, PyAny>, plan_output: &Bound<'_, PyAny>) -> PyResult { +fn inspect( + stream: &Bound<'_, PyAny>, + plan_output: &Bound<'_, PyAny>, + reasons: &HashMap>, +) -> PyResult { let entries: Vec = native_field(plan_output, "test_entries")? .try_iter()? - .map(|entry| inspection_entry(&entry?)) + .map(|entry| { + let mut entry: TestInspectionEntry = inspection_entry(&entry?)?; + entry.selection_reasons = reasons.get(&entry.label).cloned().unwrap_or_default(); + Ok(entry) + }) .collect::>()?; let warnings: Vec = native_field(plan_output, "warnings")? .try_iter()? @@ -410,6 +433,50 @@ fn inspect(stream: &Bound<'_, PyAny>, plan_output: &Bound<'_, PyAny>) -> PyResul Ok(code) } +/// Keep the planned tests that execute something changed, with each one's reasons by label. +fn select_changed<'py>( + (pipeline, stream, discovered): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + project_dir: &std::path::Path, + base: BaseComparison, + explain: bool, +) -> PyResult> { + let py: Python<'py> = pipeline.py(); + let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; + let changed: ChangedProject = changed_project((&project, discovered), project_dir, base)?; + let (tests, models) = tests_and_models(&project)?; + let plan_output: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; + let mut kept: Vec> = Vec::new(); + let mut selected: Vec = Vec::new(); + let mut reasons: HashMap> = HashMap::new(); + let mut total: usize = 0; + for entry in native_field(&plan_output, "test_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + total += 1; + let item: ExecutedItem = changed.test_item(&entry, (&tests, &models))?; + let entry_reasons: Vec = changed.reasons(&item); + if entry_reasons.is_empty() { + continue; + } + let label: String = inspection_entry(&entry)?.label; + reasons.insert(label.clone(), entry_reasons.clone()); + selected.push(SelectedItem { + label, + reasons: entry_reasons, + }); + kept.push(entry); + } + changed.report(stream, ("test", "tests"), (total, &selected, explain))?; + let plan_output: Bound<'py, PyAny> = replaced( + &plan_output, + &[("test_entries", PyTuple::new(py, kept)?.into_any())], + )?; + Ok(( + replaced(pipeline, &[("plan_output", plan_output)])?, + reasons, + changed.json((total, &selected)), + )) +} + /// Write the ready header and run the tests with nested progress; returns the results. fn run_tests<'py>( invocation: &Bound<'py, PyAny>, @@ -551,7 +618,7 @@ pub(crate) fn timeout_record<'py>( fn complete( request: &Bound<'_, PyAny>, invocation: &Bound<'_, PyAny>, - results: &[Bound<'_, PyAny>], + (results, since_json): (&[Bound<'_, PyAny>], Option<&str>), ) -> PyResult { let py: Python<'_> = request.py(); let mut failures: Vec = Vec::new(); @@ -583,8 +650,11 @@ fn complete( )?; stream.call_method0("flush")?; let tuple: Bound<'_, PyAny> = PyTuple::new(py, results)?.into_any(); - let payload: Bound<'_, PyAny> = - PyString::new(py, &format_test_execution_json(py, tuple)?).into_any(); + let payload: Bound<'_, PyAny> = PyString::new( + py, + &with_since_section(format_test_execution_json(py, tuple)?, since_json)?, + ) + .into_any(); write_execution_json_output( py, payload, @@ -604,10 +674,39 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py &record_class(py, RecordId::TestInvocation)?, !inspect_only, )?; - let pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let project_dir: Bound<'_, PyAny> = native_field(&invocation, "effective_project_dir")?; + let since: Option = native_field(request, "since")?.extract()?; + let base: Option = match since.as_deref() { + Some(since) => Some(base_comparison( + (&stream, &project_dir), + since, + &base_variant( + native_field(request, "selected_target")?.extract()?, + &native_field(request, "cli_vars")?, + )?, + )?), + None => None, + }; + let mut pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; + let mut reasons: HashMap> = HashMap::new(); + let mut since_json: Option = None; + if let Some(base) = base { + let json: String; + (pipeline, reasons, json) = select_changed( + ( + &pipeline, + &stream, + &native_field(&invocation, "discovered_inputs")?, + ), + &project_dir.extract::()?, + base, + native_field(request, "explain")?.is_truthy()?, + )?; + since_json = Some(json); + } if inspect_only { - return inspect(&stream, &pipeline.getattr("plan_output")?); + return inspect(&stream, &pipeline.getattr("plan_output")?, &reasons); } let concurrency: i64 = resolve_concurrency( &request.getattr("concurrency")?, @@ -642,7 +741,7 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py &pipeline.getattr("plan_output")?, &PyTuple::new(py, &results)?.into_any(), )?; - complete(request, &invocation, &results) + complete(request, &invocation, (&results, since_json.as_deref())) } /// Add source and safe typed values only for parameterized test rows. diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs index 76ff959782..2dc7f9c480 100644 --- a/crates/sqlbuild-python/src/bindings/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -8,6 +8,8 @@ mod failure_cause; mod helpers; #[path = "tests/test_image.rs"] mod image; +#[path = "tests/test_since_cache.rs"] +mod since_cache; #[path = "tests/test_splice.rs"] mod splice; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index 24aa5c45cc..9df3e0dfcf 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -1,8 +1,11 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; use sqlbuild_analysis::assembly::analysis_session::models::DynamicFamily; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_config::models::ConfigValue; + +use crate::bindings::_helpers::change_selection::base_cache::cache_key_with; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; @@ -285,3 +288,27 @@ pub(crate) fn report_elsewhere() { .join() .expect("the reporting thread finishes"); } + +/// A project whose model reads `ORDERS_REGION` through `@@ENV:`. +pub(super) fn project_reading_environment() -> PathBuf { + let root: PathBuf = std::env::temp_dir() + .join(format!("sqlbuild-since-cache-{}", std::process::id())) + .join("project"); + let _ = std::fs::create_dir_all(root.join("models")); + let _ = std::fs::write( + root.join("models/orders.sql"), + "SELECT order_id FROM orders WHERE region = '@@ENV:ORDERS_REGION'\n", + ); + root +} + +/// The base cache key of `project` with environment variables `values`. +pub(super) fn cache_key_under(project: &Path, values: &[(&str, &str)]) -> String { + let lookup = |name: &str| -> Option { + values + .iter() + .find(|(candidate, _)| *candidate == name) + .map(|(_, value)| (*value).to_owned()) + }; + cache_key_with(project, (None, "{}", ""), &lookup).expect("the project reads") +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs b/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs new file mode 100644 index 0000000000..6f08b2ecb1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs @@ -0,0 +1,39 @@ +use std::path::PathBuf; + +use crate::bindings::tests::helpers::{cache_key_under, project_reading_environment}; +use crate::bindings::tests::test_types::CacheKeyTestCase; + +const BASE_ENVIRONMENT: [(&str, &str); 2] = [("ORDERS_REGION", "eu"), ("UNREAD_SETTING", "a")]; + +#[test] +fn given_referenced_environment_variable_when_keying_base_cache_then_its_value_counts() { + let project: PathBuf = project_reading_environment(); + let base_key: String = cache_key_under(&project, &BASE_ENVIRONMENT); + let test_cases = [ + CacheKeyTestCase { + description: "a referenced variable with a new value changes the key", + changed: &[("ORDERS_REGION", "us"), ("UNREAD_SETTING", "a")], + expected_same_key: false, + }, + CacheKeyTestCase { + description: "an unset referenced variable changes the key", + changed: &[("UNREAD_SETTING", "a")], + expected_same_key: false, + }, + CacheKeyTestCase { + description: "a variable the project does not read leaves the key alone", + changed: &[("ORDERS_REGION", "eu"), ("UNREAD_SETTING", "b")], + expected_same_key: true, + }, + ]; + for test_case in test_cases { + let changed_key: String = cache_key_under(&project, test_case.changed); + + assert_eq!( + changed_key == base_key, + test_case.expected_same_key, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs index e3f1a5535b..af538f698a 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -71,3 +71,10 @@ pub(crate) struct FailureCauseTestCase { pub(crate) report: fn(), pub(crate) expected_code: Option<&'static str>, } + +pub(crate) struct CacheKeyTestCase { + pub(crate) description: &'static str, + /// The environment variable values of the second lookup, by name. + pub(crate) changed: &'static [(&'static str, &'static str)], + pub(crate) expected_same_key: bool, +} diff --git a/fensu.toml b/fensu.toml index ec0575896d..c2eee5ac2d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2508,6 +2508,7 @@ module_layers = [ "build_progress/transient_lines=infra", "build_progress/transient_status=infra", "cache=infra", + "change_selection=commands", "check=commands", "cli=commands", "cli/launch_state=infra", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md index 644c73e468..687fa8b274 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -42,6 +42,31 @@ sqb scenario test [flags] | `--max-snapshot-bytes` | Override per-relation byte limit for capture | | `--max-snapshot-total-bytes` | Override total byte limit for capture | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | +| `--since ` | Run only the scenarios affected by changes since the merge base of `` and `HEAD`; `@base` reads `[git] base_ref` | +| `--explain` | With `--since`, print why each selected scenario was selected | + +### Affected scenarios + +`--since ` applies the rules of [`sqb test --since`](test.md#run-only-affected-tests) +to scenarios. A scenario is selected when its own file (its definition and fixtures) or its +captured local snapshots changed, or when any node it actually executes changed: the models it +builds, the seeds and sources it reads, the functions they call, the audits attached to its models, +and the macros, enums and constants they and the scenario use. A model the scenario replaces with a +`__ref__` fixture, a source it replaces with a `__source__` fixture, or a seed it replaces with a +`__seed__` fixture, is not executed, so only changes to its declared columns, which shape the +fixture, select the scenario. [Project-wide inputs](test.md#project-wide-inputs) such as +the connection, targets and Python dependency files select every scenario. + +`--select` and `--exclude` narrow the affected scenarios further. With `--json`, the document gains +the same `since` section as `sqb test`, listing each selected scenario with its reasons. Both +commands share the cached base compile when `sqb test` uses the default target and no `--vars`, so +`sqb test --since origin/main` and `sqb scenario test --since origin/main` in one CI job compile +the base once and resolve the same changed set: + +```bash +sqb test --since "origin/$TARGET_BRANCH" +sqb scenario test --since "origin/$TARGET_BRANCH" --explain +``` ### Selectors diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 5aeb59c0fe..f95b38544c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -6,6 +6,13 @@ Online: https://sqlbuild.com/docs/cli/test/ +## Contents + +- Usage +- Flags +- Run only affected tests +- Examples + Runs SQL unit tests, independently reported parameterized cases, and multi-model tests without building models. Useful for validating test logic independently. @@ -27,6 +34,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--exclude` | Exclude tests by name, or the tests targeting specific models | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | | `--test-timeout ` | Time limit for each test in this run, such as `30s` or `2m`, or `disabled`; overrides `TEST(timeout ...)`, the target and `[settings] test_timeout` | +| `--since ` | Run only the tests affected by changes since the merge base of `` and `HEAD` (see [Run only affected tests](#run-only-affected-tests)); `@base` reads `[git] base_ref` | +| `--explain` | With `--since`, print why each selected test was selected | Test names are globally unique, so a bare test name selects exactly that test. `test:` is the explicit form and accepts the same glob patterns as names, such as `test:orders_*`. A bare pattern @@ -51,6 +60,140 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Run only affected tests + +`--since ` runs only the tests a change can affect, which keeps CI on branch pushes from +running the whole suite. SQLBuild compiles the project as it was at the merge base of `` and +`HEAD`, straight from git objects without a checkout, and compares it with the working tree, +uncommitted edits included. A test is selected when: + +- its own file changed or is new, or +- any node it actually executes changed: the models in its chain, the seeds, sources and + functions they read, and the macros, enums, constants and audits they and the test file use + (every case of a parameterized test counts), or +- a model, source or seed it mocks changed its declared columns: names, types, nullability or + contract, which shape the mock's columns, or +- a project-wide input changed: see [Project-wide inputs](#project-wide-inputs). + +The resolution is the one `--inspect` shows, so a test that mocks a model, source, seed or macro +is not selected when only that relation's SQL, data or upstream changes, while a downstream test +that really runs it is. + +A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with +formatting and comments ignored, the resolved configuration (including what `[defaults]` and +`[path_defaults]` contribute), columns and contract, attached audits, the values of the project +variables the model reads with `@@name`, and every macro, enum, constant, audit, function and seed +the node uses, resolved through [scopes](../concepts/declaration-scopes.md). A change to the +`[settings]` that shape compiled models counts as a change to every model. Descriptions do not +count; `meta` does. + +A macro counts as changed when the Python file that defines it changes, or any project module it +imports, such as a helper under `python/`; when that code reads `ctx.vars`, a change to any project +variable counts too. An enum, constant or audit counts as changed when its declaration file +changes other than in formatting and comments. A seed counts as changed with its data or column +types, a source with its declaration or inline expression, and a function with its definition. +Loaders, tasks, assets and checks are compared by their Python modules and the project modules +they import; they appear in the change summary but no unit test runs them. + +### Project-wide inputs + +Some inputs reach every test, so a change to any of them selects every test: + +- the `[settings]` that shape compiled models; +- the adapter, `[connection]` and `[connections]` (including connection settings and adapter + extensions), `[targets]` and the target in use, and project-local adapter modules under + `adapters/`. Their values are compared by digest and never printed; +- the Python environment as the project records it: the git-tracked `pyproject.toml`, `uv.lock`, + `poetry.lock`, `pdm.lock`, `Pipfile.lock` and `requirements*.txt` in the project directory or at + the repository root, compared with line endings normalised. Untracked and ignored copies do not + count. Any edit to `pyproject.toml` counts, even one unrelated to dependencies. + +Environment variables a project reads with `${ENV:NAME}` in configuration or `@@ENV:NAME` in SQL +are part of the cache key of the base compile and of the fingerprints of the SQL that reads them; +their values are never printed. + +`--since` cannot see code outside the project: modules imported from outside the project +directory, installed package versions that the files above do not record, and environment +variables read directly from Python with `os.environ`. Run the full suite when those change. + +Added nodes count as changed. A model renamed with `migrate_from` is compared with its old +definition, so a pure rename selects nothing, while a rename that also edits the model counts as a +change. Deleted nodes select nothing and are listed in the summary. + +```bash +# Tests affected by this branch +sqb test --since origin/main + +# Read the ref from [git] base_ref in sqlbuild_project.toml +sqb test --since @base + +# Only affected tests that target stg_orders, and say why each runs +sqb test stg_orders --since origin/main --explain +``` + +Selectors still apply: `--select` and positional selectors intersect with the affected tests, and +`--exclude` removes tests. `--explain` prints the reasons for each selected test, and `--inspect` +adds them to the resolved plan as `selected:` lines: + +```text +Comparing with origin/main (merge base 1a2b3c4d5e) +Compiling base revision 1a2b3c4d5e... +Compiled base revision 1a2b3c4d5e (2.31s) +Changed since origin/main: 1 model, 1 macro changed; deleted: model legacy_orders +Selected 2 of 9 tests + orders_rollup + executes stg_orders (model SQL changed) + order_timestamps + executes macro to_utc (changed) +``` + +The base compile is cached under `target/since/` per merge-base commit, target, `--vars`, local +configuration and referenced environment variables, and the newest few are kept. `sqb test --since` +and `sqb scenario test --since` in the same CI job compile the base once when they use the same +target and `--vars`; `sqb scenario test` uses the default target, so a `sqb test --target` run +compiles the base separately. When no test is affected, the command says so and exits +successfully. + +With `--json`, the progress and selection lines go to stderr, and the JSON document on stdout gains a +`since` section: the ref and merge base, the `changed`, `renamed` and `deleted` nodes, the number +of candidate tests, and each selected test with its reasons: + +```json +"since": { + "ref": "origin/main", + "merge_base": "1a2b3c4d5e...", + "changed": [ + {"kind": "model", "name": "stg_orders", "added": false, "aspects": ["sql"], "renamed_from": null} + ], + "renamed": [], + "deleted": [{"kind": "model", "name": "legacy_orders"}], + "candidate_count": 9, + "selected": [ + {"name": "orders_rollup", "reasons": ["executes stg_orders (model SQL changed)"]} + ] +} +``` + +The base revision must compile. When it does not, `--since` fails with the compile error instead +of guessing; compare with a later ref whose project compiles, or run without `--since`. A +`--target` that the base revision does not define, such as one the branch adds, compiles the base +with its default target instead; the run says so, and because the target changed, every test is +selected. + +SQL comparisons ignore layout and comments using a generic SQL tokenizer. In dialects where `--` +starts a comment only when followed by whitespace, such as MySQL, `1--1` is read as `1` followed by +a comment, so an edit inside such an expression can go unnoticed. The base tree is written from git +objects with the repository's filters. Git LFS content is restored for files that an unanchored +`.gitattributes` pattern marks, such as `*.csv filter=lfs`, but a pattern anchored to a path, such +as `/data/*.csv` at the repository root for a project in a subdirectory, does not match the +exported paths. Such a file is an LFS pointer at the base and real content in the checkout, so it +always counts as changed: at most extra tests run. + +`` is any git ref or commit, such as the merge request's target branch from a CI variable. +`--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach +the merge base, fails with an error that says what to fetch, such as `git fetch --unshallow` or a +larger clone depth; it never silently selects nothing. + ## Examples ```bash diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 25e52f4ea7..7dce49c13e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -21,6 +21,7 @@ Online: https://sqlbuild.com/docs/concepts/project-configuration/ - Janitor - Migrations - Scenario +- Git - dbt - Skills - Project Python @@ -882,6 +883,19 @@ Local type overrides for DuckDB replay are configured per adapter dialect: See [Scenarios](scenarios.md) for details on local type overrides and capture limits. +## Git + +The branch that `--since @base` compares against: + +```toml +[git] +base_ref = "origin/main" +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `base_ref` | unset | Git ref that [`sqb test --since @base`](../cli/test.md#run-only-affected-tests) and `sqb scenario test --since @base` read. Use a ref every checkout has, such as the remote default branch. | + ## dbt Configuration for running SQLBuild alongside an existing dbt project: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index aeee4b0ddc..05da2d8c4d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2061,6 +2061,13 @@ def build_lint_expansion_context( declaration_scope: DeclarationScopeBuild | None = None, static_declaration_scope: DeclarationScopeBuild | None = None, ) -> SqlExpansionContext: ... +def write_base_definition_snapshot( + *, + project_dir: str | Path, + output_path: str | Path, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, +) -> None: ... def build_dag_json( *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None ) -> str: ... @@ -6807,6 +6814,8 @@ class ScenarioTestCommandRequest(NativeRecord): json_output: bool json_output_path: Path | None scenario_timeout: str | None + since: str | None + explain: bool def __init__( self, project_dir: Path | None = None, @@ -6825,6 +6834,8 @@ class ScenarioTestCommandRequest(NativeRecord): json_output: bool = False, json_output_path: Path | None = None, scenario_timeout: str | None = None, + since: str | None = None, + explain: bool = False, ) -> None: ... class ScenarioCaptureCommandRequest(NativeRecord): @@ -6866,6 +6877,8 @@ class TestCommandRequest(NativeRecord): case_name: str | None inspect: bool test_timeout: str | None + since: str | None + explain: bool def __init__( self, project_dir: Path | None = None, @@ -6881,6 +6894,8 @@ class TestCommandRequest(NativeRecord): case_name: str | None = None, inspect: bool = False, test_timeout: str | None = None, + since: str | None = None, + explain: bool = False, ) -> None: ... class RefactorCommandRequest(NativeRecord): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 78c5d80a83..2a96d1fdbe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -493,3 +493,81 @@ class SharedPartialFixtureE2ETestCase: enriched_column: str report_column: str expected_padding_count: int + + +@dataclass(frozen=True) +class SinceSelectionE2ETestCase: + """Edits made after the base commit and what `--since` must select and say.""" + + description: str + edits: Mapping[str, str] + command: tuple[str, ...] + expected_exit_code: int + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) + deletions: tuple[str, ...] = field(default_factory=tuple) + # The git command run after the edits; `commit --all` makes them part of HEAD. + git_after_edits: tuple[str, ...] = ("status", "--short") + + +@dataclass(frozen=True) +class SinceSharedBaseE2ETestCase: + """A test run, then a scenario run, against the same base ref.""" + + description: str + edits: Mapping[str, str] + test_command: tuple[str, ...] + scenario_command: tuple[str, ...] + expected_test_fragments: tuple[str, ...] + expected_scenario_fragments: tuple[str, ...] + unexpected_scenario_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SinceJsonOutputE2ETestCase: + """A machine-output `--since` run and where its parts must go.""" + + description: str + edits: Mapping[str, str] + command: tuple[str, ...] + expected_stderr_fragments: tuple[str, ...] + expected_test_names: tuple[str, ...] + unexpected_stdout_fragments: tuple[str, ...] + # The `since` section's selected items and their reasons, by name. + expected_selected: Mapping[str, tuple[str, ...]] + + +@dataclass(frozen=True) +class SinceShallowCloneE2ETestCase: + """A shallow clone that lacks the merge base of the compared ref.""" + + description: str + clone_arguments: tuple[str, ...] + fetch_arguments: tuple[str, ...] + command: tuple[str, ...] + expected_exit_code: int + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SinceBrokenBaseE2ETestCase: + """A base revision that does not compile, and the error `--since` must report.""" + + description: str + base_files: Mapping[str, str] + deletions: tuple[str, ...] + command: tuple[str, ...] + expected_exit_code: int + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class SinceUnwritableCacheE2ETestCase: + """A `--since` run whose base snapshot cache cannot be written.""" + + description: str + blocking_file: str + command: tuple[str, ...] + expected_stderr_fragments: tuple[str, ...] + expected_test_names: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 1819396c24..75f24e9c4d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -3,7 +3,10 @@ from __future__ import annotations import json +import os import re +import subprocess +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -2150,3 +2153,291 @@ def build_shared_partial_fixture_project_files( "SELECT 1\n" ), } + + +def build_since_project_files() -> dict[str, str]: + """Build a DuckDB orders project whose tests and scenarios execute different nodes.""" + + return { + "sqlbuild_project.toml": ( + 'name = "since_orders"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "since_orders.duckdb"\n\n' + '[defaults]\nmaterialized = "table"\n' + ), + ".gitignore": "target/\n*.duckdb\n*.duckdb.wal\n", + "models/_macros/money.py": ( + "def cents_to_dollars(expression: str) -> str:\n" + ' return f"CAST({expression} AS DOUBLE) / 100"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Raw orders.\n" + " schema: main\n table: raw_orders\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: status\n type: VARCHAR\n" + " - name: amount_cents\n type: INTEGER\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Cleaned orders.');\n\n" + 'SELECT order_id, status, amount_cents FROM __source("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals in dollars.');\n\n" + 'SELECT order_id, @cents_to_dollars("amount_cents") AS amount\n' + 'FROM __ref("stg_orders")\n' + ), + "models/open_orders.sql": ( + "MODEL (description 'Open orders.');\n\n" + "SELECT order_id FROM __ref(\"stg_orders\") WHERE status = 'open'\n" + ), + "tests/unit/test_stg_orders.sql": ( + "TEST ();\n\nWITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__stg_orders AS (SELECT 1 AS order_id, 'open' AS status)\nSELECT 1\n" + ), + "tests/unit/test_order_totals.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_totals AS (SELECT 1 AS order_id, 2.5 AS amount)\nSELECT 1\n" + ), + "tests/unit/test_open_orders.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__open_orders AS (SELECT 1 AS order_id)\nSELECT 1\n" + ), + "tests/scenarios/order_totals_end_to_end.sql": ( + "SCENARIO (description 'Order totals from raw orders');\n\nWITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__order_totals AS (SELECT 1 AS order_id, 2.5 AS amount)\nSELECT 1\n" + ), + "tests/scenarios/open_orders_from_fixture.sql": ( + "SCENARIO (description 'Open orders from a staged fixture');\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__open_orders AS (SELECT 1 AS order_id)\nSELECT 1\n" + ), + } + + +def build_since_indirect_files() -> dict[str, str]: + """Build files that reach tests indirectly: variables, defaults, helpers, seeds, functions.""" + + base: dict[str, str] = build_since_project_files() + return { + "sqlbuild_project.toml": ( + base["sqlbuild_project.toml"] + + '\n[vars]\nmin_amount = "100"\n\n[scopes]\nenforce_placement = false\n' + ), + "python/_rates.py": 'DIVISOR = "100"\n', + "models/_macros/currency.py": ( + 'def whole_cents(expression: str) -> str:\n return f"({expression} * 1)"\n' + ), + "models/_macros/money.py": ( + "from models._macros.currency import whole_cents\n" + "from python._rates import DIVISOR\n\n\n" + "def cents_to_dollars(expression: str) -> str:\n" + ' return f"CAST({whole_cents(expression)} AS DOUBLE) / {DIVISOR}"\n' + ), + "macros/taxes.py": ( + 'def with_tax(expression: str) -> str:\n return f"({expression} + 0)"\n' + ), + "macros/pricing.py": ( + "from macros.taxes import with_tax\n\n\n" + "def gross(expression: str) -> str:\n return with_tax(expression)\n" + ), + "macros/fixture_values.py": "def suffix() -> str:\n return \"''\"\n", + "models/stg_orders.sql": ( + "MODEL (\n description 'Cleaned orders.',\n" + " columns (order_id (type INTEGER), status (type VARCHAR), " + "amount_cents (type INTEGER)),\n);\n\n" + 'SELECT order_id, status, amount_cents FROM __source("raw_orders")\n' + ), + "models/order_gross.sql": ( + "MODEL (description 'Gross order amounts.');\n\n" + 'SELECT order_id, @gross("amount_cents") AS gross_cents FROM __ref("stg_orders")\n' + ), + "models/order_day.sql": ( + "MODEL (description 'The UTC day of a fixed timestamp.');\n\n" + "SELECT order_id, CAST(TIMESTAMPTZ '2024-01-01 23:30:00+00' AS DATE) AS day\n" + 'FROM __ref("stg_orders")\n' + ), + "tests/unit/test_order_gross.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_gross AS (SELECT 1 AS order_id, 250 AS gross_cents)\nSELECT 1\n" + ), + "tests/unit/test_order_day.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_day AS (SELECT 1 AS order_id, DATE '2024-01-01' AS day)\nSELECT 1\n" + ), + "tests/unit/test_status_cases.sql": ( + 'TEST (name "status_cases", parameters (status string), ' + 'cases (open_case (status "open"), closed_case (status "closed")));\n\n' + "WITH\n__source__raw_orders AS (\n" + ' SELECT 1 AS order_id, @param("status") || @suffix() AS status, ' + "250 AS amount_cents\n),\n" + "__expected__stg_orders AS (\n" + ' SELECT 1 AS order_id, @param("status") || @suffix() AS status\n)\nSELECT 1\n' + ), + "audits/generic/at_least.sql": ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @column < @minimum\n' + ), + "models/order_totals.sql": ( + "MODEL (\n description 'Order totals in dollars.',\n" + " audits [at_least (column amount, minimum 0)],\n);\n\n" + 'SELECT order_id, @cents_to_dollars("amount_cents") AS amount\n' + 'FROM __ref("stg_orders")\n' + ), + "models/big_orders.sql": ( + "MODEL (description 'Large orders.');\n\n" + 'SELECT order_id FROM __ref("stg_orders") WHERE amount_cents >= @@min_amount\n' + ), + "models/marts/order_counts.sql": ( + "MODEL (description 'Order counts.');\n\n" + 'SELECT COUNT(*) AS order_count FROM __ref("stg_orders")\n' + ), + "seeds/regions.csv": "region_id,region_name\n1,north\n", + "seeds/regions.yml": ( + "seeds:\n - name: regions\n description: Sales regions.\n columns:\n" + " - name: region_id\n type: INTEGER\n" + " - name: region_name\n type: VARCHAR\n" + ), + "functions/sql/is_open.sql": ( + "FUNCTION (\n description 'Whether an order is open.',\n" + " arguments (order_status VARCHAR),\n returns BOOLEAN,\n);\n\n" + "order_status = 'open'\n" + ), + "models/regional_orders.sql": ( + "MODEL (description 'Orders with their region.');\n\n" + "SELECT o.order_id, r.region_id\n" + 'FROM __ref("stg_orders") o CROSS JOIN __seed("regions") r\n' + ), + "models/order_flags.sql": ( + "MODEL (description 'Whether each order is open.');\n\n" + 'SELECT order_id, __udf("is_open")(status) AS is_open FROM __ref("stg_orders")\n' + ), + "models/order_archive.sql": ( + "MODEL (description 'Archived orders.');\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/archive_counts.sql": ( + "MODEL (description 'Archived order count.');\n\n" + 'SELECT COUNT(*) AS archived FROM __ref("order_archive")\n' + ), + "tests/unit/test_archive_counts.sql": ( + "TEST ();\n\nWITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__archive_counts AS (SELECT 1 AS archived)\nSELECT 1\n" + ), + "python/tasks/report.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task()\ndef report_orders(ctx):\n" + ' """Report orders."""\n\n' + " return ctx.result(payload={})\n" + ), + "tests/unit/test_big_orders.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__big_orders AS (SELECT 1 AS order_id)\nSELECT 1\n" + ), + "tests/unit/test_order_counts.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_counts AS (SELECT 1 AS order_count)\nSELECT 1\n" + ), + "tests/unit/test_regional_orders.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__seed__regions AS (SELECT 1 AS region_id, 'north' AS region_name),\n" + "__expected__regional_orders AS (SELECT 1 AS order_id, 1 AS region_id)\nSELECT 1\n" + ), + "tests/unit/test_order_flags.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_flags AS (SELECT 1 AS order_id, TRUE AS is_open)\nSELECT 1\n" + ), + "tests/scenarios/regional_orders_end_to_end.sql": ( + "SCENARIO (description 'Regional orders from raw orders and the region seed');\n\n" + "WITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__regional_orders AS (SELECT 1 AS order_id, 1 AS region_id)\nSELECT 1\n" + ), + "tests/unit/test_order_totals_from_source.sql": ( + "TEST ();\n\nWITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__order_totals AS (SELECT 1 AS order_id, 2.5 AS amount)\nSELECT 1\n" + ), + } + + +_GIT_IDENTITY: dict[str, str] = { + "GIT_AUTHOR_NAME": "Example", + "GIT_AUTHOR_EMAIL": "dev@example.com", + "GIT_COMMITTER_NAME": "Example", + "GIT_COMMITTER_EMAIL": "dev@example.com", +} + + +def run_git(*, repository: Path, arguments: tuple[str, ...]) -> str: + """Run git in `repository` with a fixed identity and return its stdout.""" + + result: subprocess.CompletedProcess[str] = subprocess.run( + ("git", "-c", "commit.gpgsign=false", *arguments), + cwd=repository, + env={**os.environ, **_GIT_IDENTITY}, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + return result.stdout.strip() + + +def commit_project_on_main(*, repository: Path) -> None: + """Commit everything on `main`, then continue on a `feature` branch.""" + + run_git(repository=repository, arguments=("init", "--quiet", "--initial-branch", "main")) + run_git(repository=repository, arguments=("add", "--all")) + run_git(repository=repository, arguments=("commit", "--quiet", "-m", "orders")) + run_git(repository=repository, arguments=("checkout", "--quiet", "-b", "feature")) + + +def write_project_files(*, project_dir: Path, files: Mapping[str, str]) -> None: + """Write each file below `project_dir`, creating parent directories.""" + + relative_path: str + contents: str + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def delete_project_files(*, project_dir: Path, relative_paths: tuple[str, ...]) -> None: + """Delete each file below `project_dir`.""" + + relative_path: str + for relative_path in relative_paths: + (project_dir / relative_path).unlink() + + +def prepare_since_repository(*, tmp_path: Path, extra_files: Mapping[str, str]) -> Path: + """Write the since project with `extra_files`, commit it on `main` and branch to `feature`.""" + + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="since_orders", + repo_files={**build_since_project_files(), **extra_files}, + ) + commit_project_on_main(repository=project) + return project diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py new file mode 100644 index 0000000000..be0a4d05a6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -0,0 +1,750 @@ +"""`sqb test --since` and `sqb scenario test --since` select what a git change can affect.""" + +import json +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + SinceBrokenBaseE2ETestCase, + SinceJsonOutputE2ETestCase, + SinceSelectionE2ETestCase, + SinceShallowCloneE2ETestCase, + SinceSharedBaseE2ETestCase, + SinceUnwritableCacheE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import ( + build_since_indirect_files, + build_since_project_files, + delete_project_files, + prepare_since_repository, + run_git, + write_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +_FILES: dict[str, str] = build_since_project_files() +_STAGING: str = "models/stg_orders.sql" +_MACRO: str = "models/_macros/money.py" +_EDITED_STAGING: str = _FILES[_STAGING].replace( + "SELECT order_id, status", "SELECT order_id, LOWER(status) AS status" +) +_EDITED_MACRO: str = _FILES[_MACRO].replace("/ 100", "/ 100.0") +_REFORMATTED_STAGING: str = ( + "MODEL (description 'Cleaned and documented orders.');\n\n" + "-- one row per order\nselect\n order_id,\n status,\n amount_cents\n" + 'from __source("raw_orders")\n' +) +_ARCHIVE: str = "models/order_archive.sql" +_ARCHIVE_SQL: str = ( + "MODEL (description 'Archived orders.');\n\nSELECT order_id FROM __ref(\"stg_orders\")\n" +) +_TEST_COMMAND: tuple[str, ...] = ("--no-color", "test", "--since", "main") +_BASE_REF_CONFIG: str = _FILES["sqlbuild_project.toml"] + '\n[git]\nbase_ref = "main"\n' +_INDIRECT: dict[str, str] = build_since_indirect_files() +_PROJECT_CONFIG: str = "sqlbuild_project.toml" +_SCENARIO_COMMAND: tuple[str, ...] = ("--no-color", "scenario", "test", "--since", "main") +_EXPLAINED_TESTS: tuple[str, ...] = (*_TEST_COMMAND, "--explain") +_EXPLAINED_SCENARIOS: tuple[str, ...] = (*_SCENARIO_COMMAND, "--explain") + + +@pytest.mark.parametrize( + "test_case", + ( + SinceSelectionE2ETestCase( + description="an uncommitted model edit selects only the test that really runs it", + edits={_STAGING: _EDITED_STAGING}, + command=(*_TEST_COMMAND, "--explain"), + expected_exit_code=0, + expected_fragments=( + "Comparing with main (merge base ", + "Compiled base revision ", + "Changed since main: 1 model changed", + "Selected 1 of 3 tests", + "test_stg_orders\n executes stg_orders (model SQL changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("test_order_totals", "test_open_orders"), + ), + SinceSelectionE2ETestCase( + description="a committed macro edit selects the test whose model calls the macro", + edits={_MACRO: _EDITED_MACRO}, + command=(*_TEST_COMMAND, "--explain"), + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 macro changed", + "test_order_totals\n executes macro cents_to_dollars (changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("test_stg_orders", "test_open_orders"), + git_after_edits=("commit", "--quiet", "--all", "-m", "edit"), + ), + SinceSelectionE2ETestCase( + description="formatting, comments and descriptions change nothing", + edits={_STAGING: _REFORMATTED_STAGING}, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: no definition changes", + "No tests are affected by the changes since main", + "PASS=0 FAIL=0 TOTAL=0", + ), + ), + SinceSelectionE2ETestCase( + description="an edited test file selects that test", + edits={ + "tests/unit/test_open_orders.sql": _FILES["tests/unit/test_open_orders.sql"] + + "-- covers open orders\n" + }, + command=(*_TEST_COMMAND, "--inspect"), + expected_exit_code=0, + expected_fragments=( + "Selected 1 of 3 tests", + "test_open_orders\n", + " selected: test file changed", + "Test plan inspection complete: 1 selected, 0 errors.", + ), + unexpected_fragments=("test_stg_orders\n", "test_order_totals\n"), + ), + SinceSelectionE2ETestCase( + description="selectors intersect with the affected tests", + edits={_STAGING: _EDITED_STAGING, _MACRO: _EDITED_MACRO}, + command=(*_TEST_COMMAND, "order_totals"), + expected_exit_code=0, + expected_fragments=("Selected 1 of 1 test\n", "PASS=1 FAIL=0 TOTAL=1"), + unexpected_fragments=("test_stg_orders",), + ), + SinceSelectionE2ETestCase( + description="exclude removes affected tests", + edits={_STAGING: _EDITED_STAGING, _MACRO: _EDITED_MACRO}, + command=(*_TEST_COMMAND, "--exclude", "test:test_order_totals"), + expected_exit_code=0, + expected_fragments=("Selected 1 of 2 tests", "PASS=1 FAIL=0 TOTAL=1"), + unexpected_fragments=("test_order_totals",), + ), + SinceSelectionE2ETestCase( + description="a deleted model is listed and selects nothing", + edits={}, + deletions=(_ARCHIVE,), + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: deleted: model order_archive", + "No tests are affected by the changes since main", + ), + ), + SinceSelectionE2ETestCase( + description="a pure rename through migrate_from selects nothing", + edits={ + "models/order_history.sql": _ARCHIVE_SQL.replace( + "MODEL (", "MODEL (migrate_from order_archive, " + ), + }, + deletions=(_ARCHIVE,), + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 model renamed", + "No tests are affected by the changes since main", + ), + unexpected_fragments=("deleted:",), + ), + SinceSelectionE2ETestCase( + description="a missing ref fails instead of selecting nothing", + edits={}, + command=("--no-color", "test", "--since", "origin/missing"), + expected_exit_code=1, + expected_fragments=( + "error[C481]: Git ref 'origin/missing' does not name a commit in this repository", + "git fetch origin ", + ), + ), + SinceSelectionE2ETestCase( + description="@base without a configured base ref fails", + edits={}, + command=("--no-color", "test", "--since", "@base"), + expected_exit_code=1, + expected_fragments=( + "error[C485]: --since @base needs [git] base_ref in sqlbuild_project.toml", + ), + ), + SinceSelectionE2ETestCase( + description="--explain needs --since", + edits={}, + command=("--no-color", "test", "--explain"), + expected_exit_code=2, + expected_fragments=("--explain requires --since",), + ), + ), + ids=lambda case: case.description, +) +def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( + tmp_path: Path, test_case: SinceSelectionE2ETestCase +) -> None: + project: Path = prepare_since_repository( + tmp_path=tmp_path, extra_files={_ARCHIVE: _ARCHIVE_SQL} + ) + write_project_files(project_dir=project, files=test_case.edits) + delete_project_files(project_dir=project, relative_paths=test_case.deletions) + run_git(repository=project, arguments=test_case.git_after_edits) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output + for fragment in test_case.unexpected_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceSelectionE2ETestCase( + description="a project variable a model reads selects that model's test", + edits={ + _PROJECT_CONFIG: _INDIRECT[_PROJECT_CONFIG].replace( + 'min_amount = "100"', 'min_amount = "200"' + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 model changed\n", + "Selected 1 of 13 tests", + "test_big_orders\n executes big_orders (model vars changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + ), + SinceSelectionE2ETestCase( + description="a path default compares the resolved model configuration", + edits={ + _PROJECT_CONFIG: _INDIRECT[_PROJECT_CONFIG] + + '\n[path_defaults.marts]\nmaterialized = "view"\n' + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Selected 1 of 13 tests", + "test_order_counts\n executes order_counts (model config changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + ), + SinceSelectionE2ETestCase( + description="a helper module a macro imports changes the macro", + edits={"python/_rates.py": 'DIVISOR = "100.0"\n'}, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 macro changed\n", + "Selected 2 of 13 tests", + "test_order_totals\n executes macro cents_to_dollars (changed)", + "test_order_totals_from_source\n executes macro cents_to_dollars (changed)", + "PASS=2 FAIL=0 TOTAL=2", + ), + ), + SinceSelectionE2ETestCase( + description="a function body change selects the test that calls it", + edits={ + "functions/sql/is_open.sql": _INDIRECT["functions/sql/is_open.sql"].replace( + "order_status = 'open'", "LOWER(order_status) = 'open'" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 function changed\n", + "Selected 1 of 13 tests", + "test_order_flags\n executes function is_open (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="an audit definition change selects tests of models it is attached to", + edits={ + "audits/generic/at_least.sql": _INDIRECT["audits/generic/at_least.sql"].replace( + "@column < @minimum", "@column <= @minimum" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 audit changed\n", + "Selected 2 of 13 tests", + "test_order_totals\n executes audit at_least (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="an upstream change selects a test that runs it but not one that mocks it", + edits={ + _STAGING: _INDIRECT[_STAGING].replace( + "SELECT order_id, status", "SELECT order_id, LOWER(status) AS status" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "test_order_totals_from_source\n executes stg_orders (model SQL changed)", + "test_archive_counts\n executes stg_orders (model SQL changed)", + ), + unexpected_fragments=(" test_order_totals\n", "test_big_orders"), + ), + SinceSelectionE2ETestCase( + description="a rename through migrate_from that also changes the SQL is a change", + edits={ + "models/order_history.sql": _INDIRECT["models/order_archive.sql"] + .replace("MODEL (", "MODEL (migrate_from order_archive, ") + .replace('__ref("stg_orders")', '__ref("stg_orders") WHERE order_id > 0'), + "models/archive_counts.sql": _INDIRECT["models/archive_counts.sql"].replace( + "order_archive", "order_history" + ), + }, + deletions=("models/order_archive.sql",), + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 2 models changed; 1 model renamed", + "test_archive_counts\n" + " executes order_history (renamed from order_archive, model SQL changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("deleted:",), + ), + SinceSelectionE2ETestCase( + description="seed data a test mocks selects nothing", + edits={"seeds/regions.csv": "region_id,region_name\n1,south\n"}, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 seed changed\n", + "No tests are affected by the changes since main", + ), + ), + SinceSelectionE2ETestCase( + description="seed data selects the scenario that loads the seed", + edits={"seeds/regions.csv": "region_id,region_name\n1,south\n"}, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=( + "Selected 1 of 3 scenarios", + "regional_orders_end_to_end\n reads seed regions (changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + ), + SinceSelectionE2ETestCase( + description="a seed column type selects the scenario that loads the seed", + edits={ + "seeds/regions.yml": _INDIRECT["seeds/regions.yml"].replace( + "type: VARCHAR", "type: TEXT" + ) + }, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=("regional_orders_end_to_end\n reads seed regions (changed)",), + ), + SinceSelectionE2ETestCase( + description="an audit definition change selects scenarios that build its model", + edits={ + "audits/generic/at_least.sql": _INDIRECT["audits/generic/at_least.sql"].replace( + "@column < @minimum", "@column <= @minimum" + ) + }, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=( + "Selected 1 of 3 scenarios", + "order_totals_end_to_end\n executes audit at_least (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a scenario fixture edit selects that scenario", + edits={ + "tests/scenarios/regional_orders_end_to_end.sql": _INDIRECT[ + "tests/scenarios/regional_orders_end_to_end.sql" + ].replace("250 AS amount_cents", "300 AS amount_cents") + }, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=("regional_orders_end_to_end\n scenario file changed",), + unexpected_fragments=("order_totals_end_to_end",), + ), + SinceSelectionE2ETestCase( + description="a mocked source's declared column type selects the tests that mock it", + edits={ + "sources/raw.yml": _FILES["sources/raw.yml"].replace( + "amount_cents\n type: INTEGER", "amount_cents\n type: BIGINT" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 source changed\n", + "Selected 5 of 13 tests", + "test_stg_orders\n mocks source raw_orders (source columns changed)", + ), + unexpected_fragments=("test_order_flags",), + ), + SinceSelectionE2ETestCase( + description="a mocked model's declared column type selects the tests that mock it", + edits={ + _STAGING: _INDIRECT[_STAGING].replace( + "amount_cents (type INTEGER)", "amount_cents (type BIGINT)" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Selected 13 of 13 tests", + "test_big_orders\n mocks stg_orders (model columns changed)", + "test_archive_counts\n executes stg_orders (model columns changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a fixture's declared column type selects the scenario that replaces it", + edits={ + _STAGING: _INDIRECT[_STAGING].replace( + "amount_cents (type INTEGER)", "amount_cents (type BIGINT)" + ) + }, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=( + "open_orders_from_fixture\n mocks stg_orders (model columns changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a macro another project-wide macro imports changes both", + edits={"macros/taxes.py": _INDIRECT["macros/taxes.py"].replace("+ 0", "+ 1 - 1")}, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 2 macros changed\n", + "Selected 1 of 13 tests", + "test_order_gross\n executes macro gross (changed)\n" + " executes macro with_tax (changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + ), + SinceSelectionE2ETestCase( + description="a scoped macro another scoped macro imports changes both", + edits={ + "models/_macros/currency.py": _INDIRECT["models/_macros/currency.py"].replace( + "* 1)", "* 1.0)" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Selected 2 of 13 tests", + "test_order_totals\n executes macro cents_to_dollars (changed)\n" + " executes macro whole_cents (changed)", + "PASS=2 FAIL=0 TOTAL=2", + ), + ), + SinceSelectionE2ETestCase( + description="a macro a parameterized test's own fixture calls selects every case", + edits={ + "macros/fixture_values.py": _INDIRECT["macros/fixture_values.py"].replace( + "\"''\"", "\"'x'\"" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Selected 2 of 13 tests", + "status_cases [open_case]\n executes macro suffix (changed)", + "status_cases [closed_case]\n executes macro suffix (changed)", + "PASS=2 FAIL=0 TOTAL=2", + ), + ), + SinceSelectionE2ETestCase( + description="a connection setting selects every test, including the one it breaks", + edits={ + _PROJECT_CONFIG: _INDIRECT[_PROJECT_CONFIG].replace( + '[connection]\ndatabase = "since_orders.duckdb"\n', + '[connection]\ndatabase = "since_orders.duckdb"\n\n' + '[connection.settings]\nTimeZone = "Asia/Tokyo"\n', + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=1, + expected_fragments=( + "Changed since main: 1 connection and target configuration changed\n", + "Selected 13 of 13 tests", + "test_order_day\n uses connection and target configuration (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a tracked Python dependency lock file selects every test", + edits={"uv.lock": "version = 1\n"}, + git_after_edits=("add", "uv.lock"), + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 Python environment changed\n", + "Selected 13 of 13 tests", + "test_order_day\n uses Python environment (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="an untracked lock file is not part of the project", + edits={"uv.lock": "version = 1\n", "requirements-local.txt": "pytest\n"}, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: no definition changes", + "No tests are affected by the changes since main", + ), + ), + SinceSelectionE2ETestCase( + description="a target the base does not define compiles the base with its default", + edits={ + _PROJECT_CONFIG: _INDIRECT[_PROJECT_CONFIG] + '\n[targets.ci]\nschema = "main"\n' + }, + command=("--no-color", "test", "--target", "ci", "--since", "main"), + expected_exit_code=0, + expected_fragments=( + "Target 'ci' is not defined at the merge base, so the base compiles with its " + "default target", + "Selected 13 of 13 tests", + "PASS=13 FAIL=0 TOTAL=13", + ), + ), + SinceSelectionE2ETestCase( + description="a Python node change is listed", + edits={ + "python/tasks/report.py": _INDIRECT["python/tasks/report.py"].replace( + "payload={}", 'payload={"orders": 1}' + ) + }, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=("Changed since main: 1 task changed\n",), + ), + ), + ids=lambda case: case.description, +) +def test_given_indirect_changes_since_ref_when_testing_then_selects_what_executes_them( + tmp_path: Path, test_case: SinceSelectionE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files=_INDIRECT) + write_project_files(project_dir=project, files=test_case.edits) + delete_project_files(project_dir=project, relative_paths=test_case.deletions) + run_git(repository=project, arguments=test_case.git_after_edits) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output + for fragment in test_case.unexpected_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceBrokenBaseE2ETestCase( + description="a base revision that does not compile fails and says what to do", + base_files={ + "models/broken.sql": ( + "MODEL (description 'Reads a model that does not exist.');\n\n" + 'SELECT order_id FROM __ref("missing_orders")\n' + ) + }, + deletions=("models/broken.sql",), + command=_TEST_COMMAND, + expected_exit_code=1, + expected_fragments=( + "error[C484]: Cannot compile the project at merge base ", + "missing_orders", + "Compare with a later ref whose project compiles, or run without --since", + ), + unexpected_fragments=("Traceback", 'File "'), + ), + ), + ids=lambda case: case.description, +) +def test_given_base_that_does_not_compile_when_testing_since_then_fails_with_guidance( + tmp_path: Path, test_case: SinceBrokenBaseE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files=test_case.base_files) + delete_project_files(project_dir=project, relative_paths=test_case.deletions) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output + for fragment in test_case.unexpected_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceSharedBaseE2ETestCase( + description="a scenario run reuses the base compile of the test run", + edits={_STAGING: _EDITED_STAGING, "sqlbuild_project.toml": _BASE_REF_CONFIG}, + test_command=("--no-color", "test", "--since", "@base"), + scenario_command=("--no-color", "scenario", "test", "--since", "@base", "--explain"), + expected_test_fragments=("Compiled base revision ", "Selected 1 of 3 tests"), + expected_scenario_fragments=( + "Using cached base revision ", + "Changed since main: 1 model changed", + "Selected 1 of 2 scenarios", + "order_totals_end_to_end\n executes stg_orders (model SQL changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_scenario_fragments=("open_orders_from_fixture",), + ), + ), + ids=lambda case: case.description, +) +def test_given_test_and_scenario_runs_when_comparing_then_both_share_one_base_compile( + tmp_path: Path, test_case: SinceSharedBaseE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files={}) + write_project_files(project_dir=project, files=test_case.edits) + + tested: CompletedProcess[str] = run_sqb(command=test_case.test_command, project_dir=project) + scenarios: CompletedProcess[str] = run_sqb( + command=test_case.scenario_command, project_dir=project + ) + + tested_output: str = tested.stdout + tested.stderr + scenario_output: str = scenarios.stdout + scenarios.stderr + assert tested.returncode == 0, tested_output + assert scenarios.returncode == 0, scenario_output + for fragment in test_case.expected_test_fragments: + assert fragment in tested_output, tested_output + for fragment in test_case.expected_scenario_fragments: + assert fragment in scenario_output, scenario_output + for fragment in test_case.unexpected_scenario_fragments: + assert fragment not in scenario_output, scenario_output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceJsonOutputE2ETestCase( + description="progress and the selection go to stderr", + edits={_STAGING: _EDITED_STAGING}, + command=("--no-color", "test", "--since", "main", "--json"), + expected_stderr_fragments=("Comparing with main", "Selected 1 of 3 tests"), + expected_test_names=("test_stg_orders",), + unexpected_stdout_fragments=("test_order_totals", "Comparing with"), + expected_selected={"test_stg_orders": ("executes stg_orders (model SQL changed)",)}, + ), + SinceJsonOutputE2ETestCase( + description="a scenario run lists each selected scenario's reasons", + edits={_STAGING: _EDITED_STAGING}, + command=("--no-color", "scenario", "test", "--since", "main", "--json"), + expected_stderr_fragments=("Comparing with main", "Selected 1 of 2 scenarios"), + expected_test_names=("order_totals_end_to_end",), + unexpected_stdout_fragments=("open_orders_from_fixture", "Comparing with"), + expected_selected={ + "order_totals_end_to_end": ("executes stg_orders (model SQL changed)",) + }, + ), + ), + ids=lambda case: case.description, +) +def test_given_json_output_when_testing_since_ref_then_stdout_stays_machine_readable( + tmp_path: Path, test_case: SinceJsonOutputE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files={}) + write_project_files(project_dir=project, files=test_case.edits) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + assert result.returncode == 0, result.stdout + result.stderr + document: str = json.dumps(json.loads(result.stdout)) + for fragment in test_case.expected_stderr_fragments: + assert fragment in result.stderr, result.stderr + for name in test_case.expected_test_names: + assert f'"{name}"' in document, document + for fragment in test_case.unexpected_stdout_fragments: + assert fragment not in result.stdout, result.stdout + reference: str = json.loads(result.stdout)["since"]["ref"] + selected: list[dict[str, str | list[str]]] = json.loads(result.stdout)["since"]["selected"] + assert reference == "main", result.stdout + assert { + str(item["name"]): tuple(item["reasons"]) for item in selected + } == test_case.expected_selected, result.stdout + + +@pytest.mark.parametrize( + "test_case", + ( + SinceShallowCloneE2ETestCase( + description="a depth-one clone of both branches lacks the merge base", + clone_arguments=("--depth", "1", "--branch", "feature", "--no-single-branch"), + fetch_arguments=("fetch", "--quiet", "--depth", "1", "origin", "main"), + command=("--no-color", "test", "--since", "origin/main"), + expected_exit_code=1, + expected_fragments=( + "error[C482]: Cannot find the merge base of 'origin/main' and HEAD", + "git fetch --unshallow", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_shallow_clone_without_merge_base_when_testing_since_then_asks_for_depth( + tmp_path: Path, test_case: SinceShallowCloneE2ETestCase +) -> None: + origin: Path = prepare_since_repository(tmp_path=tmp_path, extra_files={}) + run_git(repository=origin, arguments=("commit", "--quiet", "--allow-empty", "-m", "next")) + clone: Path = tmp_path / "clone" + run_git( + repository=tmp_path, + arguments=("clone", "--quiet", *test_case.clone_arguments, origin.as_uri(), str(clone)), + ) + run_git(repository=clone, arguments=test_case.fetch_arguments) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=clone) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceUnwritableCacheE2ETestCase( + description="a file where the cache directory belongs warns and the run continues", + blocking_file="target/since", + command=("--no-color", "test", "--since", "main", "--json"), + expected_stderr_fragments=( + "warning: Could not cache the base revision at ", + "; continuing without the cache", + "Selected 1 of 3 tests", + ), + expected_test_names=("test_stg_orders",), + ), + ), + ids=lambda case: case.description, +) +def test_given_unwritable_cache_when_testing_since_then_warns_and_succeeds( + tmp_path: Path, test_case: SinceUnwritableCacheE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files={}) + write_project_files( + project_dir=project, + files={_STAGING: _EDITED_STAGING, test_case.blocking_file: "not a directory\n"}, + ) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + assert result.returncode == 0, result.stdout + result.stderr + for fragment in test_case.expected_stderr_fragments: + assert fragment in result.stderr, result.stderr + assert result.stderr.count("warning: Could not cache") == 1, result.stderr + selected: list[dict[str, str | list[str]]] = json.loads(result.stdout)["since"]["selected"] + assert tuple(str(item["name"]) for item in selected) == test_case.expected_test_names diff --git a/website/src/content/docs/docs/cli/scenario.mdx b/website/src/content/docs/docs/cli/scenario.mdx index 082f28b154..3b7feb3e67 100644 --- a/website/src/content/docs/docs/cli/scenario.mdx +++ b/website/src/content/docs/docs/cli/scenario.mdx @@ -35,6 +35,31 @@ sqb scenario test [flags] | `--max-snapshot-bytes` | Override per-relation byte limit for capture | | `--max-snapshot-total-bytes` | Override total byte limit for capture | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | +| `--since ` | Run only the scenarios affected by changes since the merge base of `` and `HEAD`; `@base` reads `[git] base_ref` | +| `--explain` | With `--since`, print why each selected scenario was selected | + +### Affected scenarios + +`--since ` applies the rules of [`sqb test --since`](/docs/cli/test#run-only-affected-tests) +to scenarios. A scenario is selected when its own file (its definition and fixtures) or its +captured local snapshots changed, or when any node it actually executes changed: the models it +builds, the seeds and sources it reads, the functions they call, the audits attached to its models, +and the macros, enums and constants they and the scenario use. A model the scenario replaces with a +`__ref__` fixture, a source it replaces with a `__source__` fixture, or a seed it replaces with a +`__seed__` fixture, is not executed, so only changes to its declared columns, which shape the +fixture, select the scenario. [Project-wide inputs](/docs/cli/test#project-wide-inputs) such as +the connection, targets and Python dependency files select every scenario. + +`--select` and `--exclude` narrow the affected scenarios further. With `--json`, the document gains +the same `since` section as `sqb test`, listing each selected scenario with its reasons. Both +commands share the cached base compile when `sqb test` uses the default target and no `--vars`, so +`sqb test --since origin/main` and `sqb scenario test --since origin/main` in one CI job compile +the base once and resolve the same changed set: + +```bash +sqb test --since "origin/$TARGET_BRANCH" +sqb scenario test --since "origin/$TARGET_BRANCH" --explain +``` ### Selectors diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index e3aa46347d..a21e789598 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -26,6 +26,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--exclude` | Exclude tests by name, or the tests targeting specific models | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | | `--test-timeout ` | Time limit for each test in this run, such as `30s` or `2m`, or `disabled`; overrides `TEST(timeout ...)`, the target and `[settings] test_timeout` | +| `--since ` | Run only the tests affected by changes since the merge base of `` and `HEAD` (see [Run only affected tests](#run-only-affected-tests)); `@base` reads `[git] base_ref` | +| `--explain` | With `--since`, print why each selected test was selected | Test names are globally unique, so a bare test name selects exactly that test. `test:` is the explicit form and accepts the same glob patterns as names, such as `test:orders_*`. A bare pattern @@ -50,6 +52,140 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Run only affected tests + +`--since ` runs only the tests a change can affect, which keeps CI on branch pushes from +running the whole suite. SQLBuild compiles the project as it was at the merge base of `` and +`HEAD`, straight from git objects without a checkout, and compares it with the working tree, +uncommitted edits included. A test is selected when: + +- its own file changed or is new, or +- any node it actually executes changed: the models in its chain, the seeds, sources and + functions they read, and the macros, enums, constants and audits they and the test file use + (every case of a parameterized test counts), or +- a model, source or seed it mocks changed its declared columns: names, types, nullability or + contract, which shape the mock's columns, or +- a project-wide input changed: see [Project-wide inputs](#project-wide-inputs). + +The resolution is the one `--inspect` shows, so a test that mocks a model, source, seed or macro +is not selected when only that relation's SQL, data or upstream changes, while a downstream test +that really runs it is. + +A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with +formatting and comments ignored, the resolved configuration (including what `[defaults]` and +`[path_defaults]` contribute), columns and contract, attached audits, the values of the project +variables the model reads with `@@name`, and every macro, enum, constant, audit, function and seed +the node uses, resolved through [scopes](/docs/concepts/declaration-scopes). A change to the +`[settings]` that shape compiled models counts as a change to every model. Descriptions do not +count; `meta` does. + +A macro counts as changed when the Python file that defines it changes, or any project module it +imports, such as a helper under `python/`; when that code reads `ctx.vars`, a change to any project +variable counts too. An enum, constant or audit counts as changed when its declaration file +changes other than in formatting and comments. A seed counts as changed with its data or column +types, a source with its declaration or inline expression, and a function with its definition. +Loaders, tasks, assets and checks are compared by their Python modules and the project modules +they import; they appear in the change summary but no unit test runs them. + +### Project-wide inputs + +Some inputs reach every test, so a change to any of them selects every test: + +- the `[settings]` that shape compiled models; +- the adapter, `[connection]` and `[connections]` (including connection settings and adapter + extensions), `[targets]` and the target in use, and project-local adapter modules under + `adapters/`. Their values are compared by digest and never printed; +- the Python environment as the project records it: the git-tracked `pyproject.toml`, `uv.lock`, + `poetry.lock`, `pdm.lock`, `Pipfile.lock` and `requirements*.txt` in the project directory or at + the repository root, compared with line endings normalised. Untracked and ignored copies do not + count. Any edit to `pyproject.toml` counts, even one unrelated to dependencies. + +Environment variables a project reads with `${ENV:NAME}` in configuration or `@@ENV:NAME` in SQL +are part of the cache key of the base compile and of the fingerprints of the SQL that reads them; +their values are never printed. + +`--since` cannot see code outside the project: modules imported from outside the project +directory, installed package versions that the files above do not record, and environment +variables read directly from Python with `os.environ`. Run the full suite when those change. + +Added nodes count as changed. A model renamed with `migrate_from` is compared with its old +definition, so a pure rename selects nothing, while a rename that also edits the model counts as a +change. Deleted nodes select nothing and are listed in the summary. + +```bash +# Tests affected by this branch +sqb test --since origin/main + +# Read the ref from [git] base_ref in sqlbuild_project.toml +sqb test --since @base + +# Only affected tests that target stg_orders, and say why each runs +sqb test stg_orders --since origin/main --explain +``` + +Selectors still apply: `--select` and positional selectors intersect with the affected tests, and +`--exclude` removes tests. `--explain` prints the reasons for each selected test, and `--inspect` +adds them to the resolved plan as `selected:` lines: + +```text +Comparing with origin/main (merge base 1a2b3c4d5e) +Compiling base revision 1a2b3c4d5e... +Compiled base revision 1a2b3c4d5e (2.31s) +Changed since origin/main: 1 model, 1 macro changed; deleted: model legacy_orders +Selected 2 of 9 tests + orders_rollup + executes stg_orders (model SQL changed) + order_timestamps + executes macro to_utc (changed) +``` + +The base compile is cached under `target/since/` per merge-base commit, target, `--vars`, local +configuration and referenced environment variables, and the newest few are kept. `sqb test --since` +and `sqb scenario test --since` in the same CI job compile the base once when they use the same +target and `--vars`; `sqb scenario test` uses the default target, so a `sqb test --target` run +compiles the base separately. When no test is affected, the command says so and exits +successfully. + +With `--json`, the progress and selection lines go to stderr, and the JSON document on stdout gains a +`since` section: the ref and merge base, the `changed`, `renamed` and `deleted` nodes, the number +of candidate tests, and each selected test with its reasons: + +```json +"since": { + "ref": "origin/main", + "merge_base": "1a2b3c4d5e...", + "changed": [ + {"kind": "model", "name": "stg_orders", "added": false, "aspects": ["sql"], "renamed_from": null} + ], + "renamed": [], + "deleted": [{"kind": "model", "name": "legacy_orders"}], + "candidate_count": 9, + "selected": [ + {"name": "orders_rollup", "reasons": ["executes stg_orders (model SQL changed)"]} + ] +} +``` + +The base revision must compile. When it does not, `--since` fails with the compile error instead +of guessing; compare with a later ref whose project compiles, or run without `--since`. A +`--target` that the base revision does not define, such as one the branch adds, compiles the base +with its default target instead; the run says so, and because the target changed, every test is +selected. + +SQL comparisons ignore layout and comments using a generic SQL tokenizer. In dialects where `--` +starts a comment only when followed by whitespace, such as MySQL, `1--1` is read as `1` followed by +a comment, so an edit inside such an expression can go unnoticed. The base tree is written from git +objects with the repository's filters. Git LFS content is restored for files that an unanchored +`.gitattributes` pattern marks, such as `*.csv filter=lfs`, but a pattern anchored to a path, such +as `/data/*.csv` at the repository root for a project in a subdirectory, does not match the +exported paths. Such a file is an LFS pointer at the base and real content in the checkout, so it +always counts as changed: at most extra tests run. + +`` is any git ref or commit, such as the merge request's target branch from a CI variable. +`--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach +the merge base, fails with an error that says what to fetch, such as `git fetch --unshallow` or a +larger clone depth; it never silently selects nothing. + ## Examples ```bash diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index ef2438826b..2455c3febc 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -859,6 +859,19 @@ Local type overrides for DuckDB replay are configured per adapter dialect: See [Scenarios](/docs/concepts/scenarios) for details on local type overrides and capture limits. +## Git + +The branch that `--since @base` compares against: + +```toml +[git] +base_ref = "origin/main" +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `base_ref` | unset | Git ref that [`sqb test --since @base`](/docs/cli/test#run-only-affected-tests) and `sqb scenario test --since @base` read. Use a ref every checkout has, such as the remote default branch. | + ## dbt Configuration for running SQLBuild alongside an existing dbt project: