From 73c5208c8dae4e7634fa54c5b64555916e513d6a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:40:38 +0100 Subject: [PATCH 01/10] feat: select tests and scenarios affected by changes since a git ref --- .../definition_changes/_helpers/aspects.rs | 57 +++ .../src/definition_changes/_helpers/mod.rs | 1 + .../src/definition_changes/constants.rs | 58 +++ .../main/changes_summary.rs | 69 ++++ .../main/compare_definitions.rs | 88 ++++ .../main/definition_digest.rs | 24 ++ .../main/definition_snapshot_json.rs | 16 + .../src/definition_changes/main/mod.rs | 7 + .../src/definition_changes/main/node_key.rs | 6 + .../main/selection_reasons.rs | 121 ++++++ .../main/selection_report.rs | 31 ++ .../src/definition_changes/mod.rs | 10 + .../src/definition_changes/models.rs | 78 ++++ .../src/definition_changes/tests.rs | 6 + .../src/definition_changes/tests/helpers.rs | 72 ++++ .../tests/test_definition_changes.rs | 314 ++++++++++++++ .../definition_changes/tests/test_types.rs | 8 + crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/argv/_helpers/normalization.rs | 14 + .../sqlbuild-cli/src/argv/command_table.json | 44 ++ crates/sqlbuild-cli/src/argv/constants.rs | 4 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 8 +- .../src/project/_helpers/documents.rs | 15 +- .../src/project/_helpers/sections.rs | 12 + .../src/project/main/load_git_base_ref.rs | 9 + .../sqlbuild-config/src/project/main/mod.rs | 1 + .../src/project/tests/test_load_config.rs | 42 ++ .../src/git_revisions/_helpers/git_command.rs | 50 +++ .../src/git_revisions/_helpers/mod.rs | 1 + .../src/git_revisions/constants.rs | 16 + .../sqlbuild-core/src/git_revisions/errors.rs | 9 + .../src/git_revisions/main/export_git_tree.rs | 68 +++ .../src/git_revisions/main/mod.rs | 4 + .../git_revisions/main/resolve_git_base.rs | 101 +++++ .../main/resolve_since_reference.rs | 27 ++ .../src/git_revisions/main/short_commit.rs | 8 + crates/sqlbuild-core/src/git_revisions/mod.rs | 10 + .../sqlbuild-core/src/git_revisions/models.rs | 15 + .../sqlbuild-core/src/git_revisions/tests.rs | 6 + .../src/git_revisions/tests/helpers.rs | 86 ++++ .../git_revisions/tests/test_git_revisions.rs | 195 +++++++++ .../src/git_revisions/tests/test_types.rs | 14 + crates/sqlbuild-core/src/lib.rs | 1 + .../main/render_test_inspection.rs | 3 + .../sqlbuild-core/src/test_command/models.rs | 2 + .../test_command/tests/test_test_command.rs | 12 + .../_helpers/change_selection/base_compile.rs | 88 ++++ .../change_selection/base_snapshot.rs | 228 ++++++++++ .../change_selection/definition_snapshot.rs | 391 ++++++++++++++++++ .../bindings/_helpers/change_selection/mod.rs | 5 + .../_helpers/change_selection/registration.rs | 9 + .../_helpers/change_selection/selection.rs | 222 ++++++++++ .../src/bindings/_helpers/cli/registration.rs | 13 +- .../_helpers/discovery/project_config.rs | 8 + .../src/bindings/_helpers/mod.rs | 1 + .../bindings/_helpers/planner_scenario/mod.rs | 2 +- .../bindings/_helpers/records/constants.rs | 4 + .../_helpers/scenario_command/test_command.rs | 63 ++- .../_helpers/sql_test_command/test_command.rs | 94 ++++- src/sqlbuild/_native.pyi | 8 + .../src/content/docs/docs/cli/scenario.mdx | 20 + website/src/content/docs/docs/cli/test.mdx | 57 +++ .../docs/concepts/project-configuration.mdx | 13 + 63 files changed, 2951 insertions(+), 19 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/definition_changes/_helpers/aspects.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/constants.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/node_key.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/mod.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/models.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/tests.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs create mode 100644 crates/sqlbuild-config/src/project/main/load_git_base_ref.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/_helpers/git_command.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/constants.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/errors.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/main/mod.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/main/resolve_since_reference.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/main/short_commit.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/mod.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/models.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/tests.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/aspects.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/aspects.rs new file mode 100644 index 0000000000..99f896fc06 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/aspects.rs @@ -0,0 +1,57 @@ +//! Aspect comparison and the words reasons use for aspects and kinds. + +use crate::definition_changes::constants::{ASPECT_LABELS, KIND_LABELS}; +use crate::definition_changes::models::NodeDefinition; + +fn aspect_rank(aspect: &str) -> usize { + ASPECT_LABELS + .iter() + .position(|(name, _)| *name == aspect) + .unwrap_or(ASPECT_LABELS.len()) +} + +/// The aspects present on either side whose digests differ, in reason order. +pub(crate) fn changed_aspects(base: &NodeDefinition, current: &NodeDefinition) -> Vec { + let mut aspects: Vec = base + .aspects + .keys() + .chain(current.aspects.keys()) + .filter(|aspect| base.aspects.get(*aspect) != current.aspects.get(*aspect)) + .cloned() + .collect(); + aspects.sort_by(|left, right| { + aspect_rank(left) + .cmp(&aspect_rank(right)) + .then(left.cmp(right)) + }); + aspects.dedup(); + aspects +} + +/// How an aspect reads in a reason. +pub(crate) fn aspect_label(aspect: &str) -> &str { + ASPECT_LABELS + .iter() + .find(|(name, _)| *name == aspect) + .map_or(aspect, |(_, label)| label) +} + +/// The singular and plural words of a node kind. +pub(crate) fn kind_words(kind: &str) -> (&str, String) { + KIND_LABELS + .iter() + .find(|(name, _, _)| *name == kind) + .map_or_else( + || (kind, format!("{kind}s")), + |(_, singular, plural)| (*singular, (*plural).to_owned()), + ) +} + +/// `a`, `a and b`, `a, b and c`. +pub(crate) fn joined(words: &[String]) -> String { + match words { + [] => String::new(), + [only] => only.clone(), + [init @ .., last] => format!("{} and {last}", init.join(", ")), + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs new file mode 100644 index 0000000000..38239ddf94 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod aspects; diff --git a/crates/sqlbuild-analysis/src/definition_changes/constants.rs b/crates/sqlbuild-analysis/src/definition_changes/constants.rs new file mode 100644 index 0000000000..48742175e4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/constants.rs @@ -0,0 +1,58 @@ +//! Node kinds, definition aspects and the snapshot format version. + +/// The version of the stored snapshot layout and fingerprint scheme; older snapshots are recomputed. +pub const DEFINITION_SNAPSHOT_FORMAT: u32 = 1; + +pub const MODEL_KIND: &str = "model"; +pub const SEED_KIND: &str = "seed"; +pub const UDF_KIND: &str = "udf"; +pub const TABLE_FUNCTION_KIND: &str = "table_fn"; +pub const MACRO_KIND: &str = "macro"; +pub const ENUM_KIND: &str = "enum"; +pub const CONSTANT_KIND: &str = "constant"; +pub const TEST_FILE_KIND: &str = "test_file"; +pub const SCENARIO_KIND: &str = "scenario"; + +/// Normalised SQL of a model, a function body or a scenario. +pub const SQL_ASPECT: &str = "sql"; +/// Header configuration, without descriptions. +pub const CONFIG_ASPECT: &str = "config"; +/// Declared columns and contract, without descriptions. +pub const COLUMNS_ASPECT: &str = "columns"; +/// Attached audits, without descriptions. +pub const AUDITS_ASPECT: &str = "audits"; +/// The whole definition of a seed, function, macro, enum or constant. +pub const DEFINITION_ASPECT: &str = "definition"; +/// The authored file of a test or scenario. +pub const FILE_ASPECT: &str = "file"; +/// Captured local snapshots of a scenario. +pub const SNAPSHOTS_ASPECT: &str = "snapshots"; + +/// How each aspect reads in a reason, in the order reasons list them. +pub(crate) const ASPECT_LABELS: [(&str, &str); 7] = [ + (SQL_ASPECT, "SQL"), + (CONFIG_ASPECT, "config"), + (COLUMNS_ASPECT, "columns"), + (AUDITS_ASPECT, "audits"), + (DEFINITION_ASPECT, "definition"), + (FILE_ASPECT, "file"), + (SNAPSHOTS_ASPECT, "snapshots"), +]; + +/// How each node kind reads in summaries: singular, plural. +pub(crate) const KIND_LABELS: [(&str, &str, &str); 14] = [ + (MODEL_KIND, "model", "models"), + (SEED_KIND, "seed", "seeds"), + (UDF_KIND, "function", "functions"), + (TABLE_FUNCTION_KIND, "table function", "table functions"), + (MACRO_KIND, "macro", "macros"), + (ENUM_KIND, "enum", "enums"), + (CONSTANT_KIND, "constant", "constants"), + ("audit", "audit", "audits"), + ("singular_audit", "singular audit", "singular audits"), + ("schema", "schema", "schemas"), + ("sql_hook", "SQL hook", "SQL hooks"), + ("python_hook", "Python hook", "Python hooks"), + (TEST_FILE_KIND, "test file", "test files"), + (SCENARIO_KIND, "scenario", "scenarios"), +]; diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs b/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs new file mode 100644 index 0000000000..8cae4b31f1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs @@ -0,0 +1,69 @@ +//! A one-line account of what changed since the base revision. + +use std::collections::BTreeMap; + +use crate::definition_changes::_helpers::aspects::kind_words; +use crate::definition_changes::constants::KIND_LABELS; +use crate::definition_changes::models::DefinitionChanges; + +fn kind_rank(kind: &str) -> usize { + KIND_LABELS + .iter() + .position(|(name, _, _)| *name == kind) + .unwrap_or(KIND_LABELS.len()) +} + +fn counted<'a>(kinds: impl Iterator) -> Option { + let mut counts: BTreeMap<(usize, &str), usize> = BTreeMap::new(); + for kind in kinds { + *counts.entry((kind_rank(kind), kind)).or_default() += 1; + } + if counts.is_empty() { + return None; + } + let parts: Vec = counts + .into_iter() + .map(|((_, kind), count)| { + let (singular, plural) = kind_words(kind); + if count == 1 { + format!("1 {singular}") + } else { + format!("{count} {plural}") + } + }) + .collect(); + Some(parts.join(", ")) +} + +/// `2 models, 1 macro changed; 1 seed added; 1 model renamed; deleted: model old_orders`, or +/// `no definition changes`. +pub fn changes_summary(changes: &DefinitionChanges) -> String { + let mut parts: Vec = Vec::new(); + let modified = changes.changed.values().filter(|change| !change.added); + if let Some(text) = counted(modified.map(|change| change.kind.as_str())) { + parts.push(format!("{text} changed")); + } + let added = changes.changed.values().filter(|change| change.added); + if let Some(text) = counted(added.map(|change| change.kind.as_str())) { + parts.push(format!("{text} added")); + } + let renamed = changes + .renamed + .iter() + .filter_map(|(_, key)| key.split_once(':').map(|(kind, _)| kind)); + if let Some(text) = counted(renamed) { + parts.push(format!("{text} renamed")); + } + if !changes.deleted.is_empty() { + let names: Vec = changes + .deleted + .iter() + .map(|node| format!("{} {}", kind_words(&node.kind).0, node.name)) + .collect(); + parts.push(format!("deleted: {}", names.join(", "))); + } + if parts.is_empty() { + return "no definition changes".to_owned(); + } + parts.join("; ") +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs b/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs new file mode 100644 index 0000000000..1ddef9f5f3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs @@ -0,0 +1,88 @@ +//! The nodes that changed between a base and a current definition snapshot. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::definition_changes::_helpers::aspects::changed_aspects; +use crate::definition_changes::main::node_key::node_key; +use crate::definition_changes::models::{ + DefinitionChanges, DefinitionSnapshot, DeletedNode, NodeChange, NodeDefinition, +}; + +/// The base node a current node renamed through `migrate_from`, if one is unclaimed. +fn renamed_base<'a>( + current_key: &str, + node: &NodeDefinition, + (base, current): (&'a DefinitionSnapshot, &DefinitionSnapshot), + claimed: &BTreeSet, +) -> Option<(String, &'a NodeDefinition)> { + node.migrate_from.iter().find_map(|old_name| { + let old_key: String = node_key(&node.kind, old_name); + if old_key == current_key + || claimed.contains(&old_key) + || current.nodes.contains_key(&old_key) + { + return None; + } + base.nodes + .get(&old_key) + .map(|definition| (old_key, definition)) + }) +} + +/// Added nodes count as changed, deleted nodes are listed, and renames follow `migrate_from`: +/// a renamed node is changed only when its definition also changed. +pub fn compare_definitions( + base: &DefinitionSnapshot, + current: &DefinitionSnapshot, +) -> DefinitionChanges { + let mut changed: BTreeMap = BTreeMap::new(); + let mut renamed: Vec<(String, String)> = Vec::new(); + let mut claimed: BTreeSet = BTreeSet::new(); + for (key, node) in ¤t.nodes { + let change = |added: bool, aspects: Vec, renamed_from: Option| NodeChange { + kind: node.kind.clone(), + name: node.name.clone(), + added, + aspects, + renamed_from, + }; + if let Some(previous) = base.nodes.get(key) { + let aspects: Vec = changed_aspects(previous, node); + if !aspects.is_empty() { + changed.insert(key.clone(), change(false, aspects, None)); + } + continue; + } + match renamed_base(key, node, (base, current), &claimed) { + Some((old_key, previous)) => { + let aspects: Vec = changed_aspects(previous, node); + if !aspects.is_empty() { + changed.insert( + key.clone(), + change(false, aspects, Some(previous.name.clone())), + ); + } + claimed.insert(old_key.clone()); + renamed.push((old_key, key.clone())); + } + None => { + changed.insert(key.clone(), change(true, Vec::new(), None)); + } + } + } + let mut deleted: Vec = base + .nodes + .iter() + .filter(|(key, _)| !current.nodes.contains_key(*key) && !claimed.contains(*key)) + .map(|(_, node)| DeletedNode { + kind: node.kind.clone(), + name: node.name.clone(), + }) + .collect(); + deleted.sort(); + DefinitionChanges { + changed, + deleted, + renamed, + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs b/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs new file mode 100644 index 0000000000..22a70af50f --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs @@ -0,0 +1,24 @@ +//! Digests of definition parts and of normalised SQL. + +use sha2::{Digest, Sha256}; + +use crate::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; + +/// The SHA-256 hex digest of `parts`, each length-prefixed so boundaries cannot shift. +pub fn definition_digest(parts: &[&str]) -> String { + let mut digest = Sha256::new(); + for part in parts { + digest.update(part.len().to_string().as_bytes()); + digest.update(b":"); + digest.update(part.as_bytes()); + } + format!("{:x}", digest.finalize()) +} + +/// The fingerprint of `sql` with whitespace, comments and keyword case ignored. +/// +/// The generic tokenizer is used whatever the adapter, so one snapshot compares across adapters; +/// SQL it cannot tokenize is fingerprinted by its exact text. +pub fn sql_definition_digest(sql: &str) -> String { + compiler_query_fingerprint(sql, None).unwrap_or_else(|_| definition_digest(&[sql])) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs b/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs new file mode 100644 index 0000000000..891f490473 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs @@ -0,0 +1,16 @@ +//! The stored JSON form of a definition snapshot. + +use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; +use crate::definition_changes::models::DefinitionSnapshot; + +/// `snapshot` as compact JSON. +pub fn definition_snapshot_json(snapshot: &DefinitionSnapshot) -> String { + serde_json::to_string(snapshot).unwrap_or_default() +} + +/// The snapshot `text` stores, or `None` when it is unreadable or from another format version. +pub fn parse_definition_snapshot(text: &str) -> Option { + serde_json::from_str::(text) + .ok() + .filter(|snapshot| snapshot.format == DEFINITION_SNAPSHOT_FORMAT) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs new file mode 100644 index 0000000000..6cdd5334db --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs @@ -0,0 +1,7 @@ +pub mod changes_summary; +pub mod compare_definitions; +pub mod definition_digest; +pub mod definition_snapshot_json; +pub mod node_key; +pub mod selection_reasons; +pub mod selection_report; diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/node_key.rs b/crates/sqlbuild-analysis/src/definition_changes/main/node_key.rs new file mode 100644 index 0000000000..b2767a3e46 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/node_key.rs @@ -0,0 +1,6 @@ +//! The snapshot key of one node. + +/// `kind:identity`, where identity is a name, or a declaration identity for scoped declarations. +pub fn node_key(kind: &str, identity: &str) -> String { + format!("{kind}:{identity}") +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs new file mode 100644 index 0000000000..e3248221bf --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs @@ -0,0 +1,121 @@ +//! Why a test or scenario is affected by the changes since the base revision. + +use std::collections::BTreeSet; + +use crate::definition_changes::_helpers::aspects::{aspect_label, joined, kind_words}; +use crate::definition_changes::constants::{CONSTANT_KIND, ENUM_KIND, MODEL_KIND, TEST_FILE_KIND}; +use crate::definition_changes::models::{ + DefinitionChanges, DefinitionSnapshot, ExecutedItem, NodeChange, +}; + +fn aspect_phrase(change: &NodeChange) -> String { + let labels: Vec = change + .aspects + .iter() + .map(|aspect| aspect_label(aspect).to_owned()) + .collect(); + joined(&labels) +} + +/// `test file changed`, `scenario file and snapshots changed`, `test file added`. +fn own_reason(change: &NodeChange) -> String { + let (singular, _) = kind_words(&change.kind); + if change.added { + return format!("{singular} added"); + } + if change.kind == TEST_FILE_KIND { + return format!("{singular} changed"); + } + format!("{singular} {} changed", aspect_phrase(change)) +} + +/// `added`, `model SQL changed`, `renamed from old_orders, config changed`, `changed`. +fn change_detail(change: &NodeChange) -> String { + if change.added { + return "added".to_owned(); + } + let detail: String = if change.kind == MODEL_KIND { + format!("model {} changed", aspect_phrase(change)) + } else { + "changed".to_owned() + }; + match &change.renamed_from { + Some(old_name) => format!("renamed from {old_name}, {detail}"), + None => detail, + } +} + +fn executed_reason(change: &NodeChange) -> String { + let subject: String = if change.kind == MODEL_KIND { + change.name.clone() + } else { + format!("{} {}", kind_words(&change.kind).0, change.name) + }; + format!("executes {subject} ({})", change_detail(change)) +} + +fn declaration_reason(change: &NodeChange) -> String { + let verb: &str = if change.kind == ENUM_KIND || change.kind == CONSTANT_KIND { + "uses" + } else { + "executes" + }; + format!( + "{verb} {} {} ({})", + kind_words(&change.kind).0, + change.name, + change_detail(change) + ) +} + +/// Declarations `item` reaches through its own SQL and its executed nodes, transitively, +/// without the ones it mocks. +fn reached_declarations(item: &ExecutedItem, current: &DefinitionSnapshot) -> BTreeSet { + let mut pending: Vec = item.declarations.clone(); + for key in &item.executed { + if let Some(node) = current.nodes.get(key) { + pending.extend(node.uses.iter().cloned()); + } + } + let mut reached: BTreeSet = BTreeSet::new(); + while let Some(key) = pending.pop() { + if item.mocked.contains(&key) || !reached.insert(key.clone()) { + continue; + } + if let Some(node) = current.nodes.get(&key) { + pending.extend(node.uses.iter().cloned()); + } + } + reached +} + +/// One reason per changed node `item` depends on: its own definition first, then executed +/// nodes in execution order, then declarations by key. Empty when nothing it runs changed. +pub fn selection_reasons( + item: &ExecutedItem, + current: &DefinitionSnapshot, + changes: &DefinitionChanges, +) -> Vec { + let mut reasons: Vec = Vec::new(); + let mut push = |reason: String| { + if !reasons.contains(&reason) { + reasons.push(reason); + } + }; + for key in &item.own_keys { + if let Some(change) = changes.changed.get(key) { + push(own_reason(change)); + } + } + for key in &item.executed { + if let Some(change) = changes.changed.get(key) { + push(executed_reason(change)); + } + } + for key in reached_declarations(item, current) { + if let Some(change) = changes.changed.get(&key) { + push(declaration_reason(change)); + } + } + reasons +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs new file mode 100644 index 0000000000..9f26bd3a06 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs @@ -0,0 +1,31 @@ +//! The lines a `--since` run prints about what changed and what it selected. + +use crate::definition_changes::models::SelectedItem; + +/// `Changed since : …`, then `Selected N of M ` with each item's reasons when +/// `explain` is set, or the line saying nothing is affected. +pub fn selection_report( + (reference, summary): (&str, &str), + (singular, plural): (&str, &str), + (total, selected): (usize, &[SelectedItem]), + explain: bool, +) -> String { + let mut text: String = format!("Changed since {reference}: {summary}\n"); + if selected.is_empty() { + text.push_str(&format!( + "No {plural} are affected by the changes since {reference}\n" + )); + return text; + } + let noun: &str = if total == 1 { singular } else { plural }; + text.push_str(&format!("Selected {} of {total} {noun}\n", selected.len())); + if explain { + for item in selected { + text.push_str(&format!(" {}\n", item.label)); + for reason in &item.reasons { + text.push_str(&format!(" {reason}\n")); + } + } + } + text +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/mod.rs new file mode 100644 index 0000000000..48ad6791b7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/mod.rs @@ -0,0 +1,10 @@ +//! Definition fingerprints of project nodes, the changes between two snapshots of them, and the +//! reasons a test or scenario that executes changed nodes is selected. + +mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/definition_changes/models.rs b/crates/sqlbuild-analysis/src/definition_changes/models.rs new file mode 100644 index 0000000000..6df7992221 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/models.rs @@ -0,0 +1,78 @@ +//! Definition snapshots, the changes between two of them, and what a test or scenario executes. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +/// The definition fingerprint of one project node. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeDefinition { + pub kind: String, + pub name: String, + /// The digest of each part of the definition, by aspect name. + pub aspects: BTreeMap, + /// Keys of the macros, enums and constants the node uses, resolved through scopes. + #[serde(default)] + pub uses: Vec, + /// Names the node was renamed from, as its `migrate_from` declares them. + #[serde(default)] + pub migrate_from: Vec, +} + +/// Every node definition of one compiled project revision, by node key. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DefinitionSnapshot { + pub format: u32, + pub nodes: BTreeMap, +} + +/// How one node differs from the base revision. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NodeChange { + pub kind: String, + pub name: String, + /// The node did not exist at the base revision. + pub added: bool, + /// The aspects whose digests differ, in aspect order; empty for an added node. + pub aspects: Vec, + /// The base name of a node its `migrate_from` renamed. + pub renamed_from: Option, +} + +/// A base node that no longer exists. +#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct DeletedNode { + pub kind: String, + pub name: String, +} + +/// The nodes that changed between a base and a current snapshot. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DefinitionChanges { + /// Added and modified nodes, by current node key. + pub changed: BTreeMap, + /// Base nodes absent from the current revision, which select nothing. + pub deleted: Vec, + /// `(base key, current key)` of renames that `migrate_from` declares. + pub renamed: Vec<(String, String)>, +} + +/// What one test or scenario depends on, honouring what it mocks. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExecutedItem { + /// Keys of the item's own definition, such as its test file. + pub own_keys: Vec, + /// Keys of the models, seeds and functions it really executes, in execution order. + pub executed: Vec, + /// Keys of the macros, enums and constants its own SQL uses. + pub declarations: Vec, + /// Keys of declarations the item mocks, whose changes never select it. + pub mocked: BTreeSet, +} + +/// One selected test or scenario and why it was selected. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SelectedItem { + pub label: String, + pub reasons: Vec, +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests.rs b/crates/sqlbuild-analysis/src/definition_changes/tests.rs new file mode 100644 index 0000000000..ded3fb082a --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_definition_changes.rs"] +mod definition_changes; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs new file mode 100644 index 0000000000..7032f9dce2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs @@ -0,0 +1,72 @@ +use std::collections::BTreeMap; + +use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; +use crate::definition_changes::main::node_key::node_key; +use crate::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; + +/// One node with `aspects` as `(aspect, digest)` and `uses` as node keys. +pub(super) fn node( + kind: &str, + name: &str, + aspects: &[(&str, &str)], + uses: &[&str], +) -> (String, NodeDefinition) { + ( + node_key(kind, name), + NodeDefinition { + kind: kind.to_owned(), + name: name.to_owned(), + aspects: aspects + .iter() + .map(|(aspect, digest)| ((*aspect).to_owned(), (*digest).to_owned())) + .collect(), + uses: uses.iter().map(|key| (*key).to_owned()).collect(), + migrate_from: Vec::new(), + }, + ) +} + +/// `definition` renamed from `old_names`. +pub(super) fn renamed( + (key, mut definition): (String, NodeDefinition), + old_names: &[&str], +) -> (String, NodeDefinition) { + definition.migrate_from = old_names.iter().map(|name| (*name).to_owned()).collect(); + (key, definition) +} + +pub(super) fn snapshot(nodes: Vec<(String, NodeDefinition)>) -> DefinitionSnapshot { + DefinitionSnapshot { + format: DEFINITION_SNAPSHOT_FORMAT, + nodes: nodes.into_iter().collect::>(), + } +} + +/// The orders project at its base revision. +pub(super) fn base_orders() -> DefinitionSnapshot { + snapshot(vec![ + node( + "model", + "stg_orders", + &[("sql", "s1"), ("config", "c1"), ("columns", "k1")], + &["macro:to_utc"], + ), + node( + "model", + "orders", + &[("sql", "s2"), ("config", "c2")], + &["enum:order_status"], + ), + node("model", "old_customers", &[("sql", "s3")], &[]), + node("seed", "countries", &[("definition", "d1")], &[]), + node( + "macro", + "to_utc", + &[("definition", "m1")], + &["macro:to_timestamp"], + ), + node("macro", "to_timestamp", &[("definition", "m2")], &[]), + node("enum", "order_status", &[("definition", "e1")], &[]), + node("test_file", "tests/orders.sql", &[("file", "t1")], &[]), + ]) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs new file mode 100644 index 0000000000..a90877e085 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs @@ -0,0 +1,314 @@ +use std::collections::BTreeSet; + +use crate::definition_changes::main::changes_summary::changes_summary; +use crate::definition_changes::main::compare_definitions::compare_definitions; +use crate::definition_changes::main::definition_digest::sql_definition_digest; +use crate::definition_changes::main::definition_snapshot_json::{ + definition_snapshot_json, parse_definition_snapshot, +}; +use crate::definition_changes::main::selection_reasons::selection_reasons; +use crate::definition_changes::main::selection_report::selection_report; +use crate::definition_changes::models::{ + DefinitionChanges, DefinitionSnapshot, DeletedNode, ExecutedItem, NodeChange, SelectedItem, +}; +use crate::definition_changes::tests::helpers::{base_orders, node, renamed, snapshot}; +use crate::definition_changes::tests::test_types::SelectionReasonTestCase; + +fn keys(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// The base orders project with `stg_orders` SQL edited, `customers` renamed from +/// `old_customers` and `countries` deleted. +fn current_orders() -> DefinitionSnapshot { + snapshot(vec![ + node( + "model", + "stg_orders", + &[("sql", "s1-edited"), ("config", "c1"), ("columns", "k1")], + &["macro:to_utc"], + ), + node( + "model", + "orders", + &[("sql", "s2"), ("config", "c2")], + &["enum:order_status"], + ), + renamed( + node("model", "customers", &[("sql", "s3")], &[]), + &["old_customers"], + ), + node( + "macro", + "to_utc", + &[("definition", "m1")], + &["macro:to_timestamp"], + ), + node("macro", "to_timestamp", &[("definition", "m2")], &[]), + node("enum", "order_status", &[("definition", "e1")], &[]), + node("test_file", "tests/orders.sql", &[("file", "t1")], &[]), + ]) +} + +#[test] +fn given_two_snapshots_when_comparing_then_reports_changed_renamed_and_deleted_nodes() { + let changes: DefinitionChanges = compare_definitions(&base_orders(), ¤t_orders()); + + assert_eq!( + changes.changed.into_iter().collect::>(), + vec![( + "model:stg_orders".to_owned(), + NodeChange { + kind: "model".to_owned(), + name: "stg_orders".to_owned(), + added: false, + aspects: keys(&["sql"]), + renamed_from: None, + } + )] + ); + assert_eq!( + changes.renamed, + vec![( + "model:old_customers".to_owned(), + "model:customers".to_owned() + )] + ); + assert_eq!( + changes.deleted, + vec![DeletedNode { + kind: "seed".to_owned(), + name: "countries".to_owned(), + }] + ); +} + +#[test] +fn given_renamed_node_with_edited_definition_when_comparing_then_it_changed_from_old_name() { + let base = snapshot(vec![node("model", "old_customers", &[("sql", "s3")], &[])]); + let current = snapshot(vec![renamed( + node( + "model", + "customers", + &[("sql", "s4"), ("config", "c4")], + &[], + ), + &["old_customers"], + )]); + + let changes: DefinitionChanges = compare_definitions(&base, ¤t); + + let change: &NodeChange = &changes.changed["model:customers"]; + assert_eq!(change.renamed_from.as_deref(), Some("old_customers")); + assert_eq!(change.aspects, keys(&["sql", "config"])); + assert!(changes.deleted.is_empty()); +} + +#[test] +fn given_changes_when_summarising_then_counts_each_kind_and_names_deleted_nodes() { + let mut current: DefinitionSnapshot = current_orders(); + current.nodes.extend([ + node("model", "returns", &[("sql", "s9")], &[]), + node("macro", "to_timestamp", &[("definition", "m3")], &[]), + ]); + + let summary: String = changes_summary(&compare_definitions(&base_orders(), ¤t)); + + assert_eq!( + summary, + "1 model, 1 macro changed; 1 model added; 1 model renamed; deleted: seed countries" + ); + assert_eq!( + changes_summary(&compare_definitions(&base_orders(), &base_orders())), + "no definition changes" + ); +} + +fn item(own: &[&str], executed: &[&str], declarations: &[&str], mocked: &[&str]) -> ExecutedItem { + ExecutedItem { + own_keys: keys(own), + executed: keys(executed), + declarations: keys(declarations), + mocked: mocked + .iter() + .map(|key| (*key).to_owned()) + .collect::>(), + } +} + +fn with_node( + mut current: DefinitionSnapshot, + entry: (String, crate::definition_changes::models::NodeDefinition), +) -> DefinitionSnapshot { + current.nodes.insert(entry.0, entry.1); + current +} + +#[test] +fn given_executed_item_when_explaining_selection_then_lists_each_changed_dependency() { + let test_cases = [ + SelectionReasonTestCase { + description: "an executed model whose SQL changed", + current: current_orders(), + item: item( + &["test_file:tests/orders.sql"], + &["model:stg_orders", "model:orders"], + &[], + &[], + ), + expected_reasons: &["executes stg_orders (model SQL changed)"], + }, + SelectionReasonTestCase { + description: "a mocked model is not executed, so its change selects nothing", + current: current_orders(), + item: item(&["test_file:tests/orders.sql"], &["model:orders"], &[], &[]), + expected_reasons: &[], + }, + SelectionReasonTestCase { + description: "a macro changed through another macro the model calls", + current: with_node( + snapshot(base_orders().nodes.into_iter().collect()), + node("macro", "to_timestamp", &[("definition", "m3")], &[]), + ), + item: item(&[], &["model:stg_orders"], &[], &[]), + expected_reasons: &["executes macro to_timestamp (changed)"], + }, + SelectionReasonTestCase { + description: "a mocked macro hides its own and its callees' changes", + current: with_node( + snapshot(base_orders().nodes.into_iter().collect()), + node("macro", "to_timestamp", &[("definition", "m3")], &[]), + ), + item: item(&[], &["model:stg_orders"], &[], &["macro:to_utc"]), + expected_reasons: &[], + }, + SelectionReasonTestCase { + description: "an enum used by an executed model", + current: with_node( + snapshot(base_orders().nodes.into_iter().collect()), + node("enum", "order_status", &[("definition", "e2")], &[]), + ), + item: item(&[], &["model:orders"], &[], &[]), + expected_reasons: &["uses enum order_status (changed)"], + }, + SelectionReasonTestCase { + description: "the test's own file and an added model", + current: with_node( + with_node( + snapshot(base_orders().nodes.into_iter().collect()), + node("test_file", "tests/orders.sql", &[("file", "t2")], &[]), + ), + node("model", "returns", &[("sql", "s9")], &[]), + ), + item: item( + &["test_file:tests/orders.sql"], + &["model:returns"], + &[], + &[], + ), + expected_reasons: &["test file changed", "executes returns (added)"], + }, + SelectionReasonTestCase { + description: "a renamed model whose config changed", + current: snapshot(vec![renamed( + node( + "model", + "customers", + &[("sql", "s3"), ("config", "c9")], + &[], + ), + &["old_customers"], + )]), + item: item(&[], &["model:customers"], &[], &[]), + expected_reasons: &[ + "executes customers (renamed from old_customers, model config changed)", + ], + }, + SelectionReasonTestCase { + description: "a changed seed the test really reads", + current: with_node( + snapshot(base_orders().nodes.into_iter().collect()), + node("seed", "countries", &[("definition", "d2")], &[]), + ), + item: item(&[], &["model:orders", "seed:countries"], &[], &[]), + expected_reasons: &["executes seed countries (changed)"], + }, + ]; + for test_case in test_cases { + let changes: DefinitionChanges = compare_definitions(&base_orders(), &test_case.current); + + let reasons: Vec = selection_reasons(&test_case.item, &test_case.current, &changes); + + assert_eq!( + reasons, + keys(test_case.expected_reasons), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_snapshot_when_stored_then_reads_back_only_in_the_same_format() { + let snapshot: DefinitionSnapshot = base_orders(); + let stored: String = definition_snapshot_json(&snapshot); + + assert_eq!(parse_definition_snapshot(&stored), Some(snapshot)); + assert_eq!( + parse_definition_snapshot(&stored.replacen("\"format\":1", "\"format\":0", 1)), + None + ); + assert_eq!(parse_definition_snapshot("not json"), None); +} + +#[test] +fn given_sql_when_fingerprinting_definition_then_ignores_layout_and_comments_only() { + let original: String = sql_definition_digest("SELECT order_id FROM orders"); + + assert_eq!( + sql_definition_digest("select\n order_id -- the key\nfrom orders"), + original + ); + assert_ne!( + sql_definition_digest("SELECT customer_id FROM orders"), + original + ); +} + +#[test] +fn given_selected_items_when_reporting_then_counts_and_explains_only_on_request() { + let selected = [SelectedItem { + label: "orders_rollup".to_owned(), + reasons: keys(&["executes stg_orders (model SQL changed)"]), + }]; + + assert_eq!( + selection_report( + ("origin/main", "1 model changed"), + ("test", "tests"), + (9, &selected), + true + ), + "Changed since origin/main: 1 model changed\nSelected 1 of 9 tests\n orders_rollup\n \ + executes stg_orders (model SQL changed)\n" + ); + assert_eq!( + selection_report( + ("origin/main", "1 model changed"), + ("test", "tests"), + (9, &selected), + false + ), + "Changed since origin/main: 1 model changed\nSelected 1 of 9 tests\n" + ); + assert_eq!( + selection_report( + ("origin/main", "no definition changes"), + ("scenario", "scenarios"), + (3, &[]), + true + ), + "Changed since origin/main: no definition changes\nNo scenarios are affected by the \ + changes since origin/main\n" + ); +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs new file mode 100644 index 0000000000..12aaebe1c0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem}; + +pub(crate) struct SelectionReasonTestCase { + pub(crate) description: &'static str, + pub(crate) current: DefinitionSnapshot, + pub(crate) item: ExecutedItem, + pub(crate) expected_reasons: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 3f2e538f9d..af65f6af67 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -11,6 +11,7 @@ pub mod compiler; mod constants; pub mod contracts; pub mod dag_artifact; +pub mod definition_changes; pub mod diff_selection; pub mod explicit_references; pub mod fingerprint_storage; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs index f446d3496b..f0b830b251 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs @@ -110,6 +110,9 @@ impl Run<'_> { "test --inspect cannot be combined with --json or --json-output", )); } + if let Some(message) = since_error(&namespace) { + return Err(self.error(parser, message)); + } if command == JANITOR_COMMAND && namespace.value("as_target") != Value::None && truthy(&namespace.value("auto_approve")) @@ -147,6 +150,17 @@ impl Run<'_> { } } +/// The first rule `--since` and `--explain` break, on the commands that accept them. +fn since_error(namespace: &Namespace) -> Option<&'static str> { + match namespace.value("since") { + Value::None if truthy(&namespace.value("explain")) => Some("--explain requires --since"), + Value::Str(reference) if reference.trim().is_empty() => { + Some("--since requires a git ref, such as origin/main or @base") + } + _ => None, + } +} + /// `_validate_scope_args`: the first rule the scope options break. fn scope_error(parser: &ParserSpec, namespace: &Namespace) -> Option<&'static str> { let present = |name: &str| namespace.value(name) != Value::None; diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 95a778e52a..49e50ad176 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1731,6 +1731,28 @@ "help": "Show the resolved test plan", "group": 1 }, + { + "kind": "store", + "options": [ + "--since" + ], + "dest": "since", + "metavar": "REF", + "help": "run only the tests affected by changes since the merge base of REF and HEAD; @base reads [git] base_ref", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--explain" + ], + "dest": "explain", + "nargs": 0, + "const": true, + "default": false, + "help": "with --since, show why each test was selected", + "group": 1 + }, { "kind": "store", "options": [ @@ -5595,6 +5617,28 @@ "help": "Per-scenario timeout for this run, such as '10m', or 'disabled'", "group": 1 }, + { + "kind": "store", + "options": [ + "--since" + ], + "dest": "since", + "metavar": "REF", + "help": "run only the scenarios affected by changes since the merge base of REF and HEAD; @base reads [git] base_ref", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--explain" + ], + "dest": "explain", + "nargs": 0, + "const": true, + "default": false, + "help": "with --since, show why each scenario was selected", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 050122eac5..5f8f0d494b 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -63,7 +63,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 177] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 179] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -83,6 +83,8 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 177] = [ ("warehouse", NamespaceDefault::None), ("case", NamespaceDefault::None), ("inspect", NamespaceDefault::Bool(false)), + ("since", NamespaceDefault::None), + ("explain", NamespaceDefault::Bool(false)), ("target_range", NamespaceDefault::None), ("from_target", NamespaceDefault::None), ("to_target", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index da56abafec..83f89785d2 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -205,7 +205,7 @@ pub(crate) const FRESHNESS_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const TEST_FIELDS: [Field; 13] = [ +const TEST_FIELDS: [Field; 15] = [ PROJECT_DIR, NO_SQL_VALIDATION, NO_COLOR, @@ -219,6 +219,8 @@ const TEST_FIELDS: [Field; 13] = [ attr("case_name", "case"), same("inspect"), same("test_timeout"), + same("since"), + same("explain"), ]; const TEST_ARGUMENTS: [Field; 1] = [request(class("TestCommandRequest"), &TEST_FIELDS)]; pub(crate) const TEST_ROUTE: Route = routed( @@ -715,7 +717,7 @@ const SCENARIO_COMMON: [Field; 7] = [ EXCLUDE, attr("retain", "scenario_retain"), ]; -const SCENARIO_TEST_FIELDS: [Field; 16] = [ +const SCENARIO_TEST_FIELDS: [Field; 18] = [ SCENARIO_COMMON[0], SCENARIO_COMMON[1], SCENARIO_COMMON[2], @@ -732,6 +734,8 @@ const SCENARIO_TEST_FIELDS: [Field; 16] = [ JSON_OUTPUT, JSON_OUTPUT_PATH, same("scenario_timeout"), + same("since"), + same("explain"), ]; const SCENARIO_TEST_ARGUMENTS: [Field; 1] = [request( class("ScenarioTestCommandRequest"), diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 5690123c50..c5c121bc45 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -14,9 +14,9 @@ use crate::project::_helpers::help::{ }; use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ - load_constants, load_cost, load_cursors, load_dbt, load_diff, load_janitor, load_local_dbt, - load_microbatches, load_migrations, load_policy_flag, load_scenario, load_settings, - load_snapshots, setting_overrides, + load_constants, load_cost, load_cursors, load_dbt, load_diff, load_git, load_janitor, + load_local_dbt, load_microbatches, load_migrations, load_policy_flag, load_scenario, + load_settings, load_snapshots, setting_overrides, }; use crate::project::_helpers::sinks::load_sinks; use crate::project::_helpers::targets::{ @@ -276,6 +276,7 @@ fn project_fields(payload: &Entries) -> Loaded { recall_section(recall, RECALL_SECTION) .map_err(|message| ConfigFailure::at(PROJECT_FILE, message))?; } + load_git(get(payload, "git"), PROJECT_FILE)?; let mut fields = identity(payload)?; fields.extend(sections(payload)?); fields.push(("targets", targets(payload)?)); @@ -423,3 +424,11 @@ pub(crate) fn local_sql_analysis_failure( .with_help(help), ) } + +/// `[git] base_ref` of `sqlbuild_project.toml`, validated as project loading validates it. +pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { + let file = resolve(paths, false)?; + let table = load_toml_mapping(paths, file)?; + let payload = table.as_map().unwrap_or_default(); + Ok(load_git(get(payload, "git"), PROJECT_FILE)?) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 021b10dbc3..4dc70f4a5b 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -536,6 +536,18 @@ pub(crate) fn load_diff(payload: Option<&ConfigValue>, file: ConfigFile) -> Load )) } +/// `[git]`: the `base_ref` that `--since @base` compares against, when set. +pub(crate) fn load_git(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded> { + let mapping = coerce_mapping(payload, place(file, "git"))?; + validate_allowed_keys(mapping, &["base_ref"], place(file, "git"))?; + optional_str(mapping, "base_ref").map_err(|_| { + ConfigFailure::at( + file, + "git.base_ref must be a non-empty git ref such as 'origin/main'", + ) + }) +} + fn janitor_count(mapping: &Entries, key: &str, default: i64) -> Loaded { optional_int(mapping, key, default) } diff --git a/crates/sqlbuild-config/src/project/main/load_git_base_ref.rs b/crates/sqlbuild-config/src/project/main/load_git_base_ref.rs new file mode 100644 index 0000000000..63d4ff0c07 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/load_git_base_ref.rs @@ -0,0 +1,9 @@ +//! Read `[git] base_ref` from `sqlbuild_project.toml`. + +use crate::project::errors::LoadFailure; +use crate::project::models::ConfigPaths; + +/// The configured base ref that `--since @base` names, or `None` when `[git]` sets none. +pub fn load_git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { + crate::project::_helpers::documents::git_base_ref(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index a3e65d82e8..7198e4d7de 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -1,4 +1,5 @@ pub mod explicit_references_help; +pub mod load_git_base_ref; pub mod load_local_config; pub mod load_project_config; pub mod local_sql_analysis_failure; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 1f460c6232..29504cd0d1 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,3 +1,4 @@ +use crate::project::main::load_git_base_ref::load_git_base_ref; use crate::project::models::ConfigFile; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; use crate::project::tests::test_types::ProjectFailureTestCase; @@ -39,6 +40,32 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { ), ), }, + ProjectFailureTestCase { + description: "a git base ref loads", + project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n", + local: "", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "an empty git base ref is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = ''\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "git.base_ref must be a non-empty git ref such as 'origin/main'", + None, + ), + }, + ProjectFailureTestCase { + description: "a misspelt git key suggests base_ref", + project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_rf = 'origin/main'\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "git contains unknown key(s): base_rf. Allowed keys: base_ref", + Some("did you mean 'base_ref'?"), + ), + }, ProjectFailureTestCase { description: "a shutdown timeout over ten minutes is rejected", project: "name = 'orders'\nadapter = 'duckdb'\n[sinks.lifecycle]\nshutdown_timeout = '601s'\n", @@ -90,3 +117,18 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { ); } } + +#[test] +fn given_git_section_when_reading_base_ref_then_returns_configured_ref() { + let (_with, configured) = project_directory( + "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n", + "", + ); + let (_without, unset) = project_directory(PROJECT, ""); + + assert_eq!( + load_git_base_ref(&configured).ok().flatten().as_deref(), + Some("origin/main") + ); + assert_eq!(load_git_base_ref(&unset).ok().flatten(), None); +} diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/git_command.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/git_command.rs new file mode 100644 index 0000000000..f8fab4f441 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/git_command.rs @@ -0,0 +1,50 @@ +//! Run one git command in a directory and read its trimmed standard output. + +use std::path::Path; +use std::process::{Command, Output, Stdio}; + +use crate::git_revisions::constants::GIT_PROGRAM; + +/// How one git command failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum GitFailure { + /// git could not be started at all. + Unavailable(String), + /// git ran and exited unsuccessfully, with its trimmed standard error. + Failed(String), +} + +impl GitFailure { + /// The failure text a message can quote. + pub(crate) fn detail(&self) -> &str { + match self { + Self::Unavailable(detail) | Self::Failed(detail) => detail, + } + } +} + +/// The trimmed stdout of `git ` run in `directory` with `environment` added. +pub(crate) fn git_output( + directory: &Path, + arguments: &[&str], + environment: &[(&str, &Path)], +) -> Result { + let mut command: Command = Command::new(GIT_PROGRAM); + command + .current_dir(directory) + .args(arguments) + .stdin(Stdio::null()) + .env("GIT_TERMINAL_PROMPT", "0"); + for (name, value) in environment { + command.env(name, value); + } + let output: Output = command + .output() + .map_err(|error| GitFailure::Unavailable(error.to_string()))?; + if !output.status.success() { + return Err(GitFailure::Failed( + String::from_utf8_lossy(&output.stderr).trim().to_owned(), + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs new file mode 100644 index 0000000000..364f98394c --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod git_command; diff --git a/crates/sqlbuild-core/src/git_revisions/constants.rs b/crates/sqlbuild-core/src/git_revisions/constants.rs new file mode 100644 index 0000000000..440931d504 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/constants.rs @@ -0,0 +1,16 @@ +//! The `--since` ref alias, git invocation details and error codes. + +/// The `--since` value that reads `[git] base_ref` from the project configuration. +pub const BASE_REF_ALIAS: &str = "@base"; +/// Hex digits of a commit hash shown in progress and summaries. +pub const SHORT_COMMIT_LENGTH: usize = 10; + +pub(crate) const GIT_PROGRAM: &str = "git"; +pub(crate) const INDEX_FILE_ENV: &str = "GIT_INDEX_FILE"; +pub(crate) const SHALLOW_TRUE: &str = "true"; + +pub(crate) const NOT_A_REPOSITORY_CODE: &str = "C480"; +pub(crate) const UNKNOWN_REF_CODE: &str = "C481"; +pub(crate) const NO_MERGE_BASE_CODE: &str = "C482"; +pub(crate) const EXPORT_FAILED_CODE: &str = "C483"; +pub(crate) const BASE_REF_UNSET_CODE: &str = "C485"; diff --git a/crates/sqlbuild-core/src/git_revisions/errors.rs b/crates/sqlbuild-core/src/git_revisions/errors.rs new file mode 100644 index 0000000000..130babf10e --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/errors.rs @@ -0,0 +1,9 @@ +//! A `--since` ref that cannot be resolved or read, with its code and remedy. + +/// The message, error code and help of one failed git step. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GitRevisionError { + pub message: String, + pub code: &'static str, + pub help: String, +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs b/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs new file mode 100644 index 0000000000..efd4f92c60 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs @@ -0,0 +1,68 @@ +//! Write the project's files at the base commit into a directory, without touching the checkout. + +use std::path::{Path, PathBuf}; + +use crate::git_revisions::_helpers::git_command::git_output; +use crate::git_revisions::constants::{EXPORT_FAILED_CODE, INDEX_FILE_ENV}; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +fn tree_of(base: &GitBase) -> String { + let prefix: &str = base.project_prefix.trim_end_matches('/'); + if prefix.is_empty() { + format!("{}^{{tree}}", base.merge_base) + } else { + format!("{}:{prefix}", base.merge_base) + } +} + +fn export_failed(base: &GitBase, detail: &str) -> GitRevisionError { + GitRevisionError { + message: format!( + "Cannot read the project at merge base {}: {detail}", + base.merge_base + ), + code: EXPORT_FAILED_CODE, + help: "Check that the repository is readable and complete, for example with `git fsck`" + .to_owned(), + } +} + +/// Write the project tree at `base.merge_base` into `destination`; `false` when it had no project. +/// +/// The tree is read from git objects through a private index file, so the working tree, the +/// repository's index and `HEAD` are never changed. +pub fn export_git_tree(base: &GitBase, destination: &Path) -> Result { + let root: &Path = &base.repository_root; + let tree: String = tree_of(base); + if git_output(root, &["cat-file", "-e", &tree], &[]).is_err() { + return Ok(false); + } + std::fs::create_dir_all(destination) + .map_err(|error| export_failed(base, &error.to_string()))?; + let index: PathBuf = destination.with_extension("index"); + let environment: [(&str, &Path); 1] = [(INDEX_FILE_ENV, index.as_path())]; + let mut prefix: String = destination.to_string_lossy().replace('\\', "/"); + if !prefix.ends_with('/') { + prefix.push('/'); + } + let exported: Result<(), GitRevisionError> = + git_output(root, &["read-tree", &tree], &environment) + .and_then(|_| { + git_output( + root, + &[ + "checkout-index", + "--all", + "--force", + &format!("--prefix={prefix}"), + ], + &environment, + ) + }) + .map(|_| ()) + .map_err(|failure| export_failed(base, failure.detail())); + let _ = std::fs::remove_file(&index); + exported?; + Ok(true) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/mod.rs b/crates/sqlbuild-core/src/git_revisions/main/mod.rs new file mode 100644 index 0000000000..75430cbb8e --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/mod.rs @@ -0,0 +1,4 @@ +pub mod export_git_tree; +pub mod resolve_git_base; +pub mod resolve_since_reference; +pub mod short_commit; diff --git a/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs b/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs new file mode 100644 index 0000000000..0ea69f2c2c --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs @@ -0,0 +1,101 @@ +//! The merge base of a ref and `HEAD` for the repository containing a project. + +use std::path::{Path, PathBuf}; + +use crate::git_revisions::_helpers::git_command::{GitFailure, git_output}; +use crate::git_revisions::constants::{ + NO_MERGE_BASE_CODE, NOT_A_REPOSITORY_CODE, SHALLOW_TRUE, UNKNOWN_REF_CODE, +}; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +const DEPTH_HELP: &str = "Fetch enough history to reach the merge base, for example \ + `git fetch --unshallow`, or in CI clone with a larger fetch depth"; + +fn not_a_repository(project_dir: &Path, failure: &GitFailure) -> GitRevisionError { + let message: String = match failure { + GitFailure::Unavailable(detail) => { + format!("--since needs git, which could not be run: {detail}") + } + GitFailure::Failed(_) => format!( + "--since needs a git repository, but {} is not inside one", + project_dir.display() + ), + }; + GitRevisionError { + message, + code: NOT_A_REPOSITORY_CODE, + help: "Run the command inside a git checkout of the project, or drop --since".to_owned(), + } +} + +fn is_shallow(root: &Path) -> bool { + git_output(root, &["rev-parse", "--is-shallow-repository"], &[]) + .is_ok_and(|value| value == SHALLOW_TRUE) +} + +/// Resolve `reference` and `HEAD` in the repository of `project_dir` to their merge base. +pub fn resolve_git_base(project_dir: &Path, reference: &str) -> Result { + let root: PathBuf = git_output(project_dir, &["rev-parse", "--show-toplevel"], &[]) + .map(PathBuf::from) + .map_err(|failure| not_a_repository(project_dir, &failure))?; + let prefix: String = git_output(project_dir, &["rev-parse", "--show-prefix"], &[]) + .map_err(|failure| not_a_repository(project_dir, &failure))?; + let shallow: bool = is_shallow(&root); + let commit: String = git_output( + &root, + &[ + "rev-parse", + "--verify", + "--quiet", + "--end-of-options", + &format!("{reference}^{{commit}}"), + ], + &[], + ) + .map_err(|_| GitRevisionError { + message: format!("Git ref '{reference}' does not name a commit in this repository"), + code: UNKNOWN_REF_CODE, + help: if shallow { + format!( + "This is a shallow clone; fetch the ref first, for example `git fetch origin \ + `, with enough depth to reach the merge base. {DEPTH_HELP}" + ) + } else { + "Check the spelling, or fetch the ref first, for example `git fetch origin `" + .to_owned() + }, + })?; + let merge_base: String = + git_output(&root, &["merge-base", &commit, "HEAD"], &[]).map_err(|failure| { + if shallow { + GitRevisionError { + message: format!( + "Cannot find the merge base of '{reference}' and HEAD: this is a shallow \ + clone without the shared history" + ), + code: NO_MERGE_BASE_CODE, + help: DEPTH_HELP.to_owned(), + } + } else { + let detail: &str = failure.detail(); + GitRevisionError { + message: if detail.is_empty() { + format!("'{reference}' and HEAD share no history") + } else { + format!("Cannot find the merge base of '{reference}' and HEAD: {detail}") + }, + code: NO_MERGE_BASE_CODE, + help: "Pass a ref that shares history with HEAD, such as the branch this \ + work started from" + .to_owned(), + } + } + })?; + Ok(GitBase { + reference: reference.to_owned(), + merge_base, + repository_root: root, + project_prefix: prefix, + }) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/resolve_since_reference.rs b/crates/sqlbuild-core/src/git_revisions/main/resolve_since_reference.rs new file mode 100644 index 0000000000..b9621ea9d8 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/resolve_since_reference.rs @@ -0,0 +1,27 @@ +//! The git ref a `--since` value names, reading `@base` from `[git] base_ref`. + +use crate::git_revisions::constants::{BASE_REF_ALIAS, BASE_REF_UNSET_CODE}; +use crate::git_revisions::errors::GitRevisionError; + +/// `value` itself, or the configured base ref when `value` is `@base`. +pub fn resolve_since_reference( + value: &str, + configured_base_ref: Option<&str>, +) -> Result { + let value: &str = value.trim(); + if value != BASE_REF_ALIAS { + return Ok(value.to_owned()); + } + match configured_base_ref.map(str::trim) { + Some(base_ref) if !base_ref.is_empty() => Ok(base_ref.to_owned()), + _ => Err(GitRevisionError { + message: format!( + "--since {BASE_REF_ALIAS} needs [git] base_ref in sqlbuild_project.toml" + ), + code: BASE_REF_UNSET_CODE, + help: "Add [git] base_ref = \"origin/main\" (your default branch) to \ + sqlbuild_project.toml, or pass a ref such as --since origin/main" + .to_owned(), + }), + } +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/short_commit.rs b/crates/sqlbuild-core/src/git_revisions/main/short_commit.rs new file mode 100644 index 0000000000..a00c8d49e4 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/short_commit.rs @@ -0,0 +1,8 @@ +//! The abbreviated commit hash shown in progress and summaries. + +use crate::git_revisions::constants::SHORT_COMMIT_LENGTH; + +/// The first hex digits of `commit`. +pub fn short_commit(commit: &str) -> &str { + commit.get(..SHORT_COMMIT_LENGTH).unwrap_or(commit) +} diff --git a/crates/sqlbuild-core/src/git_revisions/mod.rs b/crates/sqlbuild-core/src/git_revisions/mod.rs new file mode 100644 index 0000000000..19b88f3412 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/mod.rs @@ -0,0 +1,10 @@ +//! Git revisions a `--since` comparison reads: the merge base of a ref and `HEAD`, and its tree. + +mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/git_revisions/models.rs b/crates/sqlbuild-core/src/git_revisions/models.rs new file mode 100644 index 0000000000..4372d5a2ee --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/models.rs @@ -0,0 +1,15 @@ +//! The resolved base revision of a `--since` comparison. + +use std::path::PathBuf; + +/// The merge base of a ref and `HEAD`, and where the project lives in the repository. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GitBase { + /// The ref as the user gave it, after `@base` resolution. + pub reference: String, + /// The full hash of the merge-base commit. + pub merge_base: String, + pub repository_root: PathBuf, + /// The project directory relative to the repository root, with `/` separators, empty at the root. + pub project_prefix: String, +} diff --git a/crates/sqlbuild-core/src/git_revisions/tests.rs b/crates/sqlbuild-core/src/git_revisions/tests.rs new file mode 100644 index 0000000000..4a6c249d67 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_git_revisions.rs"] +mod git_revisions; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs b/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs new file mode 100644 index 0000000000..6ef69990c3 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs @@ -0,0 +1,86 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT_DIRECTORY: AtomicUsize = AtomicUsize::new(0); + +/// A fresh empty directory under the system temporary directory. +pub(super) fn scratch_directory(label: &str) -> PathBuf { + let directory: PathBuf = std::env::temp_dir().join(format!( + "sqlbuild-git-revisions-{label}-{}-{}", + std::process::id(), + NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).expect("the scratch directory is created"); + directory +} + +/// Run git in `directory` with a fixed identity, returning trimmed stdout. +pub(super) fn git(directory: &Path, arguments: &[&str]) -> String { + let output = Command::new("git") + .current_dir(directory) + .args([ + "-c", + "user.name=Example", + "-c", + "user.email=dev@example.com", + ]) + .args([ + "-c", + "init.defaultBranch=main", + "-c", + "commit.gpgsign=false", + ]) + .args(arguments) + .output() + .expect("git runs"); + assert!( + output.status.success(), + "git {arguments:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_owned() +} + +/// Write `contents` to `relative` under `root`, creating parent directories. +pub(super) fn write(root: &Path, relative: &str, contents: &str) { + let path: PathBuf = root.join(relative); + std::fs::create_dir_all(path.parent().expect("a parent")).expect("parents are created"); + std::fs::write(path, contents).expect("the file is written"); +} + +/// A repository whose project lives in `warehouse/`: `main` has one commit, `feature` one more. +/// +/// Returns the repository root and the hash of the `main` commit. +pub(super) fn project_repository() -> (PathBuf, String) { + let root: PathBuf = scratch_directory("repo"); + git(&root, &["init", "--quiet"]); + write( + &root, + "warehouse/sqlbuild_project.toml", + "name = \"orders\"\n", + ); + write( + &root, + "warehouse/models/orders.sql", + "SELECT 1 AS order_id\n", + ); + git(&root, &["add", "."]); + git(&root, &["commit", "--quiet", "-m", "orders"]); + let main: String = git(&root, &["rev-parse", "HEAD"]); + git(&root, &["checkout", "--quiet", "-b", "feature"]); + write( + &root, + "warehouse/models/orders.sql", + "SELECT 2 AS order_id\n", + ); + write( + &root, + "warehouse/models/customers.sql", + "SELECT 1 AS customer_id\n", + ); + git(&root, &["add", "."]); + git(&root, &["commit", "--quiet", "-m", "customers"]); + (root, main) +} diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs new file mode 100644 index 0000000000..1dd6323d49 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs @@ -0,0 +1,195 @@ +use std::path::{Path, PathBuf}; + +use crate::git_revisions::main::export_git_tree::export_git_tree; +use crate::git_revisions::main::resolve_git_base::resolve_git_base; +use crate::git_revisions::main::resolve_since_reference::resolve_since_reference; +use crate::git_revisions::main::short_commit::short_commit; +use crate::git_revisions::models::GitBase; +use crate::git_revisions::tests::helpers::{git, project_repository, scratch_directory, write}; +use crate::git_revisions::tests::test_types::{GitBaseFailureTestCase, SinceReferenceTestCase}; + +#[test] +fn given_since_value_when_resolving_then_reads_base_alias_from_config() { + let test_cases = [ + SinceReferenceTestCase { + description: "a plain ref is used as given", + value: "origin/main", + configured: Some("origin/release"), + expected: Ok("origin/main"), + }, + SinceReferenceTestCase { + description: "@base reads the configured base ref", + value: "@base", + configured: Some("origin/main"), + expected: Ok("origin/main"), + }, + SinceReferenceTestCase { + description: "@base without a configured base ref fails", + value: "@base", + configured: None, + expected: Err("C485"), + }, + SinceReferenceTestCase { + description: "@base with a blank base ref fails", + value: "@base", + configured: Some(" "), + expected: Err("C485"), + }, + ]; + for test_case in test_cases { + let resolved = resolve_since_reference(test_case.value, test_case.configured); + + assert_eq!( + resolved.as_deref().map_err(|error| error.code), + test_case.expected, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_project_in_subdirectory_when_resolving_base_then_returns_merge_base_and_prefix() { + let (root, main) = project_repository(); + + let base: GitBase = resolve_git_base(&root.join("warehouse"), "main").expect("main resolves"); + + assert_eq!(base.merge_base, main); + assert_eq!(base.project_prefix, "warehouse/"); + assert_eq!(base.reference, "main"); + assert_eq!(short_commit(&base.merge_base).len(), 10); +} + +#[test] +fn given_unresolvable_ref_when_resolving_base_then_fails_with_code() { + let (root, _) = project_repository(); + git(&root, &["checkout", "--quiet", "--orphan", "unrelated"]); + git(&root, &["commit", "--quiet", "-m", "unrelated"]); + git(&root, &["checkout", "--quiet", "feature"]); + let test_cases = [ + GitBaseFailureTestCase { + description: "a missing ref", + reference: "origin/missing", + expected_code: "C481", + expected_message_fragment: "'origin/missing' does not name a commit", + }, + GitBaseFailureTestCase { + description: "a ref without shared history", + reference: "unrelated", + expected_code: "C482", + expected_message_fragment: "'unrelated' and HEAD", + }, + ]; + for test_case in test_cases { + let error = resolve_git_base(&root.join("warehouse"), test_case.reference) + .expect_err(test_case.description); + + assert_eq!( + error.code, test_case.expected_code, + "{}", + test_case.description + ); + assert!( + error.message.contains(test_case.expected_message_fragment), + "{}: {}", + test_case.description, + error.message + ); + } +} + +#[test] +fn given_directory_outside_repository_when_resolving_base_then_fails_with_code() { + let directory: PathBuf = scratch_directory("plain"); + + let error = resolve_git_base(&directory, "main").expect_err("not a repository"); + + assert_eq!(error.code, "C480"); +} + +#[test] +fn given_shallow_clone_without_shared_history_when_resolving_base_then_asks_for_depth() { + let (root, _) = project_repository(); + let clone: PathBuf = scratch_directory("shallow"); + git( + &clone, + &[ + "clone", + "--quiet", + "--depth", + "1", + "--branch", + "feature", + "--no-single-branch", + &format!("file://{}", root.display()), + "checkout", + ], + ); + let checkout: PathBuf = clone.join("checkout"); + git( + &checkout, + &["fetch", "--quiet", "--depth", "1", "origin", "main"], + ); + + let error = resolve_git_base(&checkout.join("warehouse"), "origin/main") + .expect_err("the merge base is outside the shallow history"); + + assert_eq!(error.code, "C482"); + assert!(error.message.contains("shallow clone"), "{}", error.message); + assert!(error.help.contains("--unshallow"), "{}", error.help); +} + +fn read(path: &Path) -> Option { + std::fs::read_to_string(path).ok() +} + +#[test] +fn given_merge_base_when_exporting_tree_then_writes_base_files_and_leaves_checkout_alone() { + let (root, _) = project_repository(); + write( + &root, + "warehouse/models/orders.sql", + "SELECT 3 AS order_id\n", + ); + let status_before: String = git(&root, &["status", "--porcelain"]); + let base: GitBase = resolve_git_base(&root.join("warehouse"), "main").expect("main resolves"); + let destination: PathBuf = scratch_directory("export").join("base"); + + let exported: bool = export_git_tree(&base, &destination).expect("the tree exports"); + + assert!(exported); + assert_eq!( + read(&destination.join("models/orders.sql")).as_deref(), + Some("SELECT 1 AS order_id\n") + ); + assert_eq!(read(&destination.join("models/customers.sql")), None); + assert_eq!( + read(&root.join("warehouse/models/orders.sql")).as_deref(), + Some("SELECT 3 AS order_id\n") + ); + assert_eq!(git(&root, &["status", "--porcelain"]), status_before); + assert!(!destination.with_extension("index").exists()); +} + +#[test] +fn given_project_absent_at_merge_base_when_exporting_tree_then_reports_no_project() { + let (root, main) = project_repository(); + write( + &root, + "billing/sqlbuild_project.toml", + "name = \"billing\"\n", + ); + git(&root, &["add", "."]); + git(&root, &["commit", "--quiet", "-m", "billing"]); + let base: GitBase = GitBase { + reference: "main".to_owned(), + merge_base: main, + repository_root: root.clone(), + project_prefix: "billing/".to_owned(), + }; + + let exported: bool = export_git_tree(&base, &scratch_directory("absent").join("base")) + .expect("an absent tree is not an error"); + + assert!(!exported); +} diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs new file mode 100644 index 0000000000..1e3f3a5687 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs @@ -0,0 +1,14 @@ +pub(crate) struct SinceReferenceTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) configured: Option<&'static str>, + /// The resolved ref, or the error code. + pub(crate) expected: Result<&'static str, &'static str>, +} + +pub(crate) struct GitBaseFailureTestCase { + pub(crate) description: &'static str, + pub(crate) reference: &'static str, + pub(crate) expected_code: &'static str, + pub(crate) expected_message_fragment: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 2569bf6dc1..29ad9c99e3 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -30,6 +30,7 @@ pub mod dlt_destination; pub mod event_export; pub mod format_command; pub mod freshness_text; +pub mod git_revisions; pub mod graph_walk; pub mod identifier_text; pub mod ingestr_command; diff --git a/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs b/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs index c79d505b15..9787452733 100644 --- a/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs +++ b/crates/sqlbuild-core/src/test_command/main/render_test_inspection.rs @@ -37,6 +37,9 @@ pub fn render_test_inspection( " boundary: {mock} is replaced by __ref__{mock}\n" )); } + for reason in &entry.selection_reasons { + text.push_str(&format!(" selected: {reason}\n")); + } text.push('\n'); } let errors: usize = warnings diff --git a/crates/sqlbuild-core/src/test_command/models.rs b/crates/sqlbuild-core/src/test_command/models.rs index aa17cc236e..0c8b931617 100644 --- a/crates/sqlbuild-core/src/test_command/models.rs +++ b/crates/sqlbuild-core/src/test_command/models.rs @@ -21,6 +21,8 @@ pub struct TestInspectionEntry { pub mocked_table_functions: Vec, pub real_models: Vec, pub expected_models: Vec, + /// Why `--since` selected the test; empty without `--since`. + pub selection_reasons: Vec, } /// One plan warning: its severity value and message. diff --git a/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs b/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs index 46b2e84661..47e203b230 100644 --- a/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs +++ b/crates/sqlbuild-core/src/test_command/tests/test_test_command.rs @@ -242,6 +242,18 @@ fn given_plan_when_inspecting_then_lists_boundaries_and_counts_errors() { expected_text: "\nResolved test plan\n\norder_status [open_case]\n mocked sources: raw_orders\n mocked refs: stg\n mocked seeds: none\n mocked dbt refs: none\n mocked table functions: none\n real models: orders\n expected models: orders\n boundary: stg is replaced by __ref__stg\n\n warning: careful\n error: broken\nTest plan inspection failed: 1 selected, 1 errors.\n", expected_exit_code: 1, }, + InspectionTestCase { + description: "a test selected by --since lists its reasons", + entries: vec![TestInspectionEntry { + label: "orders_rollup".to_owned(), + real_models: vec!["stg_orders".to_owned()], + selection_reasons: vec!["executes stg_orders (model SQL changed)".to_owned()], + ..TestInspectionEntry::default() + }], + warnings: Vec::new(), + expected_text: "\nResolved test plan\n\norders_rollup\n mocked sources: none\n mocked refs: none\n mocked seeds: none\n mocked dbt refs: none\n mocked table functions: none\n real models: stg_orders\n expected models: none\n selected: executes stg_orders (model SQL changed)\n\nTest plan inspection complete: 1 selected, 0 errors.\n", + expected_exit_code: 0, + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs new file mode 100644 index 0000000000..47a653dd95 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs @@ -0,0 +1,88 @@ +//! The base-revision side of `--since`: compile an exported tree and write its snapshot. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyBool; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::definition_snapshot_json; +use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; + +use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::change_selection::definition_snapshot::definition_snapshot; +use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::pipeline_entries::run_compile_only_pipeline; +use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; + +fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +/// Compile the project at `project_dir` without SQL analysis and write its definition snapshot. +/// +/// `--since` runs this in a separate interpreter for the exported base revision. +#[pyfunction] +#[pyo3(signature = (*, project_dir, output_path, selected_target = None, cli_vars = None))] +fn write_base_definition_snapshot( + py: Python<'_>, + project_dir: PathBuf, + output_path: PathBuf, + selected_target: Option>, + cli_vars: Option>, +) -> PyResult<()> { + let none = || py.None().into_bound(py); + let directory: Bound<'_, PyAny> = project_dir.clone().into_pyobject(py)?.into_any(); + let target: Bound<'_, PyAny> = selected_target.unwrap_or_else(none); + let vars: Bound<'_, PyAny> = cli_vars.unwrap_or_else(none); + let discovered: Bound<'_, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", directory.clone())])?), + )?; + let context: Bound<'_, PyAny> = resolve_adapter_connection_context( + &discovered, + &directory, + (target.clone(), vars.clone()), + false, + )?; + let resolver: Bound<'_, PyAny> = + resolve_external_sql_reference_resolver(directory.clone(), discovered.clone())?; + let options: Bound<'_, PyAny> = new_record_with( + py, + RecordId::CompilePipelineOptions, + &[ + ("selected_target", target), + ("no_sql_validation", flag(py, true)), + ("source_deferral_enabled", flag(py, false)), + ( + "connection_config", + native_field(&context, "connection_config")?, + ), + ("cli_vars", vars), + ("external_sql_reference_resolver", resolver), + ], + )?; + let pipeline: Bound<'_, PyAny> = run_compile_only_pipeline( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered), + ("adapter", native_field(&context, "adapter")?), + ("options", options), + ("on_progress", none()), + ], + )?), + )?; + let snapshot: DefinitionSnapshot = + definition_snapshot(&native_field(&pipeline, "project")?, &project_dir)?; + std::fs::write(&output_path, definition_snapshot_json(&snapshot))?; + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_base_definition_snapshot, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs new file mode 100644 index 0000000000..b79f80f017 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -0,0 +1,228 @@ +//! The definition snapshot of the `--since` base revision: resolved from git, compiled from its +//! objects in a separate interpreter, and cached per merge-base commit. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_analysis::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; +use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::{ + definition_snapshot_json, parse_definition_snapshot, +}; +use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; +use sqlbuild_core::git_revisions::constants::BASE_REF_ALIAS; +use sqlbuild_core::git_revisions::errors::GitRevisionError; +use sqlbuild_core::git_revisions::main::export_git_tree::export_git_tree; +use sqlbuild_core::git_revisions::main::resolve_git_base::resolve_git_base; +use sqlbuild_core::git_revisions::main::resolve_since_reference::resolve_since_reference; +use sqlbuild_core::git_revisions::main::short_commit::short_commit; +use sqlbuild_core::git_revisions::models::GitBase; +use sqlbuild_core::scenario_command::errors::ScenarioCommandError; +use sqlbuild_rules::constants::NATIVE_BUILD_IDENTITY; + +use crate::bindings::_helpers::compile_command::runtime_identity::interpreter_values; +use crate::bindings::_helpers::discovery::project_config::configured_git_base_ref; +use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, sorted_dumps}; +use crate::bindings::_helpers::scenario_command::setup::{cli_user_error, flush, write}; + +const CACHE_DIRECTORY: [&str; 2] = ["target", "since"]; +const LOCAL_CONFIG_FILE: &str = "sqlbuild_local.toml"; +const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; +const BASE_COMPILE_FAILED_CODE: &str = "C484"; +const FAILURE_TAIL_LINES: usize = 20; +/// Compiles the exported base tree in a fresh interpreter, so the base revision's project Python +/// never shares modules with the working tree's. +const BASE_COMPILE_PROGRAM: &str = "import json, sys\n\ +from sqlbuild._native import write_base_definition_snapshot\n\ +write_base_definition_snapshot(project_dir=sys.argv[1], output_path=sys.argv[2], \ +selected_target=sys.argv[3] or None, cli_vars=json.loads(sys.argv[4]))\n"; + +/// The base revision and its definition snapshot. +pub(crate) struct BaseComparison { + pub(crate) base: GitBase, + pub(crate) snapshot: DefinitionSnapshot, +} + +/// The inputs that decide what the base compile produces besides its commit. +pub(crate) struct BaseVariant<'py> { + pub(crate) selected_target: Option, + pub(crate) cli_vars: Bound<'py, PyAny>, +} + +fn git_error(py: Python<'_>, error: GitRevisionError) -> PyErr { + cli_user_error( + py, + ScenarioCommandError { + code: error.code, + message: error.message, + help: Some(error.help), + }, + ) +} + +fn compile_failed(py: Python<'_>, base: &GitBase, detail: &str) -> PyErr { + cli_user_error( + py, + ScenarioCommandError { + code: BASE_COMPILE_FAILED_CODE, + message: format!( + "Cannot compile the project at merge base {} of '{}' and HEAD:\n{detail}", + short_commit(&base.merge_base), + base.reference + ), + help: Some( + "--since compares with the base revision, so it must compile; compare with a \ + later ref, or run without --since" + .to_owned(), + ), + }, + ) +} + +fn io_failed(py: Python<'_>, base: &GitBase, error: &std::io::Error) -> PyErr { + compile_failed(py, base, &error.to_string()) +} + +/// The last lines of a failed base compile's output. +fn failure_tail(output: &Output) -> String { + let text: String = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let lines: Vec<&str> = text + .lines() + .filter(|line| !line.trim().is_empty()) + .collect(); + lines[lines.len().saturating_sub(FAILURE_TAIL_LINES)..].join("\n") +} + +fn vars_json(cli_vars: &Bound<'_, PyAny>) -> PyResult { + if cli_vars.is_none() { + return Ok("{}".to_owned()); + } + sorted_dumps(cli_vars, COMPACT, true) +} + +/// Run the base compile of `tree` into `output` in a separate interpreter. +fn compile_base_tree( + py: Python<'_>, + base: &GitBase, + (tree, output): (&Path, &Path), + (selected_target, vars): (Option<&str>, &str), +) -> PyResult<()> { + let [_, executable, _, _] = interpreter_values(py)?; + let finished: std::io::Result = py.detach(|| { + Command::new(&executable) + .arg("-c") + .arg(BASE_COMPILE_PROGRAM) + .arg(tree) + .arg(output) + .arg(selected_target.unwrap_or_default()) + .arg(vars) + .current_dir(tree) + .stdin(Stdio::null()) + .output() + }); + let finished: Output = finished.map_err(|error| io_failed(py, base, &error))?; + if finished.status.success() && output.is_file() { + return Ok(()); + } + Err(compile_failed(py, base, &failure_tail(&finished))) +} + +/// Export the base tree, carry over the untracked local config, and compile it. +fn compiled_base_snapshot( + py: Python<'_>, + (base, project_dir, work): (&GitBase, &Path, &Path), + (selected_target, vars): (Option<&str>, &str), +) -> PyResult { + let tree: PathBuf = work.join("project"); + let exported: bool = export_git_tree(base, &tree).map_err(|error| git_error(py, error))?; + if !exported || !tree.join(PROJECT_CONFIG_FILE).is_file() { + return Ok(DefinitionSnapshot { + format: DEFINITION_SNAPSHOT_FORMAT, + ..DefinitionSnapshot::default() + }); + } + let local: PathBuf = project_dir.join(LOCAL_CONFIG_FILE); + if local.is_file() && !tree.join(LOCAL_CONFIG_FILE).exists() { + std::fs::copy(&local, tree.join(LOCAL_CONFIG_FILE)) + .map_err(|error| io_failed(py, base, &error))?; + } + let output: PathBuf = work.join("definitions.json"); + compile_base_tree(py, base, (&tree, &output), (selected_target, vars))?; + let text: String = + std::fs::read_to_string(&output).map_err(|error| io_failed(py, base, &error))?; + parse_definition_snapshot(&text) + .ok_or_else(|| compile_failed(py, base, "the base compile wrote an unreadable snapshot")) +} + +/// Resolve `since` to its merge base and load, or compile and cache, the base snapshot. +/// +/// Progress goes to `stream`: the merge base, then whether the base came from the cache or how +/// long its compile took. +pub(crate) fn base_comparison<'py>( + (stream, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + since: &str, + variant: &BaseVariant<'py>, +) -> PyResult { + let py: Python<'py> = stream.py(); + let directory: PathBuf = project_dir.extract()?; + let configured: Option = if since.trim() == BASE_REF_ALIAS { + configured_git_base_ref(project_dir)? + } else { + None + }; + let reference: String = resolve_since_reference(since, configured.as_deref()) + .map_err(|error| git_error(py, error))?; + let base: GitBase = py + .detach(|| resolve_git_base(&directory, &reference)) + .map_err(|error| git_error(py, error))?; + let short: &str = short_commit(&base.merge_base); + write( + stream, + &format!("Comparing with {reference} (merge base {short})\n"), + )?; + let vars: String = vars_json(&variant.cli_vars)?; + let target: Option<&str> = variant.selected_target.as_deref(); + let key: String = + definition_digest(&[NATIVE_BUILD_IDENTITY, target.unwrap_or_default(), &vars]); + let cache_dir: PathBuf = CACHE_DIRECTORY + .iter() + .fold(directory.clone(), |path, part| path.join(part)); + let cached: PathBuf = cache_dir.join(format!("{}-{}.json", base.merge_base, &key[..16])); + if let Some(snapshot) = std::fs::read_to_string(&cached) + .ok() + .and_then(|text| parse_definition_snapshot(&text)) + { + write(stream, &format!("Using cached base revision {short}\n"))?; + flush(stream)?; + return Ok(BaseComparison { base, snapshot }); + } + write(stream, &format!("Compiling base revision {short}...\n"))?; + flush(stream)?; + let started: Instant = Instant::now(); + let work: PathBuf = cache_dir.join(format!("work-{}-{}", short, std::process::id())); + let _ = std::fs::remove_dir_all(&work); + std::fs::create_dir_all(&work).map_err(|error| io_failed(py, &base, &error))?; + let compiled: PyResult = + compiled_base_snapshot(py, (&base, &directory, &work), (target, &vars)); + let _ = std::fs::remove_dir_all(&work); + let snapshot: DefinitionSnapshot = compiled?; + let staged: PathBuf = cached.with_extension(format!("{}.tmp", std::process::id())); + if std::fs::write(&staged, definition_snapshot_json(&snapshot)).is_ok() { + let _ = std::fs::rename(&staged, &cached); + } + write( + stream, + &format!( + "Compiled base revision {short} ({:.2}s)\n", + started.elapsed().as_secs_f64() + ), + )?; + flush(stream)?; + Ok(BaseComparison { base, snapshot }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs new file mode 100644 index 0000000000..35d9dc543d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -0,0 +1,391 @@ +//! The definition snapshot of a compiled project: one fingerprint per model, seed, function, +//! macro, enum, constant, test file and scenario, and which declarations each node uses. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, + PyStringMethods, PyTuple, PyTypeMethods, +}; +use serde_json::{Map, Value}; +use sqlbuild_analysis::definition_changes::constants::{ + AUDITS_ASPECT, COLUMNS_ASPECT, CONFIG_ASPECT, DEFINITION_ASPECT, DEFINITION_SNAPSHOT_FORMAT, + FILE_ASPECT, MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, SNAPSHOTS_ASPECT, SQL_ASPECT, + TEST_FILE_KIND, +}; +use sqlbuild_analysis::definition_changes::main::definition_digest::{ + definition_digest, sql_definition_digest, +}; +use sqlbuild_analysis::definition_changes::main::node_key::node_key; +use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; + +use crate::bindings::_helpers::compiled_views::function_reads; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; +use crate::bindings::_helpers::compiled_views::seed_reads; +use crate::bindings::_helpers::planner_identity::metadata::{function_local_hashes, seed_identity}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::models::NativeRecord; + +/// Fields that document a definition without changing what it does. +const DOCUMENTATION_FIELDS: [&str; 3] = ["description", "location", "meta"]; +/// Header values that rename rather than redefine a model. +const RENAME_CONFIG_KEYS: [&str; 2] = ["migrate_from", "description"]; +const MIGRATE_FROM_KEY: &str = "migrate_from"; +const SCENARIO_SNAPSHOT_ROOT: [&str; 2] = ["tests", "_scenario_snapshots"]; + +/// A stable JSON form of a Python value, without documentation fields of records. +fn canonical(value: &Bound<'_, PyAny>, skipped: &[&str]) -> PyResult { + if value.is_none() { + return Ok(Value::Null); + } + if let Ok(cell) = value.cast::() { + let record: &NativeRecord = cell.get(); + let mut fields: Map = Map::new(); + fields.insert("record".to_owned(), Value::from(record.spec.name)); + for (spec, item) in record.spec.fields.iter().zip(&record.values) { + if DOCUMENTATION_FIELDS.contains(&spec.name) || skipped.contains(&spec.name) { + continue; + } + fields.insert( + spec.name.to_owned(), + canonical(item.bind(value.py()), skipped)?, + ); + } + return Ok(Value::Object(fields)); + } + if value.is_instance_of::() { + return Ok(Value::from(value.extract::()?)); + } + if value.is_instance_of::() { + return Ok(Value::from(value.extract::()?)); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(Value::from(value.repr()?.to_string())); + } + if value.is_instance_of::() || value.is_instance_of::() { + let items: Vec = value + .try_iter()? + .map(|item| canonical(&item?, skipped)) + .collect::>()?; + return Ok(Value::Array(items)); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = value + .try_iter()? + .map(|item| Ok(canonical(&item?, skipped)?.to_string())) + .collect::>()?; + items.sort(); + return Ok(Value::from(items)); + } + if let Ok(mapping) = value.cast::() { + let mut entries: BTreeMap = BTreeMap::new(); + for (key, item) in mapping.iter() { + entries.insert(key.str()?.to_string(), canonical(&item, skipped)?); + } + return Ok(Value::Object(entries.into_iter().collect())); + } + let text: String = value.repr()?.to_string(); + if text.contains(" at 0x") { + return Ok(Value::from(value.get_type().name()?.to_string())); + } + Ok(Value::from(text)) +} + +/// The digest of `value`'s canonical form, without the top-level record fields `top_skipped`. +fn canonical_digest( + value: &Bound<'_, PyAny>, + (top_skipped, skipped): (&[&str], &[&str]), +) -> PyResult { + let mut encoded: Value = canonical(value, skipped)?; + if let Value::Object(fields) = &mut encoded { + for field in top_skipped { + fields.remove(*field); + } + } + Ok(definition_digest(&[&encoded.to_string()])) +} + +fn text_of(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string().replace('\\', "/")) +} + +/// The digest of a file's bytes, or of its absence. +fn file_digest(path: &Path) -> String { + match std::fs::read(path) { + Ok(bytes) => definition_digest(&[&String::from_utf8_lossy(&bytes)]), + Err(_) => definition_digest(&["missing"]), + } +} + +/// The digest of every file below `directory`, by relative path, or `None` when it is absent. +fn directory_digest(directory: &Path) -> Option { + let mut files: Vec<(String, String)> = Vec::new(); + let mut pending: Vec = vec![directory.to_path_buf()]; + while let Some(next) = pending.pop() { + for entry in std::fs::read_dir(&next).ok()?.flatten() { + let path: PathBuf = entry.path(); + if path.is_dir() { + pending.push(path); + } else { + let relative: String = path + .strip_prefix(directory) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + files.push((relative, file_digest(&path))); + } + } + } + files.sort(); + let parts: Vec<&str> = files + .iter() + .flat_map(|(path, digest)| [path.as_str(), digest.as_str()]) + .collect(); + Some(definition_digest(&parts)) +} + +fn node(kind: &str, name: &str, aspects: Vec<(&str, String)>) -> NodeDefinition { + NodeDefinition { + kind: kind.to_owned(), + name: name.to_owned(), + aspects: aspects + .into_iter() + .map(|(aspect, digest)| (aspect.to_owned(), digest)) + .collect(), + uses: Vec::new(), + migrate_from: Vec::new(), + } +} + +/// The node key of a scope declaration identity: `kind:name`, with `@owner` for owned ones. +pub(crate) fn declaration_key(identity: &Bound<'_, PyAny>) -> PyResult { + let kind: String = text_of(&native_field(&native_field(identity, "kind")?, "value")?)?; + let name: String = native_field(identity, "name")?.extract()?; + let owner: Option = native_field(identity, "owner")?.extract()?; + Ok(match owner { + Some(owner) => node_key(&kind, &format!("{name}@{owner}")), + None => node_key(&kind, &name), + }) +} + +/// `kind:name` of a usage consumer. +fn consumer_key(consumer: &Bound<'_, PyAny>) -> PyResult { + Ok(node_key( + &text_of(&native_field(&native_field(consumer, "kind")?, "value")?)?, + &native_field(consumer, "name")?.extract::()?, + )) +} + +/// Declaration keys each consumer uses, by consumer key, from the project's scope index. +pub(crate) fn declaration_uses( + project: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut uses: BTreeMap> = BTreeMap::new(); + let index: Bound<'_, PyAny> = native_field(project, "scope_index")?; + if index.is_none() { + return Ok(uses); + } + for usage in native_field(&index, "usages")?.try_iter()? { + let usage: Bound<'_, PyAny> = usage?; + uses.entry(consumer_key(&native_field(&usage, "consumer")?)?) + .or_default() + .insert(declaration_key(&native_field(&usage, "declaration")?)?); + } + Ok(uses) +} + +/// Scope declarations: a macro by its module source, and an enum, constant, audit, schema or +/// hook by its declaration file with formatting and comments ignored. +fn declaration_nodes( + project: &Bound<'_, PyAny>, + project_dir: &Path, + nodes: &mut BTreeMap, +) -> PyResult<()> { + let index: Bound<'_, PyAny> = native_field(project, "scope_index")?; + if index.is_none() { + return Ok(()); + } + for declaration in native_field(&index, "declarations")?.try_iter()? { + let declaration: Bound<'_, PyAny> = declaration?; + let identity: Bound<'_, PyAny> = native_field(&declaration, "identity")?; + let kind: String = text_of(&native_field(&native_field(&identity, "kind")?, "value")?)?; + let name: String = native_field(&identity, "name")?.extract()?; + let path: String = native_field(&declaration, "path")?.extract()?; + let macro_facts: Bound<'_, PyAny> = native_field(&declaration, "macro")?; + let (digest, uses): (String, Vec) = if kind == MACRO_KIND { + if macro_facts.is_none() { + continue; + } + let dependencies: Vec = + native_field(¯o_facts, "dependencies")?.extract()?; + ( + canonical_digest(¯o_facts, (&[], &[]))?, + dependencies + .iter() + .map(|dependency| node_key(MACRO_KIND, dependency)) + .collect(), + ) + } else { + let contents: String = + std::fs::read_to_string(project_dir.join(&path)).unwrap_or_default(); + (sql_definition_digest(&contents), Vec::new()) + }; + let mut definition: NodeDefinition = node(&kind, &name, vec![(DEFINITION_ASPECT, digest)]); + definition.uses = uses; + nodes.insert(declaration_key(&identity)?, definition); + } + Ok(()) +} + +fn migrate_from(config_values: &Bound<'_, PyAny>) -> PyResult> { + let Ok(values) = config_values.cast::() else { + return Ok(Vec::new()); + }; + let Some(value) = values.get_item(MIGRATE_FROM_KEY)? else { + return Ok(Vec::new()); + }; + if let Ok(name) = value.extract::() { + return Ok(vec![name]); + } + Ok(value.extract::>().unwrap_or_default()) +} + +/// A model by its SQL before macro expansion, its header configuration, its declared columns +/// and contract, and its audits. +fn model_node( + model: &Bound<'_, PyAny>, + uses: &BTreeMap>, +) -> PyResult<(String, NodeDefinition)> { + let name: String = model_reads::model_name(model)?; + let config_values: Bound<'_, PyAny> = native_field(&model_reads::config(model)?, "values")?; + let config: Bound<'_, PyDict> = PyDict::new(model.py()); + if let Ok(values) = config_values.cast::() { + for (key, value) in values.iter() { + if !RENAME_CONFIG_KEYS.contains(&key.str()?.to_str()?) { + config.set_item(key, value)?; + } + } + } + let schema: Bound<'_, PyAny> = model_reads::schema_entry(model)?; + let (columns, audits): (String, String) = if schema.is_none() { + (definition_digest(&[]), definition_digest(&[])) + } else { + let mut column_audits: Vec = + vec![canonical(&native_field(&schema, "audits")?, &[])?]; + for column in native_field(&schema, "columns")?.try_iter()? { + column_audits.push(canonical(&native_field(&column?, "audits")?, &[])?); + } + ( + canonical_digest(&schema, (&["name", "tags"], &["audits"]))?, + definition_digest(&[&Value::Array(column_audits).to_string()]), + ) + }; + let query_sql: String = native_field(model, "authored_query_sql")?.extract()?; + let mut definition: NodeDefinition = node( + MODEL_KIND, + &name, + vec![ + (SQL_ASPECT, sql_definition_digest(&query_sql)), + ( + CONFIG_ASPECT, + canonical_digest(config.as_any(), (&[], &[]))?, + ), + (COLUMNS_ASPECT, columns), + (AUDITS_ASPECT, audits), + ], + ); + let key: String = node_key(MODEL_KIND, &name); + definition.uses = uses.get(&key).into_iter().flatten().cloned().collect(); + definition.migrate_from = migrate_from(&config_values)?; + Ok((key, definition)) +} + +/// The definition snapshot of `project`, whose authored files live under `project_dir`. +pub(crate) fn definition_snapshot( + project: &Bound<'_, PyAny>, + project_dir: &Path, +) -> PyResult { + let mut nodes: BTreeMap = BTreeMap::new(); + let uses: BTreeMap> = declaration_uses(project)?; + declaration_nodes(project, project_dir, &mut nodes)?; + for model in project_reads::models(project)?.try_iter()? { + let (key, definition) = model_node(&model?, &uses)?; + nodes.insert(key, definition); + } + for seed in project_reads::seeds(project)?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + let name: String = seed_reads::name(&seed)?.extract()?; + let (version_hash, _) = seed_identity(&seed)?; + nodes.insert( + node_key(SEED_KIND, &name), + node(SEED_KIND, &name, vec![(DEFINITION_ASPECT, version_hash)]), + ); + } + let functions: Bound<'_, PyAny> = project_reads::functions(project)?; + let hashes: Bound<'_, PyDict> = function_local_hashes(&functions, None)?; + for function in functions.try_iter()? { + let function: Bound<'_, PyAny> = function?; + let key: Bound<'_, PyAny> = function_reads::key(&function)?; + let kind: String = text_of(&native_field( + &native_field(&key, "resource_type")?, + "value", + )?)?; + let name: String = function_reads::name(&function)?.extract()?; + let Some(hash) = hashes.get_item(&name)? else { + continue; + }; + let mut definition: NodeDefinition = + node(&kind, &name, vec![(DEFINITION_ASPECT, hash.extract()?)]); + for dependency in function_reads::deps(&function)?.try_iter()? { + let dependency: Bound<'_, PyAny> = dependency?; + definition.uses.push(node_key( + &text_of(&native_field( + &native_field(&dependency, "resource_type")?, + "value", + )?)?, + &native_field(&dependency, "name")?.extract::()?, + )); + } + nodes.insert(node_key(&kind, &name), definition); + } + for test in project_reads::sql_tests(project)?.try_iter()? { + let file: Bound<'_, PyAny> = native_field(&test?, "test_file")?; + let path: String = text_of(&native_field(&file, "relative_path")?)?; + nodes + .entry(node_key(TEST_FILE_KIND, &path)) + .or_insert_with(|| { + node( + TEST_FILE_KIND, + &path, + vec![(FILE_ASPECT, file_digest(&project_dir.join(&path)))], + ) + }); + } + let snapshot_root: PathBuf = SCENARIO_SNAPSHOT_ROOT + .iter() + .fold(project_dir.to_path_buf(), |path, part| path.join(part)); + for scenario in native_field(project, "sql_scenarios")?.try_iter()? { + let scenario: Bound<'_, PyAny> = scenario?; + let name: String = native_field(&scenario, "name")?.extract()?; + let path: String = text_of(&native_field( + &native_field(&scenario, "scenario_file")?, + "relative_path", + )?)?; + let mut aspects: Vec<(&str, String)> = + vec![(FILE_ASPECT, file_digest(&project_dir.join(&path)))]; + if let Some(digest) = directory_digest(&snapshot_root.join(&name)) { + aspects.push((SNAPSHOTS_ASPECT, digest)); + } + let key: String = node_key(SCENARIO_KIND, &name); + let mut definition: NodeDefinition = node(SCENARIO_KIND, &name, aspects); + definition.uses = uses.get(&key).into_iter().flatten().cloned().collect(); + nodes.insert(key, definition); + } + Ok(DefinitionSnapshot { + format: DEFINITION_SNAPSHOT_FORMAT, + nodes, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs new file mode 100644 index 0000000000..a119be50b2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod base_compile; +pub(crate) mod base_snapshot; +pub(crate) mod definition_snapshot; +pub(crate) mod registration; +pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/registration.rs new file mode 100644 index 0000000000..9c99d47c7e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/registration.rs @@ -0,0 +1,9 @@ +//! Register the base-revision compile entry that `--since` runs in a separate interpreter. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::change_selection::base_compile; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + base_compile::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs new file mode 100644 index 0000000000..f1d9b5406e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -0,0 +1,222 @@ +//! Which tests and scenarios a `--since` run selects, and why, from what each really executes. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::Path; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::definition_changes::constants::{ + MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, TABLE_FUNCTION_KIND, TEST_FILE_KIND, +}; +use sqlbuild_analysis::definition_changes::main::changes_summary::changes_summary; +use sqlbuild_analysis::definition_changes::main::compare_definitions::compare_definitions; +use sqlbuild_analysis::definition_changes::main::node_key::node_key; +use sqlbuild_analysis::definition_changes::main::selection_reasons::selection_reasons; +use sqlbuild_analysis::definition_changes::main::selection_report::selection_report; +use sqlbuild_analysis::definition_changes::models::{ + DefinitionChanges, DefinitionSnapshot, ExecutedItem, SelectedItem, +}; + +use crate::bindings::_helpers::change_selection::base_snapshot::BaseComparison; +use crate::bindings::_helpers::change_selection::definition_snapshot::{ + declaration_uses, definition_snapshot, +}; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; +use crate::bindings::_helpers::graph::keys::native_key; +use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::scenario_command::setup::{flush, write}; + +const MACRO_TEST_MODE: &str = "macro"; +const TEST_CONSUMER_KIND: &str = "test"; + +/// The changes since the base revision and what the current revision uses. +pub(crate) struct ChangedProject { + pub(crate) reference: String, + pub(crate) current: DefinitionSnapshot, + pub(crate) changes: DefinitionChanges, + uses: BTreeMap>, +} + +/// Compare the compiled working tree `project` with the base snapshot. +pub(crate) fn changed_project( + project: &Bound<'_, PyAny>, + project_dir: &Path, + base: BaseComparison, +) -> PyResult { + let current: DefinitionSnapshot = definition_snapshot(project, project_dir)?; + let changes: DefinitionChanges = compare_definitions(&base.snapshot, ¤t); + Ok(ChangedProject { + reference: base.base.reference, + current, + changes, + uses: declaration_uses(project)?, + }) +} + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() +} + +impl ChangedProject { + /// Keys of current nodes of `kind` named in `names`. + fn keys_named(&self, kind: &str, names: &[String]) -> BTreeSet { + self.current + .nodes + .iter() + .filter(|(_, node)| node.kind == kind && names.contains(&node.name)) + .map(|(key, _)| key.clone()) + .collect() + } + + fn declarations_of(&self, consumer: &str) -> Vec { + self.uses + .get(consumer) + .into_iter() + .flatten() + .cloned() + .collect() + } + + /// Why `item` is affected; empty when nothing it runs changed. + pub(crate) fn reasons(&self, item: &ExecutedItem) -> Vec { + selection_reasons(item, &self.current, &self.changes) + } + + /// Write the change summary and the selection to `stream`. + pub(crate) fn report( + &self, + stream: &Bound<'_, PyAny>, + nouns: (&str, &str), + (total, selected, explain): (usize, &[SelectedItem], bool), + ) -> PyResult<()> { + write( + stream, + &selection_report( + (&self.reference, &changes_summary(&self.changes)), + nouns, + (total, selected), + explain, + ), + )?; + flush(stream) + } + + /// What one planned unit test executes, honouring its mocks. + pub(crate) fn test_item( + &self, + entry: &Bound<'_, PyAny>, + (tests, models): ( + &HashMap>, + &HashMap>, + ), + ) -> PyResult { + let name: String = native_field(entry, "name")?.extract()?; + let mut item: ExecutedItem = ExecutedItem { + declarations: self.declarations_of(&node_key(TEST_CONSUMER_KIND, &name)), + ..ExecutedItem::default() + }; + let mocked_seeds: Vec = strings(&native_field(entry, "mock_seed_names")?)?; + let mocked_functions: Vec = + strings(&native_field(entry, "mock_table_function_names")?)?; + item.mocked + .extend(self.keys_named(TABLE_FUNCTION_KIND, &mocked_functions)); + if let Some(test) = tests.get(&name) { + let file: Bound<'_, PyAny> = native_field(test, "test_file")?; + item.own_keys.push(node_key( + TEST_FILE_KIND, + &native_field(&file, "relative_path")? + .str()? + .to_string() + .replace('\\', "/"), + )); + let payload: Bound<'_, PyAny> = native_field(test, "payload")?; + if let Ok(mocks) = native_field(&payload, "macro_mocks") + && let Ok(mocks) = mocks.cast::() + { + let names: Vec = mocks + .iter() + .map(|(key, _)| key.extract()) + .collect::>()?; + item.mocked.extend(self.keys_named(MACRO_KIND, &names)); + } + if let Ok(mode) = native_field(&payload, "mode") + && native_field(&mode, "value")?.extract::()? == MACRO_TEST_MODE + { + let tested: Vec = + strings(&native_field(&payload, "tested_resource_names")?)?; + item.declarations + .extend(self.keys_named(MACRO_KIND, &tested)); + } + } + for step in native_field(entry, "chain")?.try_iter()? { + let model_name: String = native_field(&step?, "model_name")?.extract()?; + item.executed.push(node_key(MODEL_KIND, &model_name)); + let Some(model) = models.get(&model_name) else { + continue; + }; + for dependency in model_reads::deps(model)?.try_iter()? { + let (kind, dependency_name) = native_key(&dependency?)?; + if kind == SEED_KIND && !mocked_seeds.contains(&dependency_name) { + item.executed.push(node_key(SEED_KIND, &dependency_name)); + } + } + } + for key in native_field(entry, "function_deps")?.try_iter()? { + let (kind, function_name) = native_key(&key?)?; + item.executed.push(node_key(&kind, &function_name)); + } + Ok(item) + } + + /// What one compiled scenario builds and loads, honouring its fixtures. + pub(crate) fn scenario_item( + &self, + scenario: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, + ) -> PyResult { + let name: String = native_field(scenario, "name")?.extract()?; + let key: String = node_key(SCENARIO_KIND, &name); + let (graph, _) = plan_scenario_graph(scenario, project, true, None)?; + let mut item: ExecutedItem = ExecutedItem { + own_keys: vec![key.clone()], + declarations: self.declarations_of(&key), + ..ExecutedItem::default() + }; + for model_name in strings(&native_field(&graph, "model_names")?)? { + item.executed.push(node_key(MODEL_KIND, &model_name)); + } + let fixtures: Vec = strings(&native_field(&graph, "seed_fixture_names")?)?; + for seed_name in strings(&native_field(&graph, "seed_names")?)? { + if !fixtures.contains(&seed_name) { + item.executed.push(node_key(SEED_KIND, &seed_name)); + } + } + for function in native_field(&graph, "function_deps")?.try_iter()? { + let (kind, function_name) = native_key(&function?)?; + item.executed.push(node_key(&kind, &function_name)); + } + Ok(item) + } +} + +/// The project's compiled tests and models by name. +pub(crate) fn tests_and_models<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult<( + HashMap>, + HashMap>, +)> { + let mut tests: HashMap> = HashMap::new(); + for test in project_reads::sql_tests(project)?.try_iter()? { + let test: Bound<'py, PyAny> = test?; + tests.insert(native_field(&test, "name")?.extract()?, test); + } + let mut models: HashMap> = HashMap::new(); + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + models.insert(model_reads::model_name(&model)?, model); + } + Ok((tests, models)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 84a52f27ab..f829d01f1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -5,12 +5,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{progress, progress_reporters, progress_text}; use crate::bindings::_helpers::cli::{argv, entry, launch_state, namespace, observability}; use crate::bindings::_helpers::{ - audit_command, build_command, build_progress, check, clone, command_runtime, compute_logs, - contract_command, cost_command, cost_report, cost_telemetry, dbt_command, dbt_events, - dbt_interop, debug_command, diff_report, event_exporting, execution_history, format_command, - freshness, init_command, janitor, load, output_capture, plan_command, presentation_text, - query_command, recall, result_output, scenario_command, seed_command, sql_test_command, - workspace_commands, + audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, + compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, + dbt_events, dbt_interop, debug_command, diff_report, event_exporting, execution_history, + format_command, freshness, init_command, janitor, load, output_capture, plan_command, + presentation_text, query_command, recall, result_output, scenario_command, seed_command, + sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -43,6 +43,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { workspace_commands::registration::register(module)?; check::registration::register(module)?; sql_test_command::registration::register(module)?; + change_selection::registration::register(module)?; seed_command::registration::register(module)?; audit_command::registration::register(module)?; build_command::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs index 6d105a3a13..e695e2ed5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{ use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_config::project::errors::{ConfigFailure, FailureClass, LoadFailure}; +use sqlbuild_config::project::main::load_git_base_ref::load_git_base_ref; use sqlbuild_config::project::main::load_local_config::load_local_config; use sqlbuild_config::project::main::load_project_config::load_project_config; use sqlbuild_config::project::main::local_sql_analysis_failure::local_sql_analysis_failure; @@ -173,6 +174,13 @@ pub(crate) fn load_project_config_file( } } +/// `[git] base_ref` of the project at `project_dir`, which `--since @base` names. +pub(crate) fn configured_git_base_ref(project_dir: &Bound<'_, PyAny>) -> PyResult> { + let files = files(project_dir)?; + load_git_base_ref(&files.native) + .or_else(|failure| Err(load_error(project_dir.py(), &failure, &files)?)) +} + /// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. #[pyfunction] pub(crate) fn load_local_config_file( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 3f78468b90..ae28284cad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod authored_values; pub(crate) mod build_command; pub(crate) mod build_progress; pub(crate) mod cache; +pub(crate) mod change_selection; pub(crate) mod check; pub(crate) mod cli; pub(crate) mod clone; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index 5296f7a525..3bacaebca1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -6,7 +6,7 @@ mod constants; mod entries; pub(crate) mod execution; mod fixtures; -mod graph; +pub(crate) mod graph; mod helper_scope; mod models; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index f29979c521..b406613164 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -581,6 +581,8 @@ const SCENARIO_TEST_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("json_output", FieldDefault::False), FieldSpec::new("json_output_path", FieldDefault::None), FieldSpec::new("scenario_timeout", FieldDefault::None), + FieldSpec::new("since", FieldDefault::None), + FieldSpec::new("explain", FieldDefault::False), ], }; const SCENARIO_CAPTURE_COMMAND_REQUEST: RecordSpec = RecordSpec { @@ -645,6 +647,8 @@ const TEST_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("case_name", FieldDefault::None), FieldSpec::new("inspect", FieldDefault::False), FieldSpec::new("test_timeout", FieldDefault::None), + FieldSpec::new("since", FieldDefault::None), + FieldSpec::new("explain", FieldDefault::False), ], }; const REFACTOR_COMMAND_REQUEST: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 700f55b4fe..9a94403302 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -6,6 +6,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::definition_changes::models::{ExecutedItem, SelectedItem}; use sqlbuild_core::cli_text::main::format_summary_footer::format_summary_footer; use sqlbuild_core::cli_text::models::{CliStyle, SummaryCount}; use sqlbuild_core::scenario_command::main::render_scenario_failures::render_scenario_failures; @@ -17,6 +18,10 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::change_selection::base_snapshot::{ + BaseComparison, BaseVariant, base_comparison, +}; +use crate::bindings::_helpers::change_selection::selection::{ChangedProject, changed_project}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -335,6 +340,42 @@ fn run_scenarios( Ok(if ok { 0 } else { 1 }) } +/// Keep only the scenarios that execute something changed since the base revision. +fn select_changed<'py>( + run: &TestRun<'py>, + (pipeline, scenarios): (&Bound<'py, PyAny>, Bound<'py, PyTuple>), + base: BaseComparison, + explain: bool, +) -> PyResult> { + let py: Python<'py> = pipeline.py(); + let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; + let changed: ChangedProject = changed_project( + &project, + &run.project.project_dir.extract::()?, + base, + )?; + let mut kept: Vec> = Vec::new(); + let mut selected: Vec = Vec::new(); + for scenario in scenarios.iter() { + let item: ExecutedItem = changed.scenario_item(&scenario, &project)?; + let reasons: Vec = changed.reasons(&item); + if reasons.is_empty() { + continue; + } + selected.push(SelectedItem { + label: native_field(&scenario, "name")?.extract()?, + reasons, + }); + kept.push(scenario); + } + changed.report( + &run.stream, + ("scenario", "scenarios"), + (scenarios.len(), &selected, explain), + )?; + PyTuple::new(py, kept) +} + /// Run one `sqb scenario test` request. #[pyfunction] #[pyo3(signature = (*, request))] @@ -356,6 +397,18 @@ pub(crate) fn run_scenario_test_command( run.style, ), )?; + let since: Option = native_field(request, "since")?.extract()?; + let base: Option = match since.as_deref() { + Some(since) => Some(base_comparison( + (&run.stream, &run.project.project_dir), + since, + &BaseVariant { + selected_target: None, + cli_vars: py.None().into_bound(py), + }, + )?), + None => None, + }; let no_sql_validation: bool = run.local || flag(request, "no_sql_validation")?; let compiled: PyResult> = compile( &run.project, @@ -374,8 +427,16 @@ pub(crate) fn run_scenario_test_command( finished?; let selectors: Vec = run.project.selectors.extract()?; let exclude: Vec = run.project.exclude.extract()?; - let scenarios: Bound<'_, PyTuple> = + let mut scenarios: Bound<'_, PyTuple> = selected_scenarios(&pipeline, (&selectors, &exclude, &run.project.project_dir))?; + if let Some(base) = base { + scenarios = select_changed( + &run, + (&pipeline, scenarios), + base, + flag(request, "explain")?, + )?; + } if run.local && (flag(request, "sync_snapshots")? || flag(request, "refresh")?) { let synced: LocalSync<'_> = sync_local_snapshots(request, &run, (&pipeline, &scenarios))?; if synced.exit_code != 0 { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 5e34f9a244..f62c92a4e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -1,12 +1,14 @@ //! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; +use std::path::PathBuf; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_analysis::definition_changes::models::{ExecutedItem, SelectedItem}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::test_command::main::render_test_completion::render_test_completion; use sqlbuild_core::test_command::main::render_test_inspection::render_test_inspection; @@ -28,6 +30,12 @@ use crate::bindings::_helpers::build_progress::progress_text::{ }; use crate::bindings::_helpers::build_progress::result_text::test_label; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::change_selection::base_snapshot::{ + BaseComparison, BaseVariant, base_comparison, +}; +use crate::bindings::_helpers::change_selection::selection::{ + ChangedProject, changed_project, tests_and_models, +}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; @@ -39,6 +47,7 @@ use crate::bindings::_helpers::planner_entries::command_plans::build_test_comman use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::result_output::documents::format_test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; @@ -385,14 +394,23 @@ fn inspection_entry(entry: &Bound<'_, PyAny>) -> PyResult { mocked_table_functions: strings(entry.getattr("mock_table_function_names")?)?, real_models, expected_models, + selection_reasons: Vec::new(), }) } /// Write the resolved plan without executing it; returns the exit code. -fn inspect(stream: &Bound<'_, PyAny>, plan_output: &Bound<'_, PyAny>) -> PyResult { +fn inspect( + stream: &Bound<'_, PyAny>, + plan_output: &Bound<'_, PyAny>, + reasons: &HashMap>, +) -> PyResult { let entries: Vec = native_field(plan_output, "test_entries")? .try_iter()? - .map(|entry| inspection_entry(&entry?)) + .map(|entry| { + let mut entry: TestInspectionEntry = inspection_entry(&entry?)?; + entry.selection_reasons = reasons.get(&entry.label).cloned().unwrap_or_default(); + Ok(entry) + }) .collect::>()?; let warnings: Vec = native_field(plan_output, "warnings")? .try_iter()? @@ -410,6 +428,50 @@ fn inspect(stream: &Bound<'_, PyAny>, plan_output: &Bound<'_, PyAny>) -> PyResul Ok(code) } +/// Keep only the planned tests that execute something changed since the base revision, write +/// the summary, and return the narrowed pipeline with each kept test's reasons by label. +fn select_changed<'py>( + (pipeline, stream): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + project_dir: &std::path::Path, + base: BaseComparison, + explain: bool, +) -> PyResult<(Bound<'py, PyAny>, HashMap>)> { + let py: Python<'py> = pipeline.py(); + let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; + let changed: ChangedProject = changed_project(&project, project_dir, base)?; + let (tests, models) = tests_and_models(&project)?; + let plan_output: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; + let mut kept: Vec> = Vec::new(); + let mut selected: Vec = Vec::new(); + let mut reasons: HashMap> = HashMap::new(); + let mut total: usize = 0; + for entry in native_field(&plan_output, "test_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + total += 1; + let item: ExecutedItem = changed.test_item(&entry, (&tests, &models))?; + let entry_reasons: Vec = changed.reasons(&item); + if entry_reasons.is_empty() { + continue; + } + let label: String = inspection_entry(&entry)?.label; + reasons.insert(label.clone(), entry_reasons.clone()); + selected.push(SelectedItem { + label, + reasons: entry_reasons, + }); + kept.push(entry); + } + changed.report(stream, ("test", "tests"), (total, &selected, explain))?; + let plan_output: Bound<'py, PyAny> = replaced( + &plan_output, + &[("test_entries", PyTuple::new(py, kept)?.into_any())], + )?; + Ok(( + replaced(pipeline, &[("plan_output", plan_output)])?, + reasons, + )) +} + /// Write the ready header and run the tests with nested progress; returns the results. fn run_tests<'py>( invocation: &Bound<'py, PyAny>, @@ -604,10 +666,32 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py &record_class(py, RecordId::TestInvocation)?, !inspect_only, )?; - let pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let project_dir: Bound<'_, PyAny> = native_field(&invocation, "effective_project_dir")?; + let since: Option = native_field(request, "since")?.extract()?; + let base: Option = match since.as_deref() { + Some(since) => Some(base_comparison( + (&stream, &project_dir), + since, + &BaseVariant { + selected_target: native_field(request, "selected_target")?.extract()?, + cli_vars: native_field(request, "cli_vars")?, + }, + )?), + None => None, + }; + let mut pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; + let mut reasons: HashMap> = HashMap::new(); + if let Some(base) = base { + (pipeline, reasons) = select_changed( + (&pipeline, &stream), + &project_dir.extract::()?, + base, + native_field(request, "explain")?.is_truthy()?, + )?; + } if inspect_only { - return inspect(&stream, &pipeline.getattr("plan_output")?); + return inspect(&stream, &pipeline.getattr("plan_output")?, &reasons); } let concurrency: i64 = resolve_concurrency( &request.getattr("concurrency")?, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 81c6d8ccd7..bbfc4487c5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6798,6 +6798,8 @@ class ScenarioTestCommandRequest(NativeRecord): json_output: bool json_output_path: Path | None scenario_timeout: str | None + since: str | None + explain: bool def __init__( self, project_dir: Path | None = None, @@ -6816,6 +6818,8 @@ class ScenarioTestCommandRequest(NativeRecord): json_output: bool = False, json_output_path: Path | None = None, scenario_timeout: str | None = None, + since: str | None = None, + explain: bool = False, ) -> None: ... class ScenarioCaptureCommandRequest(NativeRecord): @@ -6857,6 +6861,8 @@ class TestCommandRequest(NativeRecord): case_name: str | None inspect: bool test_timeout: str | None + since: str | None + explain: bool def __init__( self, project_dir: Path | None = None, @@ -6872,6 +6878,8 @@ class TestCommandRequest(NativeRecord): case_name: str | None = None, inspect: bool = False, test_timeout: str | None = None, + since: str | None = None, + explain: bool = False, ) -> None: ... class RefactorCommandRequest(NativeRecord): diff --git a/website/src/content/docs/docs/cli/scenario.mdx b/website/src/content/docs/docs/cli/scenario.mdx index 082f28b154..fb6008020f 100644 --- a/website/src/content/docs/docs/cli/scenario.mdx +++ b/website/src/content/docs/docs/cli/scenario.mdx @@ -35,6 +35,26 @@ sqb scenario test [flags] | `--max-snapshot-bytes` | Override per-relation byte limit for capture | | `--max-snapshot-total-bytes` | Override total byte limit for capture | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | +| `--since ` | Run only the scenarios affected by changes since the merge base of `` and `HEAD`; `@base` reads `[git] base_ref` | +| `--explain` | With `--since`, print why each selected scenario was selected | + +### Affected scenarios + +`--since ` applies the rules of [`sqb test --since`](/docs/cli/test#run-only-affected-tests) +to scenarios. A scenario is selected when its own file (its definition and fixtures) or its +captured local snapshots changed, or when any node it actually executes changed: the models it +builds, the seeds it loads, the functions they call, and the macros, enums and constants they and +the scenario use. A model the scenario replaces with a `__ref__` fixture, or a seed it replaces +with a `__seed__` fixture, is not executed, so its changes do not select the scenario. + +`--select` and `--exclude` narrow the affected scenarios further. Both commands share the cached +base compile, so `sqb test --since origin/main` and `sqb scenario test --since origin/main` in one +CI job resolve the same changed set: + +```bash +sqb test --since "origin/$TARGET_BRANCH" +sqb scenario test --since "origin/$TARGET_BRANCH" --explain +``` ### Selectors diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index e3aa46347d..abc7e709f5 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -26,6 +26,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--exclude` | Exclude tests by name, or the tests targeting specific models | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | | `--test-timeout ` | Time limit for each test in this run, such as `30s` or `2m`, or `disabled`; overrides `TEST(timeout ...)`, the target and `[settings] test_timeout` | +| `--since ` | Run only the tests affected by changes since the merge base of `` and `HEAD` (see [Run only affected tests](#run-only-affected-tests)); `@base` reads `[git] base_ref` | +| `--explain` | With `--since`, print why each selected test was selected | Test names are globally unique, so a bare test name selects exactly that test. `test:` is the explicit form and accepts the same glob patterns as names, such as `test:orders_*`. A bare pattern @@ -50,6 +52,61 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Run only affected tests + +`--since ` runs only the tests a change can affect, which keeps CI on branch pushes from +running the whole suite. SQLBuild compiles the project as it was at the merge base of `` and +`HEAD`, straight from git objects without a checkout, and compares it with the working tree, +uncommitted edits included. A test is selected when: + +- its own file changed or is new, or +- any node it actually executes changed: the models in its chain, the seeds and functions they + read, and the macros, enums and constants they use. The resolution is the one `--inspect` + shows, so a test that mocks a changed model, seed or macro is not selected, while a downstream + test that really runs it is. + +A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with +formatting and comments ignored, the header configuration, columns and contract, attached audits, +and every macro, enum, constant, function and seed the node uses, resolved through +[scopes](/docs/concepts/declaration-scopes). Descriptions do not count. A macro counts as changed +when the Python file that defines it changes. Added nodes count as changed. A model renamed with +`migrate_from` is compared with its old definition, so a pure rename selects nothing. Deleted +nodes select nothing and are listed in the summary. + +```bash +# Tests affected by this branch +sqb test --since origin/main + +# Read the ref from [git] base_ref in sqlbuild_project.toml +sqb test --since @base + +# Only affected tests that target stg_orders, and say why each runs +sqb test stg_orders --since origin/main --explain +``` + +Selectors still apply: `--select` and positional selectors intersect with the affected tests, and +`--exclude` removes tests. `--explain`, or `--inspect`, prints the reasons for each selected test: + +```text +Comparing with origin/main (merge base 1a2b3c4d5e) +Compiled base revision 1a2b3c4d5e (2.31s) +Changed since origin/main: 1 model, 1 macro changed; deleted: model legacy_orders +Selected 2 of 9 tests + orders_rollup + executes stg_orders (model SQL changed) + order_timestamps + executes macro to_utc (changed) +``` + +The base compile is cached per merge-base commit under `target/since/`, so `sqb test --since` and +`sqb scenario test --since` in the same CI job compile the base once and compare against the same +changed set. When no test is affected, the command says so and exits successfully. + +`` is any git ref or commit, such as the merge request's target branch from a CI variable. +`--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach +the merge base, fails with an error that says what to fetch, such as `git fetch --unshallow` or a +larger clone depth; it never silently selects nothing. + ## Examples ```bash diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 22e328e7ab..9b7537ea98 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -859,6 +859,19 @@ Local type overrides for DuckDB replay are configured per adapter dialect: See [Scenarios](/docs/concepts/scenarios) for details on local type overrides and capture limits. +## Git + +The branch that `--since @base` compares against: + +```toml +[git] +base_ref = "origin/main" +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `base_ref` | unset | Git ref that [`sqb test --since @base`](/docs/cli/test#run-only-affected-tests) and `sqb scenario test --since @base` read. Use a ref every checkout has, such as the remote default branch. | + ## dbt Configuration for running SQLBuild alongside an existing dbt project: From 5f8679a11cd4c2825f6e78aa0f3b3f5cf0083b47 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:12:49 +0100 Subject: [PATCH 02/10] test: cover --since selection end to end --- .../sqlbuild/references/docs/cli/scenario.md | 20 ++ .../sqlbuild/references/docs/cli/test.md | 67 ++++ .../docs/concepts/project-configuration.md | 14 + .../cli/commands/main/test/_test_types.py | 52 +++ .../cli/commands/main/test/helpers.py | 134 ++++++++ .../cli/commands/main/test/test_since.py | 304 ++++++++++++++++++ website/src/content/docs/docs/cli/test.mdx | 9 +- 7 files changed, 597 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md index 644c73e468..96a9f1905b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -42,6 +42,26 @@ sqb scenario test [flags] | `--max-snapshot-bytes` | Override per-relation byte limit for capture | | `--max-snapshot-total-bytes` | Override total byte limit for capture | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | +| `--since ` | Run only the scenarios affected by changes since the merge base of `` and `HEAD`; `@base` reads `[git] base_ref` | +| `--explain` | With `--since`, print why each selected scenario was selected | + +### Affected scenarios + +`--since ` applies the rules of [`sqb test --since`](test.md#run-only-affected-tests) +to scenarios. A scenario is selected when its own file (its definition and fixtures) or its +captured local snapshots changed, or when any node it actually executes changed: the models it +builds, the seeds it loads, the functions they call, and the macros, enums and constants they and +the scenario use. A model the scenario replaces with a `__ref__` fixture, or a seed it replaces +with a `__seed__` fixture, is not executed, so its changes do not select the scenario. + +`--select` and `--exclude` narrow the affected scenarios further. Both commands share the cached +base compile, so `sqb test --since origin/main` and `sqb scenario test --since origin/main` in one +CI job resolve the same changed set: + +```bash +sqb test --since "origin/$TARGET_BRANCH" +sqb scenario test --since "origin/$TARGET_BRANCH" --explain +``` ### Selectors diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 5aeb59c0fe..9f19afdd4f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -6,6 +6,13 @@ Online: https://sqlbuild.com/docs/cli/test/ +## Contents + +- Usage +- Flags +- Run only affected tests +- Examples + Runs SQL unit tests, independently reported parameterized cases, and multi-model tests without building models. Useful for validating test logic independently. @@ -27,6 +34,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--exclude` | Exclude tests by name, or the tests targeting specific models | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | | `--test-timeout ` | Time limit for each test in this run, such as `30s` or `2m`, or `disabled`; overrides `TEST(timeout ...)`, the target and `[settings] test_timeout` | +| `--since ` | Run only the tests affected by changes since the merge base of `` and `HEAD` (see [Run only affected tests](#run-only-affected-tests)); `@base` reads `[git] base_ref` | +| `--explain` | With `--since`, print why each selected test was selected | Test names are globally unique, so a bare test name selects exactly that test. `test:` is the explicit form and accepts the same glob patterns as names, such as `test:orders_*`. A bare pattern @@ -51,6 +60,64 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Run only affected tests + +`--since ` runs only the tests a change can affect, which keeps CI on branch pushes from +running the whole suite. SQLBuild compiles the project as it was at the merge base of `` and +`HEAD`, straight from git objects without a checkout, and compares it with the working tree, +uncommitted edits included. A test is selected when: + +- its own file changed or is new, or +- any node it actually executes changed: the models in its chain, the seeds and functions they + read, and the macros, enums and constants they use. The resolution is the one `--inspect` + shows, so a test that mocks a changed model, seed or macro is not selected, while a downstream + test that really runs it is. + +A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with +formatting and comments ignored, the header configuration, columns and contract, attached audits, +and every macro, enum, constant, function and seed the node uses, resolved through +[scopes](../concepts/declaration-scopes.md). Descriptions and `meta` do not count. A macro counts as +changed when the Python file that defines it changes, and an enum, constant or audit when its +declaration file changes other than in formatting and comments. Added nodes count as changed. A model renamed with +`migrate_from` is compared with its old definition, so a pure rename selects nothing. Deleted +nodes select nothing and are listed in the summary. + +```bash +# Tests affected by this branch +sqb test --since origin/main + +# Read the ref from [git] base_ref in sqlbuild_project.toml +sqb test --since @base + +# Only affected tests that target stg_orders, and say why each runs +sqb test stg_orders --since origin/main --explain +``` + +Selectors still apply: `--select` and positional selectors intersect with the affected tests, and +`--exclude` removes tests. `--explain` prints the reasons for each selected test, and `--inspect` +adds them to the resolved plan as `selected:` lines: + +```text +Comparing with origin/main (merge base 1a2b3c4d5e) +Compiling base revision 1a2b3c4d5e... +Compiled base revision 1a2b3c4d5e (2.31s) +Changed since origin/main: 1 model, 1 macro changed; deleted: model legacy_orders +Selected 2 of 9 tests + orders_rollup + executes stg_orders (model SQL changed) + order_timestamps + executes macro to_utc (changed) +``` + +The base compile is cached per merge-base commit under `target/since/`, so `sqb test --since` and +`sqb scenario test --since` in the same CI job compile the base once and compare against the same +changed set. When no test is affected, the command says so and exits successfully. + +`` is any git ref or commit, such as the merge request's target branch from a CI variable. +`--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach +the merge base, fails with an error that says what to fetch, such as `git fetch --unshallow` or a +larger clone depth; it never silently selects nothing. + ## Examples ```bash diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index c64f5cdca5..d726d75115 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -21,6 +21,7 @@ Online: https://sqlbuild.com/docs/concepts/project-configuration/ - Janitor - Migrations - Scenario +- Git - dbt - Skills - Project Python @@ -882,6 +883,19 @@ Local type overrides for DuckDB replay are configured per adapter dialect: See [Scenarios](scenarios.md) for details on local type overrides and capture limits. +## Git + +The branch that `--since @base` compares against: + +```toml +[git] +base_ref = "origin/main" +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `base_ref` | unset | Git ref that [`sqb test --since @base`](../cli/test.md#run-only-affected-tests) and `sqb scenario test --since @base` read. Use a ref every checkout has, such as the remote default branch. | + ## dbt Configuration for running SQLBuild alongside an existing dbt project: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 78c5d80a83..235ce6538b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -493,3 +493,55 @@ class SharedPartialFixtureE2ETestCase: enriched_column: str report_column: str expected_padding_count: int + + +@dataclass(frozen=True) +class SinceSelectionE2ETestCase: + """Edits made after the base commit and what `--since` must select and say.""" + + description: str + edits: Mapping[str, str] + command: tuple[str, ...] + expected_exit_code: int + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) + deletions: tuple[str, ...] = field(default_factory=tuple) + # The git command run after the edits; `commit --all` makes them part of HEAD. + git_after_edits: tuple[str, ...] = ("status", "--short") + + +@dataclass(frozen=True) +class SinceSharedBaseE2ETestCase: + """A test run, then a scenario run, against the same base ref.""" + + description: str + edits: Mapping[str, str] + test_command: tuple[str, ...] + scenario_command: tuple[str, ...] + expected_test_fragments: tuple[str, ...] + expected_scenario_fragments: tuple[str, ...] + unexpected_scenario_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SinceJsonOutputE2ETestCase: + """A machine-output `--since` run and where its parts must go.""" + + description: str + edits: Mapping[str, str] + command: tuple[str, ...] + expected_stderr_fragments: tuple[str, ...] + expected_test_names: tuple[str, ...] + unexpected_stdout_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SinceShallowCloneE2ETestCase: + """A shallow clone that lacks the merge base of the compared ref.""" + + description: str + clone_arguments: tuple[str, ...] + fetch_arguments: tuple[str, ...] + command: tuple[str, ...] + expected_exit_code: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 1819396c24..b78815fe2c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -3,7 +3,10 @@ from __future__ import annotations import json +import os import re +import subprocess +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -2150,3 +2153,134 @@ def build_shared_partial_fixture_project_files( "SELECT 1\n" ), } + + +def build_since_project_files() -> dict[str, str]: + """Build a DuckDB orders project whose tests and scenarios execute different nodes.""" + + return { + "sqlbuild_project.toml": ( + 'name = "since_orders"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "since_orders.duckdb"\n\n' + '[defaults]\nmaterialized = "table"\n' + ), + ".gitignore": "target/\n*.duckdb\n*.duckdb.wal\n", + "models/_macros/money.py": ( + "def cents_to_dollars(expression: str) -> str:\n" + ' return f"CAST({expression} AS DOUBLE) / 100"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Raw orders.\n" + " schema: main\n table: raw_orders\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: status\n type: VARCHAR\n" + " - name: amount_cents\n type: INTEGER\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Cleaned orders.');\n\n" + 'SELECT order_id, status, amount_cents FROM __source("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals in dollars.');\n\n" + 'SELECT order_id, @cents_to_dollars("amount_cents") AS amount\n' + 'FROM __ref("stg_orders")\n' + ), + "models/open_orders.sql": ( + "MODEL (description 'Open orders.');\n\n" + "SELECT order_id FROM __ref(\"stg_orders\") WHERE status = 'open'\n" + ), + "tests/unit/test_stg_orders.sql": ( + "TEST ();\n\nWITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__stg_orders AS (SELECT 1 AS order_id, 'open' AS status)\nSELECT 1\n" + ), + "tests/unit/test_order_totals.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_totals AS (SELECT 1 AS order_id, 2.5 AS amount)\nSELECT 1\n" + ), + "tests/unit/test_open_orders.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__open_orders AS (SELECT 1 AS order_id)\nSELECT 1\n" + ), + "tests/scenarios/order_totals_end_to_end.sql": ( + "SCENARIO (description 'Order totals from raw orders');\n\nWITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__order_totals AS (SELECT 1 AS order_id, 2.5 AS amount)\nSELECT 1\n" + ), + "tests/scenarios/open_orders_from_fixture.sql": ( + "SCENARIO (description 'Open orders from a staged fixture');\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__open_orders AS (SELECT 1 AS order_id)\nSELECT 1\n" + ), + } + + +_GIT_IDENTITY: dict[str, str] = { + "GIT_AUTHOR_NAME": "Example", + "GIT_AUTHOR_EMAIL": "dev@example.com", + "GIT_COMMITTER_NAME": "Example", + "GIT_COMMITTER_EMAIL": "dev@example.com", +} + + +def run_git(*, repository: Path, arguments: tuple[str, ...]) -> str: + """Run git in `repository` with a fixed identity and return its stdout.""" + + result: subprocess.CompletedProcess[str] = subprocess.run( + ("git", "-c", "commit.gpgsign=false", *arguments), + cwd=repository, + env={**os.environ, **_GIT_IDENTITY}, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + return result.stdout.strip() + + +def commit_project_on_main(*, repository: Path) -> None: + """Commit everything on `main`, then continue on a `feature` branch.""" + + run_git(repository=repository, arguments=("init", "--quiet", "--initial-branch", "main")) + run_git(repository=repository, arguments=("add", "--all")) + run_git(repository=repository, arguments=("commit", "--quiet", "-m", "orders")) + run_git(repository=repository, arguments=("checkout", "--quiet", "-b", "feature")) + + +def write_project_files(*, project_dir: Path, files: Mapping[str, str]) -> None: + """Write each file below `project_dir`, creating parent directories.""" + + relative_path: str + contents: str + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def delete_project_files(*, project_dir: Path, relative_paths: tuple[str, ...]) -> None: + """Delete each file below `project_dir`.""" + + relative_path: str + for relative_path in relative_paths: + (project_dir / relative_path).unlink() + + +def prepare_since_repository(*, tmp_path: Path, extra_files: Mapping[str, str]) -> Path: + """Write the since project with `extra_files`, commit it on `main` and branch to `feature`.""" + + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="since_orders", + repo_files={**build_since_project_files(), **extra_files}, + ) + commit_project_on_main(repository=project) + return project diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py new file mode 100644 index 0000000000..3872eb2958 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -0,0 +1,304 @@ +"""`sqb test --since` and `sqb scenario test --since` select what a git change can affect.""" + +import json +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + SinceJsonOutputE2ETestCase, + SinceSelectionE2ETestCase, + SinceShallowCloneE2ETestCase, + SinceSharedBaseE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import ( + build_since_project_files, + delete_project_files, + prepare_since_repository, + run_git, + write_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +_FILES: dict[str, str] = build_since_project_files() +_STAGING: str = "models/stg_orders.sql" +_MACRO: str = "models/_macros/money.py" +_EDITED_STAGING: str = _FILES[_STAGING].replace( + "SELECT order_id, status", "SELECT order_id, LOWER(status) AS status" +) +_EDITED_MACRO: str = _FILES[_MACRO].replace("/ 100", "/ 100.0") +_REFORMATTED_STAGING: str = ( + "MODEL (description 'Cleaned and documented orders.');\n\n" + "-- one row per order\nselect\n order_id,\n status,\n amount_cents\n" + 'from __source("raw_orders")\n' +) +_ARCHIVE: str = "models/order_archive.sql" +_ARCHIVE_SQL: str = ( + "MODEL (description 'Archived orders.');\n\nSELECT order_id FROM __ref(\"stg_orders\")\n" +) +_TEST_COMMAND: tuple[str, ...] = ("--no-color", "test", "--since", "main") +_BASE_REF_CONFIG: str = _FILES["sqlbuild_project.toml"] + '\n[git]\nbase_ref = "main"\n' + + +@pytest.mark.parametrize( + "test_case", + ( + SinceSelectionE2ETestCase( + description="an uncommitted model edit selects only the test that really runs it", + edits={_STAGING: _EDITED_STAGING}, + command=(*_TEST_COMMAND, "--explain"), + expected_exit_code=0, + expected_fragments=( + "Comparing with main (merge base ", + "Compiled base revision ", + "Changed since main: 1 model changed", + "Selected 1 of 3 tests", + "test_stg_orders\n executes stg_orders (model SQL changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("test_order_totals", "test_open_orders"), + ), + SinceSelectionE2ETestCase( + description="a committed macro edit selects the test whose model calls the macro", + edits={_MACRO: _EDITED_MACRO}, + command=(*_TEST_COMMAND, "--explain"), + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 macro changed", + "test_order_totals\n executes macro cents_to_dollars (changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("test_stg_orders", "test_open_orders"), + git_after_edits=("commit", "--quiet", "--all", "-m", "edit"), + ), + SinceSelectionE2ETestCase( + description="formatting, comments and descriptions change nothing", + edits={_STAGING: _REFORMATTED_STAGING}, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: no definition changes", + "No tests are affected by the changes since main", + "PASS=0 FAIL=0 TOTAL=0", + ), + ), + SinceSelectionE2ETestCase( + description="an edited test file selects that test", + edits={ + "tests/unit/test_open_orders.sql": _FILES["tests/unit/test_open_orders.sql"] + + "-- covers open orders\n" + }, + command=(*_TEST_COMMAND, "--inspect"), + expected_exit_code=0, + expected_fragments=( + "Selected 1 of 3 tests", + "test_open_orders\n", + " selected: test file changed", + "Test plan inspection complete: 1 selected, 0 errors.", + ), + unexpected_fragments=("test_stg_orders\n", "test_order_totals\n"), + ), + SinceSelectionE2ETestCase( + description="selectors intersect with the affected tests", + edits={_STAGING: _EDITED_STAGING, _MACRO: _EDITED_MACRO}, + command=(*_TEST_COMMAND, "order_totals"), + expected_exit_code=0, + expected_fragments=("Selected 1 of 1 test\n", "PASS=1 FAIL=0 TOTAL=1"), + unexpected_fragments=("test_stg_orders",), + ), + SinceSelectionE2ETestCase( + description="exclude removes affected tests", + edits={_STAGING: _EDITED_STAGING, _MACRO: _EDITED_MACRO}, + command=(*_TEST_COMMAND, "--exclude", "test:test_order_totals"), + expected_exit_code=0, + expected_fragments=("Selected 1 of 2 tests", "PASS=1 FAIL=0 TOTAL=1"), + unexpected_fragments=("test_order_totals",), + ), + SinceSelectionE2ETestCase( + description="a deleted model is listed and selects nothing", + edits={}, + deletions=(_ARCHIVE,), + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: deleted: model order_archive", + "No tests are affected by the changes since main", + ), + ), + SinceSelectionE2ETestCase( + description="a pure rename through migrate_from selects nothing", + edits={ + "models/order_history.sql": _ARCHIVE_SQL.replace( + "MODEL (", "MODEL (migrate_from order_archive, " + ), + }, + deletions=(_ARCHIVE,), + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 model renamed", + "No tests are affected by the changes since main", + ), + unexpected_fragments=("deleted:",), + ), + SinceSelectionE2ETestCase( + description="a missing ref fails instead of selecting nothing", + edits={}, + command=("--no-color", "test", "--since", "origin/missing"), + expected_exit_code=1, + expected_fragments=( + "error[C481]: Git ref 'origin/missing' does not name a commit in this repository", + "git fetch origin ", + ), + ), + SinceSelectionE2ETestCase( + description="@base without a configured base ref fails", + edits={}, + command=("--no-color", "test", "--since", "@base"), + expected_exit_code=1, + expected_fragments=( + "error[C485]: --since @base needs [git] base_ref in sqlbuild_project.toml", + ), + ), + SinceSelectionE2ETestCase( + description="--explain needs --since", + edits={}, + command=("--no-color", "test", "--explain"), + expected_exit_code=2, + expected_fragments=("--explain requires --since",), + ), + ), + ids=lambda case: case.description, +) +def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( + tmp_path: Path, test_case: SinceSelectionE2ETestCase +) -> None: + project: Path = prepare_since_repository( + tmp_path=tmp_path, extra_files={_ARCHIVE: _ARCHIVE_SQL} + ) + write_project_files(project_dir=project, files=test_case.edits) + delete_project_files(project_dir=project, relative_paths=test_case.deletions) + run_git(repository=project, arguments=test_case.git_after_edits) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output + for fragment in test_case.unexpected_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceSharedBaseE2ETestCase( + description="a scenario run reuses the base compile of the test run", + edits={_STAGING: _EDITED_STAGING, "sqlbuild_project.toml": _BASE_REF_CONFIG}, + test_command=("--no-color", "test", "--since", "@base"), + scenario_command=("--no-color", "scenario", "test", "--since", "@base", "--explain"), + expected_test_fragments=("Compiled base revision ", "Selected 1 of 3 tests"), + expected_scenario_fragments=( + "Using cached base revision ", + "Changed since main: 1 model changed", + "Selected 1 of 2 scenarios", + "order_totals_end_to_end\n executes stg_orders (model SQL changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_scenario_fragments=("open_orders_from_fixture",), + ), + ), + ids=lambda case: case.description, +) +def test_given_test_and_scenario_runs_when_comparing_then_both_share_one_base_compile( + tmp_path: Path, test_case: SinceSharedBaseE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files={}) + write_project_files(project_dir=project, files=test_case.edits) + + tested: CompletedProcess[str] = run_sqb(command=test_case.test_command, project_dir=project) + scenarios: CompletedProcess[str] = run_sqb( + command=test_case.scenario_command, project_dir=project + ) + + tested_output: str = tested.stdout + tested.stderr + scenario_output: str = scenarios.stdout + scenarios.stderr + assert tested.returncode == 0, tested_output + assert scenarios.returncode == 0, scenario_output + for fragment in test_case.expected_test_fragments: + assert fragment in tested_output, tested_output + for fragment in test_case.expected_scenario_fragments: + assert fragment in scenario_output, scenario_output + for fragment in test_case.unexpected_scenario_fragments: + assert fragment not in scenario_output, scenario_output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceJsonOutputE2ETestCase( + description="progress and the selection go to stderr", + edits={_STAGING: _EDITED_STAGING}, + command=("--no-color", "test", "--since", "main", "--json"), + expected_stderr_fragments=("Comparing with main", "Selected 1 of 3 tests"), + expected_test_names=("test_stg_orders",), + unexpected_stdout_fragments=("test_order_totals", "Comparing with"), + ), + ), + ids=lambda case: case.description, +) +def test_given_json_output_when_testing_since_ref_then_stdout_stays_machine_readable( + tmp_path: Path, test_case: SinceJsonOutputE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files={}) + write_project_files(project_dir=project, files=test_case.edits) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + assert result.returncode == 0, result.stdout + result.stderr + document: str = json.dumps(json.loads(result.stdout)) + for fragment in test_case.expected_stderr_fragments: + assert fragment in result.stderr, result.stderr + for name in test_case.expected_test_names: + assert f'"{name}"' in document, document + for fragment in test_case.unexpected_stdout_fragments: + assert fragment not in result.stdout, result.stdout + + +@pytest.mark.parametrize( + "test_case", + ( + SinceShallowCloneE2ETestCase( + description="a depth-one clone of both branches lacks the merge base", + clone_arguments=("--depth", "1", "--branch", "feature", "--no-single-branch"), + fetch_arguments=("fetch", "--quiet", "--depth", "1", "origin", "main"), + command=("--no-color", "test", "--since", "origin/main"), + expected_exit_code=1, + expected_fragments=( + "error[C482]: Cannot find the merge base of 'origin/main' and HEAD", + "git fetch --unshallow", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_shallow_clone_without_merge_base_when_testing_since_then_asks_for_depth( + tmp_path: Path, test_case: SinceShallowCloneE2ETestCase +) -> None: + origin: Path = prepare_since_repository(tmp_path=tmp_path, extra_files={}) + run_git(repository=origin, arguments=("commit", "--quiet", "--allow-empty", "-m", "next")) + clone: Path = tmp_path / "clone" + run_git( + repository=tmp_path, + arguments=("clone", "--quiet", *test_case.clone_arguments, origin.as_uri(), str(clone)), + ) + run_git(repository=clone, arguments=test_case.fetch_arguments) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=clone) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index abc7e709f5..e5b8394bdc 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -68,8 +68,9 @@ uncommitted edits included. A test is selected when: A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with formatting and comments ignored, the header configuration, columns and contract, attached audits, and every macro, enum, constant, function and seed the node uses, resolved through -[scopes](/docs/concepts/declaration-scopes). Descriptions do not count. A macro counts as changed -when the Python file that defines it changes. Added nodes count as changed. A model renamed with +[scopes](/docs/concepts/declaration-scopes). Descriptions and `meta` do not count. A macro counts as +changed when the Python file that defines it changes, and an enum, constant or audit when its +declaration file changes other than in formatting and comments. Added nodes count as changed. A model renamed with `migrate_from` is compared with its old definition, so a pure rename selects nothing. Deleted nodes select nothing and are listed in the summary. @@ -85,10 +86,12 @@ sqb test stg_orders --since origin/main --explain ``` Selectors still apply: `--select` and positional selectors intersect with the affected tests, and -`--exclude` removes tests. `--explain`, or `--inspect`, prints the reasons for each selected test: +`--exclude` removes tests. `--explain` prints the reasons for each selected test, and `--inspect` +adds them to the resolved plan as `selected:` lines: ```text Comparing with origin/main (merge base 1a2b3c4d5e) +Compiling base revision 1a2b3c4d5e... Compiled base revision 1a2b3c4d5e (2.31s) Changed since origin/main: 1 model, 1 macro changed; deleted: model legacy_orders Selected 2 of 9 tests From 45ec11494b7f2918c6745304de45fd447e808ab3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:13:01 +0100 Subject: [PATCH 03/10] refactor: satisfy structure rules for --since selection --- .../src/definition_changes/_helpers/mod.rs | 1 + .../definition_changes/_helpers/reasons.rs | 90 ++++ .../main/changes_summary.rs | 15 +- .../main/compare_definitions.rs | 16 +- .../main/definition_digest.rs | 12 +- .../main/definition_snapshot_json.rs | 8 - .../src/definition_changes/main/mod.rs | 2 + .../main/parse_definition_snapshot.rs | 12 + .../main/selection_reasons.rs | 93 +--- .../main/selection_report.rs | 3 +- .../main/sql_definition_digest.rs | 12 + .../src/definition_changes/mod.rs | 3 +- .../src/definition_changes/tests/helpers.rs | 66 ++- .../tests/test_definition_changes.rs | 414 ++++++++++-------- .../definition_changes/tests/test_types.rs | 45 +- .../src/project/tests/test_load_config.rs | 35 +- .../src/project/tests/test_types.rs | 6 + .../src/git_revisions/constants.rs | 3 + .../src/git_revisions/main/export_git_tree.rs | 5 +- .../git_revisions/main/resolve_git_base.rs | 5 +- .../src/git_revisions/tests/helpers.rs | 9 +- .../git_revisions/tests/test_git_revisions.rs | 214 ++++++--- .../src/git_revisions/tests/test_types.rs | 23 +- .../_helpers/change_selection/base_compile.rs | 34 +- .../change_selection/base_snapshot.rs | 50 ++- .../change_selection/definition_snapshot.rs | 45 +- .../_helpers/change_selection/selection.rs | 17 +- .../_helpers/scenario_command/test_command.rs | 7 +- .../_helpers/sql_test_command/test_command.rs | 18 +- fensu.toml | 1 + src/sqlbuild/_native.pyi | 7 + 31 files changed, 793 insertions(+), 478 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/parse_definition_snapshot.rs create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/sql_definition_digest.rs diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs index 38239ddf94..57a4708da1 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod aspects; +pub(crate) mod reasons; diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs new file mode 100644 index 0000000000..4069409ced --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs @@ -0,0 +1,90 @@ +//! The reason texts of selected tests and the declarations an executed item reaches. + +use std::collections::BTreeSet; + +use crate::definition_changes::_helpers::aspects::{aspect_label, joined, kind_words}; +use crate::definition_changes::constants::{CONSTANT_KIND, ENUM_KIND, MODEL_KIND, TEST_FILE_KIND}; +use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem, NodeChange}; + +fn aspect_phrase(change: &NodeChange) -> String { + let labels: Vec = change + .aspects + .iter() + .map(|aspect| aspect_label(aspect).to_owned()) + .collect(); + joined(&labels) +} + +/// `test file changed`, `scenario file and snapshots changed`, `test file added`. +pub(crate) fn own_reason(change: &NodeChange) -> String { + let (singular, _) = kind_words(&change.kind); + if change.added { + return format!("{singular} added"); + } + if change.kind == TEST_FILE_KIND { + return format!("{singular} changed"); + } + format!("{singular} {} changed", aspect_phrase(change)) +} + +/// `added`, `model SQL changed`, `renamed from old_orders, config changed`, `changed`. +fn change_detail(change: &NodeChange) -> String { + if change.added { + return "added".to_owned(); + } + let detail: String = if change.kind == MODEL_KIND { + format!("model {} changed", aspect_phrase(change)) + } else { + "changed".to_owned() + }; + match &change.renamed_from { + Some(old_name) => format!("renamed from {old_name}, {detail}"), + None => detail, + } +} + +pub(crate) fn executed_reason(change: &NodeChange) -> String { + let subject: String = if change.kind == MODEL_KIND { + change.name.clone() + } else { + format!("{} {}", kind_words(&change.kind).0, change.name) + }; + format!("executes {subject} ({})", change_detail(change)) +} + +pub(crate) fn declaration_reason(change: &NodeChange) -> String { + let verb: &str = if change.kind == ENUM_KIND || change.kind == CONSTANT_KIND { + "uses" + } else { + "executes" + }; + format!( + "{verb} {} {} ({})", + kind_words(&change.kind).0, + change.name, + change_detail(change) + ) +} + +/// Declarations `item` reaches, transitively through its SQL and executed nodes, minus its mocks. +pub(crate) fn reached_declarations( + item: &ExecutedItem, + current: &DefinitionSnapshot, +) -> BTreeSet { + let mut pending: Vec = item.declarations.clone(); + for key in &item.executed { + if let Some(node) = current.nodes.get(key) { + pending.extend(node.uses.iter().cloned()); + } + } + let mut reached: BTreeSet = BTreeSet::new(); + while let Some(key) = pending.pop() { + if item.mocked.contains(&key) || !reached.insert(key.clone()) { + continue; + } + if let Some(node) = current.nodes.get(&key) { + pending.extend(node.uses.iter().cloned()); + } + } + reached +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs b/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs index 8cae4b31f1..cac6c5d918 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/changes_summary.rs @@ -35,8 +35,7 @@ fn counted<'a>(kinds: impl Iterator) -> Option { Some(parts.join(", ")) } -/// `2 models, 1 macro changed; 1 seed added; 1 model renamed; deleted: model old_orders`, or -/// `no definition changes`. +/// `2 models, 1 macro changed; 1 seed added; deleted: model old_orders`, or `no definition changes`. pub fn changes_summary(changes: &DefinitionChanges) -> String { let mut parts: Vec = Vec::new(); let modified = changes.changed.values().filter(|change| !change.added); @@ -47,11 +46,13 @@ pub fn changes_summary(changes: &DefinitionChanges) -> String { if let Some(text) = counted(added.map(|change| change.kind.as_str())) { parts.push(format!("{text} added")); } - let renamed = changes - .renamed - .iter() - .filter_map(|(_, key)| key.split_once(':').map(|(kind, _)| kind)); - if let Some(text) = counted(renamed) { + let mut renamed: Vec<&str> = Vec::new(); + for (_, key) in &changes.renamed { + if let Some((kind, _)) = key.split_once(':') { + renamed.push(kind); + } + } + if let Some(text) = counted(renamed.into_iter()) { parts.push(format!("{text} renamed")); } if !changes.deleted.is_empty() { diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs b/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs index 1ddef9f5f3..81fe3874ab 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/compare_definitions.rs @@ -15,22 +15,22 @@ fn renamed_base<'a>( (base, current): (&'a DefinitionSnapshot, &DefinitionSnapshot), claimed: &BTreeSet, ) -> Option<(String, &'a NodeDefinition)> { - node.migrate_from.iter().find_map(|old_name| { + for old_name in &node.migrate_from { let old_key: String = node_key(&node.kind, old_name); if old_key == current_key || claimed.contains(&old_key) || current.nodes.contains_key(&old_key) { - return None; + continue; + } + if let Some(definition) = base.nodes.get(&old_key) { + return Some((old_key, definition)); } - base.nodes - .get(&old_key) - .map(|definition| (old_key, definition)) - }) + } + None } -/// Added nodes count as changed, deleted nodes are listed, and renames follow `migrate_from`: -/// a renamed node is changed only when its definition also changed. +/// Added nodes are changed, deleted ones listed; a `migrate_from` rename changes only with its definition. pub fn compare_definitions( base: &DefinitionSnapshot, current: &DefinitionSnapshot, diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs b/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs index 22a70af50f..6c84c8b477 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/definition_digest.rs @@ -1,9 +1,7 @@ -//! Digests of definition parts and of normalised SQL. +//! The digest of definition parts. use sha2::{Digest, Sha256}; -use crate::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; - /// The SHA-256 hex digest of `parts`, each length-prefixed so boundaries cannot shift. pub fn definition_digest(parts: &[&str]) -> String { let mut digest = Sha256::new(); @@ -14,11 +12,3 @@ pub fn definition_digest(parts: &[&str]) -> String { } format!("{:x}", digest.finalize()) } - -/// The fingerprint of `sql` with whitespace, comments and keyword case ignored. -/// -/// The generic tokenizer is used whatever the adapter, so one snapshot compares across adapters; -/// SQL it cannot tokenize is fingerprinted by its exact text. -pub fn sql_definition_digest(sql: &str) -> String { - compiler_query_fingerprint(sql, None).unwrap_or_else(|_| definition_digest(&[sql])) -} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs b/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs index 891f490473..9b283e84ef 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/definition_snapshot_json.rs @@ -1,16 +1,8 @@ //! The stored JSON form of a definition snapshot. -use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; use crate::definition_changes::models::DefinitionSnapshot; /// `snapshot` as compact JSON. pub fn definition_snapshot_json(snapshot: &DefinitionSnapshot) -> String { serde_json::to_string(snapshot).unwrap_or_default() } - -/// The snapshot `text` stores, or `None` when it is unreadable or from another format version. -pub fn parse_definition_snapshot(text: &str) -> Option { - serde_json::from_str::(text) - .ok() - .filter(|snapshot| snapshot.format == DEFINITION_SNAPSHOT_FORMAT) -} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs index 6cdd5334db..8daf29af03 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs @@ -3,5 +3,7 @@ pub mod compare_definitions; pub mod definition_digest; pub mod definition_snapshot_json; pub mod node_key; +pub mod parse_definition_snapshot; pub mod selection_reasons; pub mod selection_report; +pub mod sql_definition_digest; diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/parse_definition_snapshot.rs b/crates/sqlbuild-analysis/src/definition_changes/main/parse_definition_snapshot.rs new file mode 100644 index 0000000000..62e1cd972a --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/parse_definition_snapshot.rs @@ -0,0 +1,12 @@ +//! Read a stored definition snapshot. + +use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; +use crate::definition_changes::models::DefinitionSnapshot; + +/// The snapshot `text` stores, or `None` when it is unreadable or from another format version. +pub fn parse_definition_snapshot(text: &str) -> Option { + match serde_json::from_str::(text) { + Ok(snapshot) if snapshot.format == DEFINITION_SNAPSHOT_FORMAT => Some(snapshot), + Ok(_) | Err(_) => None, + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs index e3248221bf..f3f39c5416 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs @@ -1,96 +1,11 @@ //! Why a test or scenario is affected by the changes since the base revision. -use std::collections::BTreeSet; - -use crate::definition_changes::_helpers::aspects::{aspect_label, joined, kind_words}; -use crate::definition_changes::constants::{CONSTANT_KIND, ENUM_KIND, MODEL_KIND, TEST_FILE_KIND}; -use crate::definition_changes::models::{ - DefinitionChanges, DefinitionSnapshot, ExecutedItem, NodeChange, +use crate::definition_changes::_helpers::reasons::{ + declaration_reason, executed_reason, own_reason, reached_declarations, }; +use crate::definition_changes::models::{DefinitionChanges, DefinitionSnapshot, ExecutedItem}; -fn aspect_phrase(change: &NodeChange) -> String { - let labels: Vec = change - .aspects - .iter() - .map(|aspect| aspect_label(aspect).to_owned()) - .collect(); - joined(&labels) -} - -/// `test file changed`, `scenario file and snapshots changed`, `test file added`. -fn own_reason(change: &NodeChange) -> String { - let (singular, _) = kind_words(&change.kind); - if change.added { - return format!("{singular} added"); - } - if change.kind == TEST_FILE_KIND { - return format!("{singular} changed"); - } - format!("{singular} {} changed", aspect_phrase(change)) -} - -/// `added`, `model SQL changed`, `renamed from old_orders, config changed`, `changed`. -fn change_detail(change: &NodeChange) -> String { - if change.added { - return "added".to_owned(); - } - let detail: String = if change.kind == MODEL_KIND { - format!("model {} changed", aspect_phrase(change)) - } else { - "changed".to_owned() - }; - match &change.renamed_from { - Some(old_name) => format!("renamed from {old_name}, {detail}"), - None => detail, - } -} - -fn executed_reason(change: &NodeChange) -> String { - let subject: String = if change.kind == MODEL_KIND { - change.name.clone() - } else { - format!("{} {}", kind_words(&change.kind).0, change.name) - }; - format!("executes {subject} ({})", change_detail(change)) -} - -fn declaration_reason(change: &NodeChange) -> String { - let verb: &str = if change.kind == ENUM_KIND || change.kind == CONSTANT_KIND { - "uses" - } else { - "executes" - }; - format!( - "{verb} {} {} ({})", - kind_words(&change.kind).0, - change.name, - change_detail(change) - ) -} - -/// Declarations `item` reaches through its own SQL and its executed nodes, transitively, -/// without the ones it mocks. -fn reached_declarations(item: &ExecutedItem, current: &DefinitionSnapshot) -> BTreeSet { - let mut pending: Vec = item.declarations.clone(); - for key in &item.executed { - if let Some(node) = current.nodes.get(key) { - pending.extend(node.uses.iter().cloned()); - } - } - let mut reached: BTreeSet = BTreeSet::new(); - while let Some(key) = pending.pop() { - if item.mocked.contains(&key) || !reached.insert(key.clone()) { - continue; - } - if let Some(node) = current.nodes.get(&key) { - pending.extend(node.uses.iter().cloned()); - } - } - reached -} - -/// One reason per changed node `item` depends on: its own definition first, then executed -/// nodes in execution order, then declarations by key. Empty when nothing it runs changed. +/// One reason per changed dependency of `item`, own definition first; empty when nothing changed. pub fn selection_reasons( item: &ExecutedItem, current: &DefinitionSnapshot, diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs index 9f26bd3a06..617e67e589 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_report.rs @@ -2,8 +2,7 @@ use crate::definition_changes::models::SelectedItem; -/// `Changed since : …`, then `Selected N of M ` with each item's reasons when -/// `explain` is set, or the line saying nothing is affected. +/// The change summary, then the selected count with reasons when `explain` is set. pub fn selection_report( (reference, summary): (&str, &str), (singular, plural): (&str, &str), diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/sql_definition_digest.rs b/crates/sqlbuild-analysis/src/definition_changes/main/sql_definition_digest.rs new file mode 100644 index 0000000000..0e206f9ef8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/sql_definition_digest.rs @@ -0,0 +1,12 @@ +//! The digest of SQL with whitespace, comments and keyword case ignored. + +use crate::definition_changes::main::definition_digest::definition_digest; +use crate::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; + +/// The generic-tokenizer fingerprint of `sql`, or the digest of its text when it cannot tokenize. +pub fn sql_definition_digest(sql: &str) -> String { + match compiler_query_fingerprint(sql, None) { + Ok(fingerprint) => fingerprint, + Err(_) => definition_digest(&[sql]), + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/mod.rs index 48ad6791b7..b4c56eaa3e 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/mod.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/mod.rs @@ -1,5 +1,4 @@ -//! Definition fingerprints of project nodes, the changes between two snapshots of them, and the -//! reasons a test or scenario that executes changed nodes is selected. +//! Definition fingerprints of project nodes, their changes, and why tests that run them are selected. mod _helpers; pub mod constants; diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs index 7032f9dce2..a52c17691d 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs @@ -1,8 +1,8 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; use crate::definition_changes::main::node_key::node_key; -use crate::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; +use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem, NodeDefinition}; /// One node with `aspects` as `(aspect, digest)` and `uses` as node keys. pub(super) fn node( @@ -70,3 +70,65 @@ pub(super) fn base_orders() -> DefinitionSnapshot { node("test_file", "tests/orders.sql", &[("file", "t1")], &[]), ]) } + +pub(super) fn keys(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// The base orders project with `stg_orders` SQL edited, `customers` renamed and `countries` gone. +pub(super) fn current_orders() -> DefinitionSnapshot { + snapshot(vec![ + node( + "model", + "stg_orders", + &[("sql", "s1-edited"), ("config", "c1"), ("columns", "k1")], + &["macro:to_utc"], + ), + node( + "model", + "orders", + &[("sql", "s2"), ("config", "c2")], + &["enum:order_status"], + ), + renamed( + node("model", "customers", &[("sql", "s3")], &[]), + &["old_customers"], + ), + node( + "macro", + "to_utc", + &[("definition", "m1")], + &["macro:to_timestamp"], + ), + node("macro", "to_timestamp", &[("definition", "m2")], &[]), + node("enum", "order_status", &[("definition", "e1")], &[]), + node("test_file", "tests/orders.sql", &[("file", "t1")], &[]), + ]) +} + +/// What an item executes, by node keys. +pub(super) fn item( + own: &[&str], + executed: &[&str], + declarations: &[&str], + mocked: &[&str], +) -> ExecutedItem { + ExecutedItem { + own_keys: keys(own), + executed: keys(executed), + declarations: keys(declarations), + mocked: mocked + .iter() + .map(|key| (*key).to_owned()) + .collect::>(), + } +} + +/// `current` with `entry` added or replaced. +pub(super) fn with_node( + mut current: DefinitionSnapshot, + entry: (String, NodeDefinition), +) -> DefinitionSnapshot { + current.nodes.insert(entry.0, entry.1); + current +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs index a90877e085..e4ddbe6e17 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs @@ -1,149 +1,137 @@ -use std::collections::BTreeSet; - use crate::definition_changes::main::changes_summary::changes_summary; use crate::definition_changes::main::compare_definitions::compare_definitions; -use crate::definition_changes::main::definition_digest::sql_definition_digest; -use crate::definition_changes::main::definition_snapshot_json::{ - definition_snapshot_json, parse_definition_snapshot, -}; +use crate::definition_changes::main::definition_snapshot_json::definition_snapshot_json; +use crate::definition_changes::main::parse_definition_snapshot::parse_definition_snapshot; use crate::definition_changes::main::selection_reasons::selection_reasons; use crate::definition_changes::main::selection_report::selection_report; -use crate::definition_changes::models::{ - DefinitionChanges, DefinitionSnapshot, DeletedNode, ExecutedItem, NodeChange, SelectedItem, +use crate::definition_changes::main::sql_definition_digest::sql_definition_digest; +use crate::definition_changes::models::{DefinitionChanges, SelectedItem}; +use crate::definition_changes::tests::helpers::{ + base_orders, current_orders, item, keys, node, renamed, snapshot, with_node, +}; +use crate::definition_changes::tests::test_types::{ + CompareTestCase, SelectionReasonTestCase, SelectionReportTestCase, SnapshotJsonTestCase, + SqlDigestTestCase, }; -use crate::definition_changes::tests::helpers::{base_orders, node, renamed, snapshot}; -use crate::definition_changes::tests::test_types::SelectionReasonTestCase; - -fn keys(values: &[&str]) -> Vec { - values.iter().map(|value| (*value).to_owned()).collect() -} - -/// The base orders project with `stg_orders` SQL edited, `customers` renamed from -/// `old_customers` and `countries` deleted. -fn current_orders() -> DefinitionSnapshot { - snapshot(vec![ - node( - "model", - "stg_orders", - &[("sql", "s1-edited"), ("config", "c1"), ("columns", "k1")], - &["macro:to_utc"], - ), - node( - "model", - "orders", - &[("sql", "s2"), ("config", "c2")], - &["enum:order_status"], - ), - renamed( - node("model", "customers", &[("sql", "s3")], &[]), - &["old_customers"], - ), - node( - "macro", - "to_utc", - &[("definition", "m1")], - &["macro:to_timestamp"], - ), - node("macro", "to_timestamp", &[("definition", "m2")], &[]), - node("enum", "order_status", &[("definition", "e1")], &[]), - node("test_file", "tests/orders.sql", &[("file", "t1")], &[]), - ]) -} #[test] fn given_two_snapshots_when_comparing_then_reports_changed_renamed_and_deleted_nodes() { - let changes: DefinitionChanges = compare_definitions(&base_orders(), ¤t_orders()); - - assert_eq!( - changes.changed.into_iter().collect::>(), - vec![( - "model:stg_orders".to_owned(), - NodeChange { - kind: "model".to_owned(), - name: "stg_orders".to_owned(), - added: false, - aspects: keys(&["sql"]), - renamed_from: None, - } - )] - ); - assert_eq!( - changes.renamed, - vec![( - "model:old_customers".to_owned(), - "model:customers".to_owned() - )] - ); - assert_eq!( - changes.deleted, - vec![DeletedNode { - kind: "seed".to_owned(), - name: "countries".to_owned(), - }] - ); -} - -#[test] -fn given_renamed_node_with_edited_definition_when_comparing_then_it_changed_from_old_name() { - let base = snapshot(vec![node("model", "old_customers", &[("sql", "s3")], &[])]); - let current = snapshot(vec![renamed( - node( - "model", - "customers", - &[("sql", "s4"), ("config", "c4")], - &[], - ), - &["old_customers"], - )]); - - let changes: DefinitionChanges = compare_definitions(&base, ¤t); - - let change: &NodeChange = &changes.changed["model:customers"]; - assert_eq!(change.renamed_from.as_deref(), Some("old_customers")); - assert_eq!(change.aspects, keys(&["sql", "config"])); - assert!(changes.deleted.is_empty()); -} - -#[test] -fn given_changes_when_summarising_then_counts_each_kind_and_names_deleted_nodes() { - let mut current: DefinitionSnapshot = current_orders(); - current.nodes.extend([ - node("model", "returns", &[("sql", "s9")], &[]), - node("macro", "to_timestamp", &[("definition", "m3")], &[]), - ]); - - let summary: String = changes_summary(&compare_definitions(&base_orders(), ¤t)); - - assert_eq!( - summary, - "1 model, 1 macro changed; 1 model added; 1 model renamed; deleted: seed countries" - ); - assert_eq!( - changes_summary(&compare_definitions(&base_orders(), &base_orders())), - "no definition changes" - ); -} + let test_cases = [ + CompareTestCase { + description: "an edit, a pure rename and a deletion", + base: base_orders(), + current: current_orders(), + expected_changes: &[("model:stg_orders", &["sql"], false, None)], + expected_renamed: &[("model:old_customers", "model:customers")], + expected_deleted: &["seed countries"], + expected_summary: "1 model changed; 1 model renamed; deleted: seed countries", + }, + CompareTestCase { + description: "a rename whose definition also changed", + base: snapshot(vec![node("model", "old_customers", &[("sql", "s3")], &[])]), + current: snapshot(vec![renamed( + node( + "model", + "customers", + &[("sql", "s4"), ("config", "c4")], + &[], + ), + &["old_customers"], + )]), + expected_changes: &[( + "model:customers", + &["sql", "config"], + false, + Some("old_customers"), + )], + expected_renamed: &[("model:old_customers", "model:customers")], + expected_deleted: &[], + expected_summary: "1 model changed; 1 model renamed", + }, + CompareTestCase { + description: "added nodes count as changed and kinds are counted", + base: base_orders(), + current: with_node( + with_node( + current_orders(), + node("model", "returns", &[("sql", "s9")], &[]), + ), + node("macro", "to_timestamp", &[("definition", "m3")], &[]), + ), + expected_changes: &[ + ("macro:to_timestamp", &["definition"], false, None), + ("model:returns", &[], true, None), + ("model:stg_orders", &["sql"], false, None), + ], + expected_renamed: &[("model:old_customers", "model:customers")], + expected_deleted: &["seed countries"], + expected_summary: "1 model, 1 macro changed; 1 model added; 1 model renamed; \ + deleted: seed countries", + }, + CompareTestCase { + description: "identical snapshots", + base: base_orders(), + current: base_orders(), + expected_changes: &[], + expected_renamed: &[], + expected_deleted: &[], + expected_summary: "no definition changes", + }, + ]; + for test_case in test_cases { + let changes: DefinitionChanges = compare_definitions(&test_case.base, &test_case.current); -fn item(own: &[&str], executed: &[&str], declarations: &[&str], mocked: &[&str]) -> ExecutedItem { - ExecutedItem { - own_keys: keys(own), - executed: keys(executed), - declarations: keys(declarations), - mocked: mocked + let observed: Vec<(String, Vec, bool, Option)> = changes + .changed + .iter() + .map(|(key, change)| { + ( + key.clone(), + change.aspects.clone(), + change.added, + change.renamed_from.clone(), + ) + }) + .collect(); + let expected: Vec<(String, Vec, bool, Option)> = test_case + .expected_changes + .iter() + .map(|(key, aspects, added, renamed_from)| { + ( + (*key).to_owned(), + keys(aspects), + *added, + renamed_from.map(str::to_owned), + ) + }) + .collect(); + let deleted: Vec = changes + .deleted .iter() - .map(|key| (*key).to_owned()) - .collect::>(), + .map(|node| format!("{} {}", node.kind, node.name)) + .collect(); + let renamed: Vec<(String, String)> = test_case + .expected_renamed + .iter() + .map(|(old, new)| ((*old).to_owned(), (*new).to_owned())) + .collect(); + assert_eq!(observed, expected, "{}", test_case.description); + assert_eq!(changes.renamed, renamed, "{}", test_case.description); + assert_eq!( + deleted, + keys(test_case.expected_deleted), + "{}", + test_case.description + ); + assert_eq!( + changes_summary(&changes), + test_case.expected_summary, + "{}", + test_case.description + ); } } -fn with_node( - mut current: DefinitionSnapshot, - entry: (String, crate::definition_changes::models::NodeDefinition), -) -> DefinitionSnapshot { - current.nodes.insert(entry.0, entry.1); - current -} - #[test] fn given_executed_item_when_explaining_selection_then_lists_each_changed_dependency() { let test_cases = [ @@ -167,7 +155,7 @@ fn given_executed_item_when_explaining_selection_then_lists_each_changed_depende SelectionReasonTestCase { description: "a macro changed through another macro the model calls", current: with_node( - snapshot(base_orders().nodes.into_iter().collect()), + base_orders(), node("macro", "to_timestamp", &[("definition", "m3")], &[]), ), item: item(&[], &["model:stg_orders"], &[], &[]), @@ -176,7 +164,7 @@ fn given_executed_item_when_explaining_selection_then_lists_each_changed_depende SelectionReasonTestCase { description: "a mocked macro hides its own and its callees' changes", current: with_node( - snapshot(base_orders().nodes.into_iter().collect()), + base_orders(), node("macro", "to_timestamp", &[("definition", "m3")], &[]), ), item: item(&[], &["model:stg_orders"], &[], &["macro:to_utc"]), @@ -185,7 +173,7 @@ fn given_executed_item_when_explaining_selection_then_lists_each_changed_depende SelectionReasonTestCase { description: "an enum used by an executed model", current: with_node( - snapshot(base_orders().nodes.into_iter().collect()), + base_orders(), node("enum", "order_status", &[("definition", "e2")], &[]), ), item: item(&[], &["model:orders"], &[], &[]), @@ -195,7 +183,7 @@ fn given_executed_item_when_explaining_selection_then_lists_each_changed_depende description: "the test's own file and an added model", current: with_node( with_node( - snapshot(base_orders().nodes.into_iter().collect()), + base_orders(), node("test_file", "tests/orders.sql", &[("file", "t2")], &[]), ), node("model", "returns", &[("sql", "s9")], &[]), @@ -227,7 +215,7 @@ fn given_executed_item_when_explaining_selection_then_lists_each_changed_depende SelectionReasonTestCase { description: "a changed seed the test really reads", current: with_node( - snapshot(base_orders().nodes.into_iter().collect()), + base_orders(), node("seed", "countries", &[("definition", "d2")], &[]), ), item: item(&[], &["model:orders", "seed:countries"], &[], &[]), @@ -249,66 +237,110 @@ fn given_executed_item_when_explaining_selection_then_lists_each_changed_depende } #[test] -fn given_snapshot_when_stored_then_reads_back_only_in_the_same_format() { - let snapshot: DefinitionSnapshot = base_orders(); - let stored: String = definition_snapshot_json(&snapshot); +fn given_selected_items_when_reporting_then_counts_and_explains_only_on_request() { + let selected = || { + vec![SelectedItem { + label: "orders_rollup".to_owned(), + reasons: keys(&["executes stg_orders (model SQL changed)"]), + }] + }; + let test_cases = [ + SelectionReportTestCase { + description: "explain lists each item's reasons", + summary: "1 model changed", + nouns: ("test", "tests"), + total: 9, + selected: selected(), + explain: true, + expected_text: "Changed since origin/main: 1 model changed\nSelected 1 of 9 tests\n \ + orders_rollup\n executes stg_orders (model SQL changed)\n", + }, + SelectionReportTestCase { + description: "without explain only counts are shown", + summary: "1 model changed", + nouns: ("test", "tests"), + total: 9, + selected: selected(), + explain: false, + expected_text: "Changed since origin/main: 1 model changed\nSelected 1 of 9 tests\n", + }, + SelectionReportTestCase { + description: "nothing affected says so", + summary: "no definition changes", + nouns: ("scenario", "scenarios"), + total: 3, + selected: Vec::new(), + explain: true, + expected_text: "Changed since origin/main: no definition changes\nNo scenarios are \ + affected by the changes since origin/main\n", + }, + ]; + for test_case in test_cases { + let text: String = selection_report( + ("origin/main", test_case.summary), + test_case.nouns, + (test_case.total, &test_case.selected), + test_case.explain, + ); - assert_eq!(parse_definition_snapshot(&stored), Some(snapshot)); - assert_eq!( - parse_definition_snapshot(&stored.replacen("\"format\":1", "\"format\":0", 1)), - None - ); - assert_eq!(parse_definition_snapshot("not json"), None); + assert_eq!(text, test_case.expected_text, "{}", test_case.description); + } } #[test] -fn given_sql_when_fingerprinting_definition_then_ignores_layout_and_comments_only() { - let original: String = sql_definition_digest("SELECT order_id FROM orders"); +fn given_stored_snapshot_when_reading_then_accepts_only_the_same_format() { + let test_cases = [ + SnapshotJsonTestCase { + description: "a stored snapshot reads back", + stored: |text| text, + expected_snapshot: Some(base_orders()), + }, + SnapshotJsonTestCase { + description: "another format version is recomputed", + stored: |text| text.replacen("\"format\":1", "\"format\":0", 1), + expected_snapshot: None, + }, + SnapshotJsonTestCase { + description: "unreadable text is recomputed", + stored: |_| "not json".to_owned(), + expected_snapshot: None, + }, + ]; + for test_case in test_cases { + let text: String = (test_case.stored)(definition_snapshot_json(&base_orders())); - assert_eq!( - sql_definition_digest("select\n order_id -- the key\nfrom orders"), - original - ); - assert_ne!( - sql_definition_digest("SELECT customer_id FROM orders"), - original - ); + assert_eq!( + parse_definition_snapshot(&text), + test_case.expected_snapshot, + "{}", + test_case.description + ); + } } #[test] -fn given_selected_items_when_reporting_then_counts_and_explains_only_on_request() { - let selected = [SelectedItem { - label: "orders_rollup".to_owned(), - reasons: keys(&["executes stg_orders (model SQL changed)"]), - }]; +fn given_sql_when_fingerprinting_definition_then_ignores_layout_and_comments_only() { + let original: String = sql_definition_digest("SELECT order_id FROM orders"); + let test_cases = [ + SqlDigestTestCase { + description: "layout, comments and keyword case are ignored", + sql: "select\n order_id -- the key\nfrom orders", + expected_same_as_original: true, + }, + SqlDigestTestCase { + description: "a different column is a change", + sql: "SELECT customer_id FROM orders", + expected_same_as_original: false, + }, + ]; + for test_case in test_cases { + let digest: String = sql_definition_digest(test_case.sql); - assert_eq!( - selection_report( - ("origin/main", "1 model changed"), - ("test", "tests"), - (9, &selected), - true - ), - "Changed since origin/main: 1 model changed\nSelected 1 of 9 tests\n orders_rollup\n \ - executes stg_orders (model SQL changed)\n" - ); - assert_eq!( - selection_report( - ("origin/main", "1 model changed"), - ("test", "tests"), - (9, &selected), - false - ), - "Changed since origin/main: 1 model changed\nSelected 1 of 9 tests\n" - ); - assert_eq!( - selection_report( - ("origin/main", "no definition changes"), - ("scenario", "scenarios"), - (3, &[]), - true - ), - "Changed since origin/main: no definition changes\nNo scenarios are affected by the \ - changes since origin/main\n" - ); + assert_eq!( + digest == original, + test_case.expected_same_as_original, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs index 12aaebe1c0..f274c599d8 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs @@ -1,4 +1,24 @@ -use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem}; +use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem, SelectedItem}; + +/// One expected changed node: key, changed aspects, added, renamed-from name. +pub(crate) type ExpectedChange = ( + &'static str, + &'static [&'static str], + bool, + Option<&'static str>, +); + +pub(crate) struct CompareTestCase { + pub(crate) description: &'static str, + pub(crate) base: DefinitionSnapshot, + pub(crate) current: DefinitionSnapshot, + pub(crate) expected_changes: &'static [ExpectedChange], + /// `(base key, current key)`. + pub(crate) expected_renamed: &'static [(&'static str, &'static str)], + /// `kind name` of each deleted node. + pub(crate) expected_deleted: &'static [&'static str], + pub(crate) expected_summary: &'static str, +} pub(crate) struct SelectionReasonTestCase { pub(crate) description: &'static str, @@ -6,3 +26,26 @@ pub(crate) struct SelectionReasonTestCase { pub(crate) item: ExecutedItem, pub(crate) expected_reasons: &'static [&'static str], } + +pub(crate) struct SelectionReportTestCase { + pub(crate) description: &'static str, + pub(crate) summary: &'static str, + pub(crate) nouns: (&'static str, &'static str), + pub(crate) total: usize, + pub(crate) selected: Vec, + pub(crate) explain: bool, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct SnapshotJsonTestCase { + pub(crate) description: &'static str, + /// Turns the stored text of the base orders snapshot into the text to read back. + pub(crate) stored: fn(String) -> String, + pub(crate) expected_snapshot: Option, +} + +pub(crate) struct SqlDigestTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_same_as_original: bool, +} diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 29504cd0d1..1bcced4f42 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,7 +1,7 @@ use crate::project::main::load_git_base_ref::load_git_base_ref; use crate::project::models::ConfigFile; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; -use crate::project::tests::test_types::ProjectFailureTestCase; +use crate::project::tests::test_types::{GitBaseRefTestCase, ProjectFailureTestCase}; const PROJECT: &str = "name = 'orders'\nadapter = 'duckdb'\n"; @@ -120,15 +120,28 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { #[test] fn given_git_section_when_reading_base_ref_then_returns_configured_ref() { - let (_with, configured) = project_directory( - "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n", - "", - ); - let (_without, unset) = project_directory(PROJECT, ""); + let test_cases = [ + GitBaseRefTestCase { + description: "a configured base ref", + project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n", + expected_base_ref: Some("origin/main"), + }, + GitBaseRefTestCase { + description: "no git section", + project: PROJECT, + expected_base_ref: None, + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); - assert_eq!( - load_git_base_ref(&configured).ok().flatten().as_deref(), - Some("origin/main") - ); - assert_eq!(load_git_base_ref(&unset).ok().flatten(), None); + let base_ref = load_git_base_ref(&paths).expect(test_case.description); + + assert_eq!( + base_ref.as_deref(), + test_case.expected_base_ref, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index a3f1dd6441..09c6e090ef 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -13,3 +13,9 @@ pub(super) struct GuidanceTestCase { pub(super) expected_note: &'static str, pub(super) expected_help: &'static str, } + +pub(crate) struct GitBaseRefTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_base_ref: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/git_revisions/constants.rs b/crates/sqlbuild-core/src/git_revisions/constants.rs index 440931d504..0f375b0a09 100644 --- a/crates/sqlbuild-core/src/git_revisions/constants.rs +++ b/crates/sqlbuild-core/src/git_revisions/constants.rs @@ -14,3 +14,6 @@ pub(crate) const UNKNOWN_REF_CODE: &str = "C481"; pub(crate) const NO_MERGE_BASE_CODE: &str = "C482"; pub(crate) const EXPORT_FAILED_CODE: &str = "C483"; pub(crate) const BASE_REF_UNSET_CODE: &str = "C485"; +pub(crate) const DEPTH_HELP: &str = "Fetch enough history to reach the merge base, for example \ + `git fetch --unshallow`, or in CI clone with a larger fetch \ + depth"; diff --git a/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs b/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs index efd4f92c60..bb7f2f12ac 100644 --- a/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs +++ b/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs @@ -28,10 +28,7 @@ fn export_failed(base: &GitBase, detail: &str) -> GitRevisionError { } } -/// Write the project tree at `base.merge_base` into `destination`; `false` when it had no project. -/// -/// The tree is read from git objects through a private index file, so the working tree, the -/// repository's index and `HEAD` are never changed. +/// Write the base project tree to `destination` through a private index; `false` without a project. pub fn export_git_tree(base: &GitBase, destination: &Path) -> Result { let root: &Path = &base.repository_root; let tree: String = tree_of(base); diff --git a/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs b/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs index 0ea69f2c2c..f6d449d56d 100644 --- a/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs +++ b/crates/sqlbuild-core/src/git_revisions/main/resolve_git_base.rs @@ -4,14 +4,11 @@ use std::path::{Path, PathBuf}; use crate::git_revisions::_helpers::git_command::{GitFailure, git_output}; use crate::git_revisions::constants::{ - NO_MERGE_BASE_CODE, NOT_A_REPOSITORY_CODE, SHALLOW_TRUE, UNKNOWN_REF_CODE, + DEPTH_HELP, NO_MERGE_BASE_CODE, NOT_A_REPOSITORY_CODE, SHALLOW_TRUE, UNKNOWN_REF_CODE, }; use crate::git_revisions::errors::GitRevisionError; use crate::git_revisions::models::GitBase; -const DEPTH_HELP: &str = "Fetch enough history to reach the merge base, for example \ - `git fetch --unshallow`, or in CI clone with a larger fetch depth"; - fn not_a_repository(project_dir: &Path, failure: &GitFailure) -> GitRevisionError { let message: String = match failure { GitFailure::Unavailable(detail) => { diff --git a/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs b/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs index 6ef69990c3..de11e65211 100644 --- a/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs +++ b/crates/sqlbuild-core/src/git_revisions/tests/helpers.rs @@ -50,9 +50,7 @@ pub(super) fn write(root: &Path, relative: &str, contents: &str) { std::fs::write(path, contents).expect("the file is written"); } -/// A repository whose project lives in `warehouse/`: `main` has one commit, `feature` one more. -/// -/// Returns the repository root and the hash of the `main` commit. +/// A repository with a project in `warehouse/` and branches `main` and `feature`; root and `main`. pub(super) fn project_repository() -> (PathBuf, String) { let root: PathBuf = scratch_directory("repo"); git(&root, &["init", "--quiet"]); @@ -84,3 +82,8 @@ pub(super) fn project_repository() -> (PathBuf, String) { git(&root, &["commit", "--quiet", "-m", "customers"]); (root, main) } + +/// The text of `path`, or `None` when it is absent. +pub(super) fn read(path: &Path) -> Option { + std::fs::read_to_string(path).ok() +} diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs index 1dd6323d49..0dfa1b1ae5 100644 --- a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs @@ -1,12 +1,16 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use crate::git_revisions::main::export_git_tree::export_git_tree; use crate::git_revisions::main::resolve_git_base::resolve_git_base; use crate::git_revisions::main::resolve_since_reference::resolve_since_reference; -use crate::git_revisions::main::short_commit::short_commit; use crate::git_revisions::models::GitBase; -use crate::git_revisions::tests::helpers::{git, project_repository, scratch_directory, write}; -use crate::git_revisions::tests::test_types::{GitBaseFailureTestCase, SinceReferenceTestCase}; +use crate::git_revisions::tests::helpers::{ + git, project_repository, read, scratch_directory, write, +}; +use crate::git_revisions::tests::test_types::{ + AbsentProjectTestCase, ExportTreeTestCase, GitBaseFailureTestCase, GitBaseTestCase, + SinceReferenceTestCase, +}; #[test] fn given_since_value_when_resolving_then_reads_base_alias_from_config() { @@ -15,25 +19,25 @@ fn given_since_value_when_resolving_then_reads_base_alias_from_config() { description: "a plain ref is used as given", value: "origin/main", configured: Some("origin/release"), - expected: Ok("origin/main"), + expected_reference: Ok("origin/main"), }, SinceReferenceTestCase { description: "@base reads the configured base ref", value: "@base", configured: Some("origin/main"), - expected: Ok("origin/main"), + expected_reference: Ok("origin/main"), }, SinceReferenceTestCase { description: "@base without a configured base ref fails", value: "@base", configured: None, - expected: Err("C485"), + expected_reference: Err("C485"), }, SinceReferenceTestCase { description: "@base with a blank base ref fails", value: "@base", configured: Some(" "), - expected: Err("C485"), + expected_reference: Err("C485"), }, ]; for test_case in test_cases { @@ -41,7 +45,7 @@ fn given_since_value_when_resolving_then_reads_base_alias_from_config() { assert_eq!( resolved.as_deref().map_err(|error| error.code), - test_case.expected, + test_case.expected_reference, "{}", test_case.description ); @@ -49,15 +53,34 @@ fn given_since_value_when_resolving_then_reads_base_alias_from_config() { } #[test] -fn given_project_in_subdirectory_when_resolving_base_then_returns_merge_base_and_prefix() { +fn given_project_directory_when_resolving_base_then_returns_merge_base_and_prefix() { let (root, main) = project_repository(); + let test_cases = [ + GitBaseTestCase { + description: "a project in a subdirectory", + project_directory: "warehouse", + reference: "main", + expected_prefix: "warehouse/", + }, + GitBaseTestCase { + description: "a nested directory of the project", + project_directory: "warehouse/models", + reference: "main", + expected_prefix: "warehouse/models/", + }, + ]; + for test_case in test_cases { + let base: GitBase = + resolve_git_base(&root.join(test_case.project_directory), test_case.reference) + .expect(test_case.description); - let base: GitBase = resolve_git_base(&root.join("warehouse"), "main").expect("main resolves"); - - assert_eq!(base.merge_base, main); - assert_eq!(base.project_prefix, "warehouse/"); - assert_eq!(base.reference, "main"); - assert_eq!(short_commit(&base.merge_base).len(), 10); + assert_eq!(base.merge_base, main, "{}", test_case.description); + assert_eq!( + base.project_prefix, test_case.expected_prefix, + "{}", + test_case.description + ); + } } #[test] @@ -72,12 +95,14 @@ fn given_unresolvable_ref_when_resolving_base_then_fails_with_code() { reference: "origin/missing", expected_code: "C481", expected_message_fragment: "'origin/missing' does not name a commit", + expected_help_fragment: "git fetch origin ", }, GitBaseFailureTestCase { description: "a ref without shared history", reference: "unrelated", expected_code: "C482", expected_message_fragment: "'unrelated' and HEAD", + expected_help_fragment: "shares history with HEAD", }, ]; for test_case in test_cases { @@ -90,10 +115,12 @@ fn given_unresolvable_ref_when_resolving_base_then_fails_with_code() { test_case.description ); assert!( - error.message.contains(test_case.expected_message_fragment), - "{}: {}", + error.message.contains(test_case.expected_message_fragment) + && error.help.contains(test_case.expected_help_fragment), + "{}: {} ({})", test_case.description, - error.message + error.message, + error.help ); } } @@ -101,16 +128,37 @@ fn given_unresolvable_ref_when_resolving_base_then_fails_with_code() { #[test] fn given_directory_outside_repository_when_resolving_base_then_fails_with_code() { let directory: PathBuf = scratch_directory("plain"); + let test_cases = [GitBaseFailureTestCase { + description: "a plain directory", + reference: "main", + expected_code: "C480", + expected_message_fragment: "is not inside one", + expected_help_fragment: "drop --since", + }]; + for test_case in test_cases { + let error = + resolve_git_base(&directory, test_case.reference).expect_err(test_case.description); - let error = resolve_git_base(&directory, "main").expect_err("not a repository"); - - assert_eq!(error.code, "C480"); + assert_eq!( + error.code, test_case.expected_code, + "{}", + test_case.description + ); + assert!( + error.message.contains(test_case.expected_message_fragment) + && error.help.contains(test_case.expected_help_fragment), + "{}: {}", + test_case.description, + error.message + ); + } } #[test] fn given_shallow_clone_without_shared_history_when_resolving_base_then_asks_for_depth() { let (root, _) = project_repository(); let clone: PathBuf = scratch_directory("shallow"); + let origin: String = format!("file://{}", root.display()); git( &clone, &[ @@ -121,7 +169,7 @@ fn given_shallow_clone_without_shared_history_when_resolving_base_then_asks_for_ "--branch", "feature", "--no-single-branch", - &format!("file://{}", root.display()), + &origin, "checkout", ], ); @@ -130,17 +178,31 @@ fn given_shallow_clone_without_shared_history_when_resolving_base_then_asks_for_ &checkout, &["fetch", "--quiet", "--depth", "1", "origin", "main"], ); + let test_cases = [GitBaseFailureTestCase { + description: "the merge base is outside the shallow history", + reference: "origin/main", + expected_code: "C482", + expected_message_fragment: "shallow clone", + expected_help_fragment: "--unshallow", + }]; + for test_case in test_cases { + let error = resolve_git_base(&checkout.join("warehouse"), test_case.reference) + .expect_err(test_case.description); - let error = resolve_git_base(&checkout.join("warehouse"), "origin/main") - .expect_err("the merge base is outside the shallow history"); - - assert_eq!(error.code, "C482"); - assert!(error.message.contains("shallow clone"), "{}", error.message); - assert!(error.help.contains("--unshallow"), "{}", error.help); -} - -fn read(path: &Path) -> Option { - std::fs::read_to_string(path).ok() + assert_eq!( + error.code, test_case.expected_code, + "{}", + test_case.description + ); + assert!( + error.message.contains(test_case.expected_message_fragment) + && error.help.contains(test_case.expected_help_fragment), + "{}: {} ({})", + test_case.description, + error.message, + error.help + ); + } } #[test] @@ -154,21 +216,47 @@ fn given_merge_base_when_exporting_tree_then_writes_base_files_and_leaves_checko let status_before: String = git(&root, &["status", "--porcelain"]); let base: GitBase = resolve_git_base(&root.join("warehouse"), "main").expect("main resolves"); let destination: PathBuf = scratch_directory("export").join("base"); - let exported: bool = export_git_tree(&base, &destination).expect("the tree exports"); - assert!(exported); - assert_eq!( - read(&destination.join("models/orders.sql")).as_deref(), - Some("SELECT 1 AS order_id\n") - ); - assert_eq!(read(&destination.join("models/customers.sql")), None); - assert_eq!( - read(&root.join("warehouse/models/orders.sql")).as_deref(), - Some("SELECT 3 AS order_id\n") - ); assert_eq!(git(&root, &["status", "--porcelain"]), status_before); - assert!(!destination.with_extension("index").exists()); + let test_cases = [ + ExportTreeTestCase { + description: "an edited model is exported as committed at the base", + relative_path: "models/orders.sql", + expected_exported: Some("SELECT 1 AS order_id\n"), + expected_working_tree: Some("SELECT 3 AS order_id\n"), + }, + ExportTreeTestCase { + description: "a model added after the base is not exported", + relative_path: "models/customers.sql", + expected_exported: None, + expected_working_tree: Some("SELECT 1 AS customer_id\n"), + }, + ExportTreeTestCase { + description: "the private index file is removed", + relative_path: "../base.index", + expected_exported: None, + expected_working_tree: None, + }, + ]; + for test_case in test_cases { + let exported_text: Option = read(&destination.join(test_case.relative_path)); + let working_text: Option = + read(&root.join("warehouse").join(test_case.relative_path)); + + assert_eq!( + exported_text.as_deref(), + test_case.expected_exported, + "{}", + test_case.description + ); + assert_eq!( + working_text.as_deref(), + test_case.expected_working_tree, + "{}", + test_case.description + ); + } } #[test] @@ -181,15 +269,33 @@ fn given_project_absent_at_merge_base_when_exporting_tree_then_reports_no_projec ); git(&root, &["add", "."]); git(&root, &["commit", "--quiet", "-m", "billing"]); - let base: GitBase = GitBase { - reference: "main".to_owned(), - merge_base: main, - repository_root: root.clone(), - project_prefix: "billing/".to_owned(), - }; + let test_cases = [ + AbsentProjectTestCase { + description: "a project added after the base", + project_prefix: "billing/", + expected_exported: false, + }, + AbsentProjectTestCase { + description: "a project present at the base", + project_prefix: "warehouse/", + expected_exported: true, + }, + ]; + for test_case in test_cases { + let base: GitBase = GitBase { + reference: "main".to_owned(), + merge_base: main.clone(), + repository_root: root.clone(), + project_prefix: test_case.project_prefix.to_owned(), + }; - let exported: bool = export_git_tree(&base, &scratch_directory("absent").join("base")) - .expect("an absent tree is not an error"); + let exported: bool = export_git_tree(&base, &scratch_directory("absent").join("base")) + .expect(test_case.description); - assert!(!exported); + assert_eq!( + exported, test_case.expected_exported, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs index 1e3f3a5687..7cc85ec284 100644 --- a/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs @@ -3,7 +3,14 @@ pub(crate) struct SinceReferenceTestCase { pub(crate) value: &'static str, pub(crate) configured: Option<&'static str>, /// The resolved ref, or the error code. - pub(crate) expected: Result<&'static str, &'static str>, + pub(crate) expected_reference: Result<&'static str, &'static str>, +} + +pub(crate) struct GitBaseTestCase { + pub(crate) description: &'static str, + pub(crate) project_directory: &'static str, + pub(crate) reference: &'static str, + pub(crate) expected_prefix: &'static str, } pub(crate) struct GitBaseFailureTestCase { @@ -11,4 +18,18 @@ pub(crate) struct GitBaseFailureTestCase { pub(crate) reference: &'static str, pub(crate) expected_code: &'static str, pub(crate) expected_message_fragment: &'static str, + pub(crate) expected_help_fragment: &'static str, +} + +pub(crate) struct ExportTreeTestCase { + pub(crate) description: &'static str, + pub(crate) relative_path: &'static str, + pub(crate) expected_exported: Option<&'static str>, + pub(crate) expected_working_tree: Option<&'static str>, +} + +pub(crate) struct AbsentProjectTestCase { + pub(crate) description: &'static str, + pub(crate) project_prefix: &'static str, + pub(crate) expected_exported: bool, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs index 47a653dd95..1a514c0537 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs @@ -2,13 +2,15 @@ use std::path::PathBuf; -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyBool; +use pyo3::prelude::{ + Bound, IntoPyObject, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::definition_snapshot_json; use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; -use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::change_selection::definition_snapshot::definition_snapshot; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; @@ -22,22 +24,28 @@ fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { PyBool::new(py, value).to_owned().into_any() } -/// Compile the project at `project_dir` without SQL analysis and write its definition snapshot. -/// -/// `--since` runs this in a separate interpreter for the exported base revision. +/// Compile an exported base tree without SQL analysis and write its definition snapshot. #[pyfunction] -#[pyo3(signature = (*, project_dir, output_path, selected_target = None, cli_vars = None))] +#[pyo3(signature = (**options))] fn write_base_definition_snapshot( py: Python<'_>, - project_dir: PathBuf, - output_path: PathBuf, - selected_target: Option>, - cli_vars: Option>, + options: Option>, ) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "write_base_definition_snapshot", + options, + ( + &["project_dir", "output_path"], + &["selected_target", "cli_vars"], + ), + )?; let none = || py.None().into_bound(py); + let project_dir: PathBuf = arguments.value("project_dir")?.extract()?; + let output_path: PathBuf = arguments.value("output_path")?.extract()?; let directory: Bound<'_, PyAny> = project_dir.clone().into_pyobject(py)?.into_any(); - let target: Bound<'_, PyAny> = selected_target.unwrap_or_else(none); - let vars: Bound<'_, PyAny> = cli_vars.unwrap_or_else(none); + let target: Bound<'_, PyAny> = arguments.value("selected_target")?; + let vars: Bound<'_, PyAny> = arguments.value("cli_vars")?; let discovered: Bound<'_, PyAny> = discover_project_inputs( py, Some(keywords(py, &[("project_dir", directory.clone())])?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index b79f80f017..2272fe96c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -1,5 +1,4 @@ -//! The definition snapshot of the `--since` base revision: resolved from git, compiled from its -//! objects in a separate interpreter, and cached per merge-base commit. +//! The `--since` base snapshot: resolved from git, compiled in a child interpreter, cached per commit. use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; @@ -8,9 +7,8 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_analysis::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; -use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::{ - definition_snapshot_json, parse_definition_snapshot, -}; +use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::definition_snapshot_json; +use sqlbuild_analysis::definition_changes::main::parse_definition_snapshot::parse_definition_snapshot; use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; use sqlbuild_core::git_revisions::constants::BASE_REF_ALIAS; use sqlbuild_core::git_revisions::errors::GitRevisionError; @@ -32,8 +30,7 @@ const LOCAL_CONFIG_FILE: &str = "sqlbuild_local.toml"; const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; const BASE_COMPILE_FAILED_CODE: &str = "C484"; const FAILURE_TAIL_LINES: usize = 20; -/// Compiles the exported base tree in a fresh interpreter, so the base revision's project Python -/// never shares modules with the working tree's. +/// Compiles the exported base tree in a fresh interpreter, apart from the working tree's modules. const BASE_COMPILE_PROGRAM: &str = "import json, sys\n\ from sqlbuild._native import write_base_definition_snapshot\n\ write_base_definition_snapshot(project_dir=sys.argv[1], output_path=sys.argv[2], \ @@ -46,9 +43,21 @@ pub(crate) struct BaseComparison { } /// The inputs that decide what the base compile produces besides its commit. -pub(crate) struct BaseVariant<'py> { +pub(crate) struct BaseVariant { pub(crate) selected_target: Option, - pub(crate) cli_vars: Bound<'py, PyAny>, + /// `--vars` as sorted compact JSON. + pub(crate) vars_json: String, +} + +/// The base variant of a command's `--target` and `--vars`. +pub(crate) fn base_variant( + selected_target: Option, + cli_vars: &Bound<'_, PyAny>, +) -> PyResult { + Ok(BaseVariant { + selected_target, + vars_json: vars_json(cli_vars)?, + }) } fn git_error(py: Python<'_>, error: GitRevisionError) -> PyErr { @@ -161,13 +170,10 @@ fn compiled_base_snapshot( } /// Resolve `since` to its merge base and load, or compile and cache, the base snapshot. -/// -/// Progress goes to `stream`: the merge base, then whether the base came from the cache or how -/// long its compile took. pub(crate) fn base_comparison<'py>( (stream, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), since: &str, - variant: &BaseVariant<'py>, + variant: &BaseVariant, ) -> PyResult { let py: Python<'py> = stream.py(); let directory: PathBuf = project_dir.extract()?; @@ -186,18 +192,18 @@ pub(crate) fn base_comparison<'py>( stream, &format!("Comparing with {reference} (merge base {short})\n"), )?; - let vars: String = vars_json(&variant.cli_vars)?; + let vars: &str = &variant.vars_json; let target: Option<&str> = variant.selected_target.as_deref(); - let key: String = - definition_digest(&[NATIVE_BUILD_IDENTITY, target.unwrap_or_default(), &vars]); + let key: String = definition_digest(&[NATIVE_BUILD_IDENTITY, target.unwrap_or_default(), vars]); let cache_dir: PathBuf = CACHE_DIRECTORY .iter() .fold(directory.clone(), |path, part| path.join(part)); let cached: PathBuf = cache_dir.join(format!("{}-{}.json", base.merge_base, &key[..16])); - if let Some(snapshot) = std::fs::read_to_string(&cached) - .ok() - .and_then(|text| parse_definition_snapshot(&text)) - { + let stored: Option = match std::fs::read_to_string(&cached) { + Ok(text) => parse_definition_snapshot(&text), + Err(_) => None, + }; + if let Some(snapshot) = stored { write(stream, &format!("Using cached base revision {short}\n"))?; flush(stream)?; return Ok(BaseComparison { base, snapshot }); @@ -209,11 +215,11 @@ pub(crate) fn base_comparison<'py>( let _ = std::fs::remove_dir_all(&work); std::fs::create_dir_all(&work).map_err(|error| io_failed(py, &base, &error))?; let compiled: PyResult = - compiled_base_snapshot(py, (&base, &directory, &work), (target, &vars)); + compiled_base_snapshot(py, (&base, &directory, &work), (target, vars)); let _ = std::fs::remove_dir_all(&work); let snapshot: DefinitionSnapshot = compiled?; let staged: PathBuf = cached.with_extension(format!("{}.tmp", std::process::id())); - if std::fs::write(&staged, definition_snapshot_json(&snapshot)).is_ok() { + if let Ok(()) = std::fs::write(&staged, definition_snapshot_json(&snapshot)) { let _ = std::fs::rename(&staged, &cached); } write( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 35d9dc543d..6df31ee0f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -1,5 +1,4 @@ -//! The definition snapshot of a compiled project: one fingerprint per model, seed, function, -//! macro, enum, constant, test file and scenario, and which declarations each node uses. +//! The definition snapshot of a compiled project: one fingerprint per node and its declarations. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -15,10 +14,9 @@ use sqlbuild_analysis::definition_changes::constants::{ FILE_ASPECT, MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, SNAPSHOTS_ASPECT, SQL_ASPECT, TEST_FILE_KIND, }; -use sqlbuild_analysis::definition_changes::main::definition_digest::{ - definition_digest, sql_definition_digest, -}; +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; use sqlbuild_analysis::definition_changes::main::node_key::node_key; +use sqlbuild_analysis::definition_changes::main::sql_definition_digest::sql_definition_digest; use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; use crate::bindings::_helpers::compiled_views::function_reads; @@ -125,7 +123,11 @@ fn directory_digest(directory: &Path) -> Option { let mut files: Vec<(String, String)> = Vec::new(); let mut pending: Vec = vec![directory.to_path_buf()]; while let Some(next) = pending.pop() { - for entry in std::fs::read_dir(&next).ok()?.flatten() { + let entries: std::fs::ReadDir = match std::fs::read_dir(&next) { + Ok(entries) => entries, + Err(_) => return None, + }; + for entry in entries.flatten() { let path: PathBuf = entry.path(); if path.is_dir() { pending.push(path); @@ -197,16 +199,15 @@ pub(crate) fn declaration_uses( Ok(uses) } -/// Scope declarations: a macro by its module source, and an enum, constant, audit, schema or -/// hook by its declaration file with formatting and comments ignored. +/// Scope declarations: a macro by its module source, any other by its normalised declaration file. fn declaration_nodes( project: &Bound<'_, PyAny>, project_dir: &Path, - nodes: &mut BTreeMap, -) -> PyResult<()> { +) -> PyResult> { + let mut nodes: BTreeMap = BTreeMap::new(); let index: Bound<'_, PyAny> = native_field(project, "scope_index")?; if index.is_none() { - return Ok(()); + return Ok(nodes); } for declaration in native_field(&index, "declarations")?.try_iter()? { let declaration: Bound<'_, PyAny> = declaration?; @@ -229,15 +230,17 @@ fn declaration_nodes( .collect(), ) } else { - let contents: String = - std::fs::read_to_string(project_dir.join(&path)).unwrap_or_default(); - (sql_definition_digest(&contents), Vec::new()) + let digest: String = match std::fs::read_to_string(project_dir.join(&path)) { + Ok(contents) => sql_definition_digest(&contents), + Err(_) => definition_digest(&["missing"]), + }; + (digest, Vec::new()) }; let mut definition: NodeDefinition = node(&kind, &name, vec![(DEFINITION_ASPECT, digest)]); definition.uses = uses; nodes.insert(declaration_key(&identity)?, definition); } - Ok(()) + Ok(nodes) } fn migrate_from(config_values: &Bound<'_, PyAny>) -> PyResult> { @@ -247,14 +250,13 @@ fn migrate_from(config_values: &Bound<'_, PyAny>) -> PyResult> { let Some(value) = values.get_item(MIGRATE_FROM_KEY)? else { return Ok(Vec::new()); }; - if let Ok(name) = value.extract::() { - return Ok(vec![name]); + if value.is_instance_of::() { + return Ok(vec![value.extract()?]); } - Ok(value.extract::>().unwrap_or_default()) + value.try_iter()?.map(|name| name?.extract()).collect() } -/// A model by its SQL before macro expansion, its header configuration, its declared columns -/// and contract, and its audits. +/// A model by its SQL before macro expansion, header configuration, columns, contract and audits. fn model_node( model: &Bound<'_, PyAny>, uses: &BTreeMap>, @@ -308,9 +310,8 @@ pub(crate) fn definition_snapshot( project: &Bound<'_, PyAny>, project_dir: &Path, ) -> PyResult { - let mut nodes: BTreeMap = BTreeMap::new(); let uses: BTreeMap> = declaration_uses(project)?; - declaration_nodes(project, project_dir, &mut nodes)?; + let mut nodes: BTreeMap = declaration_nodes(project, project_dir)?; for model in project_reads::models(project)?.try_iter()? { let (key, definition) = model_node(&model?, &uses)?; nodes.insert(key, definition); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index f1d9b5406e..730c725f5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -29,6 +29,9 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::scenario_command::setup::{flush, write}; const MACRO_TEST_MODE: &str = "macro"; + +/// Compiled tests or models by name. +pub(crate) type ByName<'py> = HashMap>; const TEST_CONSUMER_KIND: &str = "test"; /// The changes since the base revision and what the current revision uses. @@ -107,10 +110,7 @@ impl ChangedProject { pub(crate) fn test_item( &self, entry: &Bound<'_, PyAny>, - (tests, models): ( - &HashMap>, - &HashMap>, - ), + (tests, models): (&ByName<'_>, &ByName<'_>), ) -> PyResult { let name: String = native_field(entry, "name")?.extract()?; let mut item: ExecutedItem = ExecutedItem { @@ -204,16 +204,13 @@ impl ChangedProject { /// The project's compiled tests and models by name. pub(crate) fn tests_and_models<'py>( project: &Bound<'py, PyAny>, -) -> PyResult<( - HashMap>, - HashMap>, -)> { - let mut tests: HashMap> = HashMap::new(); +) -> PyResult<(ByName<'py>, ByName<'py>)> { + let mut tests: ByName<'py> = HashMap::new(); for test in project_reads::sql_tests(project)?.try_iter()? { let test: Bound<'py, PyAny> = test?; tests.insert(native_field(&test, "name")?.extract()?, test); } - let mut models: HashMap> = HashMap::new(); + let mut models: ByName<'py> = HashMap::new(); for model in project_reads::models(project)?.try_iter()? { let model: Bound<'py, PyAny> = model?; models.insert(model_reads::model_name(&model)?, model); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 9a94403302..f6505c9354 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -19,7 +19,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::change_selection::base_snapshot::{ - BaseComparison, BaseVariant, base_comparison, + BaseComparison, base_comparison, base_variant, }; use crate::bindings::_helpers::change_selection::selection::{ChangedProject, changed_project}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; @@ -402,10 +402,7 @@ pub(crate) fn run_scenario_test_command( Some(since) => Some(base_comparison( (&run.stream, &run.project.project_dir), since, - &BaseVariant { - selected_target: None, - cli_vars: py.None().into_bound(py), - }, + &base_variant(None, &py.None().into_bound(py))?, )?), None => None, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index f62c92a4e6..41a0612479 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -31,7 +31,7 @@ use crate::bindings::_helpers::build_progress::progress_text::{ use crate::bindings::_helpers::build_progress::result_text::test_label; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::change_selection::base_snapshot::{ - BaseComparison, BaseVariant, base_comparison, + BaseComparison, base_comparison, base_variant, }; use crate::bindings::_helpers::change_selection::selection::{ ChangedProject, changed_project, tests_and_models, @@ -58,6 +58,9 @@ use crate::bindings::_helpers::sql_test_command::timeouts::{ }; const PASS_OUTCOME: &str = "pass"; + +/// Why `--since` selected each test, by inspection label. +type SelectionReasons = HashMap>; const EXPECT_LABEL: &str = "expect"; const UNKNOWN_GROUP: &str = "(unknown)"; const EXECUTION_COMMAND: &str = "sqb test"; @@ -428,14 +431,13 @@ fn inspect( Ok(code) } -/// Keep only the planned tests that execute something changed since the base revision, write -/// the summary, and return the narrowed pipeline with each kept test's reasons by label. +/// Keep the planned tests that execute something changed, with each one's reasons by label. fn select_changed<'py>( (pipeline, stream): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), project_dir: &std::path::Path, base: BaseComparison, explain: bool, -) -> PyResult<(Bound<'py, PyAny>, HashMap>)> { +) -> PyResult<(Bound<'py, PyAny>, SelectionReasons)> { let py: Python<'py> = pipeline.py(); let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; let changed: ChangedProject = changed_project(&project, project_dir, base)?; @@ -673,10 +675,10 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py Some(since) => Some(base_comparison( (&stream, &project_dir), since, - &BaseVariant { - selected_target: native_field(request, "selected_target")?.extract()?, - cli_vars: native_field(request, "cli_vars")?, - }, + &base_variant( + native_field(request, "selected_target")?.extract()?, + &native_field(request, "cli_vars")?, + )?, )?), None => None, }; diff --git a/fensu.toml b/fensu.toml index 7acee3cbaf..dea68505b2 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2514,6 +2514,7 @@ module_layers = [ "build_progress/transient_lines=infra", "build_progress/transient_status=infra", "cache=infra", + "change_selection=commands", "check=commands", "cli=commands", "cli/launch_state=infra", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bbfc4487c5..3599071812 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2061,6 +2061,13 @@ def build_lint_expansion_context( declaration_scope: DeclarationScopeBuild | None = None, static_declaration_scope: DeclarationScopeBuild | None = None, ) -> SqlExpansionContext: ... +def write_base_definition_snapshot( + *, + project_dir: str | Path, + output_path: str | Path, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, +) -> None: ... def build_dag_json( *, graph: ProjectGraph, project_name: str, python_graph: PythonNodeGraph | None = None ) -> str: ... From 6d0227fe6290ae5b784c85cd67e411fe5807edff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:31:17 +0100 Subject: [PATCH 04/10] feat: fingerprint variables, resolved config, sources and Python imports for --since --- .../definition_changes/_helpers/reasons.rs | 19 +- .../src/definition_changes/constants.rs | 21 +- .../src/definition_changes/main/mod.rs | 1 + .../definition_changes/main/selection_json.rs | 53 ++ .../tests/test_definition_changes.rs | 62 ++- .../definition_changes/tests/test_types.rs | 6 + .../src/rule_evidence/_helpers/closure.rs | 38 ++ .../src/rule_evidence/_helpers/mod.rs | 1 + .../src/rule_evidence/_helpers/paths.rs | 22 +- .../src/rule_evidence/main/import_closure.rs | 37 +- .../src/rule_evidence/main/mod.rs | 1 + .../main/project_import_closure.rs | 19 + .../src/rule_evidence/tests/helpers.rs | 12 + .../rule_evidence/tests/test_rule_evidence.rs | 56 ++- .../src/rule_evidence/tests/test_types.rs | 7 + .../_helpers/change_selection/base_compile.rs | 9 +- .../change_selection/base_snapshot.rs | 14 +- .../change_selection/definition_snapshot.rs | 455 ++++++++++-------- .../_helpers/change_selection/fingerprints.rs | 296 ++++++++++++ .../bindings/_helpers/change_selection/mod.rs | 1 + .../_helpers/change_selection/selection.rs | 99 +++- .../_helpers/scenario_command/test_command.rs | 61 ++- .../_helpers/sql_test_command/test_command.rs | 44 +- .../sqlbuild/references/docs/cli/scenario.md | 16 +- .../sqlbuild/references/docs/cli/test.md | 56 ++- .../cli/commands/main/test/_test_types.py | 14 + .../cli/commands/main/test/helpers.py | 107 ++++ .../cli/commands/main/test/test_since.py | 276 +++++++++++ .../src/content/docs/docs/cli/scenario.mdx | 16 +- website/src/content/docs/docs/cli/test.mdx | 56 ++- 30 files changed, 1553 insertions(+), 322 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/definition_changes/main/selection_json.rs create mode 100644 crates/sqlbuild-pysource/src/rule_evidence/_helpers/closure.rs create mode 100644 crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs index 4069409ced..bffb50b7f3 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs @@ -3,7 +3,9 @@ use std::collections::BTreeSet; use crate::definition_changes::_helpers::aspects::{aspect_label, joined, kind_words}; -use crate::definition_changes::constants::{CONSTANT_KIND, ENUM_KIND, MODEL_KIND, TEST_FILE_KIND}; +use crate::definition_changes::constants::{ + CONSTANT_KIND, ENUM_KIND, MODEL_KIND, SEED_KIND, SETTINGS_KIND, SOURCE_KIND, TEST_FILE_KIND, +}; use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem, NodeChange}; fn aspect_phrase(change: &NodeChange) -> String { @@ -43,16 +45,29 @@ fn change_detail(change: &NodeChange) -> String { } } +/// `executes stg_orders (model SQL changed)`, `reads seed countries (changed)`. pub(crate) fn executed_reason(change: &NodeChange) -> String { let subject: String = if change.kind == MODEL_KIND { change.name.clone() } else { format!("{} {}", kind_words(&change.kind).0, change.name) }; - format!("executes {subject} ({})", change_detail(change)) + let verb: &str = if change.kind == SEED_KIND || change.kind == SOURCE_KIND { + "reads" + } else { + "executes" + }; + format!("{verb} {subject} ({})", change_detail(change)) } +/// `executes macro to_utc (changed)`, `uses enum status (changed)`, `uses project settings (changed)`. pub(crate) fn declaration_reason(change: &NodeChange) -> String { + if change.kind == SETTINGS_KIND { + return format!("uses project settings ({})", change_detail(change)); + } + if change.kind == SOURCE_KIND { + return executed_reason(change); + } let verb: &str = if change.kind == ENUM_KIND || change.kind == CONSTANT_KIND { "uses" } else { diff --git a/crates/sqlbuild-analysis/src/definition_changes/constants.rs b/crates/sqlbuild-analysis/src/definition_changes/constants.rs index 48742175e4..c8c87d29b3 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/constants.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/constants.rs @@ -1,7 +1,7 @@ //! Node kinds, definition aspects and the snapshot format version. /// The version of the stored snapshot layout and fingerprint scheme; older snapshots are recomputed. -pub const DEFINITION_SNAPSHOT_FORMAT: u32 = 1; +pub const DEFINITION_SNAPSHOT_FORMAT: u32 = 2; pub const MODEL_KIND: &str = "model"; pub const SEED_KIND: &str = "seed"; @@ -12,6 +12,11 @@ pub const ENUM_KIND: &str = "enum"; pub const CONSTANT_KIND: &str = "constant"; pub const TEST_FILE_KIND: &str = "test_file"; pub const SCENARIO_KIND: &str = "scenario"; +pub const SOURCE_KIND: &str = "source"; +pub const MATERIALIZATION_KIND: &str = "materialization"; +pub const SETTINGS_KIND: &str = "settings"; +/// Python nodes by decorator. +pub const PYTHON_NODE_KINDS: [&str; 4] = ["loader", "task", "asset", "check"]; /// Normalised SQL of a model, a function body or a scenario. pub const SQL_ASPECT: &str = "sql"; @@ -27,9 +32,11 @@ pub const DEFINITION_ASPECT: &str = "definition"; pub const FILE_ASPECT: &str = "file"; /// Captured local snapshots of a scenario. pub const SNAPSHOTS_ASPECT: &str = "snapshots"; +/// Values of the project variables a definition reads with `@@name`. +pub const VARS_ASPECT: &str = "vars"; /// How each aspect reads in a reason, in the order reasons list them. -pub(crate) const ASPECT_LABELS: [(&str, &str); 7] = [ +pub(crate) const ASPECT_LABELS: [(&str, &str); 8] = [ (SQL_ASPECT, "SQL"), (CONFIG_ASPECT, "config"), (COLUMNS_ASPECT, "columns"), @@ -37,11 +44,14 @@ pub(crate) const ASPECT_LABELS: [(&str, &str); 7] = [ (DEFINITION_ASPECT, "definition"), (FILE_ASPECT, "file"), (SNAPSHOTS_ASPECT, "snapshots"), + (VARS_ASPECT, "vars"), ]; /// How each node kind reads in summaries: singular, plural. -pub(crate) const KIND_LABELS: [(&str, &str, &str); 14] = [ +pub(crate) const KIND_LABELS: [(&str, &str, &str); 21] = [ + (SETTINGS_KIND, "project settings", "project settings"), (MODEL_KIND, "model", "models"), + (SOURCE_KIND, "source", "sources"), (SEED_KIND, "seed", "seeds"), (UDF_KIND, "function", "functions"), (TABLE_FUNCTION_KIND, "table function", "table functions"), @@ -53,6 +63,11 @@ pub(crate) const KIND_LABELS: [(&str, &str, &str); 14] = [ ("schema", "schema", "schemas"), ("sql_hook", "SQL hook", "SQL hooks"), ("python_hook", "Python hook", "Python hooks"), + (MATERIALIZATION_KIND, "materialization", "materializations"), + ("loader", "loader", "loaders"), + ("task", "task", "tasks"), + ("asset", "asset", "assets"), + ("check", "check", "checks"), (TEST_FILE_KIND, "test file", "test files"), (SCENARIO_KIND, "scenario", "scenarios"), ]; diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs index 8daf29af03..4faa4cb4ae 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs @@ -4,6 +4,7 @@ pub mod definition_digest; pub mod definition_snapshot_json; pub mod node_key; pub mod parse_definition_snapshot; +pub mod selection_json; pub mod selection_reasons; pub mod selection_report; pub mod sql_definition_digest; diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_json.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_json.rs new file mode 100644 index 0000000000..550624991f --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_json.rs @@ -0,0 +1,53 @@ +//! The `since` section of a `--since` run's JSON document. + +use serde_json::{Value, json}; + +use crate::definition_changes::models::{DefinitionChanges, SelectedItem}; + +/// The ref, merge base, changed, renamed and deleted nodes, and each selected item with its reasons. +pub fn selection_json( + (reference, merge_base): (&str, &str), + changes: &DefinitionChanges, + (total, selected): (usize, &[SelectedItem]), +) -> Value { + let changed: Vec = changes + .changed + .values() + .map(|change| { + json!({ + "kind": change.kind, + "name": change.name, + "added": change.added, + "aspects": change.aspects, + "renamed_from": change.renamed_from, + }) + }) + .collect(); + let renamed: Vec = changes + .renamed + .iter() + .map(|(from, to)| { + let (kind, old_name) = from.split_once(':').unwrap_or_default(); + let new_name: &str = to.split_once(':').map_or(to.as_str(), |(_, name)| name); + json!({ "kind": kind, "from": old_name, "to": new_name }) + }) + .collect(); + let deleted: Vec = changes + .deleted + .iter() + .map(|node| json!({ "kind": node.kind, "name": node.name })) + .collect(); + let items: Vec = selected + .iter() + .map(|item| json!({ "name": item.label, "reasons": item.reasons })) + .collect(); + json!({ + "ref": reference, + "merge_base": merge_base, + "changed": changed, + "renamed": renamed, + "deleted": deleted, + "candidate_count": total, + "selected": items, + }) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs index e4ddbe6e17..cf50a1a8ac 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs @@ -1,7 +1,11 @@ +use serde_json::json; + +use crate::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; use crate::definition_changes::main::changes_summary::changes_summary; use crate::definition_changes::main::compare_definitions::compare_definitions; use crate::definition_changes::main::definition_snapshot_json::definition_snapshot_json; use crate::definition_changes::main::parse_definition_snapshot::parse_definition_snapshot; +use crate::definition_changes::main::selection_json::selection_json; use crate::definition_changes::main::selection_reasons::selection_reasons; use crate::definition_changes::main::selection_report::selection_report; use crate::definition_changes::main::sql_definition_digest::sql_definition_digest; @@ -10,8 +14,8 @@ use crate::definition_changes::tests::helpers::{ base_orders, current_orders, item, keys, node, renamed, snapshot, with_node, }; use crate::definition_changes::tests::test_types::{ - CompareTestCase, SelectionReasonTestCase, SelectionReportTestCase, SnapshotJsonTestCase, - SqlDigestTestCase, + CompareTestCase, SelectionJsonTestCase, SelectionReasonTestCase, SelectionReportTestCase, + SnapshotJsonTestCase, SqlDigestTestCase, }; #[test] @@ -219,7 +223,7 @@ fn given_executed_item_when_explaining_selection_then_lists_each_changed_depende node("seed", "countries", &[("definition", "d2")], &[]), ), item: item(&[], &["model:orders", "seed:countries"], &[], &[]), - expected_reasons: &["executes seed countries (changed)"], + expected_reasons: &["reads seed countries (changed)"], }, ]; for test_case in test_cases { @@ -297,7 +301,13 @@ fn given_stored_snapshot_when_reading_then_accepts_only_the_same_format() { }, SnapshotJsonTestCase { description: "another format version is recomputed", - stored: |text| text.replacen("\"format\":1", "\"format\":0", 1), + stored: |text| { + text.replacen( + &format!("\"format\":{DEFINITION_SNAPSHOT_FORMAT}"), + "\"format\":0", + 1, + ) + }, expected_snapshot: None, }, SnapshotJsonTestCase { @@ -344,3 +354,47 @@ fn given_sql_when_fingerprinting_definition_then_ignores_layout_and_comments_onl ); } } + +#[test] +fn given_changes_and_selection_when_writing_json_then_lists_nodes_and_reasons() { + let test_cases = [SelectionJsonTestCase { + description: "changed, renamed and deleted nodes with each selected item's reasons", + selected: vec![SelectedItem { + label: "orders_rollup".to_owned(), + reasons: keys(&["executes stg_orders (model SQL changed)"]), + }], + expected_json: json!({ + "ref": "origin/main", + "merge_base": "abc123", + "changed": [{ + "kind": "model", + "name": "stg_orders", + "added": false, + "aspects": ["sql"], + "renamed_from": null, + }], + "renamed": [{ "kind": "model", "from": "old_customers", "to": "customers" }], + "deleted": [{ "kind": "seed", "name": "countries" }], + "candidate_count": 9, + "selected": [{ + "name": "orders_rollup", + "reasons": ["executes stg_orders (model SQL changed)"], + }], + }), + }]; + for test_case in test_cases { + let changes: DefinitionChanges = compare_definitions(&base_orders(), ¤t_orders()); + + let document: serde_json::Value = selection_json( + ("origin/main", "abc123"), + &changes, + (9, &test_case.selected), + ); + + assert_eq!( + document, test_case.expected_json, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs index f274c599d8..a9e6045877 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_types.rs @@ -49,3 +49,9 @@ pub(crate) struct SqlDigestTestCase { pub(crate) sql: &'static str, pub(crate) expected_same_as_original: bool, } + +pub(crate) struct SelectionJsonTestCase { + pub(crate) description: &'static str, + pub(crate) selected: Vec, + pub(crate) expected_json: serde_json::Value, +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/closure.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/closure.rs new file mode 100644 index 0000000000..56fbc22e87 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/closure.rs @@ -0,0 +1,38 @@ +//! The local modules one Python module executes through its imports, transitively. + +use std::collections::BTreeSet; +use std::convert::Infallible; +use std::path::{Path, PathBuf}; + +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::rule_evidence::_helpers::imports::read_source; +use crate::rule_evidence::_helpers::paths::{local_import_paths, package_inits}; + +/// Every module and package initializer under `root` that `source_path` executes, sorted. +pub(crate) fn module_closure( + source_path: &Path, + (project, root): (&Path, &Path), + package: Option<&str>, + python_version: (u8, u8), +) -> Vec { + let mut helpers: BTreeSet = BTreeSet::new(); + let Ok(()) = breadth_first_reach([source_path.to_path_buf()], None, &mut helpers, |current| { + let mut candidates: BTreeSet = package_inits(current, root).into_iter().collect(); + let parsed: Option = + parse_cpython_module(&read_source(current), python_version); + if let Some(parsed) = &parsed { + let roots: Vec> = + parsed.module.body.iter().map(CpythonNode::Stmt).collect(); + for node in walk_breadth_first(roots) { + candidates.extend(local_import_paths(&node, current, (project, root), package)); + } + } + candidates.remove(source_path); + Ok::<_, Infallible>(candidates) + }); + helpers.into_iter().collect() +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs index af1a7d23cc..c4524381f9 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod cases; +pub(crate) mod closure; pub(crate) mod evidence; pub(crate) mod imports; pub(crate) mod owners; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs index c929921954..c51cf22770 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs @@ -6,9 +6,7 @@ use std::path::{Component, Path, PathBuf}; use ruff_python_ast::Stmt; use crate::cpython_tree::models::CpythonNode; -use crate::rule_evidence::constants::{ - INIT_FILE_NAME, PYTHON_SUFFIX, RULES_DIRECTORY_NAME, STAR_IMPORT_NAME, -}; +use crate::rule_evidence::constants::{INIT_FILE_NAME, PYTHON_SUFFIX, STAR_IMPORT_NAME}; use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM as INIT_MODULE_NAME; /// `Path.resolve()`: symlinks resolved through the existing prefix, the rest kept lexically. @@ -67,22 +65,24 @@ pub(crate) fn package_inits(path: &Path, rules_root: &Path) -> Vec { inits } -/// Every repository-owned rules module one import statement can execute. +/// Every module under `rules_root` one import statement can execute. +/// +/// Absolute imports resolve from `project`; with `package` set, only imports of that top-level +/// package count. pub(crate) fn local_import_paths( node: &CpythonNode<'_>, source_path: &Path, - rules_root: &Path, + (project, rules_root): (&Path, &Path), + package: Option<&str>, ) -> Vec { - let project: PathBuf = rules_root - .parent() - .map_or(PathBuf::new(), Path::to_path_buf); + let project: PathBuf = project.to_path_buf(); let mut targets: Vec<(PathBuf, Vec)> = Vec::new(); match node { CpythonNode::Stmt(Stmt::Import(import)) => { for alias in &import.names { let parts: Vec = alias.name.as_str().split('.').map(str::to_owned).collect(); - if parts[0] == RULES_DIRECTORY_NAME { + if package.is_none_or(|package| parts[0] == package) { targets.push((project.clone(), parts)); } } @@ -100,7 +100,9 @@ pub(crate) fn local_import_paths( for _ in 1..import.level { base = base.parent().map_or(base.clone(), Path::to_path_buf); } - } else if module_parts.first().map(String::as_str) != Some(RULES_DIRECTORY_NAME) { + } else if package + .is_some_and(|package| module_parts.first().map(String::as_str) != Some(package)) + { return Vec::new(); } targets.push((base.clone(), module_parts.clone())); diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs index 05ceadd2f0..1bdbd3a7f9 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/import_closure.rs @@ -1,16 +1,9 @@ //! `import_closure`: the repository-owned helper modules one rule module imports. -use std::collections::BTreeSet; -use std::convert::Infallible; use std::path::{Path, PathBuf}; -use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; - -use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; -use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; -use crate::cpython_tree::models::{CpythonNode, ParsedSource}; -use crate::rule_evidence::_helpers::imports::read_source; -use crate::rule_evidence::_helpers::paths::{local_import_paths, package_inits, resolve}; +use crate::rule_evidence::_helpers::closure::module_closure; +use crate::rule_evidence::_helpers::paths::resolve; use crate::rule_evidence::constants::RULES_DIRECTORY_NAME; /// Every rules module and package initializer `source_path` executes, sorted; paths are resolved. @@ -20,21 +13,13 @@ pub fn import_closure( python_version: (u8, u8), ) -> Vec { let rules_root: PathBuf = resolve(&project_dir.join(RULES_DIRECTORY_NAME)); - let mut helpers: BTreeSet = BTreeSet::new(); - let Ok(()) = breadth_first_reach([source_path.to_path_buf()], None, &mut helpers, |current| { - let mut candidates: BTreeSet = - package_inits(current, &rules_root).into_iter().collect(); - let parsed: Option = - parse_cpython_module(&read_source(current), python_version); - if let Some(parsed) = &parsed { - let roots: Vec> = - parsed.module.body.iter().map(CpythonNode::Stmt).collect(); - for node in walk_breadth_first(roots) { - candidates.extend(local_import_paths(&node, current, &rules_root)); - } - } - candidates.remove(source_path); - Ok::<_, Infallible>(candidates) - }); - helpers.into_iter().collect() + let project: PathBuf = rules_root + .parent() + .map_or(PathBuf::new(), Path::to_path_buf); + module_closure( + source_path, + (&project, &rules_root), + Some(RULES_DIRECTORY_NAME), + python_version, + ) } diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs index 449b4c6551..db81695ac7 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs @@ -2,4 +2,5 @@ pub mod allowed_custom_rule_import; pub mod disallowed_import; pub mod implementation_fingerprint; pub mod import_closure; +pub mod project_import_closure; pub mod test_evidence; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs new file mode 100644 index 0000000000..47b2c75e73 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs @@ -0,0 +1,19 @@ +//! `project_import_closure`: the project modules one Python file imports, transitively. + +use std::path::{Path, PathBuf}; + +use crate::rule_evidence::_helpers::closure::module_closure; +use crate::rule_evidence::_helpers::paths::resolve; + +/// Every module and package initializer under `project_dir` that `source_path` executes, sorted. +/// +/// Imports resolve from the project directory, the way project Python imports `macros.currency` +/// or `python._helpers`; standard-library and installed modules are not part of the closure. +pub fn project_import_closure( + source_path: &Path, + project_dir: &Path, + python_version: (u8, u8), +) -> Vec { + let root: PathBuf = resolve(project_dir); + module_closure(&resolve(source_path), (&root, &root), None, python_version) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs index b1599b4202..2111e866c7 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/helpers.rs @@ -29,3 +29,15 @@ fn write(path: &Path, contents: &str) { let _ = std::fs::create_dir_all(path.parent().unwrap_or(path)); let _ = std::fs::write(path, contents); } + +/// A fresh project directory named `name` holding `files`. +pub(crate) fn project_with(name: &str, files: &[(&str, &str)]) -> PathBuf { + let root: PathBuf = std::env::temp_dir() + .join(format!("sqlbuild-project-closure-{}", std::process::id())) + .join(name); + let _ = std::fs::remove_dir_all(&root); + for (path, contents) in files { + write(&root.join(path), contents); + } + std::fs::canonicalize(&root).unwrap_or(root) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs index 5035eb0930..32e2da73e9 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_rule_evidence.rs @@ -1,9 +1,10 @@ use std::path::{Path, PathBuf}; use crate::rule_evidence::main::import_closure::import_closure; +use crate::rule_evidence::main::project_import_closure::project_import_closure; use crate::rule_evidence::main::test_evidence::test_evidence; -use crate::rule_evidence::tests::helpers::rule_project; -use crate::rule_evidence::tests::test_types::EvidenceTestCase; +use crate::rule_evidence::tests::helpers::{project_with, rule_project}; +use crate::rule_evidence::tests::test_types::{EvidenceTestCase, ProjectClosureTestCase}; const PYTHON: (u8, u8) = (3, 12); @@ -74,3 +75,54 @@ fn given_rule_project_when_reading_evidence_then_matches_python_closure_and_case ); } } + +#[test] +fn given_project_module_when_reading_project_closure_then_lists_imported_project_files() { + let test_cases = [ + ProjectClosureTestCase { + description: "helper modules across directories, transitively", + files: vec![ + ( + "macros/money.py", + "import os\nfrom macros._rates import RATE\n\ndef cents(x):\n return x\n", + ), + ( + "macros/_rates.py", + "from python import shared\nRATE = 100\n", + ), + ("python/__init__.py", ""), + ("python/shared.py", "VALUE = 1\n"), + ("python/unused.py", "VALUE = 2\n"), + ], + source: "macros/money.py", + expected_closure: vec!["macros/_rates.py", "python/__init__.py", "python/shared.py"], + }, + ProjectClosureTestCase { + description: "a module importing nothing local has an empty closure", + files: vec![("macros/plain.py", "import json\n")], + source: "macros/plain.py", + expected_closure: vec![], + }, + ]; + + for test_case in test_cases { + let root: PathBuf = project_with(test_case.description, &test_case.files); + let closure: Vec = + project_import_closure(&root.join(test_case.source), &root, PYTHON); + let relative: Vec<&Path> = closure + .iter() + .map(|path| path.strip_prefix(&root).unwrap_or(path)) + .collect(); + + assert_eq!( + relative, + test_case + .expected_closure + .iter() + .map(Path::new) + .collect::>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs index fe7802a5cc..3e8802056c 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs @@ -20,3 +20,10 @@ pub(crate) struct DisallowedImportTestCase { crate::rule_evidence::models::UnparsedSource, >, } + +pub(crate) struct ProjectClosureTestCase { + pub(crate) description: &'static str, + pub(crate) files: Vec<(&'static str, &'static str)>, + pub(crate) source: &'static str, + pub(crate) expected_closure: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs index 1a514c0537..249ab5b07c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs @@ -78,15 +78,18 @@ fn write_base_definition_snapshot( Some(keywords( py, &[ - ("discovered_inputs", discovered), + ("discovered_inputs", discovered.clone()), ("adapter", native_field(&context, "adapter")?), ("options", options), ("on_progress", none()), ], )?), )?; - let snapshot: DefinitionSnapshot = - definition_snapshot(&native_field(&pipeline, "project")?, &project_dir)?; + let snapshot: DefinitionSnapshot = definition_snapshot( + &native_field(&pipeline, "project")?, + &discovered, + &project_dir, + )?; std::fs::write(&output_path, definition_snapshot_json(&snapshot))?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index 2272fe96c4..b757b7f647 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -82,8 +82,9 @@ fn compile_failed(py: Python<'_>, base: &GitBase, detail: &str) -> PyErr { base.reference ), help: Some( - "--since compares with the base revision, so it must compile; compare with a \ - later ref, or run without --since" + "--since compares with the project as it was at the merge base, so that \ + revision must compile. Compare with a later ref whose project compiles, or run \ + without --since to run everything" .to_owned(), ), }, @@ -194,7 +195,14 @@ pub(crate) fn base_comparison<'py>( )?; let vars: &str = &variant.vars_json; let target: Option<&str> = variant.selected_target.as_deref(); - let key: String = definition_digest(&[NATIVE_BUILD_IDENTITY, target.unwrap_or_default(), vars]); + let local_config: String = + std::fs::read_to_string(directory.join(LOCAL_CONFIG_FILE)).unwrap_or_default(); + let key: String = definition_digest(&[ + NATIVE_BUILD_IDENTITY, + target.unwrap_or_default(), + vars, + &local_config, + ]); let cache_dir: PathBuf = CACHE_DIRECTORY .iter() .fold(directory.clone(), |path, part| path.join(part)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 6df31ee0f6..5eca035147 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -4,150 +4,59 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{ - PyBool, PyDict, PyDictMethods, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, - PyStringMethods, PyTuple, PyTypeMethods, -}; -use serde_json::{Map, Value}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use serde_json::Value; use sqlbuild_analysis::definition_changes::constants::{ AUDITS_ASPECT, COLUMNS_ASPECT, CONFIG_ASPECT, DEFINITION_ASPECT, DEFINITION_SNAPSHOT_FORMAT, - FILE_ASPECT, MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, SNAPSHOTS_ASPECT, SQL_ASPECT, - TEST_FILE_KIND, + FILE_ASPECT, MACRO_KIND, MATERIALIZATION_KIND, MODEL_KIND, PYTHON_NODE_KINDS, SCENARIO_KIND, + SEED_KIND, SETTINGS_KIND, SNAPSHOTS_ASPECT, SOURCE_KIND, SQL_ASPECT, TEST_FILE_KIND, + VARS_ASPECT, }; use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; use sqlbuild_analysis::definition_changes::main::node_key::node_key; use sqlbuild_analysis::definition_changes::main::sql_definition_digest::sql_definition_digest; use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; +use crate::bindings::_helpers::change_selection::fingerprints::{ + Fingerprinting, directory_digest, file_digest, text_of, +}; use crate::bindings::_helpers::compiled_views::function_reads; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::seed_reads; +use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::planner_identity::metadata::{function_local_hashes, seed_identity}; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::models::NativeRecord; -/// Fields that document a definition without changing what it does. -const DOCUMENTATION_FIELDS: [&str; 3] = ["description", "location", "meta"]; /// Header values that rename rather than redefine a model. const RENAME_CONFIG_KEYS: [&str; 2] = ["migrate_from", "description"]; const MIGRATE_FROM_KEY: &str = "migrate_from"; +const MATERIALIZED_KEY: &str = "materialized"; +/// Which header keys were authored, rather than inherited from defaults, does not change a model. +const AUTHORED_KEYS_FIELD: &str = "model_header_keys"; +const SOURCE_EXPRESSION_FIELD: &str = "expression"; const SCENARIO_SNAPSHOT_ROOT: [&str; 2] = ["tests", "_scenario_snapshots"]; - -/// A stable JSON form of a Python value, without documentation fields of records. -fn canonical(value: &Bound<'_, PyAny>, skipped: &[&str]) -> PyResult { - if value.is_none() { - return Ok(Value::Null); - } - if let Ok(cell) = value.cast::() { - let record: &NativeRecord = cell.get(); - let mut fields: Map = Map::new(); - fields.insert("record".to_owned(), Value::from(record.spec.name)); - for (spec, item) in record.spec.fields.iter().zip(&record.values) { - if DOCUMENTATION_FIELDS.contains(&spec.name) || skipped.contains(&spec.name) { - continue; - } - fields.insert( - spec.name.to_owned(), - canonical(item.bind(value.py()), skipped)?, - ); - } - return Ok(Value::Object(fields)); - } - if value.is_instance_of::() { - return Ok(Value::from(value.extract::()?)); - } - if value.is_instance_of::() { - return Ok(Value::from(value.extract::()?)); - } - if value.is_instance_of::() || value.is_instance_of::() { - return Ok(Value::from(value.repr()?.to_string())); - } - if value.is_instance_of::() || value.is_instance_of::() { - let items: Vec = value - .try_iter()? - .map(|item| canonical(&item?, skipped)) - .collect::>()?; - return Ok(Value::Array(items)); - } - if value.is_instance_of::() || value.is_instance_of::() { - let mut items: Vec = value - .try_iter()? - .map(|item| Ok(canonical(&item?, skipped)?.to_string())) - .collect::>()?; - items.sort(); - return Ok(Value::from(items)); - } - if let Ok(mapping) = value.cast::() { - let mut entries: BTreeMap = BTreeMap::new(); - for (key, item) in mapping.iter() { - entries.insert(key.str()?.to_string(), canonical(&item, skipped)?); - } - return Ok(Value::Object(entries.into_iter().collect())); - } - let text: String = value.repr()?.to_string(); - if text.contains(" at 0x") { - return Ok(Value::from(value.get_type().name()?.to_string())); - } - Ok(Value::from(text)) -} - -/// The digest of `value`'s canonical form, without the top-level record fields `top_skipped`. -fn canonical_digest( - value: &Bound<'_, PyAny>, - (top_skipped, skipped): (&[&str], &[&str]), -) -> PyResult { - let mut encoded: Value = canonical(value, skipped)?; - if let Value::Object(fields) = &mut encoded { - for field in top_skipped { - fields.remove(*field); - } - } - Ok(definition_digest(&[&encoded.to_string()])) -} - -fn text_of(value: &Bound<'_, PyAny>) -> PyResult { - Ok(value.str()?.to_string().replace('\\', "/")) -} - -/// The digest of a file's bytes, or of its absence. -fn file_digest(path: &Path) -> String { - match std::fs::read(path) { - Ok(bytes) => definition_digest(&[&String::from_utf8_lossy(&bytes)]), - Err(_) => definition_digest(&["missing"]), - } -} - -/// The digest of every file below `directory`, by relative path, or `None` when it is absent. -fn directory_digest(directory: &Path) -> Option { - let mut files: Vec<(String, String)> = Vec::new(); - let mut pending: Vec = vec![directory.to_path_buf()]; - while let Some(next) = pending.pop() { - let entries: std::fs::ReadDir = match std::fs::read_dir(&next) { - Ok(entries) => entries, - Err(_) => return None, - }; - for entry in entries.flatten() { - let path: PathBuf = entry.path(); - if path.is_dir() { - pending.push(path); - } else { - let relative: String = path - .strip_prefix(directory) - .unwrap_or(&path) - .to_string_lossy() - .replace('\\', "/"); - files.push((relative, file_digest(&path))); - } - } - } - files.sort(); - let parts: Vec<&str> = files - .iter() - .flat_map(|(path, digest)| [path.as_str(), digest.as_str()]) - .collect(); - Some(definition_digest(&parts)) -} +const SETTINGS_NAME: &str = "project"; +/// Settings that tune how a run executes or validates, not what a node computes. +const RUNTIME_SETTINGS: [&str; 9] = [ + "sql_analysis", + "require_sql_analysis", + "concurrency", + "microbatch_concurrency", + "test_timeout", + "test_timeout_seconds", + "scenario_timeout", + "scenario_timeout_seconds", + "delta_compile", +]; +const PYTHON_SUFFIX: &str = ".py"; +/// Discovered Python node collections, in `PYTHON_NODE_KINDS` order. +const PYTHON_NODE_COLLECTIONS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; fn node(kind: &str, name: &str, aspects: Vec<(&str, String)>) -> NodeDefinition { NodeDefinition { @@ -199,10 +108,26 @@ pub(crate) fn declaration_uses( Ok(uses) } -/// Scope declarations: a macro by its module source, any other by its normalised declaration file. +/// The digest of a declaration file: Python with its imports, anything else as normalised SQL. +fn declaration_file_digest(fingerprints: &Fingerprinting, path: &str) -> String { + if path.ends_with(PYTHON_SUFFIX) { + return fingerprints.python_module_digest(Path::new(path)); + } + match std::fs::read_to_string(fingerprints.project_dir.join(path)) { + Ok(contents) => { + let mut parts: Vec = vec![sql_definition_digest(&contents)]; + parts.extend(fingerprints.referenced_vars_digest(&contents)); + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + definition_digest(&borrowed) + } + Err(_) => definition_digest(&["missing"]), + } +} + +/// Scope declarations: a macro by its facts and Python imports, any other by its declaration file. fn declaration_nodes( project: &Bound<'_, PyAny>, - project_dir: &Path, + fingerprints: &Fingerprinting, ) -> PyResult> { let mut nodes: BTreeMap = BTreeMap::new(); let index: Bound<'_, PyAny> = native_field(project, "scope_index")?; @@ -223,18 +148,17 @@ fn declaration_nodes( let dependencies: Vec = native_field(¯o_facts, "dependencies")?.extract()?; ( - canonical_digest(¯o_facts, (&[], &[]))?, + definition_digest(&[ + &fingerprints.canonical_digest(¯o_facts, (&[], &[]))?, + &declaration_file_digest(fingerprints, &path), + ]), dependencies .iter() .map(|dependency| node_key(MACRO_KIND, dependency)) .collect(), ) } else { - let digest: String = match std::fs::read_to_string(project_dir.join(&path)) { - Ok(contents) => sql_definition_digest(&contents), - Err(_) => definition_digest(&["missing"]), - }; - (digest, Vec::new()) + (declaration_file_digest(fingerprints, &path), Vec::new()) }; let mut definition: NodeDefinition = node(&kind, &name, vec![(DEFINITION_ASPECT, digest)]); definition.uses = uses; @@ -256,75 +180,133 @@ fn migrate_from(config_values: &Bound<'_, PyAny>) -> PyResult> { value.try_iter()?.map(|name| name?.extract()).collect() } -/// A model by its SQL before macro expansion, header configuration, columns, contract and audits. +/// The resolved configuration of a model, defaults and path defaults included, without renames. +fn config_digest(fingerprints: &Fingerprinting, config: &Bound<'_, PyAny>) -> PyResult { + let mut encoded: Value = fingerprints.canonical(config, &[AUTHORED_KEYS_FIELD])?; + if let Value::Object(fields) = &mut encoded + && let Some(Value::Object(values)) = fields.get_mut("values") + { + for key in RENAME_CONFIG_KEYS { + values.remove(key); + } + } + Ok(definition_digest(&[&encoded.to_string()])) +} + +/// A model by its SQL before macro expansion, resolved configuration, columns, contract, audits +/// and the variables its file reads. fn model_node( model: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, uses: &BTreeMap>, ) -> PyResult<(String, NodeDefinition)> { let name: String = model_reads::model_name(model)?; - let config_values: Bound<'_, PyAny> = native_field(&model_reads::config(model)?, "values")?; - let config: Bound<'_, PyDict> = PyDict::new(model.py()); - if let Ok(values) = config_values.cast::() { - for (key, value) in values.iter() { - if !RENAME_CONFIG_KEYS.contains(&key.str()?.to_str()?) { - config.set_item(key, value)?; - } - } - } + let config: Bound<'_, PyAny> = model_reads::config(model)?; + let config_values: Bound<'_, PyAny> = native_field(&config, "values")?; let schema: Bound<'_, PyAny> = model_reads::schema_entry(model)?; let (columns, audits): (String, String) = if schema.is_none() { (definition_digest(&[]), definition_digest(&[])) } else { let mut column_audits: Vec = - vec![canonical(&native_field(&schema, "audits")?, &[])?]; + vec![fingerprints.canonical(&native_field(&schema, "audits")?, &[])?]; for column in native_field(&schema, "columns")?.try_iter()? { - column_audits.push(canonical(&native_field(&column?, "audits")?, &[])?); + column_audits.push(fingerprints.canonical(&native_field(&column?, "audits")?, &[])?); } ( - canonical_digest(&schema, (&["name", "tags"], &["audits"]))?, + fingerprints.canonical_digest(&schema, (&["name", "tags"], &["audits"]))?, definition_digest(&[&Value::Array(column_audits).to_string()]), ) }; let query_sql: String = native_field(model, "authored_query_sql")?.extract()?; - let mut definition: NodeDefinition = node( - MODEL_KIND, - &name, - vec![ - (SQL_ASPECT, sql_definition_digest(&query_sql)), - ( - CONFIG_ASPECT, - canonical_digest(config.as_any(), (&[], &[]))?, - ), - (COLUMNS_ASPECT, columns), - (AUDITS_ASPECT, audits), - ], - ); + let authored_sql: String = native_field(model, "authored_sql")?.extract()?; + let mut aspects: Vec<(&str, String)> = vec![ + (SQL_ASPECT, sql_definition_digest(&query_sql)), + (CONFIG_ASPECT, config_digest(fingerprints, &config)?), + (COLUMNS_ASPECT, columns), + (AUDITS_ASPECT, audits), + ]; + if let Some(digest) = fingerprints.referenced_vars_digest(&authored_sql) { + aspects.push((VARS_ASPECT, digest)); + } + let mut definition: NodeDefinition = node(MODEL_KIND, &name, aspects); let key: String = node_key(MODEL_KIND, &name); definition.uses = uses.get(&key).into_iter().flatten().cloned().collect(); + definition.uses.push(node_key(SETTINGS_KIND, SETTINGS_NAME)); + if let Ok(values) = config_values.cast::() + && let Some(materialized) = values.get_item(MATERIALIZED_KEY)? + && let Ok(materialized) = materialized.cast::() + { + definition + .uses + .push(node_key(MATERIALIZATION_KIND, materialized.to_str()?)); + } definition.migrate_from = migrate_from(&config_values)?; Ok((key, definition)) } -/// The definition snapshot of `project`, whose authored files live under `project_dir`. -pub(crate) fn definition_snapshot( - project: &Bound<'_, PyAny>, - project_dir: &Path, -) -> PyResult { - let uses: BTreeMap> = declaration_uses(project)?; - let mut nodes: BTreeMap = declaration_nodes(project, project_dir)?; - for model in project_reads::models(project)?.try_iter()? { - let (key, definition) = model_node(&model?, &uses)?; - nodes.insert(key, definition); +/// A source by its declaration, with an inline expression's formatting ignored. +fn source_node( + source: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult<(String, NodeDefinition)> { + let name: String = source_reads::name(source)?.extract()?; + let mut encoded: Value = fingerprints.canonical(&source_reads::source_entry(source)?, &[])?; + if let Value::Object(fields) = &mut encoded + && let Some(Value::String(expression)) = fields.get(SOURCE_EXPRESSION_FIELD) + { + let normalised: String = sql_definition_digest(expression); + fields.insert(SOURCE_EXPRESSION_FIELD.to_owned(), Value::from(normalised)); } - for seed in project_reads::seeds(project)?.try_iter()? { - let seed: Bound<'_, PyAny> = seed?; - let name: String = seed_reads::name(&seed)?.extract()?; - let (version_hash, _) = seed_identity(&seed)?; - nodes.insert( - node_key(SEED_KIND, &name), - node(SEED_KIND, &name, vec![(DEFINITION_ASPECT, version_hash)]), - ); + Ok(( + node_key(SOURCE_KIND, &name), + node( + SOURCE_KIND, + &name, + vec![( + DEFINITION_ASPECT, + definition_digest(&[&encoded.to_string()]), + )], + ), + )) +} + +/// Loaders, tasks, assets, checks and materializations by their Python modules and imports. +fn python_nodes( + discovered: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, + nodes: &mut BTreeMap, +) -> PyResult<()> { + let collections = PYTHON_NODE_COLLECTIONS + .iter() + .zip(PYTHON_NODE_KINDS) + .chain([(&"materialization_files", MATERIALIZATION_KIND)]); + for (collection, kind) in collections { + let Ok(items) = discovered.getattr(*collection) else { + continue; + }; + for item in items.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let name: String = item.getattr("name")?.extract()?; + let path: PathBuf = item.getattr("file_path")?.extract()?; + nodes.insert( + node_key(kind, &name), + node( + kind, + &name, + vec![(DEFINITION_ASPECT, fingerprints.python_module_digest(&path))], + ), + ); + } } + Ok(()) +} + +/// User-defined and table functions by their definitions, Python ones with their imports. +fn function_nodes( + project: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, + nodes: &mut BTreeMap, +) -> PyResult<()> { let functions: Bound<'_, PyAny> = project_reads::functions(project)?; let hashes: Bound<'_, PyDict> = function_local_hashes(&functions, None)?; for function in functions.try_iter()? { @@ -338,8 +320,18 @@ pub(crate) fn definition_snapshot( let Some(hash) = hashes.get_item(&name)? else { continue; }; - let mut definition: NodeDefinition = - node(&kind, &name, vec![(DEFINITION_ASPECT, hash.extract()?)]); + let mut digest: String = hash.extract()?; + let source_file: Bound<'_, PyAny> = function_reads::source_file_path(&function)?; + if !source_file.is_none() { + let path: String = text_of(&source_file)?; + if path.ends_with(PYTHON_SUFFIX) { + digest = definition_digest(&[ + &digest, + &fingerprints.python_module_digest(Path::new(&path)), + ]); + } + } + let mut definition: NodeDefinition = node(&kind, &name, vec![(DEFINITION_ASPECT, digest)]); for dependency in function_reads::deps(&function)?.try_iter()? { let dependency: Bound<'_, PyAny> = dependency?; definition.uses.push(node_key( @@ -350,24 +342,50 @@ pub(crate) fn definition_snapshot( &native_field(&dependency, "name")?.extract::()?, )); } + definition.uses.push(node_key(SETTINGS_KIND, SETTINGS_NAME)); nodes.insert(node_key(&kind, &name), definition); } + Ok(()) +} + +/// Each unit test file by its text and the variables it reads. +fn test_file_nodes( + project: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, + nodes: &mut BTreeMap, +) -> PyResult<()> { for test in project_reads::sql_tests(project)?.try_iter()? { let file: Bound<'_, PyAny> = native_field(&test?, "test_file")?; let path: String = text_of(&native_field(&file, "relative_path")?)?; - nodes - .entry(node_key(TEST_FILE_KIND, &path)) - .or_insert_with(|| { - node( - TEST_FILE_KIND, - &path, - vec![(FILE_ASPECT, file_digest(&project_dir.join(&path)))], - ) - }); + let key: String = node_key(TEST_FILE_KIND, &path); + if nodes.contains_key(&key) { + continue; + } + let contents: String = native_field(&file, "contents")?.extract()?; + let mut aspects: Vec<(&str, String)> = vec![( + FILE_ASPECT, + file_digest(&fingerprints.project_dir.join(&path)), + )]; + if let Some(digest) = fingerprints.referenced_vars_digest(&contents) { + aspects.push((VARS_ASPECT, digest)); + } + nodes.insert(key, node(TEST_FILE_KIND, &path, aspects)); } + Ok(()) +} + +/// Each scenario by its file, captured snapshots and the variables it reads. +fn scenario_nodes( + project: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, + uses: &BTreeMap>, + nodes: &mut BTreeMap, +) -> PyResult<()> { let snapshot_root: PathBuf = SCENARIO_SNAPSHOT_ROOT .iter() - .fold(project_dir.to_path_buf(), |path, part| path.join(part)); + .fold(fingerprints.project_dir.clone(), |path, part| { + path.join(part) + }); for scenario in native_field(project, "sql_scenarios")?.try_iter()? { let scenario: Bound<'_, PyAny> = scenario?; let name: String = native_field(&scenario, "name")?.extract()?; @@ -375,16 +393,71 @@ pub(crate) fn definition_snapshot( &native_field(&scenario, "scenario_file")?, "relative_path", )?)?; - let mut aspects: Vec<(&str, String)> = - vec![(FILE_ASPECT, file_digest(&project_dir.join(&path)))]; + let file: PathBuf = fingerprints.project_dir.join(&path); + let mut aspects: Vec<(&str, String)> = vec![(FILE_ASPECT, file_digest(&file))]; if let Some(digest) = directory_digest(&snapshot_root.join(&name)) { aspects.push((SNAPSHOTS_ASPECT, digest)); } + let contents: String = std::fs::read_to_string(&file).unwrap_or_default(); + if let Some(digest) = fingerprints.referenced_vars_digest(&contents) { + aspects.push((VARS_ASPECT, digest)); + } let key: String = node_key(SCENARIO_KIND, &name); let mut definition: NodeDefinition = node(SCENARIO_KIND, &name, aspects); definition.uses = uses.get(&key).into_iter().flatten().cloned().collect(); nodes.insert(key, definition); } + Ok(()) +} + +/// The definition snapshot of `project`, discovered as `discovered` under `project_dir`. +pub(crate) fn definition_snapshot( + project: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, + project_dir: &Path, +) -> PyResult { + let fingerprints: Fingerprinting = Fingerprinting::new( + project, + project_dir, + &native_field(project, "effective_vars")?, + )?; + let uses: BTreeMap> = declaration_uses(project)?; + let mut nodes: BTreeMap = declaration_nodes(project, &fingerprints)?; + nodes.insert( + node_key(SETTINGS_KIND, SETTINGS_NAME), + node( + SETTINGS_KIND, + SETTINGS_NAME, + vec![( + DEFINITION_ASPECT, + fingerprints.canonical_digest( + &native_field(project, "settings")?, + (&RUNTIME_SETTINGS, &[]), + )?, + )], + ), + ); + for model in project_reads::models(project)?.try_iter()? { + let (key, definition) = model_node(&model?, &fingerprints, &uses)?; + nodes.insert(key, definition); + } + for source in project_reads::sources(project)?.try_iter()? { + let (key, definition) = source_node(&source?, &fingerprints)?; + nodes.insert(key, definition); + } + for seed in project_reads::seeds(project)?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + let name: String = seed_reads::name(&seed)?.extract()?; + let (version_hash, _) = seed_identity(&seed)?; + nodes.insert( + node_key(SEED_KIND, &name), + node(SEED_KIND, &name, vec![(DEFINITION_ASPECT, version_hash)]), + ); + } + function_nodes(project, &fingerprints, &mut nodes)?; + python_nodes(discovered, &fingerprints, &mut nodes)?; + test_file_nodes(project, &fingerprints, &mut nodes)?; + scenario_nodes(project, &fingerprints, &uses, &mut nodes)?; Ok(DefinitionSnapshot { format: DEFINITION_SNAPSHOT_FORMAT, nodes, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs new file mode 100644 index 0000000000..67566db87f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs @@ -0,0 +1,296 @@ +//! Stable digests of compiled records, authored files, project variables and Python modules. + +use std::cell::RefCell; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, PyTuple, + PyTypeMethods, +}; +use serde_json::{Map, Value}; +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; +use sqlbuild_pysource::rule_evidence::main::project_import_closure::project_import_closure; + +use crate::bindings::_helpers::records::models::NativeRecord; + +/// Fields that document a definition without changing what it does. +const DOCUMENTATION_FIELDS: [&str; 2] = ["description", "location"]; +const DATACLASS_FIELDS: &str = "__dataclass_fields__"; +/// Python source that mentions this can read project variables through `ctx.vars`. +const VARS_ATTRIBUTE: &str = "vars"; +const VARIABLE_SIGIL: &str = "@@"; +const ENVIRONMENT_VARIABLE: &str = "ENV"; + +/// What fingerprints of one project revision need: where it lives and its variables. +pub(crate) struct Fingerprinting { + pub(crate) project_dir: PathBuf, + /// Spellings of the project directory, stripped from values so both revisions compare. + roots: Vec, + /// Effective project variables as canonical JSON, by name. + vars: BTreeMap, + python_version: (u8, u8), + /// Python module digests already computed, by module path; many macros share one file. + module_digests: RefCell>, +} + +impl Fingerprinting { + pub(crate) fn new( + project: &Bound<'_, PyAny>, + project_dir: &Path, + effective_vars: &Bound<'_, PyAny>, + ) -> PyResult { + let py: Python<'_> = project.py(); + let mut roots: Vec = Vec::new(); + for root in [ + std::fs::canonicalize(project_dir).ok(), + Some(project_dir.to_path_buf()), + ] + .into_iter() + .flatten() + { + let text: String = format!("{}/", root.to_string_lossy().replace('\\', "/")); + if !roots.contains(&text) { + roots.push(text); + } + } + roots.sort_by_key(|root| std::cmp::Reverse(root.len())); + let mut fingerprinting: Self = Self { + project_dir: project_dir.to_path_buf(), + roots, + vars: BTreeMap::new(), + python_version: (py.version_info().major, py.version_info().minor), + module_digests: RefCell::new(BTreeMap::new()), + }; + if let Ok(values) = effective_vars.cast::() { + for (name, value) in values.iter() { + let encoded: String = fingerprinting.canonical(&value, &[])?.to_string(); + fingerprinting.vars.insert(name.str()?.to_string(), encoded); + } + } + Ok(fingerprinting) + } + + fn relative(&self, text: &str) -> String { + let mut text: String = text.replace('\\', "/"); + for root in &self.roots { + text = text.replace(root.as_str(), ""); + } + text + } + + /// A stable JSON form of a Python value, without documentation fields or `skipped` fields. + pub(crate) fn canonical(&self, value: &Bound<'_, PyAny>, skipped: &[&str]) -> PyResult { + if value.is_none() { + return Ok(Value::Null); + } + if let Ok(cell) = value.cast::() { + let record: &NativeRecord = cell.get(); + let mut fields: Map = Map::new(); + fields.insert("record".to_owned(), Value::from(record.spec.name)); + for (spec, item) in record.spec.fields.iter().zip(&record.values) { + if DOCUMENTATION_FIELDS.contains(&spec.name) || skipped.contains(&spec.name) { + continue; + } + fields.insert( + spec.name.to_owned(), + self.canonical(item.bind(value.py()), skipped)?, + ); + } + return Ok(Value::Object(fields)); + } + if value.is_instance_of::() { + return Ok(Value::from(value.extract::()?)); + } + if value.is_instance_of::() { + return Ok(Value::from(self.relative(&value.extract::()?))); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(Value::from(value.repr()?.to_string())); + } + if value.is_instance_of::() || value.is_instance_of::() { + let items: Vec = value + .try_iter()? + .map(|item| self.canonical(&item?, skipped)) + .collect::>()?; + return Ok(Value::Array(items)); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = value + .try_iter()? + .map(|item| Ok(self.canonical(&item?, skipped)?.to_string())) + .collect::>()?; + items.sort(); + return Ok(Value::from(items)); + } + if let Ok(mapping) = value.cast::() { + let mut entries: BTreeMap = BTreeMap::new(); + for (key, item) in mapping.iter() { + entries.insert(key.str()?.to_string(), self.canonical(&item, skipped)?); + } + return Ok(Value::Object(entries.into_iter().collect())); + } + if let Ok(declared) = value.getattr(DATACLASS_FIELDS) + && let Ok(declared) = declared.cast::() + { + let mut fields: Map = Map::new(); + fields.insert( + "record".to_owned(), + Value::from(value.get_type().name()?.to_string()), + ); + for name in declared.keys() { + let name: String = name.extract()?; + if DOCUMENTATION_FIELDS.contains(&name.as_str()) || skipped.contains(&name.as_str()) + { + continue; + } + let item: Value = self.canonical(&value.getattr(name.as_str())?, skipped)?; + fields.insert(name, item); + } + return Ok(Value::Object(fields)); + } + let text: String = value.repr()?.to_string(); + if text.contains(" at 0x") { + return Ok(Value::from(value.get_type().name()?.to_string())); + } + Ok(Value::from(self.relative(&text))) + } + + /// The digest of `value`'s canonical form, without the top-level record fields `top_skipped`. + pub(crate) fn canonical_digest( + &self, + value: &Bound<'_, PyAny>, + (top_skipped, skipped): (&[&str], &[&str]), + ) -> PyResult { + let mut encoded: Value = self.canonical(value, skipped)?; + if let Value::Object(fields) = &mut encoded { + for field in top_skipped { + fields.remove(*field); + } + } + Ok(definition_digest(&[&encoded.to_string()])) + } + + /// The digest of the variables `names`, each by its effective value or its absence. + fn vars_digest<'a>(&self, names: impl Iterator) -> String { + let mut parts: Vec<&str> = Vec::new(); + for name in names { + parts.push(name); + parts.push(self.vars.get(name).map_or("missing", String::as_str)); + } + definition_digest(&parts) + } + + /// The digest of the variables `text` reads with `@@name`, or `None` when it reads none. + pub(crate) fn referenced_vars_digest(&self, text: &str) -> Option { + let names: BTreeSet = referenced_vars(text); + (!names.is_empty()).then(|| self.vars_digest(names.iter())) + } + + /// The digest of a Python file and every project module it imports, by relative path. + /// + /// Code that mentions `vars` may read any project variable, so its digest covers them all. + pub(crate) fn python_module_digest(&self, path: &Path) -> String { + let source: PathBuf = if path.is_absolute() { + path.to_path_buf() + } else { + self.project_dir.join(path) + }; + if let Some(digest) = self.module_digests.borrow().get(&source) { + return digest.clone(); + } + let mut files: Vec = vec![source.clone()]; + files.extend(project_import_closure( + &source, + &self.project_dir, + self.python_version, + )); + let mut parts: Vec = Vec::new(); + let mut reads_vars: bool = false; + for file in files { + let contents: String = std::fs::read(&file) + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .unwrap_or_else(|_| "missing".to_owned()); + reads_vars |= contents.contains(VARS_ATTRIBUTE); + parts.push(self.relative(&file.to_string_lossy())); + parts.push(definition_digest(&[&contents])); + } + if reads_vars { + parts.push(self.vars_digest(self.vars.keys())); + } + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + let digest: String = definition_digest(&borrowed); + self.module_digests + .borrow_mut() + .insert(source, digest.clone()); + digest + } +} + +/// Names of the project variables `text` reads with `@@name`, without runtime `@@@` and `@@ENV:`. +fn referenced_vars(text: &str) -> BTreeSet { + let mut names: BTreeSet = BTreeSet::new(); + let mut rest: &str = text; + while let Some(index) = rest.find(VARIABLE_SIGIL) { + let preceded: bool = rest[..index].ends_with('@'); + let after: &str = &rest[index + VARIABLE_SIGIL.len()..]; + rest = after.trim_start_matches('@'); + if preceded || after.starts_with('@') { + continue; + } + let name: String = after + .chars() + .take_while(|character| character.is_ascii_alphanumeric() || *character == '_') + .collect(); + if name.is_empty() || (name == ENVIRONMENT_VARIABLE && after[name.len()..].starts_with(':')) + { + continue; + } + names.insert(name); + } + names +} + +pub(crate) fn text_of(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string().replace('\\', "/")) +} + +/// The digest of a file's bytes, or of its absence. +pub(crate) fn file_digest(path: &Path) -> String { + match std::fs::read(path) { + Ok(bytes) => definition_digest(&[&String::from_utf8_lossy(&bytes)]), + Err(_) => definition_digest(&["missing"]), + } +} + +/// The digest of every file below `directory`, by relative path, or `None` when it is absent. +pub(crate) fn directory_digest(directory: &Path) -> Option { + let mut files: Vec<(String, String)> = Vec::new(); + let mut pending: Vec = vec![directory.to_path_buf()]; + while let Some(next) = pending.pop() { + let entries: std::fs::ReadDir = match std::fs::read_dir(&next) { + Ok(entries) => entries, + Err(_) => return None, + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + if path.is_dir() { + pending.push(path); + } else { + let relative: String = path + .strip_prefix(directory) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + files.push((relative, file_digest(&path))); + } + } + } + files.sort(); + let parts: Vec<&str> = files + .iter() + .flat_map(|(path, digest)| [path.as_str(), digest.as_str()]) + .collect(); + Some(definition_digest(&parts)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs index a119be50b2..e49fa47121 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod base_compile; pub(crate) mod base_snapshot; pub(crate) mod definition_snapshot; +pub(crate) mod fingerprints; pub(crate) mod registration; pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 730c725f5e..7869159025 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -3,14 +3,16 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::Path; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::definition_changes::constants::{ - MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, TABLE_FUNCTION_KIND, TEST_FILE_KIND, + MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, + TEST_FILE_KIND, }; use sqlbuild_analysis::definition_changes::main::changes_summary::changes_summary; use sqlbuild_analysis::definition_changes::main::compare_definitions::compare_definitions; use sqlbuild_analysis::definition_changes::main::node_key::node_key; +use sqlbuild_analysis::definition_changes::main::selection_json::selection_json; use sqlbuild_analysis::definition_changes::main::selection_reasons::selection_reasons; use sqlbuild_analysis::definition_changes::main::selection_report::selection_report; use sqlbuild_analysis::definition_changes::models::{ @@ -29,6 +31,11 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::scenario_command::setup::{flush, write}; const MACRO_TEST_MODE: &str = "macro"; +const JSON_MODULE: &str = "json"; +/// The JSON document key of the changes and selection of a `--since` run. +const SINCE_SECTION: &str = "since"; +/// The relation call a test or scenario file reads a source with. +const SOURCE_CALL: &str = "__source("; /// Compiled tests or models by name. pub(crate) type ByName<'py> = HashMap>; @@ -37,21 +44,23 @@ const TEST_CONSUMER_KIND: &str = "test"; /// The changes since the base revision and what the current revision uses. pub(crate) struct ChangedProject { pub(crate) reference: String, + merge_base: String, pub(crate) current: DefinitionSnapshot, pub(crate) changes: DefinitionChanges, uses: BTreeMap>, } -/// Compare the compiled working tree `project` with the base snapshot. +/// Compare the compiled working tree `project`, discovered as `discovered`, with the base snapshot. pub(crate) fn changed_project( - project: &Bound<'_, PyAny>, + (project, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), project_dir: &Path, base: BaseComparison, ) -> PyResult { - let current: DefinitionSnapshot = definition_snapshot(project, project_dir)?; + let current: DefinitionSnapshot = definition_snapshot(project, discovered, project_dir)?; let changes: DefinitionChanges = compare_definitions(&base.snapshot, ¤t); Ok(ChangedProject { reference: base.base.reference, + merge_base: base.base.merge_base, current, changes, uses: declaration_uses(project)?, @@ -62,6 +71,37 @@ fn strings(value: &Bound<'_, PyAny>) -> PyResult> { value.try_iter()?.map(|item| item?.extract()).collect() } +/// Names of the sources `sql` reads with `__source("name")`. +fn source_calls(sql: &str) -> Vec { + let mut names: Vec = Vec::new(); + for (index, _) in sql.match_indices(SOURCE_CALL) { + let rest: &str = sql[index + SOURCE_CALL.len()..].trim_start(); + let Some(quote) = rest + .chars() + .next() + .filter(|quote| *quote == '"' || *quote == '\'') + else { + continue; + }; + if let Some(end) = rest[1..].find(quote) { + names.push(rest[1..=end].to_owned()); + } + } + names +} + +/// The sources `model` reads, other than `mocked` ones, as node keys. +fn read_sources(model: &Bound<'_, PyAny>, mocked: &[String]) -> PyResult> { + let mut keys: Vec = Vec::new(); + for dependency in model_reads::deps(model)?.try_iter()? { + let (kind, name) = native_key(&dependency?)?; + if kind == SOURCE_KIND && !mocked.contains(&name) { + keys.push(node_key(SOURCE_KIND, &name)); + } + } + Ok(keys) +} + impl ChangedProject { /// Keys of current nodes of `kind` named in `names`. fn keys_named(&self, kind: &str, names: &[String]) -> BTreeSet { @@ -87,6 +127,16 @@ impl ChangedProject { selection_reasons(item, &self.current, &self.changes) } + /// The `since` section of the run's JSON document. + pub(crate) fn json(&self, (total, selected): (usize, &[SelectedItem])) -> String { + selection_json( + (&self.reference, &self.merge_base), + &self.changes, + (total, selected), + ) + .to_string() + } + /// Write the change summary and the selection to `stream`. pub(crate) fn report( &self, @@ -118,6 +168,7 @@ impl ChangedProject { ..ExecutedItem::default() }; let mocked_seeds: Vec = strings(&native_field(entry, "mock_seed_names")?)?; + let mocked_sources: Vec = strings(&native_field(entry, "mock_source_names")?)?; let mocked_functions: Vec = strings(&native_field(entry, "mock_table_function_names")?)?; item.mocked @@ -131,6 +182,12 @@ impl ChangedProject { .to_string() .replace('\\', "/"), )); + let contents: String = native_field(&file, "contents")?.extract()?; + for name in source_calls(&contents) { + if !mocked_sources.contains(&name) { + item.executed.push(node_key(SOURCE_KIND, &name)); + } + } let payload: Bound<'_, PyAny> = native_field(test, "payload")?; if let Ok(mocks) = native_field(&payload, "macro_mocks") && let Ok(mocks) = mocks.cast::() @@ -156,6 +213,7 @@ impl ChangedProject { let Some(model) = models.get(&model_name) else { continue; }; + item.executed.extend(read_sources(model, &mocked_sources)?); for dependency in model_reads::deps(model)?.try_iter()? { let (kind, dependency_name) = native_key(&dependency?)?; if kind == SEED_KIND && !mocked_seeds.contains(&dependency_name) { @@ -174,7 +232,7 @@ impl ChangedProject { pub(crate) fn scenario_item( &self, scenario: &Bound<'_, PyAny>, - project: &Bound<'_, PyAny>, + (project, models): (&Bound<'_, PyAny>, &ByName<'_>), ) -> PyResult { let name: String = native_field(scenario, "name")?.extract()?; let key: String = node_key(SCENARIO_KIND, &name); @@ -184,8 +242,17 @@ impl ChangedProject { declarations: self.declarations_of(&key), ..ExecutedItem::default() }; + let source_fixtures: Vec = strings(&native_field(&graph, "source_fixture_names")?)?; + for name in source_calls(&native_field(scenario, "sql_body")?.extract::()?) { + if !source_fixtures.contains(&name) { + item.executed.push(node_key(SOURCE_KIND, &name)); + } + } for model_name in strings(&native_field(&graph, "model_names")?)? { item.executed.push(node_key(MODEL_KIND, &model_name)); + if let Some(model) = models.get(&model_name) { + item.executed.extend(read_sources(model, &source_fixtures)?); + } } let fixtures: Vec = strings(&native_field(&graph, "seed_fixture_names")?)?; for seed_name in strings(&native_field(&graph, "seed_names")?)? { @@ -201,6 +268,26 @@ impl ChangedProject { } } +/// `payload`, a JSON document, with the `since` section added when the run used `--since`. +pub(crate) fn with_since_section( + py: Python<'_>, + payload: String, + since: Option<&str>, +) -> PyResult { + let Some(since) = since else { + return Ok(payload); + }; + let json: Bound<'_, PyModule> = py.import(JSON_MODULE)?; + let document: Bound<'_, PyAny> = json.call_method1("loads", (payload,))?; + document.set_item(SINCE_SECTION, json.call_method1("loads", (since,))?)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("indent", 2)?; + let text: String = json + .call_method("dumps", (document,), Some(&options))? + .extract()?; + Ok(format!("{text}\n")) +} + /// The project's compiled tests and models by name. pub(crate) fn tests_and_models<'py>( project: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index f6505c9354..c4bbb4c2a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -21,7 +21,9 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::change_selection::base_snapshot::{ BaseComparison, base_comparison, base_variant, }; -use crate::bindings::_helpers::change_selection::selection::{ChangedProject, changed_project}; +use crate::bindings::_helpers::change_selection::selection::{ + ChangedProject, changed_project, tests_and_models, with_since_section, +}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -58,6 +60,8 @@ pub(crate) struct TestRun<'py> { pub(crate) stream: Bound<'py, PyAny>, pub(crate) style: CliStyle, pub(crate) concurrency: i64, + /// The `since` JSON section of a `--since` run. + pub(crate) since_json: Option, } fn flag(request: &Bound<'_, PyAny>, name: &str) -> PyResult { @@ -128,6 +132,7 @@ fn prepare<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = PyString::new( py, - &format_scenario_execution_json( + &with_since_section( py, - Some(keywords( + format_scenario_execution_json( py, - &[ - ("results", results.clone().into_any()), - ("local", boolean(py, run.local)), - ( - "run_namespace", - optional_text(py, namespace.value.as_deref()), - ), - ("namespace_source", text(py, &namespace.source)), - ( - "duration_ms", - pyo3::types::PyFloat::new(py, elapsed_seconds * 1000.0).into_any(), - ), - ], - )?), + Some(keywords( + py, + &[ + ("results", results.clone().into_any()), + ("local", boolean(py, run.local)), + ( + "run_namespace", + optional_text(py, namespace.value.as_deref()), + ), + ("namespace_source", text(py, &namespace.source)), + ( + "duration_ms", + pyo3::types::PyFloat::new(py, elapsed_seconds * 1000.0).into_any(), + ), + ], + )?), + )?, + run.since_json.as_deref(), )?, ) .into_any(); @@ -346,18 +355,19 @@ fn select_changed<'py>( (pipeline, scenarios): (&Bound<'py, PyAny>, Bound<'py, PyTuple>), base: BaseComparison, explain: bool, -) -> PyResult> { +) -> PyResult<(Bound<'py, PyTuple>, String)> { let py: Python<'py> = pipeline.py(); let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; let changed: ChangedProject = changed_project( - &project, + (&project, &run.project.discovered), &run.project.project_dir.extract::()?, base, )?; + let (_, models) = tests_and_models(&project)?; let mut kept: Vec> = Vec::new(); let mut selected: Vec = Vec::new(); for scenario in scenarios.iter() { - let item: ExecutedItem = changed.scenario_item(&scenario, &project)?; + let item: ExecutedItem = changed.scenario_item(&scenario, (&project, &models))?; let reasons: Vec = changed.reasons(&item); if reasons.is_empty() { continue; @@ -373,7 +383,10 @@ fn select_changed<'py>( ("scenario", "scenarios"), (scenarios.len(), &selected, explain), )?; - PyTuple::new(py, kept) + Ok(( + PyTuple::new(py, kept)?, + changed.json((scenarios.len(), &selected)), + )) } /// Run one `sqb scenario test` request. @@ -383,7 +396,7 @@ pub(crate) fn run_scenario_test_command( py: Python<'_>, request: &Bound<'_, PyAny>, ) -> PyResult { - let run: TestRun<'_> = prepare(py, request)?; + let mut run: TestRun<'_> = prepare(py, request)?; let planning: Bound<'_, PyAny> = write_headers( &run.stream, &run.project, @@ -427,12 +440,14 @@ pub(crate) fn run_scenario_test_command( let mut scenarios: Bound<'_, PyTuple> = selected_scenarios(&pipeline, (&selectors, &exclude, &run.project.project_dir))?; if let Some(base) = base { - scenarios = select_changed( + let since_json: String; + (scenarios, since_json) = select_changed( &run, (&pipeline, scenarios), base, flag(request, "explain")?, )?; + run.since_json = Some(since_json); } if run.local && (flag(request, "sync_snapshots")? || flag(request, "refresh")?) { let synced: LocalSync<'_> = sync_local_snapshots(request, &run, (&pipeline, &scenarios))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 41a0612479..7189864a3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -34,7 +34,7 @@ use crate::bindings::_helpers::change_selection::base_snapshot::{ BaseComparison, base_comparison, base_variant, }; use crate::bindings::_helpers::change_selection::selection::{ - ChangedProject, changed_project, tests_and_models, + ChangedProject, changed_project, tests_and_models, with_since_section, }; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; @@ -61,6 +61,13 @@ const PASS_OUTCOME: &str = "pass"; /// Why `--since` selected each test, by inspection label. type SelectionReasons = HashMap>; + +/// What a `--since` selection kept: the narrowed pipeline, reasons by label, and its JSON section. +struct ChangedSelection<'py> { + pipeline: Bound<'py, PyAny>, + reasons: SelectionReasons, + json: String, +} const EXPECT_LABEL: &str = "expect"; const UNKNOWN_GROUP: &str = "(unknown)"; const EXECUTION_COMMAND: &str = "sqb test"; @@ -433,14 +440,14 @@ fn inspect( /// Keep the planned tests that execute something changed, with each one's reasons by label. fn select_changed<'py>( - (pipeline, stream): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (pipeline, stream, discovered): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), project_dir: &std::path::Path, base: BaseComparison, explain: bool, -) -> PyResult<(Bound<'py, PyAny>, SelectionReasons)> { +) -> PyResult> { let py: Python<'py> = pipeline.py(); let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; - let changed: ChangedProject = changed_project(&project, project_dir, base)?; + let changed: ChangedProject = changed_project((&project, discovered), project_dir, base)?; let (tests, models) = tests_and_models(&project)?; let plan_output: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; let mut kept: Vec> = Vec::new(); @@ -468,10 +475,11 @@ fn select_changed<'py>( &plan_output, &[("test_entries", PyTuple::new(py, kept)?.into_any())], )?; - Ok(( - replaced(pipeline, &[("plan_output", plan_output)])?, + Ok(ChangedSelection { + pipeline: replaced(pipeline, &[("plan_output", plan_output)])?, reasons, - )) + json: changed.json((total, &selected)), + }) } /// Write the ready header and run the tests with nested progress; returns the results. @@ -615,7 +623,7 @@ pub(crate) fn timeout_record<'py>( fn complete( request: &Bound<'_, PyAny>, invocation: &Bound<'_, PyAny>, - results: &[Bound<'_, PyAny>], + (results, since_json): (&[Bound<'_, PyAny>], Option<&str>), ) -> PyResult { let py: Python<'_> = request.py(); let mut failures: Vec = Vec::new(); @@ -647,8 +655,11 @@ fn complete( )?; stream.call_method0("flush")?; let tuple: Bound<'_, PyAny> = PyTuple::new(py, results)?.into_any(); - let payload: Bound<'_, PyAny> = - PyString::new(py, &format_test_execution_json(py, tuple)?).into_any(); + let payload: Bound<'_, PyAny> = PyString::new( + py, + &with_since_section(py, format_test_execution_json(py, tuple)?, since_json)?, + ) + .into_any(); write_execution_json_output( py, payload, @@ -684,13 +695,20 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py }; let mut pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; let mut reasons: HashMap> = HashMap::new(); + let mut since_json: Option = None; if let Some(base) = base { - (pipeline, reasons) = select_changed( - (&pipeline, &stream), + let selection: ChangedSelection<'_> = select_changed( + ( + &pipeline, + &stream, + &native_field(&invocation, "discovered_inputs")?, + ), &project_dir.extract::()?, base, native_field(request, "explain")?.is_truthy()?, )?; + (pipeline, reasons) = (selection.pipeline, selection.reasons); + since_json = Some(selection.json); } if inspect_only { return inspect(&stream, &pipeline.getattr("plan_output")?, &reasons); @@ -728,7 +746,7 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py &pipeline.getattr("plan_output")?, &PyTuple::new(py, &results)?.into_any(), )?; - complete(request, &invocation, &results) + complete(request, &invocation, (&results, since_json.as_deref())) } /// Add source and safe typed values only for parameterized test rows. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md index 96a9f1905b..2d64161902 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -50,13 +50,15 @@ sqb scenario test [flags] `--since ` applies the rules of [`sqb test --since`](test.md#run-only-affected-tests) to scenarios. A scenario is selected when its own file (its definition and fixtures) or its captured local snapshots changed, or when any node it actually executes changed: the models it -builds, the seeds it loads, the functions they call, and the macros, enums and constants they and -the scenario use. A model the scenario replaces with a `__ref__` fixture, or a seed it replaces -with a `__seed__` fixture, is not executed, so its changes do not select the scenario. - -`--select` and `--exclude` narrow the affected scenarios further. Both commands share the cached -base compile, so `sqb test --since origin/main` and `sqb scenario test --since origin/main` in one -CI job resolve the same changed set: +builds, the seeds and sources it reads, the functions they call, the audits attached to its models, +and the macros, enums and constants they and the scenario use. A model the scenario replaces with a +`__ref__` fixture, a source it replaces with a `__source__` fixture, or a seed it replaces with a +`__seed__` fixture, is not executed, so its changes do not select the scenario. + +`--select` and `--exclude` narrow the affected scenarios further. With `--json`, the document gains +the same `since` section as `sqb test`, listing each selected scenario with its reasons. Both +commands share the cached base compile, so `sqb test --since origin/main` and +`sqb scenario test --since origin/main` in one CI job resolve the same changed set: ```bash sqb test --since "origin/$TARGET_BRANCH" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 9f19afdd4f..fc27c5c08b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -68,19 +68,30 @@ running the whole suite. SQLBuild compiles the project as it was at the merge ba uncommitted edits included. A test is selected when: - its own file changed or is new, or -- any node it actually executes changed: the models in its chain, the seeds and functions they - read, and the macros, enums and constants they use. The resolution is the one `--inspect` - shows, so a test that mocks a changed model, seed or macro is not selected, while a downstream - test that really runs it is. +- any node it actually executes changed: the models in its chain, the seeds, sources and + functions they read, and the macros, enums, constants and audits they use. The resolution is the + one `--inspect` shows, so a test that mocks a changed model, source, seed or macro is not + selected, while a downstream test that really runs it is. A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with -formatting and comments ignored, the header configuration, columns and contract, attached audits, -and every macro, enum, constant, function and seed the node uses, resolved through -[scopes](../concepts/declaration-scopes.md). Descriptions and `meta` do not count. A macro counts as -changed when the Python file that defines it changes, and an enum, constant or audit when its -declaration file changes other than in formatting and comments. Added nodes count as changed. A model renamed with -`migrate_from` is compared with its old definition, so a pure rename selects nothing. Deleted -nodes select nothing and are listed in the summary. +formatting and comments ignored, the resolved configuration (including what `[defaults]` and +`[path_defaults]` contribute), columns and contract, attached audits, the values of the project +variables the model reads with `@@name`, and every macro, enum, constant, audit, function and seed +the node uses, resolved through [scopes](../concepts/declaration-scopes.md). A change to the +`[settings]` that shape compiled models counts as a change to every model. Descriptions do not +count; `meta` does. + +A macro counts as changed when the Python file that defines it changes, or any project module it +imports, such as a helper under `python/`; when that code reads `ctx.vars`, a change to any project +variable counts too. An enum, constant or audit counts as changed when its declaration file +changes other than in formatting and comments. A seed counts as changed with its data or column +types, a source with its declaration or inline expression, and a function with its definition. +Loaders, tasks, assets and checks are compared by their Python modules and the project modules +they import; they appear in the change summary but no unit test runs them. + +Added nodes count as changed. A model renamed with `migrate_from` is compared with its old +definition, so a pure rename selects nothing, while a rename that also edits the model counts as a +change. Deleted nodes select nothing and are listed in the summary. ```bash # Tests affected by this branch @@ -113,6 +124,29 @@ The base compile is cached per merge-base commit under `target/since/`, so `sqb `sqb scenario test --since` in the same CI job compile the base once and compare against the same changed set. When no test is affected, the command says so and exits successfully. +With `--json`, the progress and selection lines go to stderr, and the JSON document on stdout gains a +`since` section: the ref and merge base, the `changed`, `renamed` and `deleted` nodes, the number +of candidate tests, and each selected test with its reasons: + +```json +"since": { + "ref": "origin/main", + "merge_base": "1a2b3c4d5e...", + "changed": [ + {"kind": "model", "name": "stg_orders", "added": false, "aspects": ["sql"], "renamed_from": null} + ], + "renamed": [], + "deleted": [{"kind": "model", "name": "legacy_orders"}], + "candidate_count": 9, + "selected": [ + {"name": "orders_rollup", "reasons": ["executes stg_orders (model SQL changed)"]} + ] +} +``` + +The base revision must compile. When it does not, `--since` fails with the compile error instead +of guessing; compare with a later ref whose project compiles, or run without `--since`. + `` is any git ref or commit, such as the merge request's target branch from a CI variable. `--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach the merge base, fails with an error that says what to fetch, such as `git fetch --unshallow` or a diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 235ce6538b..9f80f5a422 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -533,6 +533,8 @@ class SinceJsonOutputE2ETestCase: expected_stderr_fragments: tuple[str, ...] expected_test_names: tuple[str, ...] unexpected_stdout_fragments: tuple[str, ...] + # The `since` section's selected items and their reasons, by name. + expected_selected: Mapping[str, tuple[str, ...]] @dataclass(frozen=True) @@ -545,3 +547,15 @@ class SinceShallowCloneE2ETestCase: command: tuple[str, ...] expected_exit_code: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SinceBrokenBaseE2ETestCase: + """A base revision that does not compile, and the error `--since` must report.""" + + description: str + base_files: Mapping[str, str] + deletions: tuple[str, ...] + command: tuple[str, ...] + expected_exit_code: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index b78815fe2c..f8b93628bc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -2223,6 +2223,113 @@ def build_since_project_files() -> dict[str, str]: } +def build_since_indirect_files() -> dict[str, str]: + """Build files that reach tests indirectly: variables, defaults, helpers, seeds and functions.""" + + base: dict[str, str] = build_since_project_files() + return { + "sqlbuild_project.toml": base["sqlbuild_project.toml"] + "\n[vars]\nmin_amount = \"100\"\n", + "python/_rates.py": 'DIVISOR = "100"\n', + "models/_macros/money.py": ( + "from python._rates import DIVISOR\n\n\n" + "def cents_to_dollars(expression: str) -> str:\n" + ' return f"CAST({expression} AS DOUBLE) / {DIVISOR}"\n' + ), + "audits/generic/at_least.sql": ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @column < @minimum\n' + ), + "models/order_totals.sql": ( + "MODEL (\n description 'Order totals in dollars.',\n" + " audits [at_least (column amount, minimum 0)],\n);\n\n" + 'SELECT order_id, @cents_to_dollars("amount_cents") AS amount\n' + 'FROM __ref("stg_orders")\n' + ), + "models/big_orders.sql": ( + "MODEL (description 'Large orders.');\n\n" + 'SELECT order_id FROM __ref("stg_orders") WHERE amount_cents >= @@min_amount\n' + ), + "models/marts/order_counts.sql": ( + "MODEL (description 'Order counts.');\n\n" + 'SELECT COUNT(*) AS order_count FROM __ref("stg_orders")\n' + ), + "seeds/regions.csv": "region_id,region_name\n1,north\n", + "seeds/regions.yml": ( + "seeds:\n - name: regions\n description: Sales regions.\n columns:\n" + " - name: region_id\n type: INTEGER\n" + " - name: region_name\n type: VARCHAR\n" + ), + "functions/sql/is_open.sql": ( + "FUNCTION (\n description 'Whether an order is open.',\n arguments (order_status VARCHAR),\n returns BOOLEAN,\n);\n\n" + "order_status = 'open'\n" + ), + "models/regional_orders.sql": ( + "MODEL (description 'Orders with their region.');\n\n" + "SELECT o.order_id, r.region_id\n" + 'FROM __ref("stg_orders") o CROSS JOIN __seed("regions") r\n' + ), + "models/order_flags.sql": ( + "MODEL (description 'Whether each order is open.');\n\n" + 'SELECT order_id, __udf("is_open")(status) AS is_open FROM __ref("stg_orders")\n' + ), + "models/order_archive.sql": ( + "MODEL (description 'Archived orders.');\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/archive_counts.sql": ( + "MODEL (description 'Archived order count.');\n\n" + 'SELECT COUNT(*) AS archived FROM __ref("order_archive")\n' + ), + "tests/unit/test_archive_counts.sql": ( + "TEST ();\n\nWITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__archive_counts AS (SELECT 1 AS archived)\nSELECT 1\n" + ), + "python/tasks/report.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task()\ndef report_orders(ctx):\n" + ' """Report orders."""\n\n' + " return ctx.result(payload={})\n" + ), + "tests/unit/test_big_orders.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__big_orders AS (SELECT 1 AS order_id)\nSELECT 1\n" + ), + "tests/unit/test_order_counts.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_counts AS (SELECT 1 AS order_count)\nSELECT 1\n" + ), + "tests/unit/test_regional_orders.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__seed__regions AS (SELECT 1 AS region_id, 'north' AS region_name),\n" + "__expected__regional_orders AS (SELECT 1 AS order_id, 1 AS region_id)\nSELECT 1\n" + ), + "tests/unit/test_order_flags.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_flags AS (SELECT 1 AS order_id, TRUE AS is_open)\nSELECT 1\n" + ), + "tests/scenarios/regional_orders_end_to_end.sql": ( + "SCENARIO (description 'Regional orders from raw orders and the region seed');\n\n" + "WITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__regional_orders AS (SELECT 1 AS order_id, 1 AS region_id)\nSELECT 1\n" + ), + "tests/unit/test_order_totals_from_source.sql": ( + "TEST ();\n\nWITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents\n" + "),\n" + "__expected__order_totals AS (SELECT 1 AS order_id, 2.5 AS amount)\nSELECT 1\n" + ), + } + + _GIT_IDENTITY: dict[str, str] = { "GIT_AUTHOR_NAME": "Example", "GIT_AUTHOR_EMAIL": "dev@example.com", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index 3872eb2958..7de5423d0e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -7,12 +7,14 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + SinceBrokenBaseE2ETestCase, SinceJsonOutputE2ETestCase, SinceSelectionE2ETestCase, SinceShallowCloneE2ETestCase, SinceSharedBaseE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import ( + build_since_indirect_files, build_since_project_files, delete_project_files, prepare_since_repository, @@ -191,6 +193,262 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( assert fragment not in output, output +_INDIRECT: dict[str, str] = build_since_indirect_files() +_PROJECT_CONFIG: str = "sqlbuild_project.toml" +_SCENARIO_COMMAND: tuple[str, ...] = ("--no-color", "scenario", "test", "--since", "main") +_EXPLAINED_TESTS: tuple[str, ...] = (*_TEST_COMMAND, "--explain") +_EXPLAINED_SCENARIOS: tuple[str, ...] = (*_SCENARIO_COMMAND, "--explain") + + +@pytest.mark.parametrize( + "test_case", + ( + SinceSelectionE2ETestCase( + description="a project variable a model reads selects that model's test", + edits={ + _PROJECT_CONFIG: _INDIRECT[_PROJECT_CONFIG].replace( + 'min_amount = "100"', 'min_amount = "200"' + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 model changed\n", + "Selected 1 of 9 tests", + "test_big_orders\n executes big_orders (model vars changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + ), + SinceSelectionE2ETestCase( + description="a path default compares the resolved model configuration", + edits={ + _PROJECT_CONFIG: _INDIRECT[_PROJECT_CONFIG] + + '\n[path_defaults.marts]\nmaterialized = "view"\n' + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Selected 1 of 9 tests", + "test_order_counts\n executes order_counts (model config changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + ), + SinceSelectionE2ETestCase( + description="a helper module a macro imports changes the macro", + edits={"python/_rates.py": 'DIVISOR = "100.0"\n'}, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 macro changed\n", + "Selected 2 of 9 tests", + "test_order_totals\n executes macro cents_to_dollars (changed)", + "test_order_totals_from_source\n executes macro cents_to_dollars (changed)", + "PASS=2 FAIL=0 TOTAL=2", + ), + ), + SinceSelectionE2ETestCase( + description="a function body change selects the test that calls it", + edits={ + "functions/sql/is_open.sql": _INDIRECT["functions/sql/is_open.sql"].replace( + "order_status = 'open'", "LOWER(order_status) = 'open'" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 function changed\n", + "Selected 1 of 9 tests", + "test_order_flags\n executes function is_open (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="an audit definition change selects tests of models it is attached to", + edits={ + "audits/generic/at_least.sql": _INDIRECT["audits/generic/at_least.sql"].replace( + "@column < @minimum", "@column <= @minimum" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 audit changed\n", + "Selected 2 of 9 tests", + "test_order_totals\n executes audit at_least (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="an upstream change selects a test that runs it but not one that mocks it", + edits={_STAGING: _EDITED_STAGING}, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "test_order_totals_from_source\n executes stg_orders (model SQL changed)", + "test_archive_counts\n executes stg_orders (model SQL changed)", + ), + unexpected_fragments=(" test_order_totals\n", "test_big_orders"), + ), + SinceSelectionE2ETestCase( + description="a rename through migrate_from that also changes the SQL is a change", + edits={ + "models/order_history.sql": _INDIRECT["models/order_archive.sql"] + .replace("MODEL (", "MODEL (migrate_from order_archive, ") + .replace('__ref("stg_orders")', '__ref("stg_orders") WHERE order_id > 0'), + "models/archive_counts.sql": _INDIRECT["models/archive_counts.sql"].replace( + "order_archive", "order_history" + ), + }, + deletions=("models/order_archive.sql",), + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 2 models changed; 1 model renamed", + "test_archive_counts\n" + " executes order_history (renamed from order_archive, model SQL changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("deleted:",), + ), + SinceSelectionE2ETestCase( + description="seed data a test mocks selects nothing", + edits={"seeds/regions.csv": "region_id,region_name\n1,south\n"}, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 seed changed\n", + "No tests are affected by the changes since main", + ), + ), + SinceSelectionE2ETestCase( + description="seed data selects the scenario that loads the seed", + edits={"seeds/regions.csv": "region_id,region_name\n1,south\n"}, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=( + "Selected 1 of 3 scenarios", + "regional_orders_end_to_end\n reads seed regions (changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + ), + SinceSelectionE2ETestCase( + description="a seed column type selects the scenario that loads the seed", + edits={ + "seeds/regions.yml": _INDIRECT["seeds/regions.yml"].replace( + "type: VARCHAR", "type: TEXT" + ) + }, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=("regional_orders_end_to_end\n reads seed regions (changed)",), + ), + SinceSelectionE2ETestCase( + description="an audit definition change selects scenarios that build its model", + edits={ + "audits/generic/at_least.sql": _INDIRECT["audits/generic/at_least.sql"].replace( + "@column < @minimum", "@column <= @minimum" + ) + }, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=( + "Selected 1 of 3 scenarios", + "order_totals_end_to_end\n executes audit at_least (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a scenario fixture edit selects that scenario", + edits={ + "tests/scenarios/regional_orders_end_to_end.sql": _INDIRECT[ + "tests/scenarios/regional_orders_end_to_end.sql" + ].replace("250 AS amount_cents", "300 AS amount_cents") + }, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=("regional_orders_end_to_end\n scenario file changed",), + unexpected_fragments=("order_totals_end_to_end",), + ), + SinceSelectionE2ETestCase( + description="a source definition that every test mocks selects nothing", + edits={ + "sources/raw.yml": _FILES["sources/raw.yml"].replace( + "amount_cents\n type: INTEGER", "amount_cents\n type: BIGINT" + ) + }, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 source changed\n", + "No tests are affected by the changes since main", + ), + ), + SinceSelectionE2ETestCase( + description="a Python node change is listed", + edits={ + "python/tasks/report.py": _INDIRECT["python/tasks/report.py"].replace( + "payload={}", 'payload={"orders": 1}' + ) + }, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=("Changed since main: 1 task changed\n",), + ), + ), + ids=lambda case: case.description, +) +def test_given_indirect_changes_since_ref_when_testing_then_selects_what_executes_them( + tmp_path: Path, test_case: SinceSelectionE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files=_INDIRECT) + write_project_files(project_dir=project, files=test_case.edits) + delete_project_files(project_dir=project, relative_paths=test_case.deletions) + run_git(repository=project, arguments=test_case.git_after_edits) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output + for fragment in test_case.unexpected_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceBrokenBaseE2ETestCase( + description="a base revision that does not compile fails and says what to do", + base_files={ + "models/broken.sql": ( + "MODEL (description 'Reads a model that does not exist.');\n\n" + 'SELECT order_id FROM __ref("missing_orders")\n' + ) + }, + deletions=("models/broken.sql",), + command=_TEST_COMMAND, + expected_exit_code=1, + expected_fragments=( + "error[C484]: Cannot compile the project at merge base ", + "missing_orders", + "Compare with a later ref whose project compiles, or run without --since", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_base_that_does_not_compile_when_testing_since_then_fails_with_guidance( + tmp_path: Path, test_case: SinceBrokenBaseE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files=test_case.base_files) + delete_project_files(project_dir=project, relative_paths=test_case.deletions) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output + + @pytest.mark.parametrize( "test_case", ( @@ -245,6 +503,18 @@ def test_given_test_and_scenario_runs_when_comparing_then_both_share_one_base_co expected_stderr_fragments=("Comparing with main", "Selected 1 of 3 tests"), expected_test_names=("test_stg_orders",), unexpected_stdout_fragments=("test_order_totals", "Comparing with"), + expected_selected={"test_stg_orders": ("executes stg_orders (model SQL changed)",)}, + ), + SinceJsonOutputE2ETestCase( + description="a scenario run lists each selected scenario's reasons", + edits={_STAGING: _EDITED_STAGING}, + command=("--no-color", "scenario", "test", "--since", "main", "--json"), + expected_stderr_fragments=("Comparing with main", "Selected 1 of 2 scenarios"), + expected_test_names=("order_totals_end_to_end",), + unexpected_stdout_fragments=("open_orders_from_fixture", "Comparing with"), + expected_selected={ + "order_totals_end_to_end": ("executes stg_orders (model SQL changed)",) + }, ), ), ids=lambda case: case.description, @@ -265,6 +535,12 @@ def test_given_json_output_when_testing_since_ref_then_stdout_stays_machine_read assert f'"{name}"' in document, document for fragment in test_case.unexpected_stdout_fragments: assert fragment not in result.stdout, result.stdout + reference: str = json.loads(result.stdout)["since"]["ref"] + selected: list[dict[str, str | list[str]]] = json.loads(result.stdout)["since"]["selected"] + assert reference == "main", result.stdout + assert { + str(item["name"]): tuple(item["reasons"]) for item in selected + } == test_case.expected_selected, result.stdout @pytest.mark.parametrize( diff --git a/website/src/content/docs/docs/cli/scenario.mdx b/website/src/content/docs/docs/cli/scenario.mdx index fb6008020f..37c8e9f5db 100644 --- a/website/src/content/docs/docs/cli/scenario.mdx +++ b/website/src/content/docs/docs/cli/scenario.mdx @@ -43,13 +43,15 @@ sqb scenario test [flags] `--since ` applies the rules of [`sqb test --since`](/docs/cli/test#run-only-affected-tests) to scenarios. A scenario is selected when its own file (its definition and fixtures) or its captured local snapshots changed, or when any node it actually executes changed: the models it -builds, the seeds it loads, the functions they call, and the macros, enums and constants they and -the scenario use. A model the scenario replaces with a `__ref__` fixture, or a seed it replaces -with a `__seed__` fixture, is not executed, so its changes do not select the scenario. - -`--select` and `--exclude` narrow the affected scenarios further. Both commands share the cached -base compile, so `sqb test --since origin/main` and `sqb scenario test --since origin/main` in one -CI job resolve the same changed set: +builds, the seeds and sources it reads, the functions they call, the audits attached to its models, +and the macros, enums and constants they and the scenario use. A model the scenario replaces with a +`__ref__` fixture, a source it replaces with a `__source__` fixture, or a seed it replaces with a +`__seed__` fixture, is not executed, so its changes do not select the scenario. + +`--select` and `--exclude` narrow the affected scenarios further. With `--json`, the document gains +the same `since` section as `sqb test`, listing each selected scenario with its reasons. Both +commands share the cached base compile, so `sqb test --since origin/main` and +`sqb scenario test --since origin/main` in one CI job resolve the same changed set: ```bash sqb test --since "origin/$TARGET_BRANCH" diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index e5b8394bdc..a319bb890c 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -60,19 +60,30 @@ running the whole suite. SQLBuild compiles the project as it was at the merge ba uncommitted edits included. A test is selected when: - its own file changed or is new, or -- any node it actually executes changed: the models in its chain, the seeds and functions they - read, and the macros, enums and constants they use. The resolution is the one `--inspect` - shows, so a test that mocks a changed model, seed or macro is not selected, while a downstream - test that really runs it is. +- any node it actually executes changed: the models in its chain, the seeds, sources and + functions they read, and the macros, enums, constants and audits they use. The resolution is the + one `--inspect` shows, so a test that mocks a changed model, source, seed or macro is not + selected, while a downstream test that really runs it is. A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with -formatting and comments ignored, the header configuration, columns and contract, attached audits, -and every macro, enum, constant, function and seed the node uses, resolved through -[scopes](/docs/concepts/declaration-scopes). Descriptions and `meta` do not count. A macro counts as -changed when the Python file that defines it changes, and an enum, constant or audit when its -declaration file changes other than in formatting and comments. Added nodes count as changed. A model renamed with -`migrate_from` is compared with its old definition, so a pure rename selects nothing. Deleted -nodes select nothing and are listed in the summary. +formatting and comments ignored, the resolved configuration (including what `[defaults]` and +`[path_defaults]` contribute), columns and contract, attached audits, the values of the project +variables the model reads with `@@name`, and every macro, enum, constant, audit, function and seed +the node uses, resolved through [scopes](/docs/concepts/declaration-scopes). A change to the +`[settings]` that shape compiled models counts as a change to every model. Descriptions do not +count; `meta` does. + +A macro counts as changed when the Python file that defines it changes, or any project module it +imports, such as a helper under `python/`; when that code reads `ctx.vars`, a change to any project +variable counts too. An enum, constant or audit counts as changed when its declaration file +changes other than in formatting and comments. A seed counts as changed with its data or column +types, a source with its declaration or inline expression, and a function with its definition. +Loaders, tasks, assets and checks are compared by their Python modules and the project modules +they import; they appear in the change summary but no unit test runs them. + +Added nodes count as changed. A model renamed with `migrate_from` is compared with its old +definition, so a pure rename selects nothing, while a rename that also edits the model counts as a +change. Deleted nodes select nothing and are listed in the summary. ```bash # Tests affected by this branch @@ -105,6 +116,29 @@ The base compile is cached per merge-base commit under `target/since/`, so `sqb `sqb scenario test --since` in the same CI job compile the base once and compare against the same changed set. When no test is affected, the command says so and exits successfully. +With `--json`, the progress and selection lines go to stderr, and the JSON document on stdout gains a +`since` section: the ref and merge base, the `changed`, `renamed` and `deleted` nodes, the number +of candidate tests, and each selected test with its reasons: + +```json +"since": { + "ref": "origin/main", + "merge_base": "1a2b3c4d5e...", + "changed": [ + {"kind": "model", "name": "stg_orders", "added": false, "aspects": ["sql"], "renamed_from": null} + ], + "renamed": [], + "deleted": [{"kind": "model", "name": "legacy_orders"}], + "candidate_count": 9, + "selected": [ + {"name": "orders_rollup", "reasons": ["executes stg_orders (model SQL changed)"]} + ] +} +``` + +The base revision must compile. When it does not, `--since` fails with the compile error instead +of guessing; compare with a later ref whose project compiles, or run without `--since`. + `` is any git ref or commit, such as the merge request's target branch from a CI variable. `--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach the merge base, fails with an error that says what to fetch, such as `git fetch --unshallow` or a From 9ccac508aa2a8587874da90d51c1de4bae3488c5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:21:02 +0100 Subject: [PATCH 05/10] fix: satisfy policy checks for --since fingerprints --- .../src/rule_evidence/_helpers/paths.rs | 5 +- .../main/project_import_closure.rs | 5 +- .../change_selection/base_snapshot.rs | 7 +- .../change_selection/definition_snapshot.rs | 69 +++++++++---------- .../_helpers/change_selection/fingerprints.rs | 38 +++------- .../_helpers/change_selection/selection.rs | 32 ++++----- .../_helpers/scenario_command/test_command.rs | 1 - .../_helpers/sql_test_command/test_command.rs | 23 +++---- .../cli/commands/main/test/helpers.py | 7 +- .../cli/commands/main/test/test_since.py | 12 ++-- 10 files changed, 84 insertions(+), 115 deletions(-) diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs index c51cf22770..f22f797a7e 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs @@ -65,10 +65,7 @@ pub(crate) fn package_inits(path: &Path, rules_root: &Path) -> Vec { inits } -/// Every module under `rules_root` one import statement can execute. -/// -/// Absolute imports resolve from `project`; with `package` set, only imports of that top-level -/// package count. +/// Modules under `rules_root` one import executes, from `project` and only `package` when set. pub(crate) fn local_import_paths( node: &CpythonNode<'_>, source_path: &Path, diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs index 47b2c75e73..6a74ff0773 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/project_import_closure.rs @@ -5,10 +5,7 @@ use std::path::{Path, PathBuf}; use crate::rule_evidence::_helpers::closure::module_closure; use crate::rule_evidence::_helpers::paths::resolve; -/// Every module and package initializer under `project_dir` that `source_path` executes, sorted. -/// -/// Imports resolve from the project directory, the way project Python imports `macros.currency` -/// or `python._helpers`; standard-library and installed modules are not part of the closure. +/// Project modules and package initializers `source_path` imports, resolved from `project_dir`. pub fn project_import_closure( source_path: &Path, project_dir: &Path, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index b757b7f647..3b895666f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -195,8 +195,11 @@ pub(crate) fn base_comparison<'py>( )?; let vars: &str = &variant.vars_json; let target: Option<&str> = variant.selected_target.as_deref(); - let local_config: String = - std::fs::read_to_string(directory.join(LOCAL_CONFIG_FILE)).unwrap_or_default(); + let local_config: String = match std::fs::read_to_string(directory.join(LOCAL_CONFIG_FILE)) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(io_failed(py, &base, &error)), + }; let key: String = definition_digest(&[ NATIVE_BUILD_IDENTITY, target.unwrap_or_default(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 5eca035147..d12a8a4eb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -124,7 +124,7 @@ fn declaration_file_digest(fingerprints: &Fingerprinting, path: &str) -> String } } -/// Scope declarations: a macro by its facts and Python imports, any other by its declaration file. +/// Scope declarations by their file; a Python macro also by its facts and macro dependencies. fn declaration_nodes( project: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, @@ -141,10 +141,8 @@ fn declaration_nodes( let name: String = native_field(&identity, "name")?.extract()?; let path: String = native_field(&declaration, "path")?.extract()?; let macro_facts: Bound<'_, PyAny> = native_field(&declaration, "macro")?; - let (digest, uses): (String, Vec) = if kind == MACRO_KIND { - if macro_facts.is_none() { - continue; - } + let (digest, uses): (String, Vec) = if kind == MACRO_KIND && !macro_facts.is_none() + { let dependencies: Vec = native_field(¯o_facts, "dependencies")?.extract()?; ( @@ -193,8 +191,7 @@ fn config_digest(fingerprints: &Fingerprinting, config: &Bound<'_, PyAny>) -> Py Ok(definition_digest(&[&encoded.to_string()])) } -/// A model by its SQL before macro expansion, resolved configuration, columns, contract, audits -/// and the variables its file reads. +/// A model by its authored SQL, resolved configuration, columns, audits and variables it reads. fn model_node( model: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, @@ -274,39 +271,37 @@ fn source_node( fn python_nodes( discovered: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, - nodes: &mut BTreeMap, -) -> PyResult<()> { +) -> PyResult> { + let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); let collections = PYTHON_NODE_COLLECTIONS .iter() .zip(PYTHON_NODE_KINDS) - .chain([(&"materialization_files", MATERIALIZATION_KIND)]); + .map(|(collection, kind)| (*collection, kind)) + .chain([("materialization_files", MATERIALIZATION_KIND)]); for (collection, kind) in collections { - let Ok(items) = discovered.getattr(*collection) else { - continue; - }; - for item in items.try_iter()? { + for item in native_field(discovered, collection)?.try_iter()? { let item: Bound<'_, PyAny> = item?; - let name: String = item.getattr("name")?.extract()?; - let path: PathBuf = item.getattr("file_path")?.extract()?; - nodes.insert( + let name: String = native_field(&item, "name")?.extract()?; + let path: PathBuf = native_field(&item, "file_path")?.extract()?; + nodes.push(( node_key(kind, &name), node( kind, &name, vec![(DEFINITION_ASPECT, fingerprints.python_module_digest(&path))], ), - ); + )); } } - Ok(()) + Ok(nodes) } /// User-defined and table functions by their definitions, Python ones with their imports. fn function_nodes( project: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, - nodes: &mut BTreeMap, -) -> PyResult<()> { +) -> PyResult> { + let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); let functions: Bound<'_, PyAny> = project_reads::functions(project)?; let hashes: Bound<'_, PyDict> = function_local_hashes(&functions, None)?; for function in functions.try_iter()? { @@ -343,17 +338,17 @@ fn function_nodes( )); } definition.uses.push(node_key(SETTINGS_KIND, SETTINGS_NAME)); - nodes.insert(node_key(&kind, &name), definition); + nodes.push((node_key(&kind, &name), definition)); } - Ok(()) + Ok(nodes) } /// Each unit test file by its text and the variables it reads. fn test_file_nodes( project: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, - nodes: &mut BTreeMap, -) -> PyResult<()> { +) -> PyResult> { + let mut nodes: BTreeMap = BTreeMap::new(); for test in project_reads::sql_tests(project)?.try_iter()? { let file: Bound<'_, PyAny> = native_field(&test?, "test_file")?; let path: String = text_of(&native_field(&file, "relative_path")?)?; @@ -371,7 +366,7 @@ fn test_file_nodes( } nodes.insert(key, node(TEST_FILE_KIND, &path, aspects)); } - Ok(()) + Ok(nodes) } /// Each scenario by its file, captured snapshots and the variables it reads. @@ -379,8 +374,8 @@ fn scenario_nodes( project: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, uses: &BTreeMap>, - nodes: &mut BTreeMap, -) -> PyResult<()> { +) -> PyResult> { + let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); let snapshot_root: PathBuf = SCENARIO_SNAPSHOT_ROOT .iter() .fold(fingerprints.project_dir.clone(), |path, part| { @@ -398,16 +393,20 @@ fn scenario_nodes( if let Some(digest) = directory_digest(&snapshot_root.join(&name)) { aspects.push((SNAPSHOTS_ASPECT, digest)); } - let contents: String = std::fs::read_to_string(&file).unwrap_or_default(); + let contents: String = match std::fs::read_to_string(&file) { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(error.into()), + }; if let Some(digest) = fingerprints.referenced_vars_digest(&contents) { aspects.push((VARS_ASPECT, digest)); } let key: String = node_key(SCENARIO_KIND, &name); let mut definition: NodeDefinition = node(SCENARIO_KIND, &name, aspects); definition.uses = uses.get(&key).into_iter().flatten().cloned().collect(); - nodes.insert(key, definition); + nodes.push((key, definition)); } - Ok(()) + Ok(nodes) } /// The definition snapshot of `project`, discovered as `discovered` under `project_dir`. @@ -454,10 +453,10 @@ pub(crate) fn definition_snapshot( node(SEED_KIND, &name, vec![(DEFINITION_ASPECT, version_hash)]), ); } - function_nodes(project, &fingerprints, &mut nodes)?; - python_nodes(discovered, &fingerprints, &mut nodes)?; - test_file_nodes(project, &fingerprints, &mut nodes)?; - scenario_nodes(project, &fingerprints, &uses, &mut nodes)?; + nodes.extend(function_nodes(project, &fingerprints)?); + nodes.extend(python_nodes(discovered, &fingerprints)?); + nodes.extend(test_file_nodes(project, &fingerprints)?); + nodes.extend(scenario_nodes(project, &fingerprints, &uses)?); Ok(DefinitionSnapshot { format: DEFINITION_SNAPSHOT_FORMAT, nodes, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs index 67566db87f..0003170939 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::records::models::NativeRecord; /// Fields that document a definition without changing what it does. const DOCUMENTATION_FIELDS: [&str; 2] = ["description", "location"]; -const DATACLASS_FIELDS: &str = "__dataclass_fields__"; /// Python source that mentions this can read project variables through `ctx.vars`. const VARS_ATTRIBUTE: &str = "vars"; const VARIABLE_SIGIL: &str = "@@"; @@ -43,12 +42,14 @@ impl Fingerprinting { ) -> PyResult { let py: Python<'_> = project.py(); let mut roots: Vec = Vec::new(); - for root in [ - std::fs::canonicalize(project_dir).ok(), - Some(project_dir.to_path_buf()), - ] - .into_iter() - .flatten() + let canonical: Option = match std::fs::canonicalize(project_dir) { + Ok(resolved) => Some(resolved), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + for root in canonical + .into_iter() + .chain(std::iter::once(project_dir.to_path_buf())) { let text: String = format!("{}/", root.to_string_lossy().replace('\\', "/")); if !roots.contains(&text) { @@ -131,25 +132,6 @@ impl Fingerprinting { } return Ok(Value::Object(entries.into_iter().collect())); } - if let Ok(declared) = value.getattr(DATACLASS_FIELDS) - && let Ok(declared) = declared.cast::() - { - let mut fields: Map = Map::new(); - fields.insert( - "record".to_owned(), - Value::from(value.get_type().name()?.to_string()), - ); - for name in declared.keys() { - let name: String = name.extract()?; - if DOCUMENTATION_FIELDS.contains(&name.as_str()) || skipped.contains(&name.as_str()) - { - continue; - } - let item: Value = self.canonical(&value.getattr(name.as_str())?, skipped)?; - fields.insert(name, item); - } - return Ok(Value::Object(fields)); - } let text: String = value.repr()?.to_string(); if text.contains(" at 0x") { return Ok(Value::from(value.get_type().name()?.to_string())); @@ -188,9 +170,7 @@ impl Fingerprinting { (!names.is_empty()).then(|| self.vars_digest(names.iter())) } - /// The digest of a Python file and every project module it imports, by relative path. - /// - /// Code that mentions `vars` may read any project variable, so its digest covers them all. + /// A Python file and its imported project modules; code mentioning `vars` covers every variable. pub(crate) fn python_module_digest(&self, path: &Path) -> String { let source: PathBuf = if path.is_absolute() { path.to_path_buf() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 7869159025..920d1974dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -3,8 +3,10 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::Path; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::Value; use sqlbuild_analysis::definition_changes::constants::{ MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, TEST_FILE_KIND, @@ -31,7 +33,6 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::scenario_command::setup::{flush, write}; const MACRO_TEST_MODE: &str = "macro"; -const JSON_MODULE: &str = "json"; /// The JSON document key of the changes and selection of a `--since` run. const SINCE_SECTION: &str = "since"; /// The relation call a test or scenario file reads a source with. @@ -269,23 +270,22 @@ impl ChangedProject { } /// `payload`, a JSON document, with the `since` section added when the run used `--since`. -pub(crate) fn with_since_section( - py: Python<'_>, - payload: String, - since: Option<&str>, -) -> PyResult { +pub(crate) fn with_since_section(payload: String, since: Option<&str>) -> PyResult { let Some(since) = since else { return Ok(payload); }; - let json: Bound<'_, PyModule> = py.import(JSON_MODULE)?; - let document: Bound<'_, PyAny> = json.call_method1("loads", (payload,))?; - document.set_item(SINCE_SECTION, json.call_method1("loads", (since,))?)?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("indent", 2)?; - let text: String = json - .call_method("dumps", (document,), Some(&options))? - .extract()?; - Ok(format!("{text}\n")) + let invalid = |error: serde_json::Error| PyValueError::new_err(error.to_string()); + let mut document: Value = serde_json::from_str(&payload).map_err(invalid)?; + if let Value::Object(fields) = &mut document { + fields.insert( + SINCE_SECTION.to_owned(), + serde_json::from_str(since).map_err(invalid)?, + ); + } + Ok(format!( + "{}\n", + serde_json::to_string_pretty(&document).map_err(invalid)? + )) } /// The project's compiled tests and models by name. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index c4bbb4c2a3..0f4c8df59c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -167,7 +167,6 @@ fn finish( let payload: Bound<'_, PyAny> = PyString::new( py, &with_since_section( - py, format_scenario_execution_json( py, Some(keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 7189864a3f..aff77775db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -61,13 +61,8 @@ const PASS_OUTCOME: &str = "pass"; /// Why `--since` selected each test, by inspection label. type SelectionReasons = HashMap>; - /// What a `--since` selection kept: the narrowed pipeline, reasons by label, and its JSON section. -struct ChangedSelection<'py> { - pipeline: Bound<'py, PyAny>, - reasons: SelectionReasons, - json: String, -} +type ChangedSelection<'py> = (Bound<'py, PyAny>, SelectionReasons, String); const EXPECT_LABEL: &str = "expect"; const UNKNOWN_GROUP: &str = "(unknown)"; const EXECUTION_COMMAND: &str = "sqb test"; @@ -475,11 +470,11 @@ fn select_changed<'py>( &plan_output, &[("test_entries", PyTuple::new(py, kept)?.into_any())], )?; - Ok(ChangedSelection { - pipeline: replaced(pipeline, &[("plan_output", plan_output)])?, + Ok(( + replaced(pipeline, &[("plan_output", plan_output)])?, reasons, - json: changed.json((total, &selected)), - }) + changed.json((total, &selected)), + )) } /// Write the ready header and run the tests with nested progress; returns the results. @@ -657,7 +652,7 @@ fn complete( let tuple: Bound<'_, PyAny> = PyTuple::new(py, results)?.into_any(); let payload: Bound<'_, PyAny> = PyString::new( py, - &with_since_section(py, format_test_execution_json(py, tuple)?, since_json)?, + &with_since_section(format_test_execution_json(py, tuple)?, since_json)?, ) .into_any(); write_execution_json_output( @@ -697,7 +692,8 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py let mut reasons: HashMap> = HashMap::new(); let mut since_json: Option = None; if let Some(base) = base { - let selection: ChangedSelection<'_> = select_changed( + let json: String; + (pipeline, reasons, json) = select_changed( ( &pipeline, &stream, @@ -707,8 +703,7 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py base, native_field(request, "explain")?.is_truthy()?, )?; - (pipeline, reasons) = (selection.pipeline, selection.reasons); - since_json = Some(selection.json); + since_json = Some(json); } if inspect_only { return inspect(&stream, &pipeline.getattr("plan_output")?, &reasons); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index f8b93628bc..781f8d1c8e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -2224,11 +2224,11 @@ def build_since_project_files() -> dict[str, str]: def build_since_indirect_files() -> dict[str, str]: - """Build files that reach tests indirectly: variables, defaults, helpers, seeds and functions.""" + """Build files that reach tests indirectly: variables, defaults, helpers, seeds, functions.""" base: dict[str, str] = build_since_project_files() return { - "sqlbuild_project.toml": base["sqlbuild_project.toml"] + "\n[vars]\nmin_amount = \"100\"\n", + "sqlbuild_project.toml": base["sqlbuild_project.toml"] + '\n[vars]\nmin_amount = "100"\n', "python/_rates.py": 'DIVISOR = "100"\n', "models/_macros/money.py": ( "from python._rates import DIVISOR\n\n\n" @@ -2259,7 +2259,8 @@ def build_since_indirect_files() -> dict[str, str]: " - name: region_name\n type: VARCHAR\n" ), "functions/sql/is_open.sql": ( - "FUNCTION (\n description 'Whether an order is open.',\n arguments (order_status VARCHAR),\n returns BOOLEAN,\n);\n\n" + "FUNCTION (\n description 'Whether an order is open.',\n" + " arguments (order_status VARCHAR),\n returns BOOLEAN,\n);\n\n" "order_status = 'open'\n" ), "models/regional_orders.sql": ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index 7de5423d0e..3986d6c0fb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -41,6 +41,11 @@ ) _TEST_COMMAND: tuple[str, ...] = ("--no-color", "test", "--since", "main") _BASE_REF_CONFIG: str = _FILES["sqlbuild_project.toml"] + '\n[git]\nbase_ref = "main"\n' +_INDIRECT: dict[str, str] = build_since_indirect_files() +_PROJECT_CONFIG: str = "sqlbuild_project.toml" +_SCENARIO_COMMAND: tuple[str, ...] = ("--no-color", "scenario", "test", "--since", "main") +_EXPLAINED_TESTS: tuple[str, ...] = (*_TEST_COMMAND, "--explain") +_EXPLAINED_SCENARIOS: tuple[str, ...] = (*_SCENARIO_COMMAND, "--explain") @pytest.mark.parametrize( @@ -193,13 +198,6 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( assert fragment not in output, output -_INDIRECT: dict[str, str] = build_since_indirect_files() -_PROJECT_CONFIG: str = "sqlbuild_project.toml" -_SCENARIO_COMMAND: tuple[str, ...] = ("--no-color", "scenario", "test", "--since", "main") -_EXPLAINED_TESTS: tuple[str, ...] = (*_TEST_COMMAND, "--explain") -_EXPLAINED_SCENARIOS: tuple[str, ...] = (*_SCENARIO_COMMAND, "--explain") - - @pytest.mark.parametrize( "test_case", ( From 5d8a4e591744bcf7950ace22e0da72cfbf2b8a38 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:09:18 +0100 Subject: [PATCH 06/10] fix: track macro imports, mock shapes, case tests and project-wide inputs in --since --- .../definition_changes/_helpers/reasons.rs | 32 ++- .../src/definition_changes/constants.rs | 18 +- .../main/selection_reasons.rs | 9 +- .../src/definition_changes/models.rs | 2 + .../src/definition_changes/tests/helpers.rs | 7 + .../tests/test_definition_changes.rs | 28 ++- .../src/git_revisions/_helpers/failures.rs | 18 ++ .../src/git_revisions/_helpers/mod.rs | 1 + .../src/git_revisions/constants.rs | 11 + .../git_revisions/main/dependency_files.rs | 71 ++++++ .../src/git_revisions/main/export_git_tree.rs | 15 +- .../src/git_revisions/main/mod.rs | 1 + .../git_revisions/tests/test_git_revisions.rs | 63 +++++- .../src/git_revisions/tests/test_types.rs | 7 + .../_helpers/change_selection/base_cache.rs | 106 +++++++++ .../_helpers/change_selection/base_compile.rs | 35 ++- .../change_selection/base_snapshot.rs | 57 +++-- .../change_selection/definition_snapshot.rs | 212 ++++++++++++++---- .../change_selection/environment_nodes.rs | 56 +++++ .../_helpers/change_selection/fingerprints.rs | 44 +++- .../bindings/_helpers/change_selection/mod.rs | 2 + .../_helpers/change_selection/selection.rs | 52 ++++- .../sqlbuild/references/docs/cli/scenario.md | 9 +- .../sqlbuild/references/docs/cli/test.md | 53 ++++- .../cli/commands/main/test/_test_types.py | 1 + .../cli/commands/main/test/helpers.py | 53 ++++- .../cli/commands/main/test/test_since.py | 142 +++++++++++- .../src/content/docs/docs/cli/scenario.mdx | 9 +- website/src/content/docs/docs/cli/test.mdx | 53 ++++- 29 files changed, 1037 insertions(+), 130 deletions(-) create mode 100644 crates/sqlbuild-core/src/git_revisions/_helpers/failures.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/main/dependency_files.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs index bffb50b7f3..6cc9143e0a 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/reasons.rs @@ -4,8 +4,12 @@ use std::collections::BTreeSet; use crate::definition_changes::_helpers::aspects::{aspect_label, joined, kind_words}; use crate::definition_changes::constants::{ - CONSTANT_KIND, ENUM_KIND, MODEL_KIND, SEED_KIND, SETTINGS_KIND, SOURCE_KIND, TEST_FILE_KIND, + COLUMNS_ASPECT, CONNECTION_KIND, CONSTANT_KIND, DEPENDENCIES_KIND, ENUM_KIND, MODEL_KIND, + SEED_KIND, SETTINGS_KIND, SOURCE_KIND, TEST_FILE_KIND, }; + +/// Project-wide kinds whose reasons name the kind, not the single node. +const PROJECT_WIDE_KINDS: [&str; 3] = [SETTINGS_KIND, CONNECTION_KIND, DEPENDENCIES_KIND]; use crate::definition_changes::models::{DefinitionSnapshot, ExecutedItem, NodeChange}; fn aspect_phrase(change: &NodeChange) -> String { @@ -62,8 +66,12 @@ pub(crate) fn executed_reason(change: &NodeChange) -> String { /// `executes macro to_utc (changed)`, `uses enum status (changed)`, `uses project settings (changed)`. pub(crate) fn declaration_reason(change: &NodeChange) -> String { - if change.kind == SETTINGS_KIND { - return format!("uses project settings ({})", change_detail(change)); + if PROJECT_WIDE_KINDS.contains(&change.kind.as_str()) { + return format!( + "uses {} ({})", + kind_words(&change.kind).0, + change_detail(change) + ); } if change.kind == SOURCE_KIND { return executed_reason(change); @@ -103,3 +111,21 @@ pub(crate) fn reached_declarations( } reached } + +/// `mocks stg_orders (model columns changed)` when a mocked relation's declared shape changed. +pub(crate) fn mocked_reason(change: &NodeChange) -> Option { + if !change.added && !change.aspects.iter().any(|aspect| aspect == COLUMNS_ASPECT) { + return None; + } + let subject: String = if change.kind == MODEL_KIND { + change.name.clone() + } else { + format!("{} {}", kind_words(&change.kind).0, change.name) + }; + let detail: String = if change.added { + "added".to_owned() + } else { + format!("{} columns changed", kind_words(&change.kind).0) + }; + Some(format!("mocks {subject} ({detail})")) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/constants.rs b/crates/sqlbuild-analysis/src/definition_changes/constants.rs index c8c87d29b3..e6f2c470b1 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/constants.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/constants.rs @@ -1,7 +1,7 @@ //! Node kinds, definition aspects and the snapshot format version. /// The version of the stored snapshot layout and fingerprint scheme; older snapshots are recomputed. -pub const DEFINITION_SNAPSHOT_FORMAT: u32 = 2; +pub const DEFINITION_SNAPSHOT_FORMAT: u32 = 3; pub const MODEL_KIND: &str = "model"; pub const SEED_KIND: &str = "seed"; @@ -15,6 +15,12 @@ pub const SCENARIO_KIND: &str = "scenario"; pub const SOURCE_KIND: &str = "source"; pub const MATERIALIZATION_KIND: &str = "materialization"; pub const SETTINGS_KIND: &str = "settings"; +/// Adapter, connection and target configuration, adapter modules and referenced environment. +pub const CONNECTION_KIND: &str = "connection"; +/// Python dependency manifests and lock files of the project and its repository. +pub const DEPENDENCIES_KIND: &str = "dependencies"; +/// The one node of each project-wide kind. +pub const PROJECT_NODE_NAME: &str = "project"; /// Python nodes by decorator. pub const PYTHON_NODE_KINDS: [&str; 4] = ["loader", "task", "asset", "check"]; @@ -32,7 +38,7 @@ pub const DEFINITION_ASPECT: &str = "definition"; pub const FILE_ASPECT: &str = "file"; /// Captured local snapshots of a scenario. pub const SNAPSHOTS_ASPECT: &str = "snapshots"; -/// Values of the project variables a definition reads with `@@name`. +/// Values of the project variables and environment variables a definition reads. pub const VARS_ASPECT: &str = "vars"; /// How each aspect reads in a reason, in the order reasons list them. @@ -48,8 +54,14 @@ pub(crate) const ASPECT_LABELS: [(&str, &str); 8] = [ ]; /// How each node kind reads in summaries: singular, plural. -pub(crate) const KIND_LABELS: [(&str, &str, &str); 21] = [ +pub(crate) const KIND_LABELS: [(&str, &str, &str); 23] = [ (SETTINGS_KIND, "project settings", "project settings"), + ( + CONNECTION_KIND, + "connection and target configuration", + "connection and target configurations", + ), + (DEPENDENCIES_KIND, "Python environment", "Python environments"), (MODEL_KIND, "model", "models"), (SOURCE_KIND, "source", "sources"), (SEED_KIND, "seed", "seeds"), diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs b/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs index f3f39c5416..6ecafb8286 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/selection_reasons.rs @@ -1,11 +1,11 @@ //! Why a test or scenario is affected by the changes since the base revision. use crate::definition_changes::_helpers::reasons::{ - declaration_reason, executed_reason, own_reason, reached_declarations, + declaration_reason, executed_reason, mocked_reason, own_reason, reached_declarations, }; use crate::definition_changes::models::{DefinitionChanges, DefinitionSnapshot, ExecutedItem}; -/// One reason per changed dependency of `item`, own definition first; empty when nothing changed. +/// One reason per changed dependency of `item`, mocked relations by their declared columns only. pub fn selection_reasons( item: &ExecutedItem, current: &DefinitionSnapshot, @@ -27,6 +27,11 @@ pub fn selection_reasons( push(executed_reason(change)); } } + for key in &item.mocked_relations { + if let Some(reason) = changes.changed.get(key).and_then(mocked_reason) { + push(reason); + } + } for key in reached_declarations(item, current) { if let Some(change) = changes.changed.get(&key) { push(declaration_reason(change)); diff --git a/crates/sqlbuild-analysis/src/definition_changes/models.rs b/crates/sqlbuild-analysis/src/definition_changes/models.rs index 6df7992221..7057e2eefa 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/models.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/models.rs @@ -68,6 +68,8 @@ pub struct ExecutedItem { pub declarations: Vec, /// Keys of declarations the item mocks, whose changes never select it. pub mocked: BTreeSet, + /// Keys of relations the item mocks, whose declared columns still shape the mock. + pub mocked_relations: Vec, } /// One selected test or scenario and why it was selected. diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs index a52c17691d..eee20b0773 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/helpers.rs @@ -121,6 +121,7 @@ pub(super) fn item( .iter() .map(|key| (*key).to_owned()) .collect::>(), + mocked_relations: Vec::new(), } } @@ -132,3 +133,9 @@ pub(super) fn with_node( current.nodes.insert(entry.0, entry.1); current } + +/// `item` mocking the relations `keys`. +pub(super) fn mocking(mut item: ExecutedItem, keys: &[&str]) -> ExecutedItem { + item.mocked_relations = keys.iter().map(|key| (*key).to_owned()).collect(); + item +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs index cf50a1a8ac..bdb9583a5d 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs @@ -11,7 +11,7 @@ use crate::definition_changes::main::selection_report::selection_report; use crate::definition_changes::main::sql_definition_digest::sql_definition_digest; use crate::definition_changes::models::{DefinitionChanges, SelectedItem}; use crate::definition_changes::tests::helpers::{ - base_orders, current_orders, item, keys, node, renamed, snapshot, with_node, + base_orders, current_orders, item, keys, mocking, node, renamed, snapshot, with_node, }; use crate::definition_changes::tests::test_types::{ CompareTestCase, SelectionJsonTestCase, SelectionReasonTestCase, SelectionReportTestCase, @@ -156,6 +156,32 @@ fn given_executed_item_when_explaining_selection_then_lists_each_changed_depende item: item(&["test_file:tests/orders.sql"], &["model:orders"], &[], &[]), expected_reasons: &[], }, + SelectionReasonTestCase { + description: "a mocked model's SQL change does not select the test", + current: current_orders(), + item: mocking( + item(&["test_file:tests/orders.sql"], &["model:orders"], &[], &[]), + &["model:stg_orders"], + ), + expected_reasons: &[], + }, + SelectionReasonTestCase { + description: "a mocked model's declared columns shape the mock", + current: with_node( + base_orders(), + node( + "model", + "stg_orders", + &[("sql", "s1"), ("config", "c1"), ("columns", "k2")], + &["macro:to_utc"], + ), + ), + item: mocking( + item(&["test_file:tests/orders.sql"], &["model:orders"], &[], &[]), + &["model:stg_orders"], + ), + expected_reasons: &["mocks stg_orders (model columns changed)"], + }, SelectionReasonTestCase { description: "a macro changed through another macro the model calls", current: with_node( diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/failures.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/failures.rs new file mode 100644 index 0000000000..13c328ad89 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/failures.rs @@ -0,0 +1,18 @@ +//! The error of a base revision that cannot be read. + +use crate::git_revisions::constants::EXPORT_FAILED_CODE; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +/// The project at the merge base cannot be read, with git's `detail`. +pub(crate) fn export_failed(base: &GitBase, detail: &str) -> GitRevisionError { + GitRevisionError { + message: format!( + "Cannot read the project at merge base {}: {detail}", + base.merge_base + ), + code: EXPORT_FAILED_CODE, + help: "Check that the repository is readable and complete, for example with `git fsck`" + .to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs index 364f98394c..5e58cb78ea 100644 --- a/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod failures; pub(crate) mod git_command; diff --git a/crates/sqlbuild-core/src/git_revisions/constants.rs b/crates/sqlbuild-core/src/git_revisions/constants.rs index 0f375b0a09..c5091e5ae8 100644 --- a/crates/sqlbuild-core/src/git_revisions/constants.rs +++ b/crates/sqlbuild-core/src/git_revisions/constants.rs @@ -5,6 +5,17 @@ pub const BASE_REF_ALIAS: &str = "@base"; /// Hex digits of a commit hash shown in progress and summaries. pub const SHORT_COMMIT_LENGTH: usize = 10; +/// Python dependency manifests and lock files, by exact name. +pub(crate) const DEPENDENCY_FILE_NAMES: [&str; 5] = [ + "pyproject.toml", + "uv.lock", + "poetry.lock", + "pdm.lock", + "Pipfile.lock", +]; +/// `requirements*.txt`: the prefix and suffix of pip requirement files. +pub(crate) const REQUIREMENTS_FILE_AFFIXES: (&str, &str) = ("requirements", ".txt"); + pub(crate) const GIT_PROGRAM: &str = "git"; pub(crate) const INDEX_FILE_ENV: &str = "GIT_INDEX_FILE"; pub(crate) const SHALLOW_TRUE: &str = "true"; diff --git a/crates/sqlbuild-core/src/git_revisions/main/dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/dependency_files.rs new file mode 100644 index 0000000000..f3b1be9c09 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/dependency_files.rs @@ -0,0 +1,71 @@ +//! The Python dependency manifests and lock files of a project, at the base and in the checkout. + +use std::collections::BTreeSet; +use std::path::Path; + +use crate::git_revisions::_helpers::failures::export_failed; +use crate::git_revisions::_helpers::git_command::{GitFailure, git_output}; +use crate::git_revisions::constants::{DEPENDENCY_FILE_NAMES, REQUIREMENTS_FILE_AFFIXES}; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +fn is_dependency_file(name: &str) -> bool { + let (prefix, suffix) = REQUIREMENTS_FILE_AFFIXES; + DEPENDENCY_FILE_NAMES.contains(&name) || (name.starts_with(prefix) && name.ends_with(suffix)) +} + +/// The repository root and the project directory, as repository-relative directory prefixes. +fn directories(base: &GitBase) -> BTreeSet { + let project: &str = base.project_prefix.trim_end_matches('/'); + let mut directories: BTreeSet = BTreeSet::from([String::new()]); + if !project.is_empty() { + directories.insert(format!("{project}/")); + } + directories +} + +/// `(repository path, trimmed text)` of each dependency file at the merge base, sorted. +pub fn base_dependency_files(base: &GitBase) -> Result, GitRevisionError> { + let root: &Path = &base.repository_root; + let mut files: Vec<(String, String)> = Vec::new(); + for directory in directories(base) { + let tree: String = format!("{}:{directory}", base.merge_base); + let listing: String = match git_output(root, &["ls-tree", "--name-only", &tree], &[]) { + Ok(listing) => listing, + Err(GitFailure::Failed(_)) => continue, + Err(failure) => return Err(export_failed(base, failure.detail())), + }; + for name in listing.lines().filter(|name| is_dependency_file(name)) { + let path: String = format!("{directory}{name}"); + let blob: String = format!("{}:{path}", base.merge_base); + let text: String = git_output(root, &["show", &blob], &[]) + .map_err(|failure| export_failed(base, failure.detail()))?; + files.push((path, text)); + } + } + files.sort(); + Ok(files) +} + +/// `(repository path, trimmed text)` of each dependency file in the checkout, sorted. +pub fn current_dependency_files(base: &GitBase) -> std::io::Result> { + let mut files: Vec<(String, String)> = Vec::new(); + for directory in directories(base) { + let entries: std::fs::ReadDir = + match std::fs::read_dir(base.repository_root.join(&directory)) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + for entry in entries { + let entry: std::fs::DirEntry = entry?; + let name: String = entry.file_name().to_string_lossy().into_owned(); + if entry.path().is_file() && is_dependency_file(&name) { + let text: String = std::fs::read_to_string(entry.path())?; + files.push((format!("{directory}{name}"), text.trim().to_owned())); + } + } + } + files.sort(); + Ok(files) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs b/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs index bb7f2f12ac..4d090694d1 100644 --- a/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs +++ b/crates/sqlbuild-core/src/git_revisions/main/export_git_tree.rs @@ -2,8 +2,9 @@ use std::path::{Path, PathBuf}; +use crate::git_revisions::_helpers::failures::export_failed; use crate::git_revisions::_helpers::git_command::git_output; -use crate::git_revisions::constants::{EXPORT_FAILED_CODE, INDEX_FILE_ENV}; +use crate::git_revisions::constants::INDEX_FILE_ENV; use crate::git_revisions::errors::GitRevisionError; use crate::git_revisions::models::GitBase; @@ -16,18 +17,6 @@ fn tree_of(base: &GitBase) -> String { } } -fn export_failed(base: &GitBase, detail: &str) -> GitRevisionError { - GitRevisionError { - message: format!( - "Cannot read the project at merge base {}: {detail}", - base.merge_base - ), - code: EXPORT_FAILED_CODE, - help: "Check that the repository is readable and complete, for example with `git fsck`" - .to_owned(), - } -} - /// Write the base project tree to `destination` through a private index; `false` without a project. pub fn export_git_tree(base: &GitBase, destination: &Path) -> Result { let root: &Path = &base.repository_root; diff --git a/crates/sqlbuild-core/src/git_revisions/main/mod.rs b/crates/sqlbuild-core/src/git_revisions/main/mod.rs index 75430cbb8e..19416e7e9d 100644 --- a/crates/sqlbuild-core/src/git_revisions/main/mod.rs +++ b/crates/sqlbuild-core/src/git_revisions/main/mod.rs @@ -1,3 +1,4 @@ +pub mod dependency_files; pub mod export_git_tree; pub mod resolve_git_base; pub mod resolve_since_reference; diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs index 0dfa1b1ae5..dd108cdc14 100644 --- a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs @@ -1,5 +1,8 @@ use std::path::PathBuf; +use crate::git_revisions::main::dependency_files::{ + base_dependency_files, current_dependency_files, +}; use crate::git_revisions::main::export_git_tree::export_git_tree; use crate::git_revisions::main::resolve_git_base::resolve_git_base; use crate::git_revisions::main::resolve_since_reference::resolve_since_reference; @@ -8,8 +11,8 @@ use crate::git_revisions::tests::helpers::{ git, project_repository, read, scratch_directory, write, }; use crate::git_revisions::tests::test_types::{ - AbsentProjectTestCase, ExportTreeTestCase, GitBaseFailureTestCase, GitBaseTestCase, - SinceReferenceTestCase, + AbsentProjectTestCase, DependencyFilesTestCase, ExportTreeTestCase, GitBaseFailureTestCase, + GitBaseTestCase, SinceReferenceTestCase, }; #[test] @@ -299,3 +302,59 @@ fn given_project_absent_at_merge_base_when_exporting_tree_then_reports_no_projec ); } } + +#[test] +fn given_dependency_files_when_reading_both_sides_then_lists_root_and_project_files() { + let (root, _) = project_repository(); + write(&root, "uv.lock", "version = 1\n"); + write(&root, "warehouse/requirements-dev.txt", "pytest\n"); + write(&root, "warehouse/notes.txt", "not a dependency file\n"); + git(&root, &["add", "uv.lock"]); + git(&root, &["commit", "--quiet", "-m", "lock"]); + let current: Vec<(&str, &str)> = vec![ + ("uv.lock", "version = 1"), + ("warehouse/requirements-dev.txt", "pytest"), + ]; + let test_cases = [ + DependencyFilesTestCase { + description: "files added after the base are only in the checkout", + reference: "main", + expected_base: vec![], + expected_current: current.clone(), + }, + DependencyFilesTestCase { + description: "committed files are read from the merge base", + reference: "feature", + expected_base: vec![("uv.lock", "version = 1")], + expected_current: current.clone(), + }, + ]; + for test_case in test_cases { + let base: GitBase = + resolve_git_base(&root.join("warehouse"), test_case.reference).expect("ref resolves"); + + let base_files: Vec<(String, String)> = + base_dependency_files(&base).expect("the base reads"); + let current_files: Vec<(String, String)> = + current_dependency_files(&base).expect("the checkout reads"); + + let owned = |files: &[(&str, &str)]| -> Vec<(String, String)> { + files + .iter() + .map(|(path, text)| ((*path).to_owned(), (*text).to_owned())) + .collect() + }; + assert_eq!( + base_files, + owned(&test_case.expected_base), + "{}", + test_case.description + ); + assert_eq!( + current_files, + owned(&test_case.expected_current), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs index 7cc85ec284..3953a0c601 100644 --- a/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_types.rs @@ -33,3 +33,10 @@ pub(crate) struct AbsentProjectTestCase { pub(crate) project_prefix: &'static str, pub(crate) expected_exported: bool, } + +pub(crate) struct DependencyFilesTestCase { + pub(crate) description: &'static str, + pub(crate) reference: &'static str, + pub(crate) expected_base: Vec<(&'static str, &'static str)>, + pub(crate) expected_current: Vec<(&'static str, &'static str)>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs new file mode 100644 index 0000000000..3370f00520 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs @@ -0,0 +1,106 @@ +//! Where the base snapshot is cached, what its cache key covers, and how old entries are pruned. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; +use sqlbuild_rules::constants::NATIVE_BUILD_IDENTITY; + +use crate::bindings::_helpers::change_selection::fingerprints::{ + environment_digest, referenced_environment, +}; + +/// Cached base snapshots kept per project; older ones are removed after each compile. +const KEPT_SNAPSHOTS: usize = 8; +const SNAPSHOT_SUFFIX: &str = "json"; +/// Authored file types that can read environment variables through SQL or configuration. +const AUTHORED_SUFFIXES: [&str; 5] = ["toml", "yml", "yaml", "sql", "py"]; +/// Directories of build output, tools and dependencies, never authored project files. +const SKIPPED_DIRECTORIES: [&str; 4] = ["target", "logs", "node_modules", "__pycache__"]; + +/// Authored files below `directory`, skipping hidden, build and dependency directories. +fn authored_files(directory: &Path) -> std::io::Result> { + let mut files: Vec = Vec::new(); + let mut pending: Vec = vec![directory.to_path_buf()]; + while let Some(next) = pending.pop() { + for entry in std::fs::read_dir(&next)? { + let path: PathBuf = entry?.path(); + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + if name.starts_with('.') { + continue; + } + if path.is_dir() { + if !SKIPPED_DIRECTORIES.contains(&name.as_str()) { + pending.push(path); + } + } else if path + .extension() + .is_some_and(|suffix| AUTHORED_SUFFIXES.contains(&&*suffix.to_string_lossy())) + { + files.push(path); + } + } + } + files.sort(); + Ok(files) +} + +/// The digest of every environment variable the project's files read, by its current value. +fn environment_inputs(project_dir: &Path) -> std::io::Result { + let mut names: BTreeSet = BTreeSet::new(); + for file in authored_files(project_dir)? { + let bytes: Vec = std::fs::read(&file)?; + names.extend(referenced_environment(&String::from_utf8_lossy(&bytes))); + } + Ok(environment_digest(&names)) +} + +/// The cache key of a base compile besides its commit: build, target, vars, local config, env. +pub(crate) fn cache_key( + project_dir: &Path, + (target, vars, local_config): (Option<&str>, &str, &str), +) -> std::io::Result { + Ok(definition_digest(&[ + NATIVE_BUILD_IDENTITY, + target.unwrap_or_default(), + vars, + local_config, + &environment_inputs(project_dir)?, + ])) +} + +/// A work directory name no other run, even one reusing this process id, can choose. +pub(crate) fn work_directory_name(short: &str) -> String { + let nanos: u128 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos()); + format!("work-{short}-{}-{nanos:x}", std::process::id()) +} + +/// Remove all but the newest cached base snapshots in `cache_dir`. +pub(crate) fn prune_snapshots(cache_dir: &Path) -> std::io::Result<()> { + let mut snapshots: Vec<(SystemTime, PathBuf)> = Vec::new(); + for entry in std::fs::read_dir(cache_dir)? { + let path: PathBuf = entry?.path(); + if path.is_file() + && path + .extension() + .is_some_and(|suffix| suffix == SNAPSHOT_SUFFIX) + { + snapshots.push((std::fs::metadata(&path)?.modified()?, path)); + } + } + snapshots.sort_by(|left, right| right.cmp(left)); + for (_, path) in snapshots.into_iter().skip(KEPT_SNAPSHOTS) { + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs index 249ab5b07c..9dfb3aa2af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_compile.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use pyo3::prelude::{ Bound, IntoPyObject, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBool, PyDict}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::definition_snapshot_json; use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; @@ -20,10 +20,30 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; +/// The extension of the file beside the snapshot that holds notes for the run's output. +pub(crate) const NOTES_EXTENSION: &str = "notes"; + fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { PyBool::new(py, value).to_owned().into_any() } +/// Whether `target` is unset or defined by the project or its local configuration. +fn target_defined(discovered: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>) -> PyResult { + if target.is_none() { + return Ok(true); + } + for config in ["project_config", "local_config"] { + let targets: Bound<'_, PyAny> = + native_field(&native_field(discovered, config)?, "targets")?; + if let Ok(targets) = targets.cast::() + && targets.contains(target)? + { + return Ok(true); + } + } + Ok(false) +} + /// Compile an exported base tree without SQL analysis and write its definition snapshot. #[pyfunction] #[pyo3(signature = (**options))] @@ -50,6 +70,19 @@ fn write_base_definition_snapshot( py, Some(keywords(py, &[("project_dir", directory.clone())])?), )?; + let target: Bound<'_, PyAny> = if target_defined(&discovered, &target)? { + target + } else { + std::fs::write( + output_path.with_extension(NOTES_EXTENSION), + format!( + "Target '{}' is not defined at the merge base, so the base compiles with its \ + default target and everything that depends on the target is selected\n", + target.str()? + ), + )?; + none() + }; let context: Bound<'_, PyAny> = resolve_adapter_connection_context( &discovered, &directory, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index 3b895666f3..a384f9365c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -6,7 +6,6 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_analysis::definition_changes::constants::DEFINITION_SNAPSHOT_FORMAT; -use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; use sqlbuild_analysis::definition_changes::main::definition_snapshot_json::definition_snapshot_json; use sqlbuild_analysis::definition_changes::main::parse_definition_snapshot::parse_definition_snapshot; use sqlbuild_analysis::definition_changes::models::DefinitionSnapshot; @@ -18,8 +17,12 @@ use sqlbuild_core::git_revisions::main::resolve_since_reference::resolve_since_r use sqlbuild_core::git_revisions::main::short_commit::short_commit; use sqlbuild_core::git_revisions::models::GitBase; use sqlbuild_core::scenario_command::errors::ScenarioCommandError; -use sqlbuild_rules::constants::NATIVE_BUILD_IDENTITY; +use crate::bindings::_helpers::change_selection::base_cache::{ + cache_key, prune_snapshots, work_directory_name, +}; +use crate::bindings::_helpers::change_selection::base_compile::NOTES_EXTENSION; +use crate::bindings::_helpers::change_selection::environment_nodes::with_base_dependencies; use crate::bindings::_helpers::compile_command::runtime_identity::interpreter_values; use crate::bindings::_helpers::discovery::project_config::configured_git_base_ref; use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, sorted_dumps}; @@ -30,11 +33,19 @@ const LOCAL_CONFIG_FILE: &str = "sqlbuild_local.toml"; const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; const BASE_COMPILE_FAILED_CODE: &str = "C484"; const FAILURE_TAIL_LINES: usize = 20; -/// Compiles the exported base tree in a fresh interpreter, apart from the working tree's modules. -const BASE_COMPILE_PROGRAM: &str = "import json, sys\n\ -from sqlbuild._native import write_base_definition_snapshot\n\ -write_base_definition_snapshot(project_dir=sys.argv[1], output_path=sys.argv[2], \ -selected_target=sys.argv[3] or None, cli_vars=json.loads(sys.argv[4]))\n"; +/// Compiles the exported base tree in a separate interpreter; a failure prints only its error. +const BASE_COMPILE_PROGRAM: &str = concat!( + "import json, sys\n", + "from sqlbuild._native import write_base_definition_snapshot\n", + "try:\n", + " write_base_definition_snapshot(project_dir=sys.argv[1], output_path=sys.argv[2],\n", + " selected_target=sys.argv[3] or None, cli_vars=json.loads(sys.argv[4]))\n", + "except Exception as error:\n", + " code = getattr(error, 'code', None)\n", + " message = f'error[{code}]: {error}' if code else f'{type(error).__name__}: {error}'\n", + " print(message, file=sys.stderr)\n", + " sys.exit(1)\n", +); /// The base revision and its definition snapshot. pub(crate) struct BaseComparison { @@ -60,7 +71,7 @@ pub(crate) fn base_variant( }) } -fn git_error(py: Python<'_>, error: GitRevisionError) -> PyErr { +pub(crate) fn git_error(py: Python<'_>, error: GitRevisionError) -> PyErr { cli_user_error( py, ScenarioCommandError { @@ -118,11 +129,11 @@ fn vars_json(cli_vars: &Bound<'_, PyAny>) -> PyResult { /// Run the base compile of `tree` into `output` in a separate interpreter. fn compile_base_tree( - py: Python<'_>, - base: &GitBase, + (stream, base): (&Bound<'_, PyAny>, &GitBase), (tree, output): (&Path, &Path), (selected_target, vars): (Option<&str>, &str), ) -> PyResult<()> { + let py: Python<'_> = stream.py(); let [_, executable, _, _] = interpreter_values(py)?; let finished: std::io::Result = py.detach(|| { Command::new(&executable) @@ -138,6 +149,11 @@ fn compile_base_tree( }); let finished: Output = finished.map_err(|error| io_failed(py, base, &error))?; if finished.status.success() && output.is_file() { + match std::fs::read_to_string(output.with_extension(NOTES_EXTENSION)) { + Ok(notes) => write(stream, ¬es)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(io_failed(py, base, &error)), + } return Ok(()); } Err(compile_failed(py, base, &failure_tail(&finished))) @@ -145,10 +161,11 @@ fn compile_base_tree( /// Export the base tree, carry over the untracked local config, and compile it. fn compiled_base_snapshot( - py: Python<'_>, + stream: &Bound<'_, PyAny>, (base, project_dir, work): (&GitBase, &Path, &Path), (selected_target, vars): (Option<&str>, &str), ) -> PyResult { + let py: Python<'_> = stream.py(); let tree: PathBuf = work.join("project"); let exported: bool = export_git_tree(base, &tree).map_err(|error| git_error(py, error))?; if !exported || !tree.join(PROJECT_CONFIG_FILE).is_file() { @@ -163,7 +180,7 @@ fn compiled_base_snapshot( .map_err(|error| io_failed(py, base, &error))?; } let output: PathBuf = work.join("definitions.json"); - compile_base_tree(py, base, (&tree, &output), (selected_target, vars))?; + compile_base_tree((stream, base), (&tree, &output), (selected_target, vars))?; let text: String = std::fs::read_to_string(&output).map_err(|error| io_failed(py, base, &error))?; parse_definition_snapshot(&text) @@ -200,12 +217,8 @@ pub(crate) fn base_comparison<'py>( Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), Err(error) => return Err(io_failed(py, &base, &error)), }; - let key: String = definition_digest(&[ - NATIVE_BUILD_IDENTITY, - target.unwrap_or_default(), - vars, - &local_config, - ]); + let key: String = cache_key(&directory, (target, vars, &local_config)) + .map_err(|error| io_failed(py, &base, &error))?; let cache_dir: PathBuf = CACHE_DIRECTORY .iter() .fold(directory.clone(), |path, part| path.join(part)); @@ -217,22 +230,23 @@ pub(crate) fn base_comparison<'py>( if let Some(snapshot) = stored { write(stream, &format!("Using cached base revision {short}\n"))?; flush(stream)?; + let snapshot: DefinitionSnapshot = with_base_dependencies(py, &base, &snapshot)?; return Ok(BaseComparison { base, snapshot }); } write(stream, &format!("Compiling base revision {short}...\n"))?; flush(stream)?; let started: Instant = Instant::now(); - let work: PathBuf = cache_dir.join(format!("work-{}-{}", short, std::process::id())); - let _ = std::fs::remove_dir_all(&work); + let work: PathBuf = cache_dir.join(work_directory_name(short)); std::fs::create_dir_all(&work).map_err(|error| io_failed(py, &base, &error))?; let compiled: PyResult = - compiled_base_snapshot(py, (&base, &directory, &work), (target, vars)); + compiled_base_snapshot(stream, (&base, &directory, &work), (target, vars)); let _ = std::fs::remove_dir_all(&work); let snapshot: DefinitionSnapshot = compiled?; let staged: PathBuf = cached.with_extension(format!("{}.tmp", std::process::id())); if let Ok(()) = std::fs::write(&staged, definition_snapshot_json(&snapshot)) { let _ = std::fs::rename(&staged, &cached); } + prune_snapshots(&cache_dir).map_err(|error| io_failed(py, &base, &error))?; write( stream, &format!( @@ -241,5 +255,6 @@ pub(crate) fn base_comparison<'py>( ), )?; flush(stream)?; + let snapshot: DefinitionSnapshot = with_base_dependencies(py, &base, &snapshot)?; Ok(BaseComparison { base, snapshot }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index d12a8a4eb8..61fecd18ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -5,12 +5,12 @@ use std::path::{Path, PathBuf}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; -use serde_json::Value; +use serde_json::{Map, Value}; use sqlbuild_analysis::definition_changes::constants::{ - AUDITS_ASPECT, COLUMNS_ASPECT, CONFIG_ASPECT, DEFINITION_ASPECT, DEFINITION_SNAPSHOT_FORMAT, - FILE_ASPECT, MACRO_KIND, MATERIALIZATION_KIND, MODEL_KIND, PYTHON_NODE_KINDS, SCENARIO_KIND, - SEED_KIND, SETTINGS_KIND, SNAPSHOTS_ASPECT, SOURCE_KIND, SQL_ASPECT, TEST_FILE_KIND, - VARS_ASPECT, + AUDITS_ASPECT, COLUMNS_ASPECT, CONFIG_ASPECT, CONNECTION_KIND, DEFINITION_ASPECT, + DEFINITION_SNAPSHOT_FORMAT, DEPENDENCIES_KIND, FILE_ASPECT, MACRO_KIND, MATERIALIZATION_KIND, + MODEL_KIND, PROJECT_NODE_NAME, PYTHON_NODE_KINDS, SCENARIO_KIND, SEED_KIND, SETTINGS_KIND, + SNAPSHOTS_ASPECT, SOURCE_KIND, SQL_ASPECT, TEST_FILE_KIND, VARS_ASPECT, }; use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; use sqlbuild_analysis::definition_changes::main::node_key::node_key; @@ -36,7 +36,26 @@ const MATERIALIZED_KEY: &str = "materialized"; const AUTHORED_KEYS_FIELD: &str = "model_header_keys"; const SOURCE_EXPRESSION_FIELD: &str = "expression"; const SCENARIO_SNAPSHOT_ROOT: [&str; 2] = ["tests", "_scenario_snapshots"]; -const SETTINGS_NAME: &str = "project"; +const CONTRACT_KEY: &str = "contract"; +/// Source declaration fields that shape a source mock's columns. +const SOURCE_SHAPE_FIELDS: [&str; 3] = ["columns", "contract", "type_enforcement"]; +/// Project-local adapter modules. +const ADAPTERS_DIRECTORY: &str = "adapters"; +/// Project configuration fields that other nodes fingerprint, or that do not shape compiled SQL. +const PROJECT_CONFIG_ELSEWHERE: [&str; 11] = [ + "name", + "settings", + "vars", + "defaults", + "path_defaults", + "cost", + "diff", + "janitor", + "migrations", + "sinks", + "scenario", +]; +const LOCAL_CONFIG_ELSEWHERE: [&str; 4] = ["settings", "setting_overrides", "vars", "scenario"]; /// Settings that tune how a run executes or validates, not what a node computes. const RUNTIME_SETTINGS: [&str; 9] = [ "sql_analysis", @@ -75,11 +94,16 @@ fn node(kind: &str, name: &str, aspects: Vec<(&str, String)>) -> NodeDefinition pub(crate) fn declaration_key(identity: &Bound<'_, PyAny>) -> PyResult { let kind: String = text_of(&native_field(&native_field(identity, "kind")?, "value")?)?; let name: String = native_field(identity, "name")?.extract()?; - let owner: Option = native_field(identity, "owner")?.extract()?; - Ok(match owner { - Some(owner) => node_key(&kind, &format!("{name}@{owner}")), - None => node_key(&kind, &name), - }) + let owner: Bound<'_, PyAny> = native_field(identity, "owner")?; + if owner.is_none() { + return Ok(node_key(&kind, &name)); + } + let owner_kind: String = text_of(&native_field(&native_field(&owner, "kind")?, "value")?)?; + let owner_name: String = native_field(&owner, "name")?.extract()?; + Ok(node_key( + &kind, + &format!("{name}@{}", node_key(&owner_kind, &owner_name)), + )) } /// `kind:name` of a usage consumer. @@ -143,17 +167,16 @@ fn declaration_nodes( let macro_facts: Bound<'_, PyAny> = native_field(&declaration, "macro")?; let (digest, uses): (String, Vec) = if kind == MACRO_KIND && !macro_facts.is_none() { - let dependencies: Vec = - native_field(¯o_facts, "dependencies")?.extract()?; + let dependencies: Vec = native_field(¯o_facts, "dependencies")? + .try_iter()? + .map(|dependency| declaration_key(&dependency?)) + .collect::>()?; ( definition_digest(&[ &fingerprints.canonical_digest(¯o_facts, (&[], &[]))?, &declaration_file_digest(fingerprints, &path), ]), - dependencies - .iter() - .map(|dependency| node_key(MACRO_KIND, dependency)) - .collect(), + dependencies, ) } else { (declaration_file_digest(fingerprints, &path), Vec::new()) @@ -201,8 +224,15 @@ fn model_node( let config: Bound<'_, PyAny> = model_reads::config(model)?; let config_values: Bound<'_, PyAny> = native_field(&config, "values")?; let schema: Bound<'_, PyAny> = model_reads::schema_entry(model)?; + let contract: String = match config_values.cast::() { + Ok(values) => match values.get_item(CONTRACT_KEY)? { + Some(value) => fingerprints.canonical(&value, &[])?.to_string(), + None => String::new(), + }, + Err(_) => String::new(), + }; let (columns, audits): (String, String) = if schema.is_none() { - (definition_digest(&[]), definition_digest(&[])) + (definition_digest(&[&contract]), definition_digest(&[])) } else { let mut column_audits: Vec = vec![fingerprints.canonical(&native_field(&schema, "audits")?, &[])?]; @@ -210,7 +240,10 @@ fn model_node( column_audits.push(fingerprints.canonical(&native_field(&column?, "audits")?, &[])?); } ( - fingerprints.canonical_digest(&schema, (&["name", "tags"], &["audits"]))?, + definition_digest(&[ + &fingerprints.canonical_digest(&schema, (&["name", "tags"], &["audits"]))?, + &contract, + ]), definition_digest(&[&Value::Array(column_audits).to_string()]), ) }; @@ -228,7 +261,7 @@ fn model_node( let mut definition: NodeDefinition = node(MODEL_KIND, &name, aspects); let key: String = node_key(MODEL_KIND, &name); definition.uses = uses.get(&key).into_iter().flatten().cloned().collect(); - definition.uses.push(node_key(SETTINGS_KIND, SETTINGS_NAME)); + definition.uses.extend(project_wide_keys()); if let Ok(values) = config_values.cast::() && let Some(materialized) = values.get_item(MATERIALIZED_KEY)? && let Ok(materialized) = materialized.cast::() @@ -241,32 +274,130 @@ fn model_node( Ok((key, definition)) } -/// A source by its declaration, with an inline expression's formatting ignored. +/// A source by its declared columns, which shape mocks, and the rest of its declaration. fn source_node( source: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, ) -> PyResult<(String, NodeDefinition)> { let name: String = source_reads::name(source)?.extract()?; let mut encoded: Value = fingerprints.canonical(&source_reads::source_entry(source)?, &[])?; - if let Value::Object(fields) = &mut encoded - && let Some(Value::String(expression)) = fields.get(SOURCE_EXPRESSION_FIELD) - { - let normalised: String = sql_definition_digest(expression); - fields.insert(SOURCE_EXPRESSION_FIELD.to_owned(), Value::from(normalised)); + let mut shape: Map = Map::new(); + let mut reads: Option = None; + if let Value::Object(fields) = &mut encoded { + for field in SOURCE_SHAPE_FIELDS { + if let Some(value) = fields.remove(field) { + shape.insert(field.to_owned(), value); + } + } + if let Some(Value::String(expression)) = fields.get(SOURCE_EXPRESSION_FIELD) { + reads = fingerprints.referenced_vars_digest(expression); + let normalised: String = sql_definition_digest(expression); + fields.insert(SOURCE_EXPRESSION_FIELD.to_owned(), Value::from(normalised)); + } } + let mut aspects: Vec<(&str, String)> = vec![ + ( + DEFINITION_ASPECT, + definition_digest(&[&encoded.to_string()]), + ), + ( + COLUMNS_ASPECT, + definition_digest(&[&Value::Object(shape).to_string()]), + ), + ]; + aspects.extend(reads.map(|digest| (VARS_ASPECT, digest))); + let mut definition: NodeDefinition = node(SOURCE_KIND, &name, aspects); + definition.uses = project_wide_keys(); + Ok((node_key(SOURCE_KIND, &name), definition)) +} + +/// A seed by its data and settings, and by the declared columns that shape its mocks. +fn seed_node( + seed: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult<(String, NodeDefinition)> { + let name: String = seed_reads::name(seed)?.extract()?; + let (version_hash, _) = seed_identity(seed)?; + let schema: Bound<'_, PyAny> = seed_reads::schema_entry(seed)?; + let columns: String = if schema.is_none() { + definition_digest(&[]) + } else { + definition_digest(&[&fingerprints + .canonical(&native_field(&schema, "columns")?, &[])? + .to_string()]) + }; Ok(( - node_key(SOURCE_KIND, &name), + node_key(SEED_KIND, &name), node( - SOURCE_KIND, + SEED_KIND, &name, - vec![( - DEFINITION_ASPECT, - definition_digest(&[&encoded.to_string()]), - )], + vec![(DEFINITION_ASPECT, version_hash), (COLUMNS_ASPECT, columns)], + ), + )) +} + +/// Keys of the project-wide nodes that every compiled definition depends on. +pub(crate) fn project_wide_keys() -> Vec { + [SETTINGS_KIND, CONNECTION_KIND, DEPENDENCIES_KIND] + .iter() + .map(|kind| node_key(kind, PROJECT_NODE_NAME)) + .collect() +} + +/// The adapter, connections and targets as configured, the selected target, and adapter modules. +fn connection_node( + project: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, + fingerprints: &Fingerprinting, +) -> PyResult<(String, NodeDefinition)> { + let mut parts: Vec = vec![ + fingerprints.canonical_digest( + &native_field(discovered, "project_config")?, + (&PROJECT_CONFIG_ELSEWHERE, &[]), + )?, + fingerprints.canonical_digest( + &native_field(discovered, "local_config")?, + (&LOCAL_CONFIG_ELSEWHERE, &[]), + )?, + text_of(&native_field(project, "effective_target_name")?)?, + ]; + let adapter_file: Bound<'_, PyAny> = native_field(discovered, "adapter_file")?; + if !adapter_file.is_none() { + let path: PathBuf = native_field(&adapter_file, "file_path")?.extract()?; + parts.push(fingerprints.python_module_digest(&path)); + } + for module in python_files(&fingerprints.project_dir.join(ADAPTERS_DIRECTORY))? { + parts.push(fingerprints.python_module_digest(&module)); + } + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + Ok(( + node_key(CONNECTION_KIND, PROJECT_NODE_NAME), + node( + CONNECTION_KIND, + PROJECT_NODE_NAME, + vec![(DEFINITION_ASPECT, definition_digest(&borrowed))], ), )) } +/// The `.py` files directly in `directory`, sorted; none when it does not exist. +fn python_files(directory: &Path) -> PyResult> { + let entries: std::fs::ReadDir = match std::fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error.into()), + }; + let mut files: Vec = Vec::new(); + for entry in entries { + let path: PathBuf = entry?.path(); + if path.is_file() && path.to_string_lossy().ends_with(PYTHON_SUFFIX) { + files.push(path); + } + } + files.sort(); + Ok(files) +} + /// Loaders, tasks, assets, checks and materializations by their Python modules and imports. fn python_nodes( discovered: &Bound<'_, PyAny>, @@ -337,7 +468,7 @@ fn function_nodes( &native_field(&dependency, "name")?.extract::()?, )); } - definition.uses.push(node_key(SETTINGS_KIND, SETTINGS_NAME)); + definition.uses.extend(project_wide_keys()); nodes.push((node_key(&kind, &name), definition)); } Ok(nodes) @@ -423,10 +554,10 @@ pub(crate) fn definition_snapshot( let uses: BTreeMap> = declaration_uses(project)?; let mut nodes: BTreeMap = declaration_nodes(project, &fingerprints)?; nodes.insert( - node_key(SETTINGS_KIND, SETTINGS_NAME), + node_key(SETTINGS_KIND, PROJECT_NODE_NAME), node( SETTINGS_KIND, - SETTINGS_NAME, + PROJECT_NODE_NAME, vec![( DEFINITION_ASPECT, fingerprints.canonical_digest( @@ -436,6 +567,8 @@ pub(crate) fn definition_snapshot( )], ), ); + let (connection_key, connection) = connection_node(project, discovered, &fingerprints)?; + nodes.insert(connection_key, connection); for model in project_reads::models(project)?.try_iter()? { let (key, definition) = model_node(&model?, &fingerprints, &uses)?; nodes.insert(key, definition); @@ -445,13 +578,8 @@ pub(crate) fn definition_snapshot( nodes.insert(key, definition); } for seed in project_reads::seeds(project)?.try_iter()? { - let seed: Bound<'_, PyAny> = seed?; - let name: String = seed_reads::name(&seed)?.extract()?; - let (version_hash, _) = seed_identity(&seed)?; - nodes.insert( - node_key(SEED_KIND, &name), - node(SEED_KIND, &name, vec![(DEFINITION_ASPECT, version_hash)]), - ); + let (key, definition) = seed_node(&seed?, &fingerprints)?; + nodes.insert(key, definition); } nodes.extend(function_nodes(project, &fingerprints)?); nodes.extend(python_nodes(discovered, &fingerprints)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs new file mode 100644 index 0000000000..4b08e05fb9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs @@ -0,0 +1,56 @@ +//! Project-wide inputs compared outside the cached base snapshot: Python dependency files. + +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::definition_changes::constants::{ + DEFINITION_ASPECT, DEPENDENCIES_KIND, PROJECT_NODE_NAME, +}; +use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; +use sqlbuild_analysis::definition_changes::main::node_key::node_key; +use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; +use sqlbuild_core::git_revisions::main::dependency_files::{ + base_dependency_files, current_dependency_files, +}; +use sqlbuild_core::git_revisions::models::GitBase; + +use crate::bindings::_helpers::change_selection::base_snapshot::git_error; + +fn add_dependencies( + snapshot: &DefinitionSnapshot, + files: &[(String, String)], +) -> DefinitionSnapshot { + let parts: Vec<&str> = files + .iter() + .flat_map(|(path, text)| [path.as_str(), text.as_str()]) + .collect(); + let mut updated: DefinitionSnapshot = snapshot.clone(); + updated.nodes.insert( + node_key(DEPENDENCIES_KIND, PROJECT_NODE_NAME), + NodeDefinition { + kind: DEPENDENCIES_KIND.to_owned(), + name: PROJECT_NODE_NAME.to_owned(), + aspects: [(DEFINITION_ASPECT.to_owned(), definition_digest(&parts))].into(), + ..NodeDefinition::default() + }, + ); + updated +} + +/// `snapshot` of the base revision with its dependency files. +pub(crate) fn with_base_dependencies( + py: Python<'_>, + base: &GitBase, + snapshot: &DefinitionSnapshot, +) -> PyResult { + let files: Vec<(String, String)> = py + .detach(|| base_dependency_files(base)) + .map_err(|error| git_error(py, error))?; + Ok(add_dependencies(snapshot, &files)) +} + +/// `snapshot` of the working tree with its dependency files. +pub(crate) fn with_current_dependencies( + base: &GitBase, + snapshot: &DefinitionSnapshot, +) -> PyResult { + Ok(add_dependencies(snapshot, ¤t_dependency_files(base)?)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs index 0003170939..8fbaccfdb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs @@ -21,6 +21,8 @@ const DOCUMENTATION_FIELDS: [&str; 2] = ["description", "location"]; const VARS_ATTRIBUTE: &str = "vars"; const VARIABLE_SIGIL: &str = "@@"; const ENVIRONMENT_VARIABLE: &str = "ENV"; +/// How SQL and configuration read an environment variable. +const ENVIRONMENT_MARKERS: [&str; 2] = ["@@ENV:", "${ENV:"]; /// What fingerprints of one project revision need: where it lives and its variables. pub(crate) struct Fingerprinting { @@ -164,10 +166,17 @@ impl Fingerprinting { definition_digest(&parts) } - /// The digest of the variables `text` reads with `@@name`, or `None` when it reads none. + /// The digest of the project and environment variables `text` reads, or `None` without any. pub(crate) fn referenced_vars_digest(&self, text: &str) -> Option { let names: BTreeSet = referenced_vars(text); - (!names.is_empty()).then(|| self.vars_digest(names.iter())) + let environment: BTreeSet = referenced_environment(text); + if names.is_empty() && environment.is_empty() { + return None; + } + Some(definition_digest(&[ + &self.vars_digest(names.iter()), + &environment_digest(&environment), + ])) } /// A Python file and its imported project modules; code mentioning `vars` covers every variable. @@ -232,6 +241,37 @@ fn referenced_vars(text: &str) -> BTreeSet { names } +/// Names of the environment variables `text` reads with `@@ENV:NAME` or `${ENV:NAME}`. +pub(crate) fn referenced_environment(text: &str) -> BTreeSet { + let mut names: BTreeSet = BTreeSet::new(); + for marker in ENVIRONMENT_MARKERS { + for (index, _) in text.match_indices(marker) { + let name: String = text[index + marker.len()..] + .chars() + .take_while(|character| character.is_ascii_alphanumeric() || *character == '_') + .collect(); + if !name.is_empty() { + names.insert(name); + } + } + } + names +} + +/// The digest of the environment variables `names` by their current values, which it never shows. +pub(crate) fn environment_digest(names: &BTreeSet) -> String { + let mut parts: Vec = Vec::new(); + for name in names { + parts.push(name.clone()); + parts.push(match std::env::var_os(name) { + Some(value) => definition_digest(&[&value.to_string_lossy()]), + None => "unset".to_owned(), + }); + } + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + definition_digest(&borrowed) +} + pub(crate) fn text_of(value: &Bound<'_, PyAny>) -> PyResult { Ok(value.str()?.to_string().replace('\\', "/")) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs index e49fa47121..ab0bd562ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/mod.rs @@ -1,6 +1,8 @@ +pub(crate) mod base_cache; pub(crate) mod base_compile; pub(crate) mod base_snapshot; pub(crate) mod definition_snapshot; +pub(crate) mod environment_nodes; pub(crate) mod fingerprints; pub(crate) mod registration; pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 920d1974dc..6343a7bffe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -23,8 +23,9 @@ use sqlbuild_analysis::definition_changes::models::{ use crate::bindings::_helpers::change_selection::base_snapshot::BaseComparison; use crate::bindings::_helpers::change_selection::definition_snapshot::{ - declaration_uses, definition_snapshot, + declaration_uses, definition_snapshot, project_wide_keys, }; +use crate::bindings::_helpers::change_selection::environment_nodes::with_current_dependencies; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::keys::native_key; @@ -57,7 +58,10 @@ pub(crate) fn changed_project( project_dir: &Path, base: BaseComparison, ) -> PyResult { - let current: DefinitionSnapshot = definition_snapshot(project, discovered, project_dir)?; + let current: DefinitionSnapshot = with_current_dependencies( + &base.base, + &definition_snapshot(project, discovered, project_dir)?, + )?; let changes: DefinitionChanges = compare_definitions(&base.snapshot, ¤t); Ok(ChangedProject { reference: base.base.reference, @@ -91,6 +95,14 @@ fn source_calls(sql: &str) -> Vec { names } +/// Node keys of mocked models, sources and seeds. +fn relation_keys((refs, sources, seeds): (&[String], &[String], &[String])) -> Vec { + let models = refs.iter().map(|name| node_key(MODEL_KIND, name)); + let sources = sources.iter().map(|name| node_key(SOURCE_KIND, name)); + let seeds = seeds.iter().map(|name| node_key(SEED_KIND, name)); + models.chain(sources).chain(seeds).collect() +} + /// The sources `model` reads, other than `mocked` ones, as node keys. fn read_sources(model: &Bound<'_, PyAny>, mocked: &[String]) -> PyResult> { let mut keys: Vec = Vec::new(); @@ -164,25 +176,36 @@ impl ChangedProject { (tests, models): (&ByName<'_>, &ByName<'_>), ) -> PyResult { let name: String = native_field(entry, "name")?.extract()?; + let parent: Option = native_field(entry, "parent_name")?.extract()?; let mut item: ExecutedItem = ExecutedItem { - declarations: self.declarations_of(&node_key(TEST_CONSUMER_KIND, &name)), + declarations: project_wide_keys(), ..ExecutedItem::default() }; let mocked_seeds: Vec = strings(&native_field(entry, "mock_seed_names")?)?; let mocked_sources: Vec = strings(&native_field(entry, "mock_source_names")?)?; + let mocked_refs: Vec = strings(&native_field(entry, "mock_ref_names")?)?; let mocked_functions: Vec = strings(&native_field(entry, "mock_table_function_names")?)?; item.mocked .extend(self.keys_named(TABLE_FUNCTION_KIND, &mocked_functions)); - if let Some(test) = tests.get(&name) { + item.mocked_relations = relation_keys((&mocked_refs, &mocked_sources, &mocked_seeds)); + let mut consumers: Vec = vec![name.clone()]; + consumers.extend(parent.clone()); + let compiled: Option<&Bound<'_, PyAny>> = tests + .get(&name) + .or_else(|| parent.as_ref().and_then(|parent| tests.get(parent))); + if let Some(test) = compiled { let file: Bound<'_, PyAny> = native_field(test, "test_file")?; + let path: Bound<'_, PyAny> = native_field(&file, "relative_path")?; item.own_keys.push(node_key( TEST_FILE_KIND, - &native_field(&file, "relative_path")? - .str()? - .to_string() - .replace('\\', "/"), + &path.str()?.to_string().replace('\\', "/"), )); + consumers.push(native_field(&path, "stem")?.extract()?); + for block in native_field(&file, "blocks")?.try_iter()? { + let block_name: Option = native_field(&block?, "name")?.extract()?; + consumers.extend(block_name); + } let contents: String = native_field(&file, "contents")?.extract()?; for name in source_calls(&contents) { if !mocked_sources.contains(&name) { @@ -226,6 +249,10 @@ impl ChangedProject { let (kind, function_name) = native_key(&key?)?; item.executed.push(node_key(&kind, &function_name)); } + for consumer in consumers { + item.declarations + .extend(self.declarations_of(&node_key(TEST_CONSUMER_KIND, &consumer))); + } Ok(item) } @@ -238,12 +265,19 @@ impl ChangedProject { let name: String = native_field(scenario, "name")?.extract()?; let key: String = node_key(SCENARIO_KIND, &name); let (graph, _) = plan_scenario_graph(scenario, project, true, None)?; + let mut declarations: Vec = project_wide_keys(); + declarations.extend(self.declarations_of(&key)); let mut item: ExecutedItem = ExecutedItem { own_keys: vec![key.clone()], - declarations: self.declarations_of(&key), + declarations, ..ExecutedItem::default() }; let source_fixtures: Vec = strings(&native_field(&graph, "source_fixture_names")?)?; + item.mocked_relations = relation_keys(( + &strings(&native_field(&graph, "ref_fixture_names")?)?, + &source_fixtures, + &strings(&native_field(&graph, "seed_fixture_names")?)?, + )); for name in source_calls(&native_field(scenario, "sql_body")?.extract::()?) { if !source_fixtures.contains(&name) { item.executed.push(node_key(SOURCE_KIND, &name)); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md index 2d64161902..687fa8b274 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -53,12 +53,15 @@ captured local snapshots changed, or when any node it actually executes changed: builds, the seeds and sources it reads, the functions they call, the audits attached to its models, and the macros, enums and constants they and the scenario use. A model the scenario replaces with a `__ref__` fixture, a source it replaces with a `__source__` fixture, or a seed it replaces with a -`__seed__` fixture, is not executed, so its changes do not select the scenario. +`__seed__` fixture, is not executed, so only changes to its declared columns, which shape the +fixture, select the scenario. [Project-wide inputs](test.md#project-wide-inputs) such as +the connection, targets and Python dependency files select every scenario. `--select` and `--exclude` narrow the affected scenarios further. With `--json`, the document gains the same `since` section as `sqb test`, listing each selected scenario with its reasons. Both -commands share the cached base compile, so `sqb test --since origin/main` and -`sqb scenario test --since origin/main` in one CI job resolve the same changed set: +commands share the cached base compile when `sqb test` uses the default target and no `--vars`, so +`sqb test --since origin/main` and `sqb scenario test --since origin/main` in one CI job compile +the base once and resolve the same changed set: ```bash sqb test --since "origin/$TARGET_BRANCH" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index fc27c5c08b..24515e023b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -69,9 +69,15 @@ uncommitted edits included. A test is selected when: - its own file changed or is new, or - any node it actually executes changed: the models in its chain, the seeds, sources and - functions they read, and the macros, enums, constants and audits they use. The resolution is the - one `--inspect` shows, so a test that mocks a changed model, source, seed or macro is not - selected, while a downstream test that really runs it is. + functions they read, and the macros, enums, constants and audits they and the test file use + (every case of a parameterized test counts), or +- a model, source or seed it mocks changed its declared columns: names, types, nullability or + contract, which shape the mock's columns, or +- a project-wide input changed: see [Project-wide inputs](#project-wide-inputs). + +The resolution is the one `--inspect` shows, so a test that mocks a model, source, seed or macro +is not selected when only that relation's SQL, data or upstream changes, while a downstream test +that really runs it is. A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with formatting and comments ignored, the resolved configuration (including what `[defaults]` and @@ -89,6 +95,26 @@ types, a source with its declaration or inline expression, and a function with i Loaders, tasks, assets and checks are compared by their Python modules and the project modules they import; they appear in the change summary but no unit test runs them. +### Project-wide inputs + +Some inputs reach every test, so a change to any of them selects every test: + +- the `[settings]` that shape compiled models; +- the adapter, `[connection]` and `[connections]` (including connection settings and adapter + extensions), `[targets]` and the target in use, and project-local adapter modules under + `adapters/`. Their values are compared by digest and never printed; +- the Python environment as the project records it: `pyproject.toml`, `uv.lock`, `poetry.lock`, + `pdm.lock`, `Pipfile.lock` and `requirements*.txt` in the project directory or at the repository + root. + +Environment variables a project reads with `${ENV:NAME}` in configuration or `@@ENV:NAME` in SQL +are part of the cache key of the base compile and of the fingerprints of the SQL that reads them; +their values are never printed. + +`--since` cannot see code outside the project: modules imported from outside the project +directory, installed package versions that the files above do not record, and environment +variables read directly from Python with `os.environ`. Run the full suite when those change. + Added nodes count as changed. A model renamed with `migrate_from` is compared with its old definition, so a pure rename selects nothing, while a rename that also edits the model counts as a change. Deleted nodes select nothing and are listed in the summary. @@ -120,9 +146,12 @@ Selected 2 of 9 tests executes macro to_utc (changed) ``` -The base compile is cached per merge-base commit under `target/since/`, so `sqb test --since` and -`sqb scenario test --since` in the same CI job compile the base once and compare against the same -changed set. When no test is affected, the command says so and exits successfully. +The base compile is cached under `target/since/` per merge-base commit, target, `--vars`, local +configuration and referenced environment variables, and the newest few are kept. `sqb test --since` +and `sqb scenario test --since` in the same CI job compile the base once when they use the same +target and `--vars`; `sqb scenario test` uses the default target, so a `sqb test --target` run +compiles the base separately. When no test is affected, the command says so and exits +successfully. With `--json`, the progress and selection lines go to stderr, and the JSON document on stdout gains a `since` section: the ref and merge base, the `changed`, `renamed` and `deleted` nodes, the number @@ -145,7 +174,17 @@ of candidate tests, and each selected test with its reasons: ``` The base revision must compile. When it does not, `--since` fails with the compile error instead -of guessing; compare with a later ref whose project compiles, or run without `--since`. +of guessing; compare with a later ref whose project compiles, or run without `--since`. A +`--target` that the base revision does not define, such as one the branch adds, compiles the base +with its default target instead; the run says so, and because the target changed, every test is +selected. + +SQL comparisons ignore layout and comments using a generic SQL tokenizer. In dialects where `--` +starts a comment only when followed by whitespace, such as MySQL, `1--1` is read as `1` followed by +a comment, so an edit inside such an expression can go unnoticed. The base tree is read from git +objects without LFS smudging, including when a `.gitattributes` at the repository root marks files +in a project subdirectory. A file stored with Git LFS, such as a seed, is a pointer file at the +base and real content in the checkout, so it always counts as changed and selects what reads it. `` is any git ref or commit, such as the merge request's target branch from a CI variable. `--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 9f80f5a422..eb45445dca 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -559,3 +559,4 @@ class SinceBrokenBaseE2ETestCase: command: tuple[str, ...] expected_exit_code: int expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 781f8d1c8e..75f24e9c4d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -2228,12 +2228,61 @@ def build_since_indirect_files() -> dict[str, str]: base: dict[str, str] = build_since_project_files() return { - "sqlbuild_project.toml": base["sqlbuild_project.toml"] + '\n[vars]\nmin_amount = "100"\n', + "sqlbuild_project.toml": ( + base["sqlbuild_project.toml"] + + '\n[vars]\nmin_amount = "100"\n\n[scopes]\nenforce_placement = false\n' + ), "python/_rates.py": 'DIVISOR = "100"\n', + "models/_macros/currency.py": ( + 'def whole_cents(expression: str) -> str:\n return f"({expression} * 1)"\n' + ), "models/_macros/money.py": ( + "from models._macros.currency import whole_cents\n" "from python._rates import DIVISOR\n\n\n" "def cents_to_dollars(expression: str) -> str:\n" - ' return f"CAST({expression} AS DOUBLE) / {DIVISOR}"\n' + ' return f"CAST({whole_cents(expression)} AS DOUBLE) / {DIVISOR}"\n' + ), + "macros/taxes.py": ( + 'def with_tax(expression: str) -> str:\n return f"({expression} + 0)"\n' + ), + "macros/pricing.py": ( + "from macros.taxes import with_tax\n\n\n" + "def gross(expression: str) -> str:\n return with_tax(expression)\n" + ), + "macros/fixture_values.py": "def suffix() -> str:\n return \"''\"\n", + "models/stg_orders.sql": ( + "MODEL (\n description 'Cleaned orders.',\n" + " columns (order_id (type INTEGER), status (type VARCHAR), " + "amount_cents (type INTEGER)),\n);\n\n" + 'SELECT order_id, status, amount_cents FROM __source("raw_orders")\n' + ), + "models/order_gross.sql": ( + "MODEL (description 'Gross order amounts.');\n\n" + 'SELECT order_id, @gross("amount_cents") AS gross_cents FROM __ref("stg_orders")\n' + ), + "models/order_day.sql": ( + "MODEL (description 'The UTC day of a fixed timestamp.');\n\n" + "SELECT order_id, CAST(TIMESTAMPTZ '2024-01-01 23:30:00+00' AS DATE) AS day\n" + 'FROM __ref("stg_orders")\n' + ), + "tests/unit/test_order_gross.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_gross AS (SELECT 1 AS order_id, 250 AS gross_cents)\nSELECT 1\n" + ), + "tests/unit/test_order_day.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_day AS (SELECT 1 AS order_id, DATE '2024-01-01' AS day)\nSELECT 1\n" + ), + "tests/unit/test_status_cases.sql": ( + 'TEST (name "status_cases", parameters (status string), ' + 'cases (open_case (status "open"), closed_case (status "closed")));\n\n' + "WITH\n__source__raw_orders AS (\n" + ' SELECT 1 AS order_id, @param("status") || @suffix() AS status, ' + "250 AS amount_cents\n),\n" + "__expected__stg_orders AS (\n" + ' SELECT 1 AS order_id, @param("status") || @suffix() AS status\n)\nSELECT 1\n' ), "audits/generic/at_least.sql": ( 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @column < @minimum\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index 3986d6c0fb..3e60c04676 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -212,7 +212,7 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( expected_exit_code=0, expected_fragments=( "Changed since main: 1 model changed\n", - "Selected 1 of 9 tests", + "Selected 1 of 13 tests", "test_big_orders\n executes big_orders (model vars changed)", "PASS=1 FAIL=0 TOTAL=1", ), @@ -226,7 +226,7 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( command=_EXPLAINED_TESTS, expected_exit_code=0, expected_fragments=( - "Selected 1 of 9 tests", + "Selected 1 of 13 tests", "test_order_counts\n executes order_counts (model config changed)", "PASS=1 FAIL=0 TOTAL=1", ), @@ -238,7 +238,7 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( expected_exit_code=0, expected_fragments=( "Changed since main: 1 macro changed\n", - "Selected 2 of 9 tests", + "Selected 2 of 13 tests", "test_order_totals\n executes macro cents_to_dollars (changed)", "test_order_totals_from_source\n executes macro cents_to_dollars (changed)", "PASS=2 FAIL=0 TOTAL=2", @@ -255,7 +255,7 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( expected_exit_code=0, expected_fragments=( "Changed since main: 1 function changed\n", - "Selected 1 of 9 tests", + "Selected 1 of 13 tests", "test_order_flags\n executes function is_open (changed)", ), ), @@ -270,13 +270,17 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( expected_exit_code=0, expected_fragments=( "Changed since main: 1 audit changed\n", - "Selected 2 of 9 tests", + "Selected 2 of 13 tests", "test_order_totals\n executes audit at_least (changed)", ), ), SinceSelectionE2ETestCase( description="an upstream change selects a test that runs it but not one that mocks it", - edits={_STAGING: _EDITED_STAGING}, + edits={ + _STAGING: _INDIRECT[_STAGING].replace( + "SELECT order_id, status", "SELECT order_id, LOWER(status) AS status" + ) + }, command=_EXPLAINED_TESTS, expected_exit_code=0, expected_fragments=( @@ -365,17 +369,134 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( unexpected_fragments=("order_totals_end_to_end",), ), SinceSelectionE2ETestCase( - description="a source definition that every test mocks selects nothing", + description="a mocked source's declared column type selects the tests that mock it", edits={ "sources/raw.yml": _FILES["sources/raw.yml"].replace( "amount_cents\n type: INTEGER", "amount_cents\n type: BIGINT" ) }, - command=_TEST_COMMAND, + command=_EXPLAINED_TESTS, expected_exit_code=0, expected_fragments=( "Changed since main: 1 source changed\n", - "No tests are affected by the changes since main", + "Selected 5 of 13 tests", + "test_stg_orders\n mocks source raw_orders (source columns changed)", + ), + unexpected_fragments=("test_order_flags",), + ), + SinceSelectionE2ETestCase( + description="a mocked model's declared column type selects the tests that mock it", + edits={ + _STAGING: _INDIRECT[_STAGING].replace( + "amount_cents (type INTEGER)", "amount_cents (type BIGINT)" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Selected 13 of 13 tests", + "test_big_orders\n mocks stg_orders (model columns changed)", + "test_archive_counts\n executes stg_orders (model columns changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a fixture's declared column type selects the scenario that replaces it", + edits={ + _STAGING: _INDIRECT[_STAGING].replace( + "amount_cents (type INTEGER)", "amount_cents (type BIGINT)" + ) + }, + command=_EXPLAINED_SCENARIOS, + expected_exit_code=0, + expected_fragments=( + "open_orders_from_fixture\n mocks stg_orders (model columns changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a macro another project-wide macro imports changes both", + edits={"macros/taxes.py": _INDIRECT["macros/taxes.py"].replace("+ 0", "+ 1 - 1")}, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 2 macros changed\n", + "Selected 1 of 13 tests", + "test_order_gross\n executes macro gross (changed)\n" + " executes macro with_tax (changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + ), + SinceSelectionE2ETestCase( + description="a scoped macro another scoped macro imports changes both", + edits={ + "models/_macros/currency.py": _INDIRECT["models/_macros/currency.py"].replace( + "* 1)", "* 1.0)" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Selected 2 of 13 tests", + "test_order_totals\n executes macro cents_to_dollars (changed)\n" + " executes macro whole_cents (changed)", + "PASS=2 FAIL=0 TOTAL=2", + ), + ), + SinceSelectionE2ETestCase( + description="a macro a parameterized test's own fixture calls selects every case", + edits={ + "macros/fixture_values.py": _INDIRECT["macros/fixture_values.py"].replace( + "\"''\"", "\"'x'\"" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Selected 2 of 13 tests", + "status_cases [open_case]\n executes macro suffix (changed)", + "status_cases [closed_case]\n executes macro suffix (changed)", + "PASS=2 FAIL=0 TOTAL=2", + ), + ), + SinceSelectionE2ETestCase( + description="a connection setting selects every test, including the one it breaks", + edits={ + _PROJECT_CONFIG: _INDIRECT[_PROJECT_CONFIG].replace( + '[connection]\ndatabase = "since_orders.duckdb"\n', + '[connection]\ndatabase = "since_orders.duckdb"\n\n' + '[connection.settings]\nTimeZone = "Asia/Tokyo"\n', + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=1, + expected_fragments=( + "Changed since main: 1 connection and target configuration changed\n", + "Selected 13 of 13 tests", + "test_order_day\n uses connection and target configuration (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a Python dependency lock file selects every test", + edits={"uv.lock": "version = 1\n"}, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 Python environment changed\n", + "Selected 13 of 13 tests", + "test_order_day\n uses Python environment (changed)", + ), + ), + SinceSelectionE2ETestCase( + description="a target the base does not define compiles the base with its default", + edits={ + _PROJECT_CONFIG: _INDIRECT[_PROJECT_CONFIG] + '\n[targets.ci]\nschema = "main"\n' + }, + command=("--no-color", "test", "--target", "ci", "--since", "main"), + expected_exit_code=0, + expected_fragments=( + "Target 'ci' is not defined at the merge base, so the base compiles with its " + "default target", + "Selected 13 of 13 tests", + "PASS=13 FAIL=0 TOTAL=13", ), ), SinceSelectionE2ETestCase( @@ -429,6 +550,7 @@ def test_given_indirect_changes_since_ref_when_testing_then_selects_what_execute "missing_orders", "Compare with a later ref whose project compiles, or run without --since", ), + unexpected_fragments=("Traceback", 'File "'), ), ), ids=lambda case: case.description, @@ -445,6 +567,8 @@ def test_given_base_that_does_not_compile_when_testing_since_then_fails_with_gui assert result.returncode == test_case.expected_exit_code, output for fragment in test_case.expected_fragments: assert fragment in output, output + for fragment in test_case.unexpected_fragments: + assert fragment not in output, output @pytest.mark.parametrize( diff --git a/website/src/content/docs/docs/cli/scenario.mdx b/website/src/content/docs/docs/cli/scenario.mdx index 37c8e9f5db..3b7feb3e67 100644 --- a/website/src/content/docs/docs/cli/scenario.mdx +++ b/website/src/content/docs/docs/cli/scenario.mdx @@ -46,12 +46,15 @@ captured local snapshots changed, or when any node it actually executes changed: builds, the seeds and sources it reads, the functions they call, the audits attached to its models, and the macros, enums and constants they and the scenario use. A model the scenario replaces with a `__ref__` fixture, a source it replaces with a `__source__` fixture, or a seed it replaces with a -`__seed__` fixture, is not executed, so its changes do not select the scenario. +`__seed__` fixture, is not executed, so only changes to its declared columns, which shape the +fixture, select the scenario. [Project-wide inputs](/docs/cli/test#project-wide-inputs) such as +the connection, targets and Python dependency files select every scenario. `--select` and `--exclude` narrow the affected scenarios further. With `--json`, the document gains the same `since` section as `sqb test`, listing each selected scenario with its reasons. Both -commands share the cached base compile, so `sqb test --since origin/main` and -`sqb scenario test --since origin/main` in one CI job resolve the same changed set: +commands share the cached base compile when `sqb test` uses the default target and no `--vars`, so +`sqb test --since origin/main` and `sqb scenario test --since origin/main` in one CI job compile +the base once and resolve the same changed set: ```bash sqb test --since "origin/$TARGET_BRANCH" diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index a319bb890c..867e75e22d 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -61,9 +61,15 @@ uncommitted edits included. A test is selected when: - its own file changed or is new, or - any node it actually executes changed: the models in its chain, the seeds, sources and - functions they read, and the macros, enums, constants and audits they use. The resolution is the - one `--inspect` shows, so a test that mocks a changed model, source, seed or macro is not - selected, while a downstream test that really runs it is. + functions they read, and the macros, enums, constants and audits they and the test file use + (every case of a parameterized test counts), or +- a model, source or seed it mocks changed its declared columns: names, types, nullability or + contract, which shape the mock's columns, or +- a project-wide input changed: see [Project-wide inputs](#project-wide-inputs). + +The resolution is the one `--inspect` shows, so a test that mocks a model, source, seed or macro +is not selected when only that relation's SQL, data or upstream changes, while a downstream test +that really runs it is. A node changed when its definition fingerprint differs. The fingerprint covers the model SQL with formatting and comments ignored, the resolved configuration (including what `[defaults]` and @@ -81,6 +87,26 @@ types, a source with its declaration or inline expression, and a function with i Loaders, tasks, assets and checks are compared by their Python modules and the project modules they import; they appear in the change summary but no unit test runs them. +### Project-wide inputs + +Some inputs reach every test, so a change to any of them selects every test: + +- the `[settings]` that shape compiled models; +- the adapter, `[connection]` and `[connections]` (including connection settings and adapter + extensions), `[targets]` and the target in use, and project-local adapter modules under + `adapters/`. Their values are compared by digest and never printed; +- the Python environment as the project records it: `pyproject.toml`, `uv.lock`, `poetry.lock`, + `pdm.lock`, `Pipfile.lock` and `requirements*.txt` in the project directory or at the repository + root. + +Environment variables a project reads with `${ENV:NAME}` in configuration or `@@ENV:NAME` in SQL +are part of the cache key of the base compile and of the fingerprints of the SQL that reads them; +their values are never printed. + +`--since` cannot see code outside the project: modules imported from outside the project +directory, installed package versions that the files above do not record, and environment +variables read directly from Python with `os.environ`. Run the full suite when those change. + Added nodes count as changed. A model renamed with `migrate_from` is compared with its old definition, so a pure rename selects nothing, while a rename that also edits the model counts as a change. Deleted nodes select nothing and are listed in the summary. @@ -112,9 +138,12 @@ Selected 2 of 9 tests executes macro to_utc (changed) ``` -The base compile is cached per merge-base commit under `target/since/`, so `sqb test --since` and -`sqb scenario test --since` in the same CI job compile the base once and compare against the same -changed set. When no test is affected, the command says so and exits successfully. +The base compile is cached under `target/since/` per merge-base commit, target, `--vars`, local +configuration and referenced environment variables, and the newest few are kept. `sqb test --since` +and `sqb scenario test --since` in the same CI job compile the base once when they use the same +target and `--vars`; `sqb scenario test` uses the default target, so a `sqb test --target` run +compiles the base separately. When no test is affected, the command says so and exits +successfully. With `--json`, the progress and selection lines go to stderr, and the JSON document on stdout gains a `since` section: the ref and merge base, the `changed`, `renamed` and `deleted` nodes, the number @@ -137,7 +166,17 @@ of candidate tests, and each selected test with its reasons: ``` The base revision must compile. When it does not, `--since` fails with the compile error instead -of guessing; compare with a later ref whose project compiles, or run without `--since`. +of guessing; compare with a later ref whose project compiles, or run without `--since`. A +`--target` that the base revision does not define, such as one the branch adds, compiles the base +with its default target instead; the run says so, and because the target changed, every test is +selected. + +SQL comparisons ignore layout and comments using a generic SQL tokenizer. In dialects where `--` +starts a comment only when followed by whitespace, such as MySQL, `1--1` is read as `1` followed by +a comment, so an edit inside such an expression can go unnoticed. The base tree is read from git +objects without LFS smudging, including when a `.gitattributes` at the repository root marks files +in a project subdirectory. A file stored with Git LFS, such as a seed, is a pointer file at the +base and real content in the checkout, so it always counts as changed and selects what reads it. `` is any git ref or commit, such as the merge request's target branch from a CI variable. `--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach From 0e56a710c00351e30f560baed5a68ed891d9e31a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:02:21 +0100 Subject: [PATCH 07/10] refactor: one entry function per dependency-file module --- .../src/definition_changes/constants.rs | 6 +- .../_helpers/dependency_files.rs | 22 ++++++ .../src/git_revisions/_helpers/mod.rs | 1 + .../main/base_dependency_files.rs | 32 +++++++++ .../main/current_dependency_files.rs | 27 +++++++ .../git_revisions/main/dependency_files.rs | 71 ------------------- .../src/git_revisions/main/mod.rs | 3 +- .../git_revisions/tests/test_git_revisions.rs | 5 +- .../change_selection/environment_nodes.rs | 5 +- 9 files changed, 93 insertions(+), 79 deletions(-) create mode 100644 crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs create mode 100644 crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs delete mode 100644 crates/sqlbuild-core/src/git_revisions/main/dependency_files.rs diff --git a/crates/sqlbuild-analysis/src/definition_changes/constants.rs b/crates/sqlbuild-analysis/src/definition_changes/constants.rs index e6f2c470b1..f17cdf9140 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/constants.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/constants.rs @@ -61,7 +61,11 @@ pub(crate) const KIND_LABELS: [(&str, &str, &str); 23] = [ "connection and target configuration", "connection and target configurations", ), - (DEPENDENCIES_KIND, "Python environment", "Python environments"), + ( + DEPENDENCIES_KIND, + "Python environment", + "Python environments", + ), (MODEL_KIND, "model", "models"), (SOURCE_KIND, "source", "sources"), (SEED_KIND, "seed", "seeds"), diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs new file mode 100644 index 0000000000..0b1cec853f --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs @@ -0,0 +1,22 @@ +//! Which files record a project's Python dependencies, and where they are looked for. + +use std::collections::BTreeSet; + +use crate::git_revisions::constants::{DEPENDENCY_FILE_NAMES, REQUIREMENTS_FILE_AFFIXES}; +use crate::git_revisions::models::GitBase; + +/// Whether `name` is a Python dependency manifest or lock file. +pub(crate) fn is_dependency_file(name: &str) -> bool { + let (prefix, suffix) = REQUIREMENTS_FILE_AFFIXES; + DEPENDENCY_FILE_NAMES.contains(&name) || (name.starts_with(prefix) && name.ends_with(suffix)) +} + +/// The repository root and the project directory, as repository-relative directory prefixes. +pub(crate) fn directories(base: &GitBase) -> BTreeSet { + let project: &str = base.project_prefix.trim_end_matches('/'); + let mut directories: BTreeSet = BTreeSet::from([String::new()]); + if !project.is_empty() { + directories.insert(format!("{project}/")); + } + directories +} diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs index 5e58cb78ea..050dde0f95 100644 --- a/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod dependency_files; pub(crate) mod failures; pub(crate) mod git_command; diff --git a/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs new file mode 100644 index 0000000000..b0192ad6bc --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs @@ -0,0 +1,32 @@ +//! The Python dependency manifests and lock files of a project at the merge base. + +use std::path::Path; + +use crate::git_revisions::_helpers::dependency_files::{directories, is_dependency_file}; +use crate::git_revisions::_helpers::failures::export_failed; +use crate::git_revisions::_helpers::git_command::{GitFailure, git_output}; +use crate::git_revisions::errors::GitRevisionError; +use crate::git_revisions::models::GitBase; + +/// `(repository path, trimmed text)` of each dependency file at the merge base, sorted. +pub fn base_dependency_files(base: &GitBase) -> Result, GitRevisionError> { + let root: &Path = &base.repository_root; + let mut files: Vec<(String, String)> = Vec::new(); + for directory in directories(base) { + let tree: String = format!("{}:{directory}", base.merge_base); + let listing: String = match git_output(root, &["ls-tree", "--name-only", &tree], &[]) { + Ok(listing) => listing, + Err(GitFailure::Failed(_)) => continue, + Err(failure) => return Err(export_failed(base, failure.detail())), + }; + for name in listing.lines().filter(|name| is_dependency_file(name)) { + let path: String = format!("{directory}{name}"); + let blob: String = format!("{}:{path}", base.merge_base); + let text: String = git_output(root, &["show", &blob], &[]) + .map_err(|failure| export_failed(base, failure.detail()))?; + files.push((path, text)); + } + } + files.sort(); + Ok(files) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs new file mode 100644 index 0000000000..595d7a04d7 --- /dev/null +++ b/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs @@ -0,0 +1,27 @@ +//! The Python dependency manifests and lock files of a project in the checkout. + +use crate::git_revisions::_helpers::dependency_files::{directories, is_dependency_file}; +use crate::git_revisions::models::GitBase; + +/// `(repository path, trimmed text)` of each dependency file in the checkout, sorted. +pub fn current_dependency_files(base: &GitBase) -> std::io::Result> { + let mut files: Vec<(String, String)> = Vec::new(); + for directory in directories(base) { + let entries: std::fs::ReadDir = + match std::fs::read_dir(base.repository_root.join(&directory)) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + for entry in entries { + let entry: std::fs::DirEntry = entry?; + let name: String = entry.file_name().to_string_lossy().into_owned(); + if entry.path().is_file() && is_dependency_file(&name) { + let text: String = std::fs::read_to_string(entry.path())?; + files.push((format!("{directory}{name}"), text.trim().to_owned())); + } + } + } + files.sort(); + Ok(files) +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/dependency_files.rs deleted file mode 100644 index f3b1be9c09..0000000000 --- a/crates/sqlbuild-core/src/git_revisions/main/dependency_files.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! The Python dependency manifests and lock files of a project, at the base and in the checkout. - -use std::collections::BTreeSet; -use std::path::Path; - -use crate::git_revisions::_helpers::failures::export_failed; -use crate::git_revisions::_helpers::git_command::{GitFailure, git_output}; -use crate::git_revisions::constants::{DEPENDENCY_FILE_NAMES, REQUIREMENTS_FILE_AFFIXES}; -use crate::git_revisions::errors::GitRevisionError; -use crate::git_revisions::models::GitBase; - -fn is_dependency_file(name: &str) -> bool { - let (prefix, suffix) = REQUIREMENTS_FILE_AFFIXES; - DEPENDENCY_FILE_NAMES.contains(&name) || (name.starts_with(prefix) && name.ends_with(suffix)) -} - -/// The repository root and the project directory, as repository-relative directory prefixes. -fn directories(base: &GitBase) -> BTreeSet { - let project: &str = base.project_prefix.trim_end_matches('/'); - let mut directories: BTreeSet = BTreeSet::from([String::new()]); - if !project.is_empty() { - directories.insert(format!("{project}/")); - } - directories -} - -/// `(repository path, trimmed text)` of each dependency file at the merge base, sorted. -pub fn base_dependency_files(base: &GitBase) -> Result, GitRevisionError> { - let root: &Path = &base.repository_root; - let mut files: Vec<(String, String)> = Vec::new(); - for directory in directories(base) { - let tree: String = format!("{}:{directory}", base.merge_base); - let listing: String = match git_output(root, &["ls-tree", "--name-only", &tree], &[]) { - Ok(listing) => listing, - Err(GitFailure::Failed(_)) => continue, - Err(failure) => return Err(export_failed(base, failure.detail())), - }; - for name in listing.lines().filter(|name| is_dependency_file(name)) { - let path: String = format!("{directory}{name}"); - let blob: String = format!("{}:{path}", base.merge_base); - let text: String = git_output(root, &["show", &blob], &[]) - .map_err(|failure| export_failed(base, failure.detail()))?; - files.push((path, text)); - } - } - files.sort(); - Ok(files) -} - -/// `(repository path, trimmed text)` of each dependency file in the checkout, sorted. -pub fn current_dependency_files(base: &GitBase) -> std::io::Result> { - let mut files: Vec<(String, String)> = Vec::new(); - for directory in directories(base) { - let entries: std::fs::ReadDir = - match std::fs::read_dir(base.repository_root.join(&directory)) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => return Err(error), - }; - for entry in entries { - let entry: std::fs::DirEntry = entry?; - let name: String = entry.file_name().to_string_lossy().into_owned(); - if entry.path().is_file() && is_dependency_file(&name) { - let text: String = std::fs::read_to_string(entry.path())?; - files.push((format!("{directory}{name}"), text.trim().to_owned())); - } - } - } - files.sort(); - Ok(files) -} diff --git a/crates/sqlbuild-core/src/git_revisions/main/mod.rs b/crates/sqlbuild-core/src/git_revisions/main/mod.rs index 19416e7e9d..c841f13156 100644 --- a/crates/sqlbuild-core/src/git_revisions/main/mod.rs +++ b/crates/sqlbuild-core/src/git_revisions/main/mod.rs @@ -1,4 +1,5 @@ -pub mod dependency_files; +pub mod base_dependency_files; +pub mod current_dependency_files; pub mod export_git_tree; pub mod resolve_git_base; pub mod resolve_since_reference; diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs index dd108cdc14..6d3c2244cc 100644 --- a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs @@ -1,8 +1,7 @@ use std::path::PathBuf; -use crate::git_revisions::main::dependency_files::{ - base_dependency_files, current_dependency_files, -}; +use crate::git_revisions::main::base_dependency_files::base_dependency_files; +use crate::git_revisions::main::current_dependency_files::current_dependency_files; use crate::git_revisions::main::export_git_tree::export_git_tree; use crate::git_revisions::main::resolve_git_base::resolve_git_base; use crate::git_revisions::main::resolve_since_reference::resolve_since_reference; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs index 4b08e05fb9..fd6adacf7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs @@ -7,9 +7,8 @@ use sqlbuild_analysis::definition_changes::constants::{ use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; use sqlbuild_analysis::definition_changes::main::node_key::node_key; use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefinition}; -use sqlbuild_core::git_revisions::main::dependency_files::{ - base_dependency_files, current_dependency_files, -}; +use sqlbuild_core::git_revisions::main::base_dependency_files::base_dependency_files; +use sqlbuild_core::git_revisions::main::current_dependency_files::current_dependency_files; use sqlbuild_core::git_revisions::models::GitBase; use crate::bindings::_helpers::change_selection::base_snapshot::git_error; From 1d3b1a5b923c3063f2156ab0f6239f3b7aea7a64 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 00:58:33 +0100 Subject: [PATCH 08/10] fix: read only tracked dependency files and keep --since notes and cache tidy --- .../_helpers/dependency_files.rs | 5 ++ .../main/base_dependency_files.rs | 8 +- .../main/current_dependency_files.rs | 50 ++++++++---- .../git_revisions/tests/test_git_revisions.rs | 13 +++- .../_helpers/change_selection/base_cache.rs | 78 ++++++++++++++----- .../change_selection/base_snapshot.rs | 56 ++++++++----- .../change_selection/environment_nodes.rs | 6 +- .../_helpers/change_selection/fingerprints.rs | 17 +++- .../_helpers/change_selection/selection.rs | 1 + crates/sqlbuild-python/src/bindings/tests.rs | 2 + .../src/bindings/tests/test_since_cache.rs | 61 +++++++++++++++ .../src/bindings/tests/test_types.rs | 7 ++ .../sqlbuild/references/docs/cli/test.md | 17 ++-- .../cli/commands/main/test/test_since.py | 13 +++- website/src/content/docs/docs/cli/test.mdx | 17 ++-- 15 files changed, 274 insertions(+), 77 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs index 0b1cec853f..f27a06ef64 100644 --- a/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs @@ -20,3 +20,8 @@ pub(crate) fn directories(base: &GitBase) -> BTreeSet { } directories } + +/// `text` with CRLF line endings as LF and surrounding whitespace removed, as git stores it. +pub(crate) fn normalised(text: &str) -> String { + text.replace("\r\n", "\n").trim().to_owned() +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs index b0192ad6bc..ea8c0c4b00 100644 --- a/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs +++ b/crates/sqlbuild-core/src/git_revisions/main/base_dependency_files.rs @@ -2,13 +2,15 @@ use std::path::Path; -use crate::git_revisions::_helpers::dependency_files::{directories, is_dependency_file}; +use crate::git_revisions::_helpers::dependency_files::{ + directories, is_dependency_file, normalised, +}; use crate::git_revisions::_helpers::failures::export_failed; use crate::git_revisions::_helpers::git_command::{GitFailure, git_output}; use crate::git_revisions::errors::GitRevisionError; use crate::git_revisions::models::GitBase; -/// `(repository path, trimmed text)` of each dependency file at the merge base, sorted. +/// `(repository path, text)` of each dependency file at the merge base, sorted. pub fn base_dependency_files(base: &GitBase) -> Result, GitRevisionError> { let root: &Path = &base.repository_root; let mut files: Vec<(String, String)> = Vec::new(); @@ -24,7 +26,7 @@ pub fn base_dependency_files(base: &GitBase) -> Result, Gi let blob: String = format!("{}:{path}", base.merge_base); let text: String = git_output(root, &["show", &blob], &[]) .map_err(|failure| export_failed(base, failure.detail()))?; - files.push((path, text)); + files.push((path, normalised(&text))); } } files.sort(); diff --git a/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs index 595d7a04d7..e03b069872 100644 --- a/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs +++ b/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs @@ -1,25 +1,45 @@ -//! The Python dependency manifests and lock files of a project in the checkout. +//! The git-tracked Python dependency manifests and lock files of a project in the checkout. -use crate::git_revisions::_helpers::dependency_files::{directories, is_dependency_file}; +use std::path::Path; + +use crate::git_revisions::_helpers::dependency_files::{ + directories, is_dependency_file, normalised, +}; +use crate::git_revisions::_helpers::failures::export_failed; +use crate::git_revisions::_helpers::git_command::git_output; +use crate::git_revisions::errors::GitRevisionError; use crate::git_revisions::models::GitBase; -/// `(repository path, trimmed text)` of each dependency file in the checkout, sorted. -pub fn current_dependency_files(base: &GitBase) -> std::io::Result> { +/// `(repository path, text)` of each tracked dependency file as the checkout has it, sorted. +pub fn current_dependency_files(base: &GitBase) -> Result, GitRevisionError> { + let root: &Path = &base.repository_root; let mut files: Vec<(String, String)> = Vec::new(); for directory in directories(base) { - let entries: std::fs::ReadDir = - match std::fs::read_dir(base.repository_root.join(&directory)) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => return Err(error), + let listing: String = git_output( + root, + &[ + "ls-files", + "-z", + "--full-name", + "--", + &format!("{directory}*"), + ], + &[], + ) + .map_err(|failure| export_failed(base, failure.detail()))?; + for path in listing.split('\0').filter(|path| !path.is_empty()) { + let Some(name) = path.strip_prefix(directory.as_str()) else { + continue; }; - for entry in entries { - let entry: std::fs::DirEntry = entry?; - let name: String = entry.file_name().to_string_lossy().into_owned(); - if entry.path().is_file() && is_dependency_file(&name) { - let text: String = std::fs::read_to_string(entry.path())?; - files.push((format!("{directory}{name}"), text.trim().to_owned())); + if name.contains('/') || !is_dependency_file(name) { + continue; } + let text: String = match std::fs::read(root.join(path)) { + Ok(bytes) => String::from_utf8_lossy(&bytes).into_owned(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(export_failed(base, &error.to_string())), + }; + files.push((path.to_owned(), normalised(&text))); } } files.sort(); diff --git a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs index 6d3c2244cc..2ef025b90d 100644 --- a/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs +++ b/crates/sqlbuild-core/src/git_revisions/tests/test_git_revisions.rs @@ -306,17 +306,24 @@ fn given_project_absent_at_merge_base_when_exporting_tree_then_reports_no_projec fn given_dependency_files_when_reading_both_sides_then_lists_root_and_project_files() { let (root, _) = project_repository(); write(&root, "uv.lock", "version = 1\n"); - write(&root, "warehouse/requirements-dev.txt", "pytest\n"); + write( + &root, + "warehouse/requirements-dev.txt", + "pytest\r\nruff\r\n", + ); + write(&root, "warehouse/requirements-local.txt", "untracked\n"); write(&root, "warehouse/notes.txt", "not a dependency file\n"); git(&root, &["add", "uv.lock"]); git(&root, &["commit", "--quiet", "-m", "lock"]); + git(&root, &["add", "warehouse/requirements-dev.txt"]); let current: Vec<(&str, &str)> = vec![ ("uv.lock", "version = 1"), - ("warehouse/requirements-dev.txt", "pytest"), + ("warehouse/requirements-dev.txt", "pytest\nruff"), ]; let test_cases = [ DependencyFilesTestCase { - description: "files added after the base are only in the checkout", + description: "tracked files added after the base are only in the checkout, with LF \ + line endings; untracked ones are ignored", reference: "main", expected_base: vec![], expected_current: current.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs index 3370f00520..2e88520c54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_cache.rs @@ -2,18 +2,23 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use sqlbuild_analysis::definition_changes::main::definition_digest::definition_digest; use sqlbuild_rules::constants::NATIVE_BUILD_IDENTITY; +use crate::bindings::_helpers::change_selection::base_compile::NOTES_EXTENSION; use crate::bindings::_helpers::change_selection::fingerprints::{ - environment_digest, referenced_environment, + environment_digest_with, process_environment, referenced_environment, }; /// Cached base snapshots kept per project; older ones are removed after each compile. const KEPT_SNAPSHOTS: usize = 8; const SNAPSHOT_SUFFIX: &str = "json"; +const STAGED_SUFFIX: &str = "tmp"; +const WORK_PREFIX: &str = "work-"; +/// Work directories and staged files older than this belong to runs that did not clean up. +const STALE_AFTER: Duration = Duration::from_secs(24 * 60 * 60); /// Authored file types that can read environment variables through SQL or configuration. const AUTHORED_SUFFIXES: [&str; 5] = ["toml", "yml", "yaml", "sql", "py"]; /// Directories of build output, tools and dependencies, never authored project files. @@ -49,27 +54,39 @@ fn authored_files(directory: &Path) -> std::io::Result> { Ok(files) } -/// The digest of every environment variable the project's files read, by its current value. -fn environment_inputs(project_dir: &Path) -> std::io::Result { +/// The digest of every environment variable the project's files read, by its `lookup` value. +fn environment_inputs( + project_dir: &Path, + lookup: &dyn Fn(&str) -> Option, +) -> std::io::Result { let mut names: BTreeSet = BTreeSet::new(); for file in authored_files(project_dir)? { let bytes: Vec = std::fs::read(&file)?; names.extend(referenced_environment(&String::from_utf8_lossy(&bytes))); } - Ok(environment_digest(&names)) + Ok(environment_digest_with(&names, lookup)) } /// The cache key of a base compile besides its commit: build, target, vars, local config, env. pub(crate) fn cache_key( + project_dir: &Path, + inputs: (Option<&str>, &str, &str), +) -> std::io::Result { + cache_key_with(project_dir, inputs, &process_environment) +} + +/// `cache_key` with environment variables read through `lookup`. +pub(crate) fn cache_key_with( project_dir: &Path, (target, vars, local_config): (Option<&str>, &str, &str), + lookup: &dyn Fn(&str) -> Option, ) -> std::io::Result { Ok(definition_digest(&[ NATIVE_BUILD_IDENTITY, target.unwrap_or_default(), vars, local_config, - &environment_inputs(project_dir)?, + &environment_inputs(project_dir, lookup)?, ])) } @@ -78,29 +95,52 @@ pub(crate) fn work_directory_name(short: &str) -> String { let nanos: u128 = SystemTime::now() .duration_since(UNIX_EPOCH) .map_or(0, |elapsed| elapsed.as_nanos()); - format!("work-{short}-{}-{nanos:x}", std::process::id()) + format!("{WORK_PREFIX}{short}-{}-{nanos:x}", std::process::id()) } -/// Remove all but the newest cached base snapshots in `cache_dir`. +fn removed(result: std::io::Result<()>) -> std::io::Result<()> { + match result { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Keep the newest snapshots and their notes; drop the rest and day-old staged files and work dirs. pub(crate) fn prune_snapshots(cache_dir: &Path) -> std::io::Result<()> { + let now: SystemTime = SystemTime::now(); let mut snapshots: Vec<(SystemTime, PathBuf)> = Vec::new(); + let mut leftovers: Vec = Vec::new(); for entry in std::fs::read_dir(cache_dir)? { let path: PathBuf = entry?.path(); - if path.is_file() - && path - .extension() - .is_some_and(|suffix| suffix == SNAPSHOT_SUFFIX) - { - snapshots.push((std::fs::metadata(&path)?.modified()?, path)); + let modified: SystemTime = std::fs::metadata(&path)?.modified()?; + let stale: bool = now + .duration_since(modified) + .is_ok_and(|age| age > STALE_AFTER); + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let suffix: String = path + .extension() + .map(|suffix| suffix.to_string_lossy().into_owned()) + .unwrap_or_default(); + if path.is_dir() { + if stale && name.starts_with(WORK_PREFIX) { + removed(std::fs::remove_dir_all(&path))?; + } + } else if suffix == SNAPSHOT_SUFFIX { + snapshots.push((modified, path)); + } else if suffix == STAGED_SUFFIX && stale { + leftovers.push(path); } } snapshots.sort_by(|left, right| right.cmp(left)); for (_, path) in snapshots.into_iter().skip(KEPT_SNAPSHOTS) { - match std::fs::remove_file(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error), - } + leftovers.push(path.with_extension(NOTES_EXTENSION)); + leftovers.push(path); + } + for path in leftovers { + removed(std::fs::remove_file(&path))?; } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index a384f9365c..28fb517624 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -129,11 +129,11 @@ fn vars_json(cli_vars: &Bound<'_, PyAny>) -> PyResult { /// Run the base compile of `tree` into `output` in a separate interpreter. fn compile_base_tree( - (stream, base): (&Bound<'_, PyAny>, &GitBase), + py: Python<'_>, + base: &GitBase, (tree, output): (&Path, &Path), (selected_target, vars): (Option<&str>, &str), ) -> PyResult<()> { - let py: Python<'_> = stream.py(); let [_, executable, _, _] = interpreter_values(py)?; let finished: std::io::Result = py.detach(|| { Command::new(&executable) @@ -149,11 +149,6 @@ fn compile_base_tree( }); let finished: Output = finished.map_err(|error| io_failed(py, base, &error))?; if finished.status.success() && output.is_file() { - match std::fs::read_to_string(output.with_extension(NOTES_EXTENSION)) { - Ok(notes) => write(stream, ¬es)?, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(io_failed(py, base, &error)), - } return Ok(()); } Err(compile_failed(py, base, &failure_tail(&finished))) @@ -161,18 +156,18 @@ fn compile_base_tree( /// Export the base tree, carry over the untracked local config, and compile it. fn compiled_base_snapshot( - stream: &Bound<'_, PyAny>, + py: Python<'_>, (base, project_dir, work): (&GitBase, &Path, &Path), (selected_target, vars): (Option<&str>, &str), -) -> PyResult { - let py: Python<'_> = stream.py(); +) -> PyResult<(DefinitionSnapshot, String)> { let tree: PathBuf = work.join("project"); let exported: bool = export_git_tree(base, &tree).map_err(|error| git_error(py, error))?; if !exported || !tree.join(PROJECT_CONFIG_FILE).is_file() { - return Ok(DefinitionSnapshot { + let empty: DefinitionSnapshot = DefinitionSnapshot { format: DEFINITION_SNAPSHOT_FORMAT, ..DefinitionSnapshot::default() - }); + }; + return Ok((empty, String::new())); } let local: PathBuf = project_dir.join(LOCAL_CONFIG_FILE); if local.is_file() && !tree.join(LOCAL_CONFIG_FILE).exists() { @@ -180,11 +175,21 @@ fn compiled_base_snapshot( .map_err(|error| io_failed(py, base, &error))?; } let output: PathBuf = work.join("definitions.json"); - compile_base_tree((stream, base), (&tree, &output), (selected_target, vars))?; + compile_base_tree(py, base, (&tree, &output), (selected_target, vars))?; let text: String = std::fs::read_to_string(&output).map_err(|error| io_failed(py, base, &error))?; - parse_definition_snapshot(&text) - .ok_or_else(|| compile_failed(py, base, "the base compile wrote an unreadable snapshot")) + let snapshot: DefinitionSnapshot = parse_definition_snapshot(&text) + .ok_or_else(|| compile_failed(py, base, "the base compile wrote an unreadable snapshot"))?; + Ok((snapshot, read_notes(py, base, &output)?)) +} + +/// The notes stored beside snapshot `path`, empty when there are none. +fn read_notes(py: Python<'_>, base: &GitBase, path: &Path) -> PyResult { + match std::fs::read_to_string(path.with_extension(NOTES_EXTENSION)) { + Ok(notes) => Ok(notes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(String::new()), + Err(error) => Err(io_failed(py, base, &error)), + } } /// Resolve `since` to its merge base and load, or compile and cache, the base snapshot. @@ -229,6 +234,7 @@ pub(crate) fn base_comparison<'py>( }; if let Some(snapshot) = stored { write(stream, &format!("Using cached base revision {short}\n"))?; + write(stream, &read_notes(py, &base, &cached)?)?; flush(stream)?; let snapshot: DefinitionSnapshot = with_base_dependencies(py, &base, &snapshot)?; return Ok(BaseComparison { base, snapshot }); @@ -238,12 +244,24 @@ pub(crate) fn base_comparison<'py>( let started: Instant = Instant::now(); let work: PathBuf = cache_dir.join(work_directory_name(short)); std::fs::create_dir_all(&work).map_err(|error| io_failed(py, &base, &error))?; - let compiled: PyResult = - compiled_base_snapshot(stream, (&base, &directory, &work), (target, vars)); + let compiled: PyResult<(DefinitionSnapshot, String)> = + compiled_base_snapshot(py, (&base, &directory, &work), (target, vars)); let _ = std::fs::remove_dir_all(&work); - let snapshot: DefinitionSnapshot = compiled?; + let (snapshot, notes) = compiled?; + write(stream, ¬es)?; let staged: PathBuf = cached.with_extension(format!("{}.tmp", std::process::id())); - if let Ok(()) = std::fs::write(&staged, definition_snapshot_json(&snapshot)) { + let notes_path: PathBuf = cached.with_extension(NOTES_EXTENSION); + let notes_saved: std::io::Result<()> = if notes.is_empty() { + std::fs::remove_file(¬es_path).or_else(|error| match error.kind() { + std::io::ErrorKind::NotFound => Ok(()), + _ => Err(error), + }) + } else { + std::fs::write(¬es_path, ¬es) + }; + if notes_saved.is_ok() + && let Ok(()) = std::fs::write(&staged, definition_snapshot_json(&snapshot)) + { let _ = std::fs::rename(&staged, &cached); } prune_snapshots(&cache_dir).map_err(|error| io_failed(py, &base, &error))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs index fd6adacf7c..a675d91201 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/environment_nodes.rs @@ -48,8 +48,12 @@ pub(crate) fn with_base_dependencies( /// `snapshot` of the working tree with its dependency files. pub(crate) fn with_current_dependencies( + py: Python<'_>, base: &GitBase, snapshot: &DefinitionSnapshot, ) -> PyResult { - Ok(add_dependencies(snapshot, ¤t_dependency_files(base)?)) + let files: Vec<(String, String)> = py + .detach(|| current_dependency_files(base)) + .map_err(|error| git_error(py, error))?; + Ok(add_dependencies(snapshot, &files)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs index 8fbaccfdb3..8d983c663e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/fingerprints.rs @@ -258,13 +258,26 @@ pub(crate) fn referenced_environment(text: &str) -> BTreeSet { names } +/// The value of environment variable `name` in this process. +pub(crate) fn process_environment(name: &str) -> Option { + std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) +} + /// The digest of the environment variables `names` by their current values, which it never shows. pub(crate) fn environment_digest(names: &BTreeSet) -> String { + environment_digest_with(names, &process_environment) +} + +/// The digest of the environment variables `names` by the values `lookup` gives them. +pub(crate) fn environment_digest_with( + names: &BTreeSet, + lookup: &dyn Fn(&str) -> Option, +) -> String { let mut parts: Vec = Vec::new(); for name in names { parts.push(name.clone()); - parts.push(match std::env::var_os(name) { - Some(value) => definition_digest(&[&value.to_string_lossy()]), + parts.push(match lookup(name) { + Some(value) => definition_digest(&[&value]), None => "unset".to_owned(), }); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 6343a7bffe..33a5191efa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -59,6 +59,7 @@ pub(crate) fn changed_project( base: BaseComparison, ) -> PyResult { let current: DefinitionSnapshot = with_current_dependencies( + project.py(), &base.base, &definition_snapshot(project, discovered, project_dir)?, )?; diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs index 76ff959782..2dc7f9c480 100644 --- a/crates/sqlbuild-python/src/bindings/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -8,6 +8,8 @@ mod failure_cause; mod helpers; #[path = "tests/test_image.rs"] mod image; +#[path = "tests/test_since_cache.rs"] +mod since_cache; #[path = "tests/test_splice.rs"] mod splice; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs b/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs new file mode 100644 index 0000000000..67037ad36d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs @@ -0,0 +1,61 @@ +use std::path::{Path, PathBuf}; + +use crate::bindings::_helpers::change_selection::base_cache::cache_key_with; +use crate::bindings::tests::test_types::CacheKeyTestCase; + +const BASE_ENVIRONMENT: [(&str, &str); 2] = [("ORDERS_REGION", "eu"), ("UNREAD_SETTING", "a")]; + +fn project_reading_environment() -> PathBuf { + let root: PathBuf = std::env::temp_dir() + .join(format!("sqlbuild-since-cache-{}", std::process::id())) + .join("project"); + let _ = std::fs::create_dir_all(root.join("models")); + let _ = std::fs::write( + root.join("models/orders.sql"), + "SELECT order_id FROM orders WHERE region = '@@ENV:ORDERS_REGION'\n", + ); + root +} + +fn key(project: &Path, values: &[(&str, &str)]) -> String { + let lookup = |name: &str| -> Option { + values + .iter() + .find(|(candidate, _)| *candidate == name) + .map(|(_, value)| (*value).to_owned()) + }; + cache_key_with(project, (None, "{}", ""), &lookup).expect("the project reads") +} + +#[test] +fn given_referenced_environment_variable_when_keying_base_cache_then_its_value_counts() { + let project: PathBuf = project_reading_environment(); + let base_key: String = key(&project, &BASE_ENVIRONMENT); + let test_cases = [ + CacheKeyTestCase { + description: "a referenced variable with a new value changes the key", + changed: &[("ORDERS_REGION", "us"), ("UNREAD_SETTING", "a")], + expected_same_key: false, + }, + CacheKeyTestCase { + description: "an unset referenced variable changes the key", + changed: &[("UNREAD_SETTING", "a")], + expected_same_key: false, + }, + CacheKeyTestCase { + description: "a variable the project does not read leaves the key alone", + changed: &[("ORDERS_REGION", "eu"), ("UNREAD_SETTING", "b")], + expected_same_key: true, + }, + ]; + for test_case in test_cases { + let changed_key: String = key(&project, test_case.changed); + + assert_eq!( + changed_key == base_key, + test_case.expected_same_key, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs index e3f1a5535b..af538f698a 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -71,3 +71,10 @@ pub(crate) struct FailureCauseTestCase { pub(crate) report: fn(), pub(crate) expected_code: Option<&'static str>, } + +pub(crate) struct CacheKeyTestCase { + pub(crate) description: &'static str, + /// The environment variable values of the second lookup, by name. + pub(crate) changed: &'static [(&'static str, &'static str)], + pub(crate) expected_same_key: bool, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 24515e023b..f95b38544c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -103,9 +103,10 @@ Some inputs reach every test, so a change to any of them selects every test: - the adapter, `[connection]` and `[connections]` (including connection settings and adapter extensions), `[targets]` and the target in use, and project-local adapter modules under `adapters/`. Their values are compared by digest and never printed; -- the Python environment as the project records it: `pyproject.toml`, `uv.lock`, `poetry.lock`, - `pdm.lock`, `Pipfile.lock` and `requirements*.txt` in the project directory or at the repository - root. +- the Python environment as the project records it: the git-tracked `pyproject.toml`, `uv.lock`, + `poetry.lock`, `pdm.lock`, `Pipfile.lock` and `requirements*.txt` in the project directory or at + the repository root, compared with line endings normalised. Untracked and ignored copies do not + count. Any edit to `pyproject.toml` counts, even one unrelated to dependencies. Environment variables a project reads with `${ENV:NAME}` in configuration or `@@ENV:NAME` in SQL are part of the cache key of the base compile and of the fingerprints of the SQL that reads them; @@ -181,10 +182,12 @@ selected. SQL comparisons ignore layout and comments using a generic SQL tokenizer. In dialects where `--` starts a comment only when followed by whitespace, such as MySQL, `1--1` is read as `1` followed by -a comment, so an edit inside such an expression can go unnoticed. The base tree is read from git -objects without LFS smudging, including when a `.gitattributes` at the repository root marks files -in a project subdirectory. A file stored with Git LFS, such as a seed, is a pointer file at the -base and real content in the checkout, so it always counts as changed and selects what reads it. +a comment, so an edit inside such an expression can go unnoticed. The base tree is written from git +objects with the repository's filters. Git LFS content is restored for files that an unanchored +`.gitattributes` pattern marks, such as `*.csv filter=lfs`, but a pattern anchored to a path, such +as `/data/*.csv` at the repository root for a project in a subdirectory, does not match the +exported paths. Such a file is an LFS pointer at the base and real content in the checkout, so it +always counts as changed: at most extra tests run. `` is any git ref or commit, such as the merge request's target branch from a CI variable. `--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index 3e60c04676..5c379af80a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -475,8 +475,9 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( ), ), SinceSelectionE2ETestCase( - description="a Python dependency lock file selects every test", + description="a tracked Python dependency lock file selects every test", edits={"uv.lock": "version = 1\n"}, + git_after_edits=("add", "uv.lock"), command=_EXPLAINED_TESTS, expected_exit_code=0, expected_fragments=( @@ -485,6 +486,16 @@ def test_given_changes_since_ref_when_testing_then_runs_only_affected_tests( "test_order_day\n uses Python environment (changed)", ), ), + SinceSelectionE2ETestCase( + description="an untracked lock file is not part of the project", + edits={"uv.lock": "version = 1\n", "requirements-local.txt": "pytest\n"}, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: no definition changes", + "No tests are affected by the changes since main", + ), + ), SinceSelectionE2ETestCase( description="a target the base does not define compiles the base with its default", edits={ diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index 867e75e22d..a21e789598 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -95,9 +95,10 @@ Some inputs reach every test, so a change to any of them selects every test: - the adapter, `[connection]` and `[connections]` (including connection settings and adapter extensions), `[targets]` and the target in use, and project-local adapter modules under `adapters/`. Their values are compared by digest and never printed; -- the Python environment as the project records it: `pyproject.toml`, `uv.lock`, `poetry.lock`, - `pdm.lock`, `Pipfile.lock` and `requirements*.txt` in the project directory or at the repository - root. +- the Python environment as the project records it: the git-tracked `pyproject.toml`, `uv.lock`, + `poetry.lock`, `pdm.lock`, `Pipfile.lock` and `requirements*.txt` in the project directory or at + the repository root, compared with line endings normalised. Untracked and ignored copies do not + count. Any edit to `pyproject.toml` counts, even one unrelated to dependencies. Environment variables a project reads with `${ENV:NAME}` in configuration or `@@ENV:NAME` in SQL are part of the cache key of the base compile and of the fingerprints of the SQL that reads them; @@ -173,10 +174,12 @@ selected. SQL comparisons ignore layout and comments using a generic SQL tokenizer. In dialects where `--` starts a comment only when followed by whitespace, such as MySQL, `1--1` is read as `1` followed by -a comment, so an edit inside such an expression can go unnoticed. The base tree is read from git -objects without LFS smudging, including when a `.gitattributes` at the repository root marks files -in a project subdirectory. A file stored with Git LFS, such as a seed, is a pointer file at the -base and real content in the checkout, so it always counts as changed and selects what reads it. +a comment, so an edit inside such an expression can go unnoticed. The base tree is written from git +objects with the repository's filters. Git LFS content is restored for files that an unanchored +`.gitattributes` pattern marks, such as `*.csv filter=lfs`, but a pattern anchored to a path, such +as `/data/*.csv` at the repository root for a project in a subdirectory, does not match the +exported paths. Such a file is an LFS pointer at the base and real content in the checkout, so it +always counts as changed: at most extra tests run. `` is any git ref or commit, such as the merge request's target branch from a CI variable. `--since` always takes a value. A ref that does not exist, or a shallow clone that does not reach From 6aa55a2e2edb814e2f04a79a1384e7f1dc140d4a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:51:52 +0100 Subject: [PATCH 09/10] refactor: split checkout dependency reading and cache writes into helpers --- .../_helpers/dependency_files.rs | 43 +++++++++++++++++++ .../main/current_dependency_files.rs | 34 ++------------- .../change_selection/base_snapshot.rs | 35 ++++++++------- .../src/bindings/tests/helpers.rs | 27 ++++++++++++ .../src/bindings/tests/test_since_cache.rs | 30 ++----------- 5 files changed, 98 insertions(+), 71 deletions(-) diff --git a/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs index f27a06ef64..7121192d36 100644 --- a/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs +++ b/crates/sqlbuild-core/src/git_revisions/_helpers/dependency_files.rs @@ -1,8 +1,12 @@ //! Which files record a project's Python dependencies, and where they are looked for. use std::collections::BTreeSet; +use std::path::Path; +use crate::git_revisions::_helpers::failures::export_failed; +use crate::git_revisions::_helpers::git_command::git_output; use crate::git_revisions::constants::{DEPENDENCY_FILE_NAMES, REQUIREMENTS_FILE_AFFIXES}; +use crate::git_revisions::errors::GitRevisionError; use crate::git_revisions::models::GitBase; /// Whether `name` is a Python dependency manifest or lock file. @@ -25,3 +29,42 @@ pub(crate) fn directories(base: &GitBase) -> BTreeSet { pub(crate) fn normalised(text: &str) -> String { text.replace("\r\n", "\n").trim().to_owned() } + +/// Repository paths of the dependency files git tracks directly in `directory`. +pub(crate) fn tracked_dependency_paths( + base: &GitBase, + directory: &str, +) -> Result, GitRevisionError> { + let listing: String = git_output( + &base.repository_root, + &[ + "ls-files", + "-z", + "--full-name", + "--", + &format!("{directory}*"), + ], + &[], + ) + .map_err(|failure| export_failed(base, failure.detail()))?; + Ok(listing + .split('\0') + .filter_map(|path| { + let name: &str = path.strip_prefix(directory)?; + (!name.contains('/') && is_dependency_file(name)).then(|| path.to_owned()) + }) + .collect()) +} + +/// The normalised checkout text of repository path `path`, or `None` when it was deleted. +pub(crate) fn checkout_text( + base: &GitBase, + path: &str, +) -> Result, GitRevisionError> { + let file: &Path = &base.repository_root.join(path); + match std::fs::read(file) { + Ok(bytes) => Ok(Some(normalised(&String::from_utf8_lossy(&bytes)))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(export_failed(base, &error.to_string())), + } +} diff --git a/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs b/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs index e03b069872..bef9d44aa7 100644 --- a/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs +++ b/crates/sqlbuild-core/src/git_revisions/main/current_dependency_files.rs @@ -1,45 +1,19 @@ //! The git-tracked Python dependency manifests and lock files of a project in the checkout. -use std::path::Path; - use crate::git_revisions::_helpers::dependency_files::{ - directories, is_dependency_file, normalised, + checkout_text, directories, tracked_dependency_paths, }; -use crate::git_revisions::_helpers::failures::export_failed; -use crate::git_revisions::_helpers::git_command::git_output; use crate::git_revisions::errors::GitRevisionError; use crate::git_revisions::models::GitBase; /// `(repository path, text)` of each tracked dependency file as the checkout has it, sorted. pub fn current_dependency_files(base: &GitBase) -> Result, GitRevisionError> { - let root: &Path = &base.repository_root; let mut files: Vec<(String, String)> = Vec::new(); for directory in directories(base) { - let listing: String = git_output( - root, - &[ - "ls-files", - "-z", - "--full-name", - "--", - &format!("{directory}*"), - ], - &[], - ) - .map_err(|failure| export_failed(base, failure.detail()))?; - for path in listing.split('\0').filter(|path| !path.is_empty()) { - let Some(name) = path.strip_prefix(directory.as_str()) else { - continue; - }; - if name.contains('/') || !is_dependency_file(name) { - continue; + for path in tracked_dependency_paths(base, &directory)? { + if let Some(text) = checkout_text(base, &path)? { + files.push((path, text)); } - let text: String = match std::fs::read(root.join(path)) { - Ok(bytes) => String::from_utf8_lossy(&bytes).into_owned(), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => return Err(export_failed(base, &error.to_string())), - }; - files.push((path.to_owned(), normalised(&text))); } } files.sort(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index 28fb517624..498b869f4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -192,6 +192,25 @@ fn read_notes(py: Python<'_>, base: &GitBase, path: &Path) -> PyResult { } } +/// Cache `snapshot` at `cached` with its `notes`, staging the snapshot so readers never see half. +fn store_snapshot( + cached: &Path, + (snapshot, notes): (&DefinitionSnapshot, &str), +) -> std::io::Result<()> { + let notes_path: PathBuf = cached.with_extension(NOTES_EXTENSION); + if notes.is_empty() { + match std::fs::remove_file(¬es_path) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error), + _ => {} + } + } else { + std::fs::write(¬es_path, notes)?; + } + let staged: PathBuf = cached.with_extension(format!("{}.tmp", std::process::id())); + std::fs::write(&staged, definition_snapshot_json(snapshot))?; + std::fs::rename(&staged, cached) +} + /// Resolve `since` to its merge base and load, or compile and cache, the base snapshot. pub(crate) fn base_comparison<'py>( (stream, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -249,21 +268,7 @@ pub(crate) fn base_comparison<'py>( let _ = std::fs::remove_dir_all(&work); let (snapshot, notes) = compiled?; write(stream, ¬es)?; - let staged: PathBuf = cached.with_extension(format!("{}.tmp", std::process::id())); - let notes_path: PathBuf = cached.with_extension(NOTES_EXTENSION); - let notes_saved: std::io::Result<()> = if notes.is_empty() { - std::fs::remove_file(¬es_path).or_else(|error| match error.kind() { - std::io::ErrorKind::NotFound => Ok(()), - _ => Err(error), - }) - } else { - std::fs::write(¬es_path, ¬es) - }; - if notes_saved.is_ok() - && let Ok(()) = std::fs::write(&staged, definition_snapshot_json(&snapshot)) - { - let _ = std::fs::rename(&staged, &cached); - } + store_snapshot(&cached, (&snapshot, ¬es)).map_err(|error| io_failed(py, &base, &error))?; prune_snapshots(&cache_dir).map_err(|error| io_failed(py, &base, &error))?; write( stream, diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index 24aa5c45cc..9df3e0dfcf 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -1,8 +1,11 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; use sqlbuild_analysis::assembly::analysis_session::models::DynamicFamily; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_config::models::ConfigValue; + +use crate::bindings::_helpers::change_selection::base_cache::cache_key_with; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; @@ -285,3 +288,27 @@ pub(crate) fn report_elsewhere() { .join() .expect("the reporting thread finishes"); } + +/// A project whose model reads `ORDERS_REGION` through `@@ENV:`. +pub(super) fn project_reading_environment() -> PathBuf { + let root: PathBuf = std::env::temp_dir() + .join(format!("sqlbuild-since-cache-{}", std::process::id())) + .join("project"); + let _ = std::fs::create_dir_all(root.join("models")); + let _ = std::fs::write( + root.join("models/orders.sql"), + "SELECT order_id FROM orders WHERE region = '@@ENV:ORDERS_REGION'\n", + ); + root +} + +/// The base cache key of `project` with environment variables `values`. +pub(super) fn cache_key_under(project: &Path, values: &[(&str, &str)]) -> String { + let lookup = |name: &str| -> Option { + values + .iter() + .find(|(candidate, _)| *candidate == name) + .map(|(_, value)| (*value).to_owned()) + }; + cache_key_with(project, (None, "{}", ""), &lookup).expect("the project reads") +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs b/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs index 67037ad36d..6f08b2ecb1 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_since_cache.rs @@ -1,36 +1,14 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; -use crate::bindings::_helpers::change_selection::base_cache::cache_key_with; +use crate::bindings::tests::helpers::{cache_key_under, project_reading_environment}; use crate::bindings::tests::test_types::CacheKeyTestCase; const BASE_ENVIRONMENT: [(&str, &str); 2] = [("ORDERS_REGION", "eu"), ("UNREAD_SETTING", "a")]; -fn project_reading_environment() -> PathBuf { - let root: PathBuf = std::env::temp_dir() - .join(format!("sqlbuild-since-cache-{}", std::process::id())) - .join("project"); - let _ = std::fs::create_dir_all(root.join("models")); - let _ = std::fs::write( - root.join("models/orders.sql"), - "SELECT order_id FROM orders WHERE region = '@@ENV:ORDERS_REGION'\n", - ); - root -} - -fn key(project: &Path, values: &[(&str, &str)]) -> String { - let lookup = |name: &str| -> Option { - values - .iter() - .find(|(candidate, _)| *candidate == name) - .map(|(_, value)| (*value).to_owned()) - }; - cache_key_with(project, (None, "{}", ""), &lookup).expect("the project reads") -} - #[test] fn given_referenced_environment_variable_when_keying_base_cache_then_its_value_counts() { let project: PathBuf = project_reading_environment(); - let base_key: String = key(&project, &BASE_ENVIRONMENT); + let base_key: String = cache_key_under(&project, &BASE_ENVIRONMENT); let test_cases = [ CacheKeyTestCase { description: "a referenced variable with a new value changes the key", @@ -49,7 +27,7 @@ fn given_referenced_environment_variable_when_keying_base_cache_then_its_value_c }, ]; for test_case in test_cases { - let changed_key: String = key(&project, test_case.changed); + let changed_key: String = cache_key_under(&project, test_case.changed); assert_eq!( changed_key == base_key, From 33fe6bb5d3c2dc945df8d4525c680e25a2a07031 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:13:57 +0100 Subject: [PATCH 10/10] fix: keep running with a warning when the --since base cache cannot be written --- .../change_selection/base_snapshot.rs | 74 +++++++++++++++++-- .../cli/commands/main/test/_test_types.py | 11 +++ .../cli/commands/main/test/test_since.py | 37 ++++++++++ 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs index 498b869f4f..2b25114035 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/base_snapshot.rs @@ -1,5 +1,6 @@ //! The `--since` base snapshot: resolved from git, compiled in a child interpreter, cached per commit. +use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::time::Instant; @@ -29,6 +30,8 @@ use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, sorted_d use crate::bindings::_helpers::scenario_command::setup::{cli_user_error, flush, write}; const CACHE_DIRECTORY: [&str; 2] = ["target", "since"]; +/// Where base compiles work when the project's cache directory cannot be created. +const TEMPORARY_WORK_ROOT: &str = "sqlbuild-since"; const LOCAL_CONFIG_FILE: &str = "sqlbuild_local.toml"; const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; const BASE_COMPILE_FAILED_CODE: &str = "C484"; @@ -192,6 +195,52 @@ fn read_notes(py: Python<'_>, base: &GitBase, path: &Path) -> PyResult { } } +/// The notes cached beside snapshot `path`; unreadable notes are skipped with a warning. +fn cached_notes(path: &Path) -> String { + let notes: PathBuf = path.with_extension(NOTES_EXTENSION); + match std::fs::read_to_string(¬es) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => { + warn(&format!( + "Could not read the cached notes at {}: {error}", + notes.display() + )); + String::new() + } + } +} + +/// Write one warning line to stderr, which machine-readable stdout never carries. +fn warn(message: &str) { + let mut stderr: std::io::StderrLock<'_> = std::io::stderr().lock(); + match writeln!(stderr, "warning: {message}") { + Ok(()) | Err(_) => {} + } +} + +/// The cache only saves time, so a failure to write it is a warning and the run continues. +fn warn_uncached(path: &Path, error: &std::io::Error) { + warn(&format!( + "Could not cache the base revision at {}: {error}; continuing without the cache", + path.display() + )); +} + +/// A fresh work directory under `cache_dir`, or the system temporary directory without one. +fn work_directory(cache_dir: &Path, short: &str) -> std::io::Result { + let name: String = work_directory_name(short); + let work: PathBuf = cache_dir.join(&name); + match std::fs::create_dir_all(&work) { + Ok(()) => Ok(work), + Err(_unavailable) => { + let fallback: PathBuf = std::env::temp_dir().join(TEMPORARY_WORK_ROOT).join(&name); + std::fs::create_dir_all(&fallback)?; + Ok(fallback) + } + } +} + /// Cache `snapshot` at `cached` with its `notes`, staging the snapshot so readers never see half. fn store_snapshot( cached: &Path, @@ -207,8 +256,14 @@ fn store_snapshot( std::fs::write(¬es_path, notes)?; } let staged: PathBuf = cached.with_extension(format!("{}.tmp", std::process::id())); - std::fs::write(&staged, definition_snapshot_json(snapshot))?; - std::fs::rename(&staged, cached) + let stored: std::io::Result<()> = std::fs::write(&staged, definition_snapshot_json(snapshot)) + .and_then(|()| std::fs::rename(&staged, cached)); + if stored.is_err() { + match std::fs::remove_file(&staged) { + Ok(()) | Err(_) => {} + } + } + stored } /// Resolve `since` to its merge base and load, or compile and cache, the base snapshot. @@ -253,7 +308,7 @@ pub(crate) fn base_comparison<'py>( }; if let Some(snapshot) = stored { write(stream, &format!("Using cached base revision {short}\n"))?; - write(stream, &read_notes(py, &base, &cached)?)?; + write(stream, &cached_notes(&cached))?; flush(stream)?; let snapshot: DefinitionSnapshot = with_base_dependencies(py, &base, &snapshot)?; return Ok(BaseComparison { base, snapshot }); @@ -261,15 +316,20 @@ pub(crate) fn base_comparison<'py>( write(stream, &format!("Compiling base revision {short}...\n"))?; flush(stream)?; let started: Instant = Instant::now(); - let work: PathBuf = cache_dir.join(work_directory_name(short)); - std::fs::create_dir_all(&work).map_err(|error| io_failed(py, &base, &error))?; + let work: PathBuf = + work_directory(&cache_dir, short).map_err(|error| io_failed(py, &base, &error))?; let compiled: PyResult<(DefinitionSnapshot, String)> = compiled_base_snapshot(py, (&base, &directory, &work), (target, vars)); let _ = std::fs::remove_dir_all(&work); let (snapshot, notes) = compiled?; write(stream, ¬es)?; - store_snapshot(&cached, (&snapshot, ¬es)).map_err(|error| io_failed(py, &base, &error))?; - prune_snapshots(&cache_dir).map_err(|error| io_failed(py, &base, &error))?; + match store_snapshot(&cached, (&snapshot, ¬es)) { + Ok(()) => match prune_snapshots(&cache_dir) { + Ok(()) => {} + Err(error) => warn_uncached(&cache_dir, &error), + }, + Err(error) => warn_uncached(&cached, &error), + } write( stream, &format!( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index eb45445dca..2a96d1fdbe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -560,3 +560,14 @@ class SinceBrokenBaseE2ETestCase: expected_exit_code: int expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class SinceUnwritableCacheE2ETestCase: + """A `--since` run whose base snapshot cache cannot be written.""" + + description: str + blocking_file: str + command: tuple[str, ...] + expected_stderr_fragments: tuple[str, ...] + expected_test_names: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index 5c379af80a..be0a4d05a6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -12,6 +12,7 @@ SinceSelectionE2ETestCase, SinceShallowCloneE2ETestCase, SinceSharedBaseE2ETestCase, + SinceUnwritableCacheE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import ( build_since_indirect_files, @@ -711,3 +712,39 @@ def test_given_shallow_clone_without_merge_base_when_testing_since_then_asks_for assert result.returncode == test_case.expected_exit_code, output for fragment in test_case.expected_fragments: assert fragment in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceUnwritableCacheE2ETestCase( + description="a file where the cache directory belongs warns and the run continues", + blocking_file="target/since", + command=("--no-color", "test", "--since", "main", "--json"), + expected_stderr_fragments=( + "warning: Could not cache the base revision at ", + "; continuing without the cache", + "Selected 1 of 3 tests", + ), + expected_test_names=("test_stg_orders",), + ), + ), + ids=lambda case: case.description, +) +def test_given_unwritable_cache_when_testing_since_then_warns_and_succeeds( + tmp_path: Path, test_case: SinceUnwritableCacheE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files={}) + write_project_files( + project_dir=project, + files={_STAGING: _EDITED_STAGING, test_case.blocking_file: "not a directory\n"}, + ) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + assert result.returncode == 0, result.stdout + result.stderr + for fragment in test_case.expected_stderr_fragments: + assert fragment in result.stderr, result.stderr + assert result.stderr.count("warning: Could not cache") == 1, result.stderr + selected: list[dict[str, str | list[str]]] = json.loads(result.stdout)["since"]["selected"] + assert tuple(str(item["name"]) for item in selected) == test_case.expected_test_names