diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 2451efca3d..ca86762b22 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -133,6 +133,7 @@ impl AnalysisSession { dependency_ordered: self.dependency_ordered, cyclic: self.cyclic, legacy_models: self.legacy_models.clone(), + families: self.request.dynamic_families_by_table.clone(), }); let (schema_additions, analysis_names) = self.catalog.into_changes(); let mut request: SessionRequest = self.request; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index da66105b17..088c9ee3c9 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -199,6 +199,8 @@ pub struct RelationTables { pub cyclic: bool, /// Models whose analysis took the legacy path, which reads every relation's facts. pub legacy_models: Vec, + /// The request's dynamic column families by table, in request order. + pub families: Vec<(String, Vec)>, } /// One inferred output column; nullability is Python's `InferredNullability` value. diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs index 0ceec82d72..ed6080411f 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs @@ -30,7 +30,13 @@ use crate::project_snapshot::models::PathStamp; const NO_STORED_COMPILE: &str = "no stored compile"; const REUSE_BYPASSED: &str = "compile reuse is off"; const OUTPUT_UNREADABLE: &str = "the stored output cannot be replayed"; -const IDENTITY_CHANGED: &str = "the invocation, runtime, environment or settings changed"; +const INVOCATION_CHANGED: &str = "the command invocation changed"; +const RUNTIME_CHANGED: &str = "the Python runtime changed"; +const SEARCH_PATH_CHANGED: &str = "a Python search path entry changed"; +const ENVIRONMENT_NAMES_CHANGED: &str = "the set of tracked environment variables changed"; +const ENVIRONMENT_CHANGED: &str = "a tracked environment variable changed"; +const SETTINGS_CHANGED: &str = "provider settings inputs changed"; +const MODULES_CHANGED: &str = "an imported Python module outside the project changed"; const TARGET_CHANGED: &str = "the compiled target changed"; fn now_ns() -> i64 { @@ -115,8 +121,8 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { attempt.restamped = restamped_paths(&inputs.project_files, &attempt.snapshot); if comparison.unchanged { attempt.delta = DeltaCandidate::Unavailable(OUTPUT_UNREADABLE); - } else if !comparison.identity_unchanged { - attempt.delta = DeltaCandidate::Unavailable(IDENTITY_CHANGED); + } else if let Some(reason) = comparison.identity_change { + attempt.delta = DeltaCandidate::Unavailable(reason); } else if !target_files_unchanged( &inputs.target_files, &attempt.project_dir, @@ -146,21 +152,16 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { /// Whether a stored compile still matches, with the digests still valid either way. struct Comparison { unchanged: bool, - identity_unchanged: bool, + /// The first identity component that differs, if any. + identity_change: Option<&'static str>, digests: HashMap, digested_files: usize, } /// Compare every stored identity, carrying forward digests still valid either way. fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> Comparison { - let identity_unchanged: bool = inputs.invocation_digest == attempt.identity.invocation_digest - && inputs.runtime == attempt.identity.runtime - && inputs.search_path == attempt.search_path - && tracked_environment_names(&inputs.environment_names, rules) == inputs.environment_names - && environment_digest(&inputs.environment_names, rules) == inputs.environment_digest - && settings_inputs_digest(&inputs.settings_inputs, rules) == inputs.settings_digest - && module_stamps_unchanged(&inputs.modules); - let (files_unchanged, verified, digested_files) = if identity_unchanged { + let identity_change: Option<&'static str> = identity_change(attempt, inputs, rules); + let (files_unchanged, verified, digested_files) = if identity_change.is_none() { compare_project_files( &attempt.project_dir, &inputs.project_files, @@ -170,7 +171,7 @@ fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> (false, HashMap::new(), 0) }; Comparison { - identity_unchanged, + identity_change, unchanged: files_unchanged && target_files_unchanged( &inputs.target_files, @@ -183,6 +184,33 @@ fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> } } +/// The first stored identity component that no longer matches, in checking order. +fn identity_change( + attempt: &ReuseAttempt, + inputs: &StoredInputs, + rules: &ReuseRules, +) -> Option<&'static str> { + if inputs.invocation_digest != attempt.identity.invocation_digest { + Some(INVOCATION_CHANGED) + } else if inputs.runtime != attempt.identity.runtime { + Some(RUNTIME_CHANGED) + } else if inputs.search_path != attempt.search_path { + Some(SEARCH_PATH_CHANGED) + } else if tracked_environment_names(&inputs.environment_names, rules) + != inputs.environment_names + { + Some(ENVIRONMENT_NAMES_CHANGED) + } else if environment_digest(&inputs.environment_names, rules) != inputs.environment_digest { + Some(ENVIRONMENT_CHANGED) + } else if settings_inputs_digest(&inputs.settings_inputs, rules) != inputs.settings_digest { + Some(SETTINGS_CHANGED) + } else if !module_stamps_unchanged(&inputs.modules) { + Some(MODULES_CHANGED) + } else { + None + } +} + pub(crate) fn record( attempt: &ReuseAttempt, record: &CompileRecord, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs index 212576852e..480a828b5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs @@ -2,6 +2,7 @@ use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use crate::bindings::_helpers::compile_delta::driver::DELTA_VERIFY_ENV_VAR; use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; /// The directory under the compiler cache directory holding the native reuse store. @@ -16,8 +17,11 @@ pub(crate) const REUSE_DISABLE_VALUE: &str = "1"; pub(crate) const REUSE_LOGGER_NAME: &str = "sqlbuild.compile.reuse"; pub(crate) const TRACKED_ENVIRONMENT_PREFIXES: [&str; 2] = ["SQLBUILD_", "SQB_"]; /// Sorted; tracked-prefix variables that never invalidate a stored compile. -pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 2] = - [CAPTURE_DIR_ENV_VAR, REUSE_DISABLE_ENV_VAR]; +pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 3] = [ + CAPTURE_DIR_ENV_VAR, + REUSE_DISABLE_ENV_VAR, + DELTA_VERIFY_ENV_VAR, +]; pub(crate) const MISSING_ENVIRONMENT_VALUE: &str = "\0missing"; pub(crate) const MISSING_FILE_DIGEST: &str = "\0missing"; pub(crate) const PROJECT_CONFIG_FILENAMES: [&str; 2] = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs index 03a24163f9..f04188f3da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs @@ -4,7 +4,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use pyo3::prelude::PyResult; -use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, SessionRequest}; +use sqlbuild_analysis::assembly::analysis_session::models::{ + DynamicFamily, RelationTables, SessionRequest, +}; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_discovery::header_changes::types::HeaderKeys; use sqlbuild_rules::engine::main::retained_request::retained_request; @@ -103,6 +105,8 @@ pub(crate) struct ImageCapture { pub(crate) environment_names: Vec, /// Each discovered model's header keys, by model file path. pub(crate) header_keys: BTreeMap, + /// The audit factories each model file's header attaches, for files attaching any. + pub(crate) audit_references: BTreeMap>, /// Each model file's SQL lint cache key, in project model order. pub(crate) lint_model_keys: Option>, /// Each other SQL file's lint cache key, in walk order. @@ -193,6 +197,8 @@ pub(crate) struct DeltaInjection { pub(crate) coverage: Option>, /// Every project model in project order, with the models it reads. pub(crate) coverage_order: Vec<(String, Vec)>, + /// The audit factories each model file's header attaches, for files attaching any. + pub(crate) audit_references: BTreeMap>, } static CAPTURE: Mutex> = Mutex::new(None); @@ -328,17 +334,9 @@ pub(crate) fn injected_fact_digest(key: &[String]) -> Option { /// Give a delta's analysis session the whole project's relation facts and analysis order. pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult { - if !request.dynamic_families_by_table.is_empty() { - capture_unsupported("a model declares dynamic column families"); - } let Some(injection) = injection() else { return Ok(request); }; - if !request.dynamic_families_by_table.is_empty() { - return Err(delta_unsupported( - "a model declares dynamic column families", - )); - } let relations: &RelationTables = &injection.relations; request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); request.retain_relations = true; @@ -347,6 +345,16 @@ pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult = analysed + .iter() + .copied() + .chain(injection.rerun_names.iter().map(String::as_str)) + .collect(); + request.dynamic_families_by_table = injected_families( + &request.dynamic_families_by_table, + &relations.families, + &rerun, + )?; for (table, injected) in [ (&mut request.column_types, &relations.types), (&mut request.column_nullability, &relations.nullability), @@ -363,6 +371,32 @@ pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult)], + stored: &[(String, Vec)], + rerun: &BTreeSet<&str>, +) -> pyo3::PyResult)>> { + let families_of = |table: &[(String, Vec)], name: &str| { + table + .iter() + .find(|(table, _)| table == name) + .map(|(_, families)| families.clone()) + }; + let changed: bool = current + .iter() + .any(|(name, _)| !rerun.contains(name.as_str())) + || rerun + .iter() + .any(|name| families_of(current, name) != families_of(stored, name)); + if changed { + return Err(delta_unsupported( + "a model's dynamic column families changed", + )); + } + Ok(stored.to_vec()) +} + /// What a delta compile's reuse record keeps of the stored compile it patched. #[derive(Debug, Default)] pub(crate) struct RecordMerge { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index df7b38b822..1c779e83ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -34,6 +34,7 @@ use crate::bindings::_helpers::compile_driver::command::{ }; use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::rules::custom_evaluation::stored_custom_rule_refusal; use sqlbuild_discovery::header_changes::main::custom_rule_reads::custom_rule_reads; use sqlbuild_discovery::header_changes::main::header_change::header_change; use sqlbuild_discovery::header_changes::models::HeaderChange; @@ -57,6 +58,8 @@ const UNKNOWN_SOURCE: &str = "unknown source"; const SOURCE_KIND: &str = "source"; /// The upstream kind of a model a model reads. const MODEL_KIND: &str = "model"; +/// The custom-rule fact listing every project source. +const PROJECT_SOURCES_FACT: &str = "project.sources"; const TESTS_ROOT: &str = "tests/"; const STDERR: &str = "stderr"; const SQL_SUFFIX: &str = ".sql"; @@ -321,6 +324,7 @@ fn lent_injection( .collect(), lint_model_keys: image.lint_model_keys.take(), lint_file_keys: image.lint_file_keys.take(), + audit_references: image.audit_references.clone(), ..DeltaInjection::default() }; (image, injection) @@ -588,6 +592,17 @@ fn prepared( Ok(found) => found, Err(reason) => return Ok(Err(reason)), }; + let all_sources: bool = gate + .fact_digests + .iter() + .any(|(key, _)| lists_all_sources(key)); + if let Some(reason) = stored_custom_rule_refusal( + &project_dir, + (&rerun.model_paths, &rerun.changed), + all_sources, + ) { + return Ok(Err(reason.to_owned())); + } let limit: usize = MAX_RERUN_MODELS.max(gate.models.len() * MAX_RERUN_PERCENT / 100); if rerun.models.len() > limit { return Ok(Err(TOO_MANY_MODELS.to_owned())); @@ -647,7 +662,11 @@ fn rounds<'py>( { return stopped("the stored compile reports problems in files the edit leaves alone"); } - rerun.sources = sources_read(&image, &rerun); + rerun.sources = if reads_all_sources(&injection) { + None + } else { + sources_read(&image, &rerun) + }; let injection_for_round: DeltaInjection = for_rerun(injection, &rerun, rerun.sources.clone()); let (outcome, back) = round(request, attempt, (injection_for_round, &rerun))?; @@ -705,6 +724,19 @@ fn rounds<'py>( stopped("changed shapes reach too far") } +/// Whether a custom rule read the project's whole source list, which a delta must then keep. +fn reads_all_sources(injection: &DeltaInjection) -> bool { + injection + .fact_digests + .keys() + .any(|key| lists_all_sources(key)) +} + +/// Whether a custom-rule fact key is the project's whole source list. +fn lists_all_sources(key: &[String]) -> bool { + matches!(key, [name] if name == PROJECT_SOURCES_FACT) +} + /// The sources the rerun models, functions, audits and rerun SQL tests read; `None` for all. fn sources_read(image: &ProjectImage, rerun: &Rerun) -> Option> { if image.diagnostics.iter().any(outside_resources) { @@ -1162,6 +1194,10 @@ fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> Pr image.coverage = capture.coverage; } image.header_keys.extend(capture.header_keys); + image + .audit_references + .retain(|path, _| !rerun.model_paths.contains(path)); + image.audit_references.extend(capture.audit_references); let mut pinned: BTreeSet = std::mem::take(&mut image.pinned_sources) .into_iter() .collect(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 24322ec018..af898eccf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; +use sqlbuild_analysis::assembly::analysis_session::models::{DynamicFamily, RelationTables}; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_rules::models::RetainedRows; @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compile_context::image_codec::{Decoder, Encoder}; use sqlbuild_discovery::header_changes::types::HeaderKeys; /// Changes whenever the image's meaning changes, so an older image is never read. -pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v8"; +pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v9"; pub(crate) const GATE_SECTION: &str = "gate"; /// The upstream kind of a model another model reads. const MODEL_KIND: &str = "model"; @@ -46,6 +46,7 @@ pub(crate) struct ProjectImage { pub(crate) lint_model_keys: Option>, pub(crate) lint_file_keys: Option)>>, pub(crate) header_keys: BTreeMap, + pub(crate) audit_references: BTreeMap>, pub(crate) rules: Option, } @@ -101,8 +102,12 @@ pub(crate) struct ImageRelations { pub(crate) dependency_ordered: bool, pub(crate) cyclic: bool, pub(crate) legacy_models: Vec, + pub(crate) families: FamiliesByTable, } +/// Each table's dynamic column families, in the order the analysis request listed them. +pub(crate) type FamiliesByTable = Vec<(String, Vec)>; + impl From for ImageRelations { fn from(tables: RelationTables) -> Self { Self { @@ -113,6 +118,7 @@ impl From for ImageRelations { dependency_ordered: tables.dependency_ordered, cyclic: tables.cyclic, legacy_models: tables.legacy_models, + families: tables.families, } } } @@ -127,6 +133,7 @@ impl From for RelationTables { dependency_ordered: relations.dependency_ordered, cyclic: relations.cyclic, legacy_models: relations.legacy_models, + families: relations.families, } } } @@ -193,6 +200,7 @@ struct StoredFacts { dependency_ordered: bool, cyclic: bool, legacy_models: Vec, + families: FamiliesByTable, model_subjects: Option)>>, diagnostics: Vec, text_models: Option>, @@ -203,6 +211,7 @@ struct StoredFacts { coverage: Option>, lint_model_keys: Option>, lint_file_keys: Option)>>, + audit_references: BTreeMap>, } fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { @@ -282,6 +291,39 @@ fn stored_shapes(decoder: &mut Decoder<'_>) -> Option { }) } +fn put_families<'a>(encoder: &mut Encoder<'a>, families: &'a FamiliesByTable) { + encoder.count(families.len()); + for (table, declared) in families { + encoder.string(table); + encoder.count(declared.len()); + for family in declared { + encoder.string(&family.name); + encoder.string(&family.pivot_column); + encoder.string(&family.value_column); + encoder.string(&family.aggregate); + encoder.string(&family.data_type); + encoder.optional_string(family.name_pattern.as_deref()); + } + } +} + +fn families(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| { + let table: String = decoder.string()?; + let declared: Vec = decoder.list(|decoder| { + Some(DynamicFamily { + name: decoder.string()?, + pivot_column: decoder.string()?, + value_column: decoder.string()?, + aggregate: decoder.string()?, + data_type: decoder.string()?, + name_pattern: decoder.optional_string()?, + }) + })?; + Some((table, declared)) + }) +} + impl ImageGate { fn encode(&self) -> Vec { let mut encoder: Encoder<'_> = Encoder::default(); @@ -374,6 +416,7 @@ impl StoredFacts { encoder.flag(self.dependency_ordered); encoder.flag(self.cyclic); encoder.strings(self.legacy_models.iter().map(String::as_str)); + put_families(&mut encoder, &self.families); encoder.flag(self.model_subjects.is_some()); if let Some(subjects) = &self.model_subjects { encoder.count(subjects.len()); @@ -401,6 +444,11 @@ impl StoredFacts { encoder.string(path); encoder.strings(names.iter().map(String::as_str)); } + encoder.count(self.audit_references.len()); + for (path, names) in &self.audit_references { + encoder.string(path); + encoder.strings(names.iter().map(String::as_str)); + } encoder.bytes(&misc); Ok(encoder.finish()) } @@ -419,6 +467,7 @@ impl StoredFacts { let dependency_ordered: bool = decoder.flag()?; let cyclic: bool = decoder.flag()?; let legacy_models: Vec = decoder.strings()?; + let families: FamiliesByTable = families(&mut decoder)?; let model_subjects = if decoder.flag()? { Some(decoder.list(|decoder| { Some(( @@ -445,6 +494,10 @@ impl StoredFacts { .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? .into_iter() .collect(); + let audit_references: BTreeMap> = decoder + .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? + .into_iter() + .collect(); let StoredMisc { diagnostics, text_models, @@ -461,6 +514,7 @@ impl StoredFacts { dependency_ordered, cyclic, legacy_models, + families, model_subjects, diagnostics, text_models, @@ -471,6 +525,7 @@ impl StoredFacts { coverage, lint_model_keys, lint_file_keys, + audit_references, }) } } @@ -532,6 +587,7 @@ impl ProjectImage { lint_model_keys: capture.lint_model_keys, lint_file_keys: capture.lint_file_keys, header_keys: capture.header_keys, + audit_references: capture.audit_references, rules: capture .rules .map(|rules| ImageRules::Retained(rules.retained)), @@ -562,6 +618,7 @@ impl ProjectImage { dependency_ordered: relations.dependency_ordered, cyclic: relations.cyclic, legacy_models: relations.legacy_models, + families: relations.families, model_subjects: self.model_subjects, diagnostics: self.diagnostics, text_models: self.text_models, @@ -571,6 +628,7 @@ impl ProjectImage { test_sources: self.test_sources, lint_model_keys: self.lint_model_keys, lint_file_keys: self.lint_file_keys, + audit_references: self.audit_references, }; let rules: Option = self.rules.and_then(ImageRules::retained); let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); @@ -624,6 +682,7 @@ impl ProjectImage { dependency_ordered: facts.dependency_ordered, cyclic: facts.cyclic, legacy_models: facts.legacy_models, + families: facts.families, types: Shapes::new(), }; let model_shapes: Shapes = restored(facts.model_shapes, &types)?; @@ -646,6 +705,7 @@ impl ProjectImage { lint_model_keys: facts.lint_model_keys, lint_file_keys: facts.lint_file_keys, header_keys: gate.header_keys, + audit_references: facts.audit_references, rules, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index ba96e7ae51..cae1f4f929 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -20,6 +21,9 @@ use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; use sqlbuild_pysource::definition_locations::main::decorator_location::decorator_location; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; +use crate::bindings::_helpers::compile_context::delta::{ + DeltaInjection, capture, capturing, injection, +}; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ DiagnosticSettings, HookDescriptionRow, LoaderRow, PythonResourceRow, ResourceDiagnostics, ScenarioDescriptionRow, @@ -174,19 +178,45 @@ fn orphaned_factories<'py>(diagnostics: &Diagnostics<'_, 'py>) -> PyResult> = injection(); + let by_path: bool = injected.is_some() || capturing(); let mut referenced: HashSet = HashSet::new(); + let mut paths: HashSet = HashSet::new(); for model_file in &diagnostics.settings.model_files { - let header: Bound<'py, PyDict> = - native_field(model_file.bind(py), "header_values")?.downcast_into()?; - let Some(names) = header.get_item(AUDIT_FACTORIES_KEY)? else { - continue; - }; - let Ok(names) = names.downcast::() else { - continue; + let model_file: &Bound<'py, PyAny> = model_file.bind(py); + let path: Option = if by_path { + Some(posix( + &native_field(model_file, "relative_path")?.str()?.to_cow()?, + )) + } else { + None }; - for name in names.iter() { - if name.is_instance_of::() { - referenced.insert(name.extract()?); + let header: Bound<'py, PyDict> = + native_field(model_file, "header_values")?.downcast_into()?; + let mut names: Vec = Vec::new(); + if let Some(values) = header.get_item(AUDIT_FACTORIES_KEY)? + && let Ok(values) = values.downcast::() + { + for name in values.iter() { + if name.is_instance_of::() { + names.push(name.extract()?); + } + } + } + if let Some(path) = path { + if !names.is_empty() { + capture(|capture| { + capture.audit_references.insert(path.clone(), names.clone()); + }); + } + paths.insert(path); + } + referenced.extend(names); + } + if let Some(injected) = &injected { + for (path, names) in &injected.audit_references { + if !paths.contains(path) { + referenced.extend(names.iter().cloned()); } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 1d6cf5b789..98eac488eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -18,6 +18,9 @@ use sqlbuild_rules::custom_rule_cache::types::ReadSet; use crate::bindings::_helpers::boundary::python_calls::imported; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::cache::native_store::current_code_identity; +use crate::bindings::_helpers::compile_context::delta::{ + DeltaInjection, delta_unsupported, injection, +}; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; @@ -31,13 +34,22 @@ use crate::bindings::_helpers::rules::custom_host_payload::{ use crate::bindings::_helpers::rules::custom_hosts::{HostCancel, HostRequest, run_custom_hosts}; use crate::bindings::_helpers::rules::custom_rule_cache::CustomRuleCache; use crate::bindings::_helpers::rules::custom_rule_cache::custom_rule_identity; +use crate::bindings::_helpers::rules::custom_rule_cache::stored_rules; use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_implementation_fingerprint; use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_import_closure; use crate::bindings::_helpers::rules::fact_encoder::{Encoded, reproducible}; -use crate::bindings::_helpers::rules::fact_replay::{FactReplay, is_tree_fact}; -use crate::bindings::_helpers::rules::rule_run::bulk_cache_file; +use crate::bindings::_helpers::rules::fact_replay::{ + FactReplay, delta_safe_fact, is_tree_fact, read_text_path, +}; +use crate::bindings::_helpers::rules::rule_run::{bulk_cache_file, rules_bulk_cache_path}; const PROJECT_SUBJECT: &str = "project"; +/// Why a delta compile cannot rerun a custom rule that reads facts of models it does not compile. +const READ_MORE: &str = "a custom rule read more than the changed models"; +const PROJECT_WIDE: &str = "a project-wide custom rule must run again"; +const OUTSIDE_DELTA: &str = "a custom rule must run again for a model the delta does not compile"; +/// The custom-rule cache's subject key of project-wide rules. +const PROJECT_SUBJECT_KEY: &str = ""; const MODEL_SUBJECT: &str = "model"; /// The run one custom-rule evaluation describes. pub(crate) struct CustomRun<'a, 'py> { @@ -140,13 +152,11 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( readset_digests.push(reproducible(digests.combined(readset))?); } let plan: HostPlan = cache.plan(readset_digests); - if !plan.is_empty() && crate::bindings::_helpers::compile_context::delta::injection().is_some() + let delta: Option> = injection(); + if let Some(injection) = &delta + && let Some(reason) = delta_plan_refusal(&plan, &cache, injection) { - return Err( - crate::bindings::_helpers::compile_context::delta::delta_unsupported( - "a custom rule must run again", - ), - ); + return Err(delta_unsupported(reason)); } if !plan.is_empty() { let rules: Vec<(String, HostRuleSubject)> = custom @@ -169,6 +179,13 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( &plan, )?; let evaluations: Vec = evaluations(host_run, &mut digests, enabled)?; + if let Some(injection) = &delta + && !evaluations + .iter() + .all(|evaluation| delta_safe_evaluation(evaluation, injection)) + { + return Err(delta_unsupported(READ_MORE)); + } cache.record(py, evaluations)?; } crate::bindings::_helpers::compile_context::delta::capture(|capture| { @@ -189,6 +206,80 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( ) } +/// Why a delta can't run `plan`: it runs only its own models' rules that read what it holds. +fn delta_plan_refusal( + plan: &HostPlan, + cache: &CustomRuleCache, + injection: &DeltaInjection, +) -> Option<&'static str> { + let all_sources: bool = injection.source_names.is_none(); + for (code, subjects) in plan { + let Some(subjects) = subjects else { + return Some(PROJECT_WIDE); + }; + for subject in subjects { + if !injection.model_paths.contains(subject) { + return Some(OUTSIDE_DELTA); + } + if cache.stored_readset(code, subject).is_some_and(|readset| { + !readset + .iter() + .all(|key| delta_safe_fact(key, &injection.model_paths, all_sources)) + }) { + return Some(READ_MORE); + } + } + } + None +} + +/// Why a delta of `model_paths` would stop at custom rules, judged from their stored read sets. +pub(crate) fn stored_custom_rule_refusal( + project_dir: &Path, + (model_paths, changed): (&BTreeSet, &BTreeSet), + all_sources: bool, +) -> Option<&'static str> { + let stored = stored_rules(&rules_bulk_cache_path( + project_dir.to_path_buf(), + CUSTOM_RULES_CACHE_FILE, + )); + for rule in stored.values() { + for (subject, entry) in &rule.entries { + let Some(readset) = rule.readsets.get(entry.readset) else { + continue; + }; + if model_paths.contains(subject) { + if !readset + .iter() + .all(|key| delta_safe_fact(key, model_paths, all_sources)) + { + return Some(READ_MORE); + } + } else if readset + .iter() + .any(|key| read_text_path(key).is_some_and(|path| changed.contains(path))) + { + return Some(if subject == PROJECT_SUBJECT_KEY { + PROJECT_WIDE + } else { + OUTSIDE_DELTA + }); + } + } + } + None +} + +/// Whether a delta compile evaluated a custom rule from facts it holds in full. +fn delta_safe_evaluation(evaluation: &CustomEvaluation, injection: &DeltaInjection) -> bool { + let all_sources: bool = injection.source_names.is_none(); + evaluation.cacheable.as_ref().is_some_and(|(readset, _)| { + readset + .iter() + .all(|key| delta_safe_fact(key, &injection.model_paths, all_sources)) + }) +} + /// `CustomRulesOutcome(findings=..., cache_hits=..., cache_misses=..., custom_ms=..., ...)`. pub(crate) fn outcome<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index b971ea26ac..b83586f846 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -53,7 +53,7 @@ fn rules_error(py: Python<'_>, message: String) -> PyResult { Ok(pyo3::PyErr::from_value(class.call1((message,))?)) } -fn stored_rules(path: &std::path::Path) -> StoredRules { +pub(crate) fn stored_rules(path: &std::path::Path) -> StoredRules { let Ok(text) = std::fs::read_to_string(path) else { return StoredRules::new(); }; @@ -120,6 +120,16 @@ impl CustomRuleCache { &self.pending } + /// The read set `code` last recorded for `subject` under its current identity, if any. + pub(crate) fn stored_readset(&self, code: &str, subject: &str) -> Option<&ReadSet> { + let rule: &CustomRule = self.rules.iter().find(|rule| rule.code == code)?; + let cached = self.stored.get(code)?; + if rule.identity.as_deref() != Some(cached.identity.as_str()) { + return None; + } + cached.readsets.get(cached.entries.get(subject)?.readset) + } + /// Reuse results given each pending read set's digest; returns the subjects still to run. pub(crate) fn plan( &mut self, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs index 98bbdb4fec..a794a92a43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs @@ -1,6 +1,6 @@ //! Native replay of the compiler facts custom-rule evaluations recorded, for their digests. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, PyTypeMethods, Python}; @@ -150,6 +150,63 @@ pub(crate) fn is_tree_fact(key: &FactKey) -> bool { }) } +/// The fact a key name reads, if it names one. +fn fact_of(name: &str) -> Option { + FACTS + .iter() + .find(|(fact_name, _, _)| *fact_name == name) + .map(|(_, fact, _)| *fact) +} + +/// Whether a delta compiling `model_paths` reads `key` exactly as a full compile would. +pub(crate) fn delta_safe_fact( + key: &FactKey, + model_paths: &BTreeSet, + all_sources: bool, +) -> bool { + if is_tree_fact(key) { + return !key + .first() + .is_some_and(|name| fact_of(name) == Some(Fact::ResourcesUnder)); + } + if let [name] = key.as_slice() { + return all_sources + && FACTS + .iter() + .any(|(fact_name, fact, _)| fact_name == name && *fact == Fact::Sources); + } + let [name, path] = key.as_slice() else { + return false; + }; + FACTS.iter().any(|(fact_name, fact, arguments)| { + fact_name == name + && *arguments == Arguments::ModelPath + && matches!( + fact, + Fact::SqlForModel + | Fact::Declared + | Fact::Inferred + | Fact::Names + | Fact::Enforced + | Fact::Grain + ) + && model_paths.contains(path) + }) +} + +/// The project file whose text `key` reads: a file's text or a model's SQL. +pub(crate) fn read_text_path(key: &FactKey) -> Option<&str> { + let [name, path] = key.as_slice() else { + return None; + }; + FACTS + .iter() + .any(|(fact_name, fact, _)| { + fact_name == name && matches!(fact, Fact::ReadText | Fact::SqlForModel) + }) + .then_some(path.as_str()) +} + /// What the facts read of one compiled model. pub(crate) struct ModelEntry<'py> { pub(crate) model: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs index 44f522ea77..18e57b5f55 100644 --- a/crates/sqlbuild-python/src/bindings/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_coverage.rs"] mod coverage; +#[path = "tests/test_delta_facts.rs"] +mod delta_facts; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_image.rs"] diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index c9ec506f5a..c51ca5e950 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; +use sqlbuild_analysis::assembly::analysis_session::models::DynamicFamily; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_config::models::ConfigValue; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; @@ -67,6 +68,17 @@ pub(crate) fn image() -> ProjectImage { dependency_ordered: true, cyclic: false, legacy_models: strings(&["customers"]), + families: vec![( + "order_pivot".to_owned(), + vec![DynamicFamily { + name: "category_amounts".to_owned(), + pivot_column: "category".to_owned(), + value_column: "amount".to_owned(), + aggregate: "MAX".to_owned(), + data_type: "DOUBLE".to_owned(), + name_pattern: Some("^amount_".to_owned()), + }], + )], }, fact_digests: vec![(strings(&["model", "orders"]), "fact-digest".to_owned())], model_subjects: Some(vec![ @@ -113,6 +125,10 @@ pub(crate) fn image() -> ProjectImage { "models/orders.sql".to_owned(), pairs(&[("description", "0123")]), )]), + audit_references: BTreeMap::from([( + "models/orders.sql".to_owned(), + strings(&["order_quality"]), + )]), rules: Some(ImageRules::Retained(RetainedRows { request_json: request, rows: RULE_ROWS.to_vec(), @@ -146,6 +162,7 @@ pub(crate) fn facts(image: &ProjectImage) -> String { &image.lint_model_keys, &image.lint_file_keys, &image.header_keys, + &image.audit_references, ), ) ) diff --git a/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs b/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs new file mode 100644 index 0000000000..4b582b8eff --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs @@ -0,0 +1,77 @@ +use std::collections::BTreeSet; + +use crate::bindings::_helpers::rules::fact_replay::delta_safe_fact; +use crate::bindings::tests::helpers::{names, strings}; +use crate::bindings::tests::test_types::DeltaFactTestCase; + +#[test] +fn given_custom_rule_fact_when_a_delta_compiles_some_models_then_only_facts_it_holds_are_safe() { + let test_cases = [ + DeltaFactTestCase { + description: "the SQL of a compiled model", + key: &["sql.for_model", "models/orders.sql"], + all_sources: false, + expected_safe: true, + }, + DeltaFactTestCase { + description: "the inferred columns of a compiled model", + key: &["columns.inferred", "models/orders.sql"], + all_sources: false, + expected_safe: true, + }, + DeltaFactTestCase { + description: "the SQL of a model the delta does not compile", + key: &["sql.for_model", "models/customers.sql"], + all_sources: false, + expected_safe: false, + }, + DeltaFactTestCase { + description: "the dependencies of a compiled model", + key: &["graph.dependencies", "models/orders.sql"], + all_sources: false, + expected_safe: false, + }, + DeltaFactTestCase { + description: "the text of any project file", + key: &["tree.read_text", "models/customers.sql"], + all_sources: false, + expected_safe: true, + }, + DeltaFactTestCase { + description: "the models under a folder, listed from the project's models", + key: &["tree.resources_under", "models"], + all_sources: false, + expected_safe: false, + }, + DeltaFactTestCase { + description: "the files under a folder, listed from the filesystem", + key: &["tree.descendants", "models"], + all_sources: false, + expected_safe: true, + }, + DeltaFactTestCase { + description: "every model", + key: &["project.models"], + all_sources: true, + expected_safe: false, + }, + DeltaFactTestCase { + description: "every source when the delta kept them all", + key: &["project.sources"], + all_sources: true, + expected_safe: true, + }, + DeltaFactTestCase { + description: "every source when the delta kept only those it reads", + key: &["project.sources"], + all_sources: false, + expected_safe: false, + }, + ]; + let compiled: BTreeSet = names(&["models/orders.sql"]); + for test_case in test_cases { + let safe: bool = delta_safe_fact(&strings(test_case.key), &compiled, test_case.all_sources); + + assert_eq!(safe, test_case.expected_safe, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs index 5d4ab140dd..c5785986b5 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -57,3 +57,11 @@ pub(crate) struct UncompiledModelTestCase { pub(crate) message: &'static str, pub(crate) expected_uncompiled: bool, } + +/// A custom-rule fact key a delta compile holding some models may or may not read in full. +pub(crate) struct DeltaFactTestCase { + pub(crate) description: &'static str, + pub(crate) key: &'static [&'static str], + pub(crate) all_sources: bool, + pub(crate) expected_safe: bool, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index abf369a31b..09030d8ff4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -163,14 +163,20 @@ The compile runs in full instead, decided before any compile work starts, when: - An edited model changes what it references, or changes a header setting other than its `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch between `view` and `table` materialization. -- A custom rule reads something the edit changed, such as a model's contract or materialization. +- A project-subject custom rule reads something the edit changed, or a model-subject custom rule + must run again for a model the edit did not change, for example because it reads the text of an + edited upstream file. - The stored compile reported errors or warnings other than rule findings in model or SQL test files the edit did not change. - The stored compile used the other output format (`--json` or text), or has no project image. A few edits are found to need a full compile only part-way through, for example when a macro's -arguments change which models or sources it reads, or when the edit changes the shape of so many -models that a full compile is cheaper. +arguments change which models or sources it reads, when the edit changes a model's +`dynamic_columns` families, or when the edit changes the shape of so many models that a full +compile is cheaper. A model-subject custom rule that must run again for an edited model runs in the +delta compile when it reads only that model's SQL, columns, contract and grain, project files, and +the source list; if it reads anything else, such as the model's dependencies, the compile runs in +full. Otherwise, an edit runs the whole compile again, and a few caches under `target/cache/` shorten it: every model, test, and audit is rendered again, and contracts and semantic validation run for diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 3d196f0420..60270542b0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1164,6 +1164,26 @@ class DeltaCompileFullCopyTestCase: expected_difference: None = None +@dataclass(frozen=True) +class DeltaCompileProjectShapeTestCase: + description: str + files: tuple[tuple[str, str], ...] + edits: tuple[tuple[str, str, str], ...] + expected_delta_line: bool + stored_edits: tuple[tuple[str, str, str], ...] = () + expected_difference: None = None + + +@dataclass(frozen=True) +class DeltaCompileVerifyTestCase: + description: str + files: tuple[tuple[str, str], ...] + edits: tuple[tuple[str, str, str], ...] + env: dict[str, str] + expected_stderr_line: str + stored_edits: tuple[tuple[str, str, str], ...] = () + + @dataclass(frozen=True) class DeltaCompileSequenceTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index b532e5a73e..5a5d28869f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5287,6 +5287,24 @@ def run_delta_compile( ) +def store_shaped_project( + *, + project_dir: Path, + files: tuple[tuple[str, str], ...], + stored_edits: tuple[tuple[str, str, str], ...], +) -> None: + """Add `files` to the project, apply `stored_edits` and compile once to store it.""" + + for relative_path, contents in files: + write_project_file(project_dir, relative_path, contents) + for relative_path, old, new in stored_edits: + replace_project_text(project_dir, relative_path, old, new) + stored: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=("compile",), env={} + ) + assert "Traceback" not in stored.stderr, stored.stderr[-2000:] + + def run_full_compile(project_dir: Path, *, args: tuple[str, ...]) -> DeltaCompileOutcome: """Compile the project at `project_dir` in full, with delta compiles off.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 55c6ad429a..a9fb903033 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -11,10 +11,12 @@ from scripts.delta_compile_series._helpers.outcomes import full_compile_difference from scripts.delta_compile_series.constants import ( DELTA_COMPILE_LINE, + DELTA_VERIFIED_LINE, DOUBLE_CAST, DOUBLE_COLUMN, FLOAT_CAST, FLOAT_COLUMN, + MISMATCH_EXIT_CODE, PROJECT_CONFIG, ) from scripts.delta_compile_series.main.run_delta_compile_sequence import ( @@ -34,13 +36,16 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DeltaCompileFullCopyTestCase, DeltaCompileOptOutTestCase, + DeltaCompileProjectShapeTestCase, DeltaCompileSequenceTestCase, DeltaCompileSeriesTestCase, + DeltaCompileVerifyTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( replace_project_text, run_delta_compile, run_full_compile, + store_shaped_project, without_compile_timings, ) @@ -76,6 +81,110 @@ "[defaults]", ) _REUSE_ROOT: str = "target/cache/compiler-native-v1/project-reuse-v3" +_PIVOT_ROOT: str = f"{_SMALL_MODEL_ROOT}/batch00009" +_CATEGORY_AMOUNTS: tuple[str, str] = ( + f"{_PIVOT_ROOT}/order_category_amounts.sql", + """MODEL ( + description "Order amounts per category.", + materialized table, + contract enforced, + columns ( + customer_id (type INTEGER, description "Customer identifier."), + category (type VARCHAR, description "Order category."), + amount (type DOUBLE, description "Order amount."), + ), +); + +SELECT CAST(b.id AS INTEGER) AS customer_id, CAST('books' AS VARCHAR) AS category, + CAST(b.amount AS DOUBLE) AS amount +FROM __ref("sales000__int_clean__orders_00001") AS b +""", +) +_CATEGORY_PIVOT_PATH: str = f"{_PIVOT_ROOT}/order_category_pivot.sql" +_CATEGORY_PIVOT: tuple[str, str] = ( + _CATEGORY_PIVOT_PATH, + """MODEL ( + description "Order amounts pivoted per category.", + materialized table, + contract enforced, + columns (customer_id (type INTEGER, description "Customer identifier.")), + dynamic_columns ( + category_amounts ( + pivot_column category, + value_column amount, + aggregate MAX, + type DOUBLE + ) + ), +); + +PIVOT __ref("order_category_amounts") +ON category +USING MAX(amount) +GROUP BY customer_id +""", +) +_ORDER_QUALITY_FACTORY: tuple[str, str] = ( + "python/factories/quality.py", + """from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory + + +@audit_factory +def order_quality(): + return [ + AuditCase( + name="amount_present", + definition="not_null", + arguments={"column": "amount"}, + severity=AuditSeverity.ERROR, + ), + ] +""", +) +_ATTACH_ORDER_QUALITY: tuple[str, str, str] = ( + _ANALYSED_MODEL, + "MODEL (description", + "MODEL (audit_factories [order_quality], description", +) +_MODEL_SQL_RULE: tuple[str, str] = ( + "rules/order_sql.py", + """from sqlbuild.rules import Finding, Model, RuleContext, rule + + +@rule( + code="XSQBRBENCH002", + message="Order models must not leave TODO markers", + remediation="Resolve the TODO.", +) +def order_todo_markers(*, model: Model, ctx: RuleContext) -> list[Finding]: + authored = ctx.sql.for_model(model).authored.source + return [ctx.finding(subject=model)] if "TODO" in authored else [] +""", +) +_FOLDER_RULE: tuple[str, str] = ( + "rules/order_folders.py", + """from sqlbuild.rules import Finding, Model, RuleContext, rule + + +@rule( + code="XSQBRBENCH003", + message="Order models must share their folder with another model", + remediation="Group the model with related models.", +) +def order_folder_siblings(*, model: Model, ctx: RuleContext) -> list[Finding]: + if not model.name.endswith("_00005"): + return [] + siblings = ctx.project.tree.resources_under(model.path.parent.as_posix()) + return [ctx.finding(subject=model)] if len(siblings) < 2 else [] +""", +) +_RULE_READ_MORE: str = "delta compile fell back: a custom rule read more than the changed models" +_FAMILY_TYPE_EDIT: tuple[str, str, str] = (_CATEGORY_PIVOT_PATH, "type DOUBLE", "type VARCHAR") +_FAMILIES_CHANGED: str = "delta compile fell back: a model's dynamic column families changed" +_EXAMPLE_TRACKED_VARIABLE: str = "SQLBUILD_ORDERS_EXAMPLE" +_TRACKED_VARIABLE_ADDED: str = ( + "delta compile fell back: the set of tracked environment variables changed" +) _EDITED_MODEL: str = ( "models/sales000/intermediate/clean/batch00000/sales000__int_clean__orders_00003.sql" ) @@ -239,6 +348,126 @@ def test_given_stored_compile_when_compiling_an_edit_then_it_prints_what_a_full_ ) +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileProjectShapeTestCase( + description="a model with dynamic column families", + files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileProjectShapeTestCase( + description="an audit factory attached only by an unchanged model", + files=(_ORDER_QUALITY_FACTORY,), + stored_edits=(_ATTACH_ORDER_QUALITY,), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileProjectShapeTestCase( + description="a model-subject custom rule reading the edited model's SQL", + files=(_MODEL_SQL_RULE,), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileProjectShapeTestCase( + description="a model-subject custom rule listing models under a folder", + files=(_FOLDER_RULE,), + edits=(_BODY_EDIT,), + expected_delta_line=False, + ), + DeltaCompileProjectShapeTestCase( + description="an edit to dynamic column families compiles in full", + files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + edits=(_FAMILY_TYPE_EDIT,), + expected_delta_line=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_shape_when_compiling_an_edit_then_it_prints_what_a_full_compile_prints( + test_case: DeltaCompileProjectShapeTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_SMALL_MODEL_COUNT) + store_shaped_project( + project_dir=project_dir, files=test_case.files, stored_edits=test_case.stored_edits + ) + for relative_path, old, new in test_case.edits: + replace_project_text(project_dir, relative_path, old, new) + copy: Path = tmp_path / "full_copy" + _ = shutil.copytree(project_dir, copy, symlinks=True) + + edited: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=_TEXT_COMPILE, env={} + ) + + assert (DELTA_COMPILE_LINE in edited.stderr) == test_case.expected_delta_line, edited.stderr + assert ( + full_compile_difference( + outcome=edited, + full=run_full_compile(copy, args=_TEXT_COMPILE), + project_dir=project_dir, + full_project_dir=copy, + ) + == test_case.expected_difference + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileVerifyTestCase( + description="every shape together matches a full compile", + files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT, _ORDER_QUALITY_FACTORY, _MODEL_SQL_RULE), + stored_edits=(_ATTACH_ORDER_QUALITY,), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=DELTA_VERIFIED_LINE, + ), + DeltaCompileVerifyTestCase( + description="an edit to dynamic column families names why it fell back", + files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + edits=(_FAMILY_TYPE_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=_FAMILIES_CHANGED, + ), + DeltaCompileVerifyTestCase( + description="a rule listing models under a folder names why it fell back", + files=(_FOLDER_RULE,), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=_RULE_READ_MORE, + ), + DeltaCompileVerifyTestCase( + description="a new tracked variable names what changed", + files=(), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1", _EXAMPLE_TRACKED_VARIABLE: "1"}, + expected_stderr_line=_TRACKED_VARIABLE_ADDED, + ), + ], + ids=lambda case: case.description, +) +def test_given_verified_edit_when_compiling_then_it_verifies_or_names_why_it_fell_back( + test_case: DeltaCompileVerifyTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_SMALL_MODEL_COUNT) + store_shaped_project( + project_dir=project_dir, files=test_case.files, stored_edits=test_case.stored_edits + ) + for relative_path, old, new in test_case.edits: + replace_project_text(project_dir, relative_path, old, new) + + edited: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=_TEXT_COMPILE, env=test_case.env + ) + + assert edited.returncode != MISMATCH_EXIT_CODE, edited.stderr[-2000:] + assert test_case.expected_stderr_line in edited.stderr.splitlines(), edited.stderr[-2000:] + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index bff480f577..0973b91ad2 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -153,14 +153,20 @@ The compile runs in full instead, decided before any compile work starts, when: - An edited model changes what it references, or changes a header setting other than its `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch between `view` and `table` materialization. -- A custom rule reads something the edit changed, such as a model's contract or materialization. +- A project-subject custom rule reads something the edit changed, or a model-subject custom rule + must run again for a model the edit did not change, for example because it reads the text of an + edited upstream file. - The stored compile reported errors or warnings other than rule findings in model or SQL test files the edit did not change. - The stored compile used the other output format (`--json` or text), or has no project image. A few edits are found to need a full compile only part-way through, for example when a macro's -arguments change which models or sources it reads, or when the edit changes the shape of so many -models that a full compile is cheaper. +arguments change which models or sources it reads, when the edit changes a model's +`dynamic_columns` families, or when the edit changes the shape of so many models that a full +compile is cheaper. A model-subject custom rule that must run again for an edited model runs in the +delta compile when it reads only that model's SQL, columns, contract and grain, project files, and +the source list; if it reads anything else, such as the model's dependencies, the compile runs in +full. Otherwise, an edit runs the whole compile again, and a few caches under `target/cache/` shorten it: every model, test, and audit is rendered again, and contracts and semantic validation run for