From 088685399b62155b4ffd93700fcc49e581e28ea7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:11:31 +0100 Subject: [PATCH 1/4] fix(compile): name what changed when a delta compile falls back on identity Delta compile reported one message for any invocation, runtime, search path, environment, settings or module change. Name the first component that differs instead. SQLBUILD_VERIFY_DELTA_COMPILE was a tracked variable, so turning it on for one compile changed the reuse identity and made the very edit it was meant to verify fall back. It does not change compile output, so it no longer invalidates a stored compile. --- .../src/project_reuse/_helpers/attempt.rs | 54 ++++++++++++++----- .../_helpers/compile_command/constants.rs | 8 ++- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs index 0ceec82d72..ed6080411f 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs @@ -30,7 +30,13 @@ use crate::project_snapshot::models::PathStamp; const NO_STORED_COMPILE: &str = "no stored compile"; const REUSE_BYPASSED: &str = "compile reuse is off"; const OUTPUT_UNREADABLE: &str = "the stored output cannot be replayed"; -const IDENTITY_CHANGED: &str = "the invocation, runtime, environment or settings changed"; +const INVOCATION_CHANGED: &str = "the command invocation changed"; +const RUNTIME_CHANGED: &str = "the Python runtime changed"; +const SEARCH_PATH_CHANGED: &str = "a Python search path entry changed"; +const ENVIRONMENT_NAMES_CHANGED: &str = "the set of tracked environment variables changed"; +const ENVIRONMENT_CHANGED: &str = "a tracked environment variable changed"; +const SETTINGS_CHANGED: &str = "provider settings inputs changed"; +const MODULES_CHANGED: &str = "an imported Python module outside the project changed"; const TARGET_CHANGED: &str = "the compiled target changed"; fn now_ns() -> i64 { @@ -115,8 +121,8 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { attempt.restamped = restamped_paths(&inputs.project_files, &attempt.snapshot); if comparison.unchanged { attempt.delta = DeltaCandidate::Unavailable(OUTPUT_UNREADABLE); - } else if !comparison.identity_unchanged { - attempt.delta = DeltaCandidate::Unavailable(IDENTITY_CHANGED); + } else if let Some(reason) = comparison.identity_change { + attempt.delta = DeltaCandidate::Unavailable(reason); } else if !target_files_unchanged( &inputs.target_files, &attempt.project_dir, @@ -146,21 +152,16 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { /// Whether a stored compile still matches, with the digests still valid either way. struct Comparison { unchanged: bool, - identity_unchanged: bool, + /// The first identity component that differs, if any. + identity_change: Option<&'static str>, digests: HashMap, digested_files: usize, } /// Compare every stored identity, carrying forward digests still valid either way. fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> Comparison { - let identity_unchanged: bool = inputs.invocation_digest == attempt.identity.invocation_digest - && inputs.runtime == attempt.identity.runtime - && inputs.search_path == attempt.search_path - && tracked_environment_names(&inputs.environment_names, rules) == inputs.environment_names - && environment_digest(&inputs.environment_names, rules) == inputs.environment_digest - && settings_inputs_digest(&inputs.settings_inputs, rules) == inputs.settings_digest - && module_stamps_unchanged(&inputs.modules); - let (files_unchanged, verified, digested_files) = if identity_unchanged { + let identity_change: Option<&'static str> = identity_change(attempt, inputs, rules); + let (files_unchanged, verified, digested_files) = if identity_change.is_none() { compare_project_files( &attempt.project_dir, &inputs.project_files, @@ -170,7 +171,7 @@ fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> (false, HashMap::new(), 0) }; Comparison { - identity_unchanged, + identity_change, unchanged: files_unchanged && target_files_unchanged( &inputs.target_files, @@ -183,6 +184,33 @@ fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> } } +/// The first stored identity component that no longer matches, in checking order. +fn identity_change( + attempt: &ReuseAttempt, + inputs: &StoredInputs, + rules: &ReuseRules, +) -> Option<&'static str> { + if inputs.invocation_digest != attempt.identity.invocation_digest { + Some(INVOCATION_CHANGED) + } else if inputs.runtime != attempt.identity.runtime { + Some(RUNTIME_CHANGED) + } else if inputs.search_path != attempt.search_path { + Some(SEARCH_PATH_CHANGED) + } else if tracked_environment_names(&inputs.environment_names, rules) + != inputs.environment_names + { + Some(ENVIRONMENT_NAMES_CHANGED) + } else if environment_digest(&inputs.environment_names, rules) != inputs.environment_digest { + Some(ENVIRONMENT_CHANGED) + } else if settings_inputs_digest(&inputs.settings_inputs, rules) != inputs.settings_digest { + Some(SETTINGS_CHANGED) + } else if !module_stamps_unchanged(&inputs.modules) { + Some(MODULES_CHANGED) + } else { + None + } +} + pub(crate) fn record( attempt: &ReuseAttempt, record: &CompileRecord, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs index 212576852e..480a828b5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs @@ -2,6 +2,7 @@ use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use crate::bindings::_helpers::compile_delta::driver::DELTA_VERIFY_ENV_VAR; use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; /// The directory under the compiler cache directory holding the native reuse store. @@ -16,8 +17,11 @@ pub(crate) const REUSE_DISABLE_VALUE: &str = "1"; pub(crate) const REUSE_LOGGER_NAME: &str = "sqlbuild.compile.reuse"; pub(crate) const TRACKED_ENVIRONMENT_PREFIXES: [&str; 2] = ["SQLBUILD_", "SQB_"]; /// Sorted; tracked-prefix variables that never invalidate a stored compile. -pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 2] = - [CAPTURE_DIR_ENV_VAR, REUSE_DISABLE_ENV_VAR]; +pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 3] = [ + CAPTURE_DIR_ENV_VAR, + REUSE_DISABLE_ENV_VAR, + DELTA_VERIFY_ENV_VAR, +]; pub(crate) const MISSING_ENVIRONMENT_VALUE: &str = "\0missing"; pub(crate) const MISSING_FILE_DIGEST: &str = "\0missing"; pub(crate) const PROJECT_CONFIG_FILENAMES: [&str; 2] = From 9ad4fa6c1bf58c912fe6aad609d565a7df9ce441 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:11:41 +0100 Subject: [PATCH 2/4] perf(compile): delta-compile projects with dynamic columns, audit factories and model rules Three project shapes made every edit compile in full: - Any enforced model declaring dynamic_columns ruled out the project image. The image now keeps the analysis request's families and a delta injects them for models it does not analyse; an edit that changes a model's own families still falls back. - A delta saw only the rerun models' headers, so every audit factory attached elsewhere was reported as orphaned and the delta fell back. The image keeps each model's attached factories. - Any custom rule that had to run again ruled out the delta, so editing a model a model-subject rule reads always compiled in full. A delta now runs model-subject rules for its own models when their recorded reads are limited to those models' SQL, columns and contract, project files, and the source list (kept in full when a rule reads it). It checks the stored read sets before starting and the new ones after, and falls back otherwise. The image format moves to v9. --- .../analysis_session/_helpers/session.rs | 1 + .../src/assembly/analysis_session/models.rs | 2 + .../_helpers/compile_context/delta.rs | 52 ++++- .../bindings/_helpers/compile_delta/driver.rs | 38 +++- .../bindings/_helpers/compile_delta/image.rs | 64 +++++- .../input_loops/resource_diagnostics.rs | 50 ++++- .../_helpers/rules/custom_evaluation.rs | 107 +++++++++- .../_helpers/rules/custom_rule_cache.rs | 12 +- .../bindings/_helpers/rules/fact_replay.rs | 49 ++++- crates/sqlbuild-python/src/bindings/tests.rs | 2 + .../src/bindings/tests/helpers.rs | 17 ++ .../src/bindings/tests/test_delta_facts.rs | 65 ++++++ .../src/bindings/tests/test_types.rs | 8 + .../cli/commands/main/compile/_test_types.py | 20 ++ .../cli/commands/main/compile/helpers.py | 18 ++ .../main/compile/test_compile_delta.py | 198 ++++++++++++++++++ website/src/content/docs/docs/cli/compile.mdx | 12 +- 17 files changed, 680 insertions(+), 35 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 2451efca3d..ca86762b22 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -133,6 +133,7 @@ impl AnalysisSession { dependency_ordered: self.dependency_ordered, cyclic: self.cyclic, legacy_models: self.legacy_models.clone(), + families: self.request.dynamic_families_by_table.clone(), }); let (schema_additions, analysis_names) = self.catalog.into_changes(); let mut request: SessionRequest = self.request; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index da66105b17..088c9ee3c9 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -199,6 +199,8 @@ pub struct RelationTables { pub cyclic: bool, /// Models whose analysis took the legacy path, which reads every relation's facts. pub legacy_models: Vec, + /// The request's dynamic column families by table, in request order. + pub families: Vec<(String, Vec)>, } /// One inferred output column; nullability is Python's `InferredNullability` value. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs index 03a24163f9..f04188f3da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_context/delta.rs @@ -4,7 +4,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use pyo3::prelude::PyResult; -use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, SessionRequest}; +use sqlbuild_analysis::assembly::analysis_session::models::{ + DynamicFamily, RelationTables, SessionRequest, +}; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_discovery::header_changes::types::HeaderKeys; use sqlbuild_rules::engine::main::retained_request::retained_request; @@ -103,6 +105,8 @@ pub(crate) struct ImageCapture { pub(crate) environment_names: Vec, /// Each discovered model's header keys, by model file path. pub(crate) header_keys: BTreeMap, + /// The audit factories each model file's header attaches, for files attaching any. + pub(crate) audit_references: BTreeMap>, /// Each model file's SQL lint cache key, in project model order. pub(crate) lint_model_keys: Option>, /// Each other SQL file's lint cache key, in walk order. @@ -193,6 +197,8 @@ pub(crate) struct DeltaInjection { pub(crate) coverage: Option>, /// Every project model in project order, with the models it reads. pub(crate) coverage_order: Vec<(String, Vec)>, + /// The audit factories each model file's header attaches, for files attaching any. + pub(crate) audit_references: BTreeMap>, } static CAPTURE: Mutex> = Mutex::new(None); @@ -328,17 +334,9 @@ pub(crate) fn injected_fact_digest(key: &[String]) -> Option { /// Give a delta's analysis session the whole project's relation facts and analysis order. pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult { - if !request.dynamic_families_by_table.is_empty() { - capture_unsupported("a model declares dynamic column families"); - } let Some(injection) = injection() else { return Ok(request); }; - if !request.dynamic_families_by_table.is_empty() { - return Err(delta_unsupported( - "a model declares dynamic column families", - )); - } let relations: &RelationTables = &injection.relations; request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); request.retain_relations = true; @@ -347,6 +345,16 @@ pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult = analysed + .iter() + .copied() + .chain(injection.rerun_names.iter().map(String::as_str)) + .collect(); + request.dynamic_families_by_table = injected_families( + &request.dynamic_families_by_table, + &relations.families, + &rerun, + )?; for (table, injected) in [ (&mut request.column_types, &relations.types), (&mut request.column_nullability, &relations.nullability), @@ -363,6 +371,32 @@ pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult)], + stored: &[(String, Vec)], + rerun: &BTreeSet<&str>, +) -> pyo3::PyResult)>> { + let families_of = |table: &[(String, Vec)], name: &str| { + table + .iter() + .find(|(table, _)| table == name) + .map(|(_, families)| families.clone()) + }; + let changed: bool = current + .iter() + .any(|(name, _)| !rerun.contains(name.as_str())) + || rerun + .iter() + .any(|name| families_of(current, name) != families_of(stored, name)); + if changed { + return Err(delta_unsupported( + "a model's dynamic column families changed", + )); + } + Ok(stored.to_vec()) +} + /// What a delta compile's reuse record keeps of the stored compile it patched. #[derive(Debug, Default)] pub(crate) struct RecordMerge { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index df7b38b822..1c779e83ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -34,6 +34,7 @@ use crate::bindings::_helpers::compile_driver::command::{ }; use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::rules::custom_evaluation::stored_custom_rule_refusal; use sqlbuild_discovery::header_changes::main::custom_rule_reads::custom_rule_reads; use sqlbuild_discovery::header_changes::main::header_change::header_change; use sqlbuild_discovery::header_changes::models::HeaderChange; @@ -57,6 +58,8 @@ const UNKNOWN_SOURCE: &str = "unknown source"; const SOURCE_KIND: &str = "source"; /// The upstream kind of a model a model reads. const MODEL_KIND: &str = "model"; +/// The custom-rule fact listing every project source. +const PROJECT_SOURCES_FACT: &str = "project.sources"; const TESTS_ROOT: &str = "tests/"; const STDERR: &str = "stderr"; const SQL_SUFFIX: &str = ".sql"; @@ -321,6 +324,7 @@ fn lent_injection( .collect(), lint_model_keys: image.lint_model_keys.take(), lint_file_keys: image.lint_file_keys.take(), + audit_references: image.audit_references.clone(), ..DeltaInjection::default() }; (image, injection) @@ -588,6 +592,17 @@ fn prepared( Ok(found) => found, Err(reason) => return Ok(Err(reason)), }; + let all_sources: bool = gate + .fact_digests + .iter() + .any(|(key, _)| lists_all_sources(key)); + if let Some(reason) = stored_custom_rule_refusal( + &project_dir, + (&rerun.model_paths, &rerun.changed), + all_sources, + ) { + return Ok(Err(reason.to_owned())); + } let limit: usize = MAX_RERUN_MODELS.max(gate.models.len() * MAX_RERUN_PERCENT / 100); if rerun.models.len() > limit { return Ok(Err(TOO_MANY_MODELS.to_owned())); @@ -647,7 +662,11 @@ fn rounds<'py>( { return stopped("the stored compile reports problems in files the edit leaves alone"); } - rerun.sources = sources_read(&image, &rerun); + rerun.sources = if reads_all_sources(&injection) { + None + } else { + sources_read(&image, &rerun) + }; let injection_for_round: DeltaInjection = for_rerun(injection, &rerun, rerun.sources.clone()); let (outcome, back) = round(request, attempt, (injection_for_round, &rerun))?; @@ -705,6 +724,19 @@ fn rounds<'py>( stopped("changed shapes reach too far") } +/// Whether a custom rule read the project's whole source list, which a delta must then keep. +fn reads_all_sources(injection: &DeltaInjection) -> bool { + injection + .fact_digests + .keys() + .any(|key| lists_all_sources(key)) +} + +/// Whether a custom-rule fact key is the project's whole source list. +fn lists_all_sources(key: &[String]) -> bool { + matches!(key, [name] if name == PROJECT_SOURCES_FACT) +} + /// The sources the rerun models, functions, audits and rerun SQL tests read; `None` for all. fn sources_read(image: &ProjectImage, rerun: &Rerun) -> Option> { if image.diagnostics.iter().any(outside_resources) { @@ -1162,6 +1194,10 @@ fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> Pr image.coverage = capture.coverage; } image.header_keys.extend(capture.header_keys); + image + .audit_references + .retain(|path, _| !rerun.model_paths.contains(path)); + image.audit_references.extend(capture.audit_references); let mut pinned: BTreeSet = std::mem::take(&mut image.pinned_sources) .into_iter() .collect(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 24322ec018..af898eccf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; +use sqlbuild_analysis::assembly::analysis_session::models::{DynamicFamily, RelationTables}; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_rules::models::RetainedRows; @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compile_context::image_codec::{Decoder, Encoder}; use sqlbuild_discovery::header_changes::types::HeaderKeys; /// Changes whenever the image's meaning changes, so an older image is never read. -pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v8"; +pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v9"; pub(crate) const GATE_SECTION: &str = "gate"; /// The upstream kind of a model another model reads. const MODEL_KIND: &str = "model"; @@ -46,6 +46,7 @@ pub(crate) struct ProjectImage { pub(crate) lint_model_keys: Option>, pub(crate) lint_file_keys: Option)>>, pub(crate) header_keys: BTreeMap, + pub(crate) audit_references: BTreeMap>, pub(crate) rules: Option, } @@ -101,8 +102,12 @@ pub(crate) struct ImageRelations { pub(crate) dependency_ordered: bool, pub(crate) cyclic: bool, pub(crate) legacy_models: Vec, + pub(crate) families: FamiliesByTable, } +/// Each table's dynamic column families, in the order the analysis request listed them. +pub(crate) type FamiliesByTable = Vec<(String, Vec)>; + impl From for ImageRelations { fn from(tables: RelationTables) -> Self { Self { @@ -113,6 +118,7 @@ impl From for ImageRelations { dependency_ordered: tables.dependency_ordered, cyclic: tables.cyclic, legacy_models: tables.legacy_models, + families: tables.families, } } } @@ -127,6 +133,7 @@ impl From for RelationTables { dependency_ordered: relations.dependency_ordered, cyclic: relations.cyclic, legacy_models: relations.legacy_models, + families: relations.families, } } } @@ -193,6 +200,7 @@ struct StoredFacts { dependency_ordered: bool, cyclic: bool, legacy_models: Vec, + families: FamiliesByTable, model_subjects: Option)>>, diagnostics: Vec, text_models: Option>, @@ -203,6 +211,7 @@ struct StoredFacts { coverage: Option>, lint_model_keys: Option>, lint_file_keys: Option)>>, + audit_references: BTreeMap>, } fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { @@ -282,6 +291,39 @@ fn stored_shapes(decoder: &mut Decoder<'_>) -> Option { }) } +fn put_families<'a>(encoder: &mut Encoder<'a>, families: &'a FamiliesByTable) { + encoder.count(families.len()); + for (table, declared) in families { + encoder.string(table); + encoder.count(declared.len()); + for family in declared { + encoder.string(&family.name); + encoder.string(&family.pivot_column); + encoder.string(&family.value_column); + encoder.string(&family.aggregate); + encoder.string(&family.data_type); + encoder.optional_string(family.name_pattern.as_deref()); + } + } +} + +fn families(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| { + let table: String = decoder.string()?; + let declared: Vec = decoder.list(|decoder| { + Some(DynamicFamily { + name: decoder.string()?, + pivot_column: decoder.string()?, + value_column: decoder.string()?, + aggregate: decoder.string()?, + data_type: decoder.string()?, + name_pattern: decoder.optional_string()?, + }) + })?; + Some((table, declared)) + }) +} + impl ImageGate { fn encode(&self) -> Vec { let mut encoder: Encoder<'_> = Encoder::default(); @@ -374,6 +416,7 @@ impl StoredFacts { encoder.flag(self.dependency_ordered); encoder.flag(self.cyclic); encoder.strings(self.legacy_models.iter().map(String::as_str)); + put_families(&mut encoder, &self.families); encoder.flag(self.model_subjects.is_some()); if let Some(subjects) = &self.model_subjects { encoder.count(subjects.len()); @@ -401,6 +444,11 @@ impl StoredFacts { encoder.string(path); encoder.strings(names.iter().map(String::as_str)); } + encoder.count(self.audit_references.len()); + for (path, names) in &self.audit_references { + encoder.string(path); + encoder.strings(names.iter().map(String::as_str)); + } encoder.bytes(&misc); Ok(encoder.finish()) } @@ -419,6 +467,7 @@ impl StoredFacts { let dependency_ordered: bool = decoder.flag()?; let cyclic: bool = decoder.flag()?; let legacy_models: Vec = decoder.strings()?; + let families: FamiliesByTable = families(&mut decoder)?; let model_subjects = if decoder.flag()? { Some(decoder.list(|decoder| { Some(( @@ -445,6 +494,10 @@ impl StoredFacts { .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? .into_iter() .collect(); + let audit_references: BTreeMap> = decoder + .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? + .into_iter() + .collect(); let StoredMisc { diagnostics, text_models, @@ -461,6 +514,7 @@ impl StoredFacts { dependency_ordered, cyclic, legacy_models, + families, model_subjects, diagnostics, text_models, @@ -471,6 +525,7 @@ impl StoredFacts { coverage, lint_model_keys, lint_file_keys, + audit_references, }) } } @@ -532,6 +587,7 @@ impl ProjectImage { lint_model_keys: capture.lint_model_keys, lint_file_keys: capture.lint_file_keys, header_keys: capture.header_keys, + audit_references: capture.audit_references, rules: capture .rules .map(|rules| ImageRules::Retained(rules.retained)), @@ -562,6 +618,7 @@ impl ProjectImage { dependency_ordered: relations.dependency_ordered, cyclic: relations.cyclic, legacy_models: relations.legacy_models, + families: relations.families, model_subjects: self.model_subjects, diagnostics: self.diagnostics, text_models: self.text_models, @@ -571,6 +628,7 @@ impl ProjectImage { test_sources: self.test_sources, lint_model_keys: self.lint_model_keys, lint_file_keys: self.lint_file_keys, + audit_references: self.audit_references, }; let rules: Option = self.rules.and_then(ImageRules::retained); let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); @@ -624,6 +682,7 @@ impl ProjectImage { dependency_ordered: facts.dependency_ordered, cyclic: facts.cyclic, legacy_models: facts.legacy_models, + families: facts.families, types: Shapes::new(), }; let model_shapes: Shapes = restored(facts.model_shapes, &types)?; @@ -646,6 +705,7 @@ impl ProjectImage { lint_model_keys: facts.lint_model_keys, lint_file_keys: facts.lint_file_keys, header_keys: gate.header_keys, + audit_references: facts.audit_references, rules, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index ba96e7ae51..cae1f4f929 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -20,6 +21,9 @@ use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; use sqlbuild_pysource::definition_locations::main::decorator_location::decorator_location; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; +use crate::bindings::_helpers::compile_context::delta::{ + DeltaInjection, capture, capturing, injection, +}; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ DiagnosticSettings, HookDescriptionRow, LoaderRow, PythonResourceRow, ResourceDiagnostics, ScenarioDescriptionRow, @@ -174,19 +178,45 @@ fn orphaned_factories<'py>(diagnostics: &Diagnostics<'_, 'py>) -> PyResult> = injection(); + let by_path: bool = injected.is_some() || capturing(); let mut referenced: HashSet = HashSet::new(); + let mut paths: HashSet = HashSet::new(); for model_file in &diagnostics.settings.model_files { - let header: Bound<'py, PyDict> = - native_field(model_file.bind(py), "header_values")?.downcast_into()?; - let Some(names) = header.get_item(AUDIT_FACTORIES_KEY)? else { - continue; - }; - let Ok(names) = names.downcast::() else { - continue; + let model_file: &Bound<'py, PyAny> = model_file.bind(py); + let path: Option = if by_path { + Some(posix( + &native_field(model_file, "relative_path")?.str()?.to_cow()?, + )) + } else { + None }; - for name in names.iter() { - if name.is_instance_of::() { - referenced.insert(name.extract()?); + let header: Bound<'py, PyDict> = + native_field(model_file, "header_values")?.downcast_into()?; + let mut names: Vec = Vec::new(); + if let Some(values) = header.get_item(AUDIT_FACTORIES_KEY)? + && let Ok(values) = values.downcast::() + { + for name in values.iter() { + if name.is_instance_of::() { + names.push(name.extract()?); + } + } + } + if let Some(path) = path { + if !names.is_empty() { + capture(|capture| { + capture.audit_references.insert(path.clone(), names.clone()); + }); + } + paths.insert(path); + } + referenced.extend(names); + } + if let Some(injected) = &injected { + for (path, names) in &injected.audit_references { + if !paths.contains(path) { + referenced.extend(names.iter().cloned()); } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 1d6cf5b789..98eac488eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -18,6 +18,9 @@ use sqlbuild_rules::custom_rule_cache::types::ReadSet; use crate::bindings::_helpers::boundary::python_calls::imported; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::cache::native_store::current_code_identity; +use crate::bindings::_helpers::compile_context::delta::{ + DeltaInjection, delta_unsupported, injection, +}; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; @@ -31,13 +34,22 @@ use crate::bindings::_helpers::rules::custom_host_payload::{ use crate::bindings::_helpers::rules::custom_hosts::{HostCancel, HostRequest, run_custom_hosts}; use crate::bindings::_helpers::rules::custom_rule_cache::CustomRuleCache; use crate::bindings::_helpers::rules::custom_rule_cache::custom_rule_identity; +use crate::bindings::_helpers::rules::custom_rule_cache::stored_rules; use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_implementation_fingerprint; use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_import_closure; use crate::bindings::_helpers::rules::fact_encoder::{Encoded, reproducible}; -use crate::bindings::_helpers::rules::fact_replay::{FactReplay, is_tree_fact}; -use crate::bindings::_helpers::rules::rule_run::bulk_cache_file; +use crate::bindings::_helpers::rules::fact_replay::{ + FactReplay, delta_safe_fact, is_tree_fact, read_text_path, +}; +use crate::bindings::_helpers::rules::rule_run::{bulk_cache_file, rules_bulk_cache_path}; const PROJECT_SUBJECT: &str = "project"; +/// Why a delta compile cannot rerun a custom rule that reads facts of models it does not compile. +const READ_MORE: &str = "a custom rule read more than the changed models"; +const PROJECT_WIDE: &str = "a project-wide custom rule must run again"; +const OUTSIDE_DELTA: &str = "a custom rule must run again for a model the delta does not compile"; +/// The custom-rule cache's subject key of project-wide rules. +const PROJECT_SUBJECT_KEY: &str = ""; const MODEL_SUBJECT: &str = "model"; /// The run one custom-rule evaluation describes. pub(crate) struct CustomRun<'a, 'py> { @@ -140,13 +152,11 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( readset_digests.push(reproducible(digests.combined(readset))?); } let plan: HostPlan = cache.plan(readset_digests); - if !plan.is_empty() && crate::bindings::_helpers::compile_context::delta::injection().is_some() + let delta: Option> = injection(); + if let Some(injection) = &delta + && let Some(reason) = delta_plan_refusal(&plan, &cache, injection) { - return Err( - crate::bindings::_helpers::compile_context::delta::delta_unsupported( - "a custom rule must run again", - ), - ); + return Err(delta_unsupported(reason)); } if !plan.is_empty() { let rules: Vec<(String, HostRuleSubject)> = custom @@ -169,6 +179,13 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( &plan, )?; let evaluations: Vec = evaluations(host_run, &mut digests, enabled)?; + if let Some(injection) = &delta + && !evaluations + .iter() + .all(|evaluation| delta_safe_evaluation(evaluation, injection)) + { + return Err(delta_unsupported(READ_MORE)); + } cache.record(py, evaluations)?; } crate::bindings::_helpers::compile_context::delta::capture(|capture| { @@ -189,6 +206,80 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( ) } +/// Why a delta can't run `plan`: it runs only its own models' rules that read what it holds. +fn delta_plan_refusal( + plan: &HostPlan, + cache: &CustomRuleCache, + injection: &DeltaInjection, +) -> Option<&'static str> { + let all_sources: bool = injection.source_names.is_none(); + for (code, subjects) in plan { + let Some(subjects) = subjects else { + return Some(PROJECT_WIDE); + }; + for subject in subjects { + if !injection.model_paths.contains(subject) { + return Some(OUTSIDE_DELTA); + } + if cache.stored_readset(code, subject).is_some_and(|readset| { + !readset + .iter() + .all(|key| delta_safe_fact(key, &injection.model_paths, all_sources)) + }) { + return Some(READ_MORE); + } + } + } + None +} + +/// Why a delta of `model_paths` would stop at custom rules, judged from their stored read sets. +pub(crate) fn stored_custom_rule_refusal( + project_dir: &Path, + (model_paths, changed): (&BTreeSet, &BTreeSet), + all_sources: bool, +) -> Option<&'static str> { + let stored = stored_rules(&rules_bulk_cache_path( + project_dir.to_path_buf(), + CUSTOM_RULES_CACHE_FILE, + )); + for rule in stored.values() { + for (subject, entry) in &rule.entries { + let Some(readset) = rule.readsets.get(entry.readset) else { + continue; + }; + if model_paths.contains(subject) { + if !readset + .iter() + .all(|key| delta_safe_fact(key, model_paths, all_sources)) + { + return Some(READ_MORE); + } + } else if readset + .iter() + .any(|key| read_text_path(key).is_some_and(|path| changed.contains(path))) + { + return Some(if subject == PROJECT_SUBJECT_KEY { + PROJECT_WIDE + } else { + OUTSIDE_DELTA + }); + } + } + } + None +} + +/// Whether a delta compile evaluated a custom rule from facts it holds in full. +fn delta_safe_evaluation(evaluation: &CustomEvaluation, injection: &DeltaInjection) -> bool { + let all_sources: bool = injection.source_names.is_none(); + evaluation.cacheable.as_ref().is_some_and(|(readset, _)| { + readset + .iter() + .all(|key| delta_safe_fact(key, &injection.model_paths, all_sources)) + }) +} + /// `CustomRulesOutcome(findings=..., cache_hits=..., cache_misses=..., custom_ms=..., ...)`. pub(crate) fn outcome<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index b971ea26ac..b83586f846 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -53,7 +53,7 @@ fn rules_error(py: Python<'_>, message: String) -> PyResult { Ok(pyo3::PyErr::from_value(class.call1((message,))?)) } -fn stored_rules(path: &std::path::Path) -> StoredRules { +pub(crate) fn stored_rules(path: &std::path::Path) -> StoredRules { let Ok(text) = std::fs::read_to_string(path) else { return StoredRules::new(); }; @@ -120,6 +120,16 @@ impl CustomRuleCache { &self.pending } + /// The read set `code` last recorded for `subject` under its current identity, if any. + pub(crate) fn stored_readset(&self, code: &str, subject: &str) -> Option<&ReadSet> { + let rule: &CustomRule = self.rules.iter().find(|rule| rule.code == code)?; + let cached = self.stored.get(code)?; + if rule.identity.as_deref() != Some(cached.identity.as_str()) { + return None; + } + cached.readsets.get(cached.entries.get(subject)?.readset) + } + /// Reuse results given each pending read set's digest; returns the subjects still to run. pub(crate) fn plan( &mut self, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs index 98bbdb4fec..5846008a82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs @@ -1,6 +1,6 @@ //! Native replay of the compiler facts custom-rule evaluations recorded, for their digests. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, PyTypeMethods, Python}; @@ -150,6 +150,53 @@ pub(crate) fn is_tree_fact(key: &FactKey) -> bool { }) } +/// Whether a delta compiling `model_paths` reads `key` exactly as a full compile would. +pub(crate) fn delta_safe_fact( + key: &FactKey, + model_paths: &BTreeSet, + all_sources: bool, +) -> bool { + if is_tree_fact(key) { + return true; + } + if let [name] = key.as_slice() { + return all_sources + && FACTS + .iter() + .any(|(fact_name, fact, _)| fact_name == name && *fact == Fact::Sources); + } + let [name, path] = key.as_slice() else { + return false; + }; + FACTS.iter().any(|(fact_name, fact, arguments)| { + fact_name == name + && *arguments == Arguments::ModelPath + && matches!( + fact, + Fact::SqlForModel + | Fact::Declared + | Fact::Inferred + | Fact::Names + | Fact::Enforced + | Fact::Grain + ) + && model_paths.contains(path) + }) +} + +/// The project file whose text `key` reads: a file's text or a model's SQL. +pub(crate) fn read_text_path(key: &FactKey) -> Option<&str> { + let [name, path] = key.as_slice() else { + return None; + }; + FACTS + .iter() + .any(|(fact_name, fact, _)| { + fact_name == name && matches!(fact, Fact::ReadText | Fact::SqlForModel) + }) + .then_some(path.as_str()) +} + /// What the facts read of one compiled model. pub(crate) struct ModelEntry<'py> { pub(crate) model: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs index 44f522ea77..18e57b5f55 100644 --- a/crates/sqlbuild-python/src/bindings/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_coverage.rs"] mod coverage; +#[path = "tests/test_delta_facts.rs"] +mod delta_facts; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_image.rs"] diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index c9ec506f5a..c51ca5e950 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; +use sqlbuild_analysis::assembly::analysis_session::models::DynamicFamily; use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_config::models::ConfigValue; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; @@ -67,6 +68,17 @@ pub(crate) fn image() -> ProjectImage { dependency_ordered: true, cyclic: false, legacy_models: strings(&["customers"]), + families: vec![( + "order_pivot".to_owned(), + vec![DynamicFamily { + name: "category_amounts".to_owned(), + pivot_column: "category".to_owned(), + value_column: "amount".to_owned(), + aggregate: "MAX".to_owned(), + data_type: "DOUBLE".to_owned(), + name_pattern: Some("^amount_".to_owned()), + }], + )], }, fact_digests: vec![(strings(&["model", "orders"]), "fact-digest".to_owned())], model_subjects: Some(vec![ @@ -113,6 +125,10 @@ pub(crate) fn image() -> ProjectImage { "models/orders.sql".to_owned(), pairs(&[("description", "0123")]), )]), + audit_references: BTreeMap::from([( + "models/orders.sql".to_owned(), + strings(&["order_quality"]), + )]), rules: Some(ImageRules::Retained(RetainedRows { request_json: request, rows: RULE_ROWS.to_vec(), @@ -146,6 +162,7 @@ pub(crate) fn facts(image: &ProjectImage) -> String { &image.lint_model_keys, &image.lint_file_keys, &image.header_keys, + &image.audit_references, ), ) ) diff --git a/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs b/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs new file mode 100644 index 0000000000..4ede87353a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs @@ -0,0 +1,65 @@ +use std::collections::BTreeSet; + +use crate::bindings::_helpers::rules::fact_replay::delta_safe_fact; +use crate::bindings::tests::helpers::{names, strings}; +use crate::bindings::tests::test_types::DeltaFactTestCase; + +#[test] +fn given_custom_rule_fact_when_a_delta_compiles_some_models_then_only_facts_it_holds_are_safe() { + let test_cases = [ + DeltaFactTestCase { + description: "the SQL of a compiled model", + key: &["sql.for_model", "models/orders.sql"], + all_sources: false, + expected_safe: true, + }, + DeltaFactTestCase { + description: "the inferred columns of a compiled model", + key: &["columns.inferred", "models/orders.sql"], + all_sources: false, + expected_safe: true, + }, + DeltaFactTestCase { + description: "the SQL of a model the delta does not compile", + key: &["sql.for_model", "models/customers.sql"], + all_sources: false, + expected_safe: false, + }, + DeltaFactTestCase { + description: "the dependencies of a compiled model", + key: &["graph.dependencies", "models/orders.sql"], + all_sources: false, + expected_safe: false, + }, + DeltaFactTestCase { + description: "the text of any project file", + key: &["tree.read_text", "models/customers.sql"], + all_sources: false, + expected_safe: true, + }, + DeltaFactTestCase { + description: "every model", + key: &["project.models"], + all_sources: true, + expected_safe: false, + }, + DeltaFactTestCase { + description: "every source when the delta kept them all", + key: &["project.sources"], + all_sources: true, + expected_safe: true, + }, + DeltaFactTestCase { + description: "every source when the delta kept only those it reads", + key: &["project.sources"], + all_sources: false, + expected_safe: false, + }, + ]; + let compiled: BTreeSet = names(&["models/orders.sql"]); + for test_case in test_cases { + let safe: bool = delta_safe_fact(&strings(test_case.key), &compiled, test_case.all_sources); + + assert_eq!(safe, test_case.expected_safe, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs index 5d4ab140dd..c5785986b5 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -57,3 +57,11 @@ pub(crate) struct UncompiledModelTestCase { pub(crate) message: &'static str, pub(crate) expected_uncompiled: bool, } + +/// A custom-rule fact key a delta compile holding some models may or may not read in full. +pub(crate) struct DeltaFactTestCase { + pub(crate) description: &'static str, + pub(crate) key: &'static [&'static str], + pub(crate) all_sources: bool, + pub(crate) expected_safe: bool, +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index ebd260fa5e..d79fd3d29b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1164,6 +1164,26 @@ class DeltaCompileFullCopyTestCase: expected_difference: None = None +@dataclass(frozen=True) +class DeltaCompileProjectShapeTestCase: + description: str + files: tuple[tuple[str, str], ...] + edits: tuple[tuple[str, str, str], ...] + expected_delta_line: bool + stored_edits: tuple[tuple[str, str, str], ...] = () + expected_difference: None = None + + +@dataclass(frozen=True) +class DeltaCompileVerifyTestCase: + description: str + files: tuple[tuple[str, str], ...] + edits: tuple[tuple[str, str, str], ...] + env: dict[str, str] + expected_stderr_line: str + stored_edits: tuple[tuple[str, str, str], ...] = () + + @dataclass(frozen=True) class DeltaCompileSequenceTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index b532e5a73e..5a5d28869f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5287,6 +5287,24 @@ def run_delta_compile( ) +def store_shaped_project( + *, + project_dir: Path, + files: tuple[tuple[str, str], ...], + stored_edits: tuple[tuple[str, str, str], ...], +) -> None: + """Add `files` to the project, apply `stored_edits` and compile once to store it.""" + + for relative_path, contents in files: + write_project_file(project_dir, relative_path, contents) + for relative_path, old, new in stored_edits: + replace_project_text(project_dir, relative_path, old, new) + stored: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=("compile",), env={} + ) + assert "Traceback" not in stored.stderr, stored.stderr[-2000:] + + def run_full_compile(project_dir: Path, *, args: tuple[str, ...]) -> DeltaCompileOutcome: """Compile the project at `project_dir` in full, with delta compiles off.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 55c6ad429a..47c329cbbc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -11,10 +11,12 @@ from scripts.delta_compile_series._helpers.outcomes import full_compile_difference from scripts.delta_compile_series.constants import ( DELTA_COMPILE_LINE, + DELTA_VERIFIED_LINE, DOUBLE_CAST, DOUBLE_COLUMN, FLOAT_CAST, FLOAT_COLUMN, + MISMATCH_EXIT_CODE, PROJECT_CONFIG, ) from scripts.delta_compile_series.main.run_delta_compile_sequence import ( @@ -34,13 +36,16 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DeltaCompileFullCopyTestCase, DeltaCompileOptOutTestCase, + DeltaCompileProjectShapeTestCase, DeltaCompileSequenceTestCase, DeltaCompileSeriesTestCase, + DeltaCompileVerifyTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( replace_project_text, run_delta_compile, run_full_compile, + store_shaped_project, without_compile_timings, ) @@ -76,6 +81,92 @@ "[defaults]", ) _REUSE_ROOT: str = "target/cache/compiler-native-v1/project-reuse-v3" +_PIVOT_ROOT: str = f"{_SMALL_MODEL_ROOT}/batch00009" +_CATEGORY_AMOUNTS: tuple[str, str] = ( + f"{_PIVOT_ROOT}/order_category_amounts.sql", + """MODEL ( + description "Order amounts per category.", + materialized table, + contract enforced, + columns ( + customer_id (type INTEGER, description "Customer identifier."), + category (type VARCHAR, description "Order category."), + amount (type DOUBLE, description "Order amount."), + ), +); + +SELECT CAST(b.id AS INTEGER) AS customer_id, CAST('books' AS VARCHAR) AS category, + CAST(b.amount AS DOUBLE) AS amount +FROM __ref("sales000__int_clean__orders_00001") AS b +""", +) +_CATEGORY_PIVOT_PATH: str = f"{_PIVOT_ROOT}/order_category_pivot.sql" +_CATEGORY_PIVOT: tuple[str, str] = ( + _CATEGORY_PIVOT_PATH, + """MODEL ( + description "Order amounts pivoted per category.", + materialized table, + contract enforced, + columns (customer_id (type INTEGER, description "Customer identifier.")), + dynamic_columns ( + category_amounts ( + pivot_column category, + value_column amount, + aggregate MAX, + type DOUBLE + ) + ), +); + +PIVOT __ref("order_category_amounts") +ON category +USING MAX(amount) +GROUP BY customer_id +""", +) +_ORDER_QUALITY_FACTORY: tuple[str, str] = ( + "python/factories/quality.py", + """from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory + + +@audit_factory +def order_quality(): + return [ + AuditCase( + name="amount_present", + definition="not_null", + arguments={"column": "amount"}, + severity=AuditSeverity.ERROR, + ), + ] +""", +) +_ATTACH_ORDER_QUALITY: tuple[str, str, str] = ( + _ANALYSED_MODEL, + "MODEL (description", + "MODEL (audit_factories [order_quality], description", +) +_MODEL_SQL_RULE: tuple[str, str] = ( + "rules/order_sql.py", + """from sqlbuild.rules import Finding, Model, RuleContext, rule + + +@rule( + code="XSQBRBENCH002", + message="Order models must not leave TODO markers", + remediation="Resolve the TODO.", +) +def order_todo_markers(*, model: Model, ctx: RuleContext) -> list[Finding]: + authored = ctx.sql.for_model(model).authored.source + return [ctx.finding(subject=model)] if "TODO" in authored else [] +""", +) +_FAMILY_TYPE_EDIT: tuple[str, str, str] = (_CATEGORY_PIVOT_PATH, "type DOUBLE", "type VARCHAR") +_FAMILIES_CHANGED: str = "delta compile fell back: a model's dynamic column families changed" +_EXAMPLE_TRACKED_VARIABLE: str = "SQLBUILD_ORDERS_EXAMPLE" +_TRACKED_VARIABLE_ADDED: str = ( + "delta compile fell back: the set of tracked environment variables changed" +) _EDITED_MODEL: str = ( "models/sales000/intermediate/clean/batch00000/sales000__int_clean__orders_00003.sql" ) @@ -239,6 +330,113 @@ def test_given_stored_compile_when_compiling_an_edit_then_it_prints_what_a_full_ ) +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileProjectShapeTestCase( + description="a model with dynamic column families", + files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileProjectShapeTestCase( + description="an audit factory attached only by an unchanged model", + files=(_ORDER_QUALITY_FACTORY,), + stored_edits=(_ATTACH_ORDER_QUALITY,), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileProjectShapeTestCase( + description="a model-subject custom rule reading the edited model's SQL", + files=(_MODEL_SQL_RULE,), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileProjectShapeTestCase( + description="an edit to dynamic column families compiles in full", + files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + edits=(_FAMILY_TYPE_EDIT,), + expected_delta_line=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_shape_when_compiling_an_edit_then_it_prints_what_a_full_compile_prints( + test_case: DeltaCompileProjectShapeTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_SMALL_MODEL_COUNT) + store_shaped_project( + project_dir=project_dir, files=test_case.files, stored_edits=test_case.stored_edits + ) + for relative_path, old, new in test_case.edits: + replace_project_text(project_dir, relative_path, old, new) + copy: Path = tmp_path / "full_copy" + _ = shutil.copytree(project_dir, copy, symlinks=True) + + edited: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=_TEXT_COMPILE, env={} + ) + + assert (DELTA_COMPILE_LINE in edited.stderr) == test_case.expected_delta_line, edited.stderr + assert ( + full_compile_difference( + outcome=edited, + full=run_full_compile(copy, args=_TEXT_COMPILE), + project_dir=project_dir, + full_project_dir=copy, + ) + == test_case.expected_difference + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileVerifyTestCase( + description="every shape together matches a full compile", + files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT, _ORDER_QUALITY_FACTORY, _MODEL_SQL_RULE), + stored_edits=(_ATTACH_ORDER_QUALITY,), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=DELTA_VERIFIED_LINE, + ), + DeltaCompileVerifyTestCase( + description="an edit to dynamic column families names why it fell back", + files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + edits=(_FAMILY_TYPE_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=_FAMILIES_CHANGED, + ), + DeltaCompileVerifyTestCase( + description="a new tracked variable names what changed", + files=(), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1", _EXAMPLE_TRACKED_VARIABLE: "1"}, + expected_stderr_line=_TRACKED_VARIABLE_ADDED, + ), + ], + ids=lambda case: case.description, +) +def test_given_verified_edit_when_compiling_then_it_verifies_or_names_why_it_fell_back( + test_case: DeltaCompileVerifyTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_SMALL_MODEL_COUNT) + store_shaped_project( + project_dir=project_dir, files=test_case.files, stored_edits=test_case.stored_edits + ) + for relative_path, old, new in test_case.edits: + replace_project_text(project_dir, relative_path, old, new) + + edited: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=_TEXT_COMPILE, env=test_case.env + ) + + assert edited.returncode != MISMATCH_EXIT_CODE, edited.stderr[-2000:] + assert test_case.expected_stderr_line in edited.stderr.splitlines(), edited.stderr[-2000:] + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 0ded0472b2..8f1f1f62e7 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -135,14 +135,20 @@ The compile runs in full instead, decided before any compile work starts, when: - An edited model changes what it references, or changes a header setting other than its `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch between `view` and `table` materialization. -- A custom rule reads something the edit changed, such as a model's contract or materialization. +- A project-subject custom rule reads something the edit changed, or a model-subject custom rule + must run again for a model the edit did not change, for example because it reads the text of an + edited upstream file. - The stored compile reported errors or warnings other than rule findings in model or SQL test files the edit did not change. - The stored compile used the other output format (`--json` or text), or has no project image. A few edits are found to need a full compile only part-way through, for example when a macro's -arguments change which models or sources it reads, or when the edit changes the shape of so many -models that a full compile is cheaper. +arguments change which models or sources it reads, when the edit changes a model's +`dynamic_columns` families, or when the edit changes the shape of so many models that a full +compile is cheaper. A model-subject custom rule that must run again for an edited model runs in the +delta compile when it reads only that model's SQL, columns, contract and grain, project files, and +the source list; if it reads anything else, such as the model's dependencies, the compile runs in +full. Otherwise, an edit runs the whole compile again, and a few caches under `target/cache/` shorten it: every model, test, and audit is rendered again, and contracts and semantic validation run for From 40e7499933c2d9c79a60f7d56ce209ae828c25e3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:55:41 +0100 Subject: [PATCH 3/4] docs(compile): regenerate the docs skill compile page --- .../skills/sqlbuild/references/docs/cli/compile.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index bfffb7f32d..c276ba51a4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -144,14 +144,20 @@ The compile runs in full instead, decided before any compile work starts, when: - An edited model changes what it references, or changes a header setting other than its `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch between `view` and `table` materialization. -- A custom rule reads something the edit changed, such as a model's contract or materialization. +- A project-subject custom rule reads something the edit changed, or a model-subject custom rule + must run again for a model the edit did not change, for example because it reads the text of an + edited upstream file. - The stored compile reported errors or warnings other than rule findings in model or SQL test files the edit did not change. - The stored compile used the other output format (`--json` or text), or has no project image. A few edits are found to need a full compile only part-way through, for example when a macro's -arguments change which models or sources it reads, or when the edit changes the shape of so many -models that a full compile is cheaper. +arguments change which models or sources it reads, when the edit changes a model's +`dynamic_columns` families, or when the edit changes the shape of so many models that a full +compile is cheaper. A model-subject custom rule that must run again for an edited model runs in the +delta compile when it reads only that model's SQL, columns, contract and grain, project files, and +the source list; if it reads anything else, such as the model's dependencies, the compile runs in +full. Otherwise, an edit runs the whole compile again, and a few caches under `target/cache/` shorten it: every model, test, and audit is rendered again, and contracts and semantic validation run for From b3328f793c4d8ba03b0e54461e1002e48de26f27 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:11:19 +0100 Subject: [PATCH 4/4] fix(compile): fall back when a delta's custom rule lists models under a folder tree.resources_under is listed from the project's models, not the filesystem, and a delta compile holds only the models it reruns. A model-subject rule reading it for an edited model ran on the partial list and could report different findings than a full compile. Treat it as a read beyond the changed models so the delta falls back. --- .../bindings/_helpers/rules/fact_replay.rs | 12 ++++++- .../src/bindings/tests/test_delta_facts.rs | 12 +++++++ .../main/compile/test_compile_delta.py | 31 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs index 5846008a82..a794a92a43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs @@ -150,6 +150,14 @@ pub(crate) fn is_tree_fact(key: &FactKey) -> bool { }) } +/// The fact a key name reads, if it names one. +fn fact_of(name: &str) -> Option { + FACTS + .iter() + .find(|(fact_name, _, _)| *fact_name == name) + .map(|(_, fact, _)| *fact) +} + /// Whether a delta compiling `model_paths` reads `key` exactly as a full compile would. pub(crate) fn delta_safe_fact( key: &FactKey, @@ -157,7 +165,9 @@ pub(crate) fn delta_safe_fact( all_sources: bool, ) -> bool { if is_tree_fact(key) { - return true; + return !key + .first() + .is_some_and(|name| fact_of(name) == Some(Fact::ResourcesUnder)); } if let [name] = key.as_slice() { return all_sources diff --git a/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs b/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs index 4ede87353a..4b582b8eff 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_delta_facts.rs @@ -37,6 +37,18 @@ fn given_custom_rule_fact_when_a_delta_compiles_some_models_then_only_facts_it_h all_sources: false, expected_safe: true, }, + DeltaFactTestCase { + description: "the models under a folder, listed from the project's models", + key: &["tree.resources_under", "models"], + all_sources: false, + expected_safe: false, + }, + DeltaFactTestCase { + description: "the files under a folder, listed from the filesystem", + key: &["tree.descendants", "models"], + all_sources: false, + expected_safe: true, + }, DeltaFactTestCase { description: "every model", key: &["project.models"], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 47c329cbbc..a9fb903033 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -161,6 +161,24 @@ def order_todo_markers(*, model: Model, ctx: RuleContext) -> list[Finding]: return [ctx.finding(subject=model)] if "TODO" in authored else [] """, ) +_FOLDER_RULE: tuple[str, str] = ( + "rules/order_folders.py", + """from sqlbuild.rules import Finding, Model, RuleContext, rule + + +@rule( + code="XSQBRBENCH003", + message="Order models must share their folder with another model", + remediation="Group the model with related models.", +) +def order_folder_siblings(*, model: Model, ctx: RuleContext) -> list[Finding]: + if not model.name.endswith("_00005"): + return [] + siblings = ctx.project.tree.resources_under(model.path.parent.as_posix()) + return [ctx.finding(subject=model)] if len(siblings) < 2 else [] +""", +) +_RULE_READ_MORE: str = "delta compile fell back: a custom rule read more than the changed models" _FAMILY_TYPE_EDIT: tuple[str, str, str] = (_CATEGORY_PIVOT_PATH, "type DOUBLE", "type VARCHAR") _FAMILIES_CHANGED: str = "delta compile fell back: a model's dynamic column families changed" _EXAMPLE_TRACKED_VARIABLE: str = "SQLBUILD_ORDERS_EXAMPLE" @@ -352,6 +370,12 @@ def test_given_stored_compile_when_compiling_an_edit_then_it_prints_what_a_full_ edits=(_BODY_EDIT,), expected_delta_line=True, ), + DeltaCompileProjectShapeTestCase( + description="a model-subject custom rule listing models under a folder", + files=(_FOLDER_RULE,), + edits=(_BODY_EDIT,), + expected_delta_line=False, + ), DeltaCompileProjectShapeTestCase( description="an edit to dynamic column families compiles in full", files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), @@ -408,6 +432,13 @@ def test_given_project_shape_when_compiling_an_edit_then_it_prints_what_a_full_c env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, expected_stderr_line=_FAMILIES_CHANGED, ), + DeltaCompileVerifyTestCase( + description="a rule listing models under a folder names why it fell back", + files=(_FOLDER_RULE,), + edits=(_BODY_EDIT,), + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + expected_stderr_line=_RULE_READ_MORE, + ), DeltaCompileVerifyTestCase( description="a new tracked variable names what changed", files=(),