From 469c99097c672ec7d7104deccb1298b388664db4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:32:21 +0100 Subject: [PATCH 01/32] feat(cache): keep a project image beside recent compile reuse slots Store an image file next to each reuse slot, written between the output and the slot so the slot rename stays the commit point, and keep images only for the two most recently used slots. A missed reuse check now also reports which project files changed by content, the input a delta compile starts from. --- .../src/project_reuse/_helpers/attempt.rs | 46 +++- .../src/project_reuse/_helpers/entry.rs | 63 ++++- .../src/project_reuse/_helpers/files.rs | 62 ++++- .../src/project_reuse/_helpers/image.rs | 91 +++++++ .../src/project_reuse/_helpers/mod.rs | 1 + .../src/project_reuse/constants.rs | 4 + .../src/project_reuse/main/mod.rs | 1 + .../src/project_reuse/main/read_delta_base.rs | 8 + .../src/project_reuse/models.rs | 24 ++ .../sqlbuild-cache/src/project_reuse/tests.rs | 2 + .../src/project_reuse/tests/helpers.rs | 9 +- .../project_reuse/tests/test_project_image.rs | 242 ++++++++++++++++++ .../project_reuse/tests/test_project_reuse.rs | 4 +- .../src/project_reuse/tests/test_types.rs | 26 ++ 14 files changed, 559 insertions(+), 24 deletions(-) create mode 100644 crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs create mode 100644 crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs create mode 100644 crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs index 689056c288..94a4c79ece 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs @@ -8,19 +8,20 @@ use crate::project_reuse::_helpers::entry::{ read_output, read_slot, remove_entry, rewrite_inputs, write_entry, }; use crate::project_reuse::_helpers::files::{ - carried_forward_digests, compare_project_files, needs_refresh, restamped_paths, - stored_project_files, with_missing_digests, + carried_forward_digests, compare_project_files, needs_refresh, project_file_changes, + restamped_paths, stored_project_files, unverified_paths, with_missing_digests, }; use crate::project_reuse::_helpers::identity::{ environment_digest, module_stamps, module_stamps_unchanged, search_path_stamps, settings_inputs_digest, tracked_environment_names, }; +use crate::project_reuse::_helpers::image::{image_path, mark_used}; use crate::project_reuse::_helpers::target::{target_files_unchanged, verified_target_files}; use crate::project_reuse::_helpers::timings::compile_timings_span; use crate::project_reuse::errors::ReuseRecordError; use crate::project_reuse::models::{ - CompileRecord, RecordOutcome, RecordResult, ReuseAttempt, ReuseCheck, ReuseOutcome, ReuseRules, - SettingsInputs, StoredInputs, StoredOutput, + CompileRecord, DeltaCandidate, RecordOutcome, RecordResult, ReuseAttempt, ReuseCheck, + ReuseOutcome, ReuseRules, SettingsInputs, StoredInputs, StoredOutput, }; use crate::project_snapshot::main::snapshot_project_files::snapshot_project_files; use crate::project_snapshot::main::text_path::text_path; @@ -34,6 +35,12 @@ fn now_ns() -> i64 { }) } +const NO_STORED_COMPILE: &str = "no stored compile"; +const REUSE_BYPASSED: &str = "compile reuse is off"; +const OUTPUT_UNREADABLE: &str = "the stored output cannot be replayed"; +const IDENTITY_CHANGED: &str = "the invocation, runtime, environment or settings changed"; +const TARGET_CHANGED: &str = "the compiled target changed"; + fn bypassed(check: &ReuseCheck<'_>) -> bool { check.bypass_requested || check.rules.disabling_variables.iter().any(|(name, value)| { @@ -65,8 +72,10 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { replay: None, replay_failed: false, digested_files: 0, + delta: DeltaCandidate::Unavailable(NO_STORED_COMPILE), }; if bypassed(&check) { + attempt.delta = DeltaCandidate::Unavailable(REUSE_BYPASSED); return attempt; } attempt.outcome = ReuseOutcome::Miss; @@ -87,6 +96,7 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { return attempt; } attempt.outcome = ReuseOutcome::Hit; + mark_used(&image_path(&check.store_path)); if needs_refresh(&inputs.project_files, &attempt.snapshot) { let refreshed: StoredInputs = StoredInputs { project_files: stored_project_files( @@ -103,6 +113,29 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { return attempt; } attempt.restamped = restamped_paths(&inputs.project_files, &attempt.snapshot); + if comparison.unchanged { + attempt.delta = DeltaCandidate::Unavailable(OUTPUT_UNREADABLE); + } else if !comparison.identity_unchanged { + attempt.delta = DeltaCandidate::Unavailable(IDENTITY_CHANGED); + } else if !target_files_unchanged( + &inputs.target_files, + &attempt.project_dir, + inputs.target_tree, + rules, + ) { + attempt.delta = DeltaCandidate::Unavailable(TARGET_CHANGED); + } else { + let unverified: HashSet = + unverified_paths(&inputs.project_files, &attempt.snapshot); + let known: usize = attempt.digests.len(); + attempt.digests = with_missing_digests(&attempt, &unverified, rules.racy_window_ns); + attempt.digested_files += attempt.digests.len() - known; + attempt.delta = DeltaCandidate::Files(project_file_changes( + &inputs.project_files, + &attempt.snapshot, + (&attempt.digests, &unverified), + )); + } } let known: usize = attempt.digests.len(); attempt.digests = with_missing_digests(&attempt, &HashSet::new(), rules.racy_window_ns); @@ -113,6 +146,7 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { /// Whether a stored compile still matches, with the digests still valid either way. struct Comparison { unchanged: bool, + identity_unchanged: bool, digests: HashMap, digested_files: usize, } @@ -136,6 +170,7 @@ fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> (false, HashMap::new(), 0) }; Comparison { + identity_unchanged, unchanged: files_unchanged && target_files_unchanged( &inputs.target_files, @@ -228,8 +263,7 @@ fn store( } write_entry( &attempt.store_path, - &inputs, - &output, + (&inputs, &output, record.image.as_ref()), rules.max_stored_entries, )?; Ok(RecordResult { diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs index 4f6bf849e1..fc4b0109c7 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs @@ -4,11 +4,16 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use crate::digest::types::ContentDigest; +use crate::project_reuse::_helpers::image::{image_path, prune_images, read_image, write_image}; use crate::project_reuse::_helpers::sections::{ SectionFile, decode_sections, encode_sections, write_atomically, }; -use crate::project_reuse::constants::{REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX}; -use crate::project_reuse::models::{SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile}; +use crate::project_reuse::constants::{ + REUSE_MAX_STORED_IMAGES, REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX, +}; +use crate::project_reuse::models::{ + ImageSections, SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile, +}; use crate::project_snapshot::models::PathStamp; use crate::store::errors::StoreDecodeError; use crate::store::models::{RecordReader, RecordWriter}; @@ -20,6 +25,7 @@ const INPUTS_SECTION: &str = "inputs"; const OUTPUT_IDENTITY_SECTION: &str = "output"; const OUTPUT_META_SECTION: &str = "meta"; const STDOUT_SECTION: &str = "stdout"; +const IMAGE_IDENTITY_SECTION: &str = "image"; /// How long an output with no slot is kept, so a compile still writing its slot keeps it. const ORPHANED_OUTPUT_GRACE: Duration = Duration::from_secs(600); @@ -28,6 +34,7 @@ const ORPHANED_OUTPUT_GRACE: Duration = Duration::from_secs(600); pub(crate) struct StoredSlot { pub(crate) inputs: StoredInputs, output_identity: ContentDigest, + image_identity: Option, sections: Vec<(String, Vec)>, } @@ -54,9 +61,14 @@ fn decoded_slot(bytes: &[u8]) -> Result { let output_identity: ContentDigest = required(&file, OUTPUT_IDENTITY_SECTION)? .try_into() .map_err(|_| StoreDecodeError)?; + let image_identity: Option = match file.section(IMAGE_IDENTITY_SECTION) { + Some(bytes) => Some(bytes.try_into().map_err(|_| StoreDecodeError)?), + None => None, + }; Ok(StoredSlot { inputs, output_identity, + image_identity, sections: file .sections .into_iter() @@ -103,11 +115,23 @@ pub(crate) fn read_entry(path: &Path) -> Option<(StoredInputs, StoredOutput)> { Some((slot.inputs, output)) } -/// Replace the slot with one stored compile, then keep at most `max_entries` slots. +/// The stored inputs, output and image of the slot, or `None` when any cannot be read. +pub(crate) fn read_delta_base(path: &Path) -> Option<(StoredInputs, StoredOutput, ImageSections)> { + let slot: StoredSlot = read_slot(path)?; + let image: ImageSections = read_slot_image(path, &slot)?; + let output: StoredOutput = read_output(path, &slot)?; + Some((slot.inputs, output, image)) +} + +/// The image the slot names, or `None` when it has none or it cannot be read. +pub(crate) fn read_slot_image(path: &Path, slot: &StoredSlot) -> Option { + read_image(path, slot.image_identity.as_ref()?) +} + +/// Replace the slot with one stored compile and its image, then keep at most `max_entries` slots. pub(crate) fn write_entry( path: &Path, - inputs: &StoredInputs, - output: &StoredOutput, + (inputs, output, image): (&StoredInputs, &StoredOutput, Option<&ImageSections>), max_entries: usize, ) -> std::io::Result<()> { let meta: Vec = encode_output(output); @@ -120,14 +144,26 @@ pub(crate) fn write_entry( ); write_atomically(&output_path(path), &encoded)?; drop(encoded); - write_slot( - path, - &[ - (INPUTS_SECTION, &encode_inputs(inputs)), - (OUTPUT_IDENTITY_SECTION, &identity), - ], - )?; + let image_identity: Option = match image { + Some(sections) => Some(write_image(path, sections)?), + None => { + let _ = std::fs::remove_file(image_path(path)); + None + } + }; + let encoded_inputs: Vec = encode_inputs(inputs); + let mut sections: Vec<(&str, &[u8])> = vec![ + (INPUTS_SECTION, &encoded_inputs), + (OUTPUT_IDENTITY_SECTION, &identity), + ]; + if let Some(image_identity) = &image_identity { + sections.push((IMAGE_IDENTITY_SECTION, image_identity)); + } + write_slot(path, §ions)?; prune_entries(path, max_entries); + if let Some(directory) = path.parent() { + prune_images(directory, path, REUSE_MAX_STORED_IMAGES); + } Ok(()) } @@ -153,10 +189,11 @@ fn write_slot(path: &Path, sections: &[(&str, &[u8])]) -> std::io::Result<()> { write_atomically(path, &encoded) } -/// Remove the slot and its output. +/// Remove the slot, its output and its image. pub(crate) fn remove_entry(path: &Path) { let _ = std::fs::remove_file(path); let _ = std::fs::remove_file(output_path(path)); + let _ = std::fs::remove_file(image_path(path)); } /// Keep at most `max_entries` slots, newest first, and remove outputs no slot names any more. diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs index 69c98dbfb4..85b34391d4 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use crate::digest::main::digest_files::digest_files; use crate::digest::main::hex_digest::hex_digest; use crate::project_reuse::constants::NATIVE_DIGEST_PREFIX; -use crate::project_reuse::models::{ReuseAttempt, StoredProjectFile}; +use crate::project_reuse::models::{ProjectFileChanges, ReuseAttempt, StoredProjectFile}; use crate::project_snapshot::main::text_path::text_path; use crate::project_snapshot::models::PathStamp; @@ -202,3 +202,63 @@ pub(crate) fn pending_digest_bytes( .map(|stamp| stamp.size) .sum() } + +/// The project paths whose content, kind or presence differs from the stored compile's. +/// +/// A file whose stamp still matches is unchanged; every file in `unverified` is compared by +/// content, and one without a usable digest on either side counts as changed. +pub(crate) fn project_file_changes( + stored: &[StoredProjectFile], + current: &[PathStamp], + (digests, unverified): (&HashMap, &HashSet), +) -> ProjectFileChanges { + let previous_by_path: HashMap<&str, &StoredProjectFile> = by_path(stored); + let mut changes: ProjectFileChanges = ProjectFileChanges::default(); + let mut seen: HashSet<&str> = HashSet::with_capacity(current.len()); + for stamp in current { + let path: &str = stamp.relative_path.as_str(); + seen.insert(path); + let Some(previous) = previous_by_path.get(path) else { + changes.added.push(stamp.relative_path.clone()); + continue; + }; + if previous.stamp.kind != stamp.kind || previous.stamp.link != stamp.link { + changes.changed.push(stamp.relative_path.clone()); + continue; + } + if !hashable(stamp) || !unverified.contains(path) { + continue; + } + let current_digest: Option<&String> = digests.get(path); + if current_digest.is_none() || current_digest != previous.digest.as_ref() { + changes.changed.push(stamp.relative_path.clone()); + } + } + for file in stored { + if !seen.contains(file.stamp.relative_path.as_str()) { + changes.removed.push(file.stamp.relative_path.clone()); + } + } + changes.changed.sort(); + changes.added.sort(); + changes.removed.sort(); + changes +} + +/// Hashable current files whose stored digest may no longer apply: restamped or stored racy. +pub(crate) fn unverified_paths( + stored: &[StoredProjectFile], + current: &[PathStamp], +) -> HashSet { + let previous_by_path: HashMap<&str, &StoredProjectFile> = by_path(stored); + current + .iter() + .filter(|stamp| hashable(stamp)) + .filter(|stamp| { + previous_by_path + .get(stamp.relative_path.as_str()) + .is_some_and(|previous| previous.racy || previous.stamp != **stamp) + }) + .map(|stamp| stamp.relative_path.clone()) + .collect() +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs new file mode 100644 index 0000000000..dbb64d9079 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs @@ -0,0 +1,91 @@ +//! The project image beside a slot: the facts a delta compile patches, kept for recent slots only. + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use crate::digest::types::ContentDigest; +use crate::project_reuse::_helpers::sections::{ + SectionFile, decode_sections, encode_sections, write_atomically, +}; +use crate::project_reuse::constants::{ + REUSE_IMAGE_SUFFIX, REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX, +}; +use crate::project_reuse::models::ImageSections; + +const IMAGE_ROLE: &str = "project-reuse-image"; + +/// The image file paired with the slot file at `path`. +pub(crate) fn image_path(path: &Path) -> PathBuf { + path.with_extension(REUSE_IMAGE_SUFFIX) +} + +/// Write `sections` as the slot's image; returns the identity the slot names it by. +pub(crate) fn write_image(path: &Path, sections: &ImageSections) -> std::io::Result { + let borrowed: Vec<(&str, &[u8])> = sections + .iter() + .map(|(name, bytes)| (name.as_str(), bytes.as_slice())) + .collect(); + let (encoded, identity) = encode_sections(IMAGE_ROLE, &borrowed); + write_atomically(&image_path(path), &encoded)?; + Ok(identity) +} + +/// The image the slot names, or `None` when it is missing, damaged or another compile's. +/// +/// Reading marks the image as used, so pruning keeps the images of recently used slots. +pub(crate) fn read_image(path: &Path, identity: &ContentDigest) -> Option { + let file_path: PathBuf = image_path(path); + let bytes: Vec = std::fs::read(&file_path).ok()?; + let file: SectionFile<'_> = decode_sections(&bytes, IMAGE_ROLE).ok()?; + if file.identity != *identity { + return None; + } + let sections: ImageSections = file + .sections + .into_iter() + .map(|(name, bytes)| (name, bytes.to_vec())) + .collect(); + mark_used(&file_path); + Some(sections) +} + +/// Mark an image as just used; failing to is harmless and only affects pruning order. +pub(crate) fn mark_used(path: &Path) { + if let Ok(file) = std::fs::OpenOptions::new().append(true).open(path) { + let _ = file.set_modified(SystemTime::now()); + } +} + +/// Keep the images of the `keep_images` most recently used slots, and drop images no slot names. +pub(crate) fn prune_images(directory: &Path, keep: &Path, keep_images: usize) { + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + let keep_image: PathBuf = image_path(keep); + let mut images: Vec<(SystemTime, PathBuf)> = Vec::new(); + for path in entries.flatten().map(|entry| entry.path()) { + if !path + .extension() + .is_some_and(|suffix| suffix == REUSE_IMAGE_SUFFIX) + { + continue; + } + let slot: PathBuf = path.with_extension(REUSE_STORE_SUFFIX); + let output: PathBuf = path.with_extension(REUSE_OUTPUT_SUFFIX); + if !slot.exists() && !output.exists() { + let _ = std::fs::remove_file(&path); + continue; + } + if path == keep_image { + continue; + } + let Ok(modified) = std::fs::metadata(&path).and_then(|metadata| metadata.modified()) else { + continue; + }; + images.push((modified, path)); + } + images.sort_by(|left, right| right.cmp(left)); + for (_, stale) in images.into_iter().skip(keep_images.saturating_sub(1)) { + let _ = std::fs::remove_file(stale); + } +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs index 30f44c1302..0556229271 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod attempt; pub(crate) mod entry; pub(crate) mod files; pub(crate) mod identity; +pub(crate) mod image; pub(crate) mod sections; pub(crate) mod target; pub(crate) mod timings; diff --git a/crates/sqlbuild-cache/src/project_reuse/constants.rs b/crates/sqlbuild-cache/src/project_reuse/constants.rs index c5407b5c90..17ffaad603 100644 --- a/crates/sqlbuild-cache/src/project_reuse/constants.rs +++ b/crates/sqlbuild-cache/src/project_reuse/constants.rs @@ -2,5 +2,9 @@ pub const REUSE_STORE_SUFFIX: &str = "store"; /// Suffix of the file beside each slot file holding the stored compile's output. pub const REUSE_OUTPUT_SUFFIX: &str = "output"; +/// Suffix of the file beside a recently used slot holding the facts a delta compile patches. +pub const REUSE_IMAGE_SUFFIX: &str = "image"; +/// How many of the most recently used slots keep their image. +pub const REUSE_MAX_STORED_IMAGES: usize = 2; /// Prefix of digests computed natively, so they never equal a Python digest. pub const NATIVE_DIGEST_PREFIX: &str = "b3:"; diff --git a/crates/sqlbuild-cache/src/project_reuse/main/mod.rs b/crates/sqlbuild-cache/src/project_reuse/main/mod.rs index f6d51418fe..f45ee7b384 100644 --- a/crates/sqlbuild-cache/src/project_reuse/main/mod.rs +++ b/crates/sqlbuild-cache/src/project_reuse/main/mod.rs @@ -1,4 +1,5 @@ pub mod check_reuse; pub mod pending_digest_bytes; +pub mod read_delta_base; pub mod record_reuse; pub mod replay_stdout; diff --git a/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs new file mode 100644 index 0000000000..c9c6d55d39 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs @@ -0,0 +1,8 @@ +use std::path::Path; + +use crate::project_reuse::models::{ImageSections, StoredInputs, StoredOutput}; + +/// The stored compile at `store_path` with its image, the base a delta compile patches. +pub fn read_delta_base(store_path: &Path) -> Option<(StoredInputs, StoredOutput, ImageSections)> { + crate::project_reuse::_helpers::entry::read_delta_base(store_path) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/models.rs b/crates/sqlbuild-cache/src/project_reuse/models.rs index 7a2ffde470..2a3c116066 100644 --- a/crates/sqlbuild-cache/src/project_reuse/models.rs +++ b/crates/sqlbuild-cache/src/project_reuse/models.rs @@ -69,6 +69,26 @@ pub struct StoredInputs { pub settings_digest: String, } +/// Named sections of one project image, in stored order. +pub type ImageSections = Vec<(String, Vec)>; + +/// The project paths that differ from the stored compile's, each list sorted. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ProjectFileChanges { + pub changed: Vec, + pub added: Vec, + pub removed: Vec, +} + +/// Whether a missed check may be compiled as a delta of the stored compile, and from what. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeltaCandidate { + /// Only project file contents differ from the stored compile. + Files(ProjectFileChanges), + /// Why the stored compile cannot be the base of a delta compile. + Unavailable(&'static str), +} + /// The command output of one stored compile. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct StoredOutput { @@ -105,6 +125,8 @@ pub struct ReuseAttempt { pub replay_failed: bool, /// Project files the check read to compare or complete content digests. pub digested_files: usize, + /// Whether a miss may be compiled as a delta of the stored compile. + pub delta: DeltaCandidate, } /// What the host asks of one reuse check. @@ -137,6 +159,8 @@ pub struct CompileRecord { pub artifacts_written: bool, pub dag_path: Option, pub json_output: bool, + /// The project image to keep beside the stored compile, when one was built. + pub image: Option, } /// What one compile left at one artifact path. diff --git a/crates/sqlbuild-cache/src/project_reuse/tests.rs b/crates/sqlbuild-cache/src/project_reuse/tests.rs index 98085a469c..66a11ad735 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests.rs @@ -2,6 +2,8 @@ mod file_changes; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_project_image.rs"] +mod project_image; #[path = "tests/test_project_reuse.rs"] mod project_reuse; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs b/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs index 14b3da8483..09ff216d01 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs @@ -64,7 +64,12 @@ pub(crate) fn stored_output() -> StoredOutput { pub(crate) fn store_aged_slots(directory: &Path, slots: u64, max_entries: usize) { for slot in 0..slots { let path = directory.join(format!("slot{slot}.store")); - write_entry(&path, &stored_inputs(), &stored_output(), max_entries).expect("stored"); + write_entry( + &path, + (&stored_inputs(), &stored_output(), None), + max_entries, + ) + .expect("stored"); let stamp = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(slot + 1); std::fs::File::options() .write(true) @@ -123,7 +128,7 @@ pub(crate) fn replace_output(path: &Path) { stdout: "{\"command\": \"build\"}\n".to_owned(), ..stored_output() }; - write_entry(&other, &stored_inputs(), &output, usize::MAX).expect("other compile"); + write_entry(&other, (&stored_inputs(), &output, None), usize::MAX).expect("other compile"); std::fs::rename(output_file(&other), output_file(path)).expect("swapped output"); } diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs new file mode 100644 index 0000000000..c78b1bf4ac --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs @@ -0,0 +1,242 @@ +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use crate::project_reuse::_helpers::entry::{read_delta_base, write_entry}; +use crate::project_reuse::_helpers::files::{project_file_changes, unverified_paths}; +use crate::project_reuse::models::{ImageSections, ProjectFileChanges, StoredProjectFile}; +use crate::project_reuse::tests::helpers::{stamp, stored_inputs, stored_output}; +use crate::project_reuse::tests::test_types::{ + ChangedFileTestCase, ImagePruneTestCase, ImageReadTestCase, +}; + +fn image() -> ImageSections { + vec![ + ("facts".to_owned(), b"{\"models\": [\"orders\"]}".to_vec()), + ("rules".to_owned(), b"{}".to_vec()), + ] +} + +fn image_file(path: &Path) -> PathBuf { + path.with_extension("image") +} + +#[test] +fn given_stored_compile_with_image_when_reading_delta_base_then_only_an_intact_image_is_read() { + let test_cases = [ + ImageReadTestCase { + description: "intact image", + edit: |_| {}, + expected_read: true, + }, + ImageReadTestCase { + description: "missing image", + edit: |path| std::fs::remove_file(image_file(path)).expect("removed"), + expected_read: false, + }, + ImageReadTestCase { + description: "damaged image", + edit: |path| { + let mut bytes = std::fs::read(image_file(path)).expect("image"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + std::fs::write(image_file(path), bytes).expect("damaged"); + }, + expected_read: false, + }, + ImageReadTestCase { + description: "image of another compile", + edit: |path| { + let other = path.with_file_name("other.store"); + let mut sections = image(); + sections[0].1 = b"{\"models\": [\"customers\"]}".to_vec(); + write_entry( + &other, + (&stored_inputs(), &stored_output(), Some(§ions)), + 8, + ) + .expect("other compile"); + std::fs::copy(image_file(&other), image_file(path)).expect("replaced"); + }, + expected_read: false, + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("reuse/slot.store"); + write_entry( + &path, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored compile"); + (test_case.edit)(&path); + + let read = read_delta_base(&path); + + assert_eq!( + read, + test_case + .expected_read + .then(|| (stored_inputs(), stored_output(), image())), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_compile_without_image_when_storing_again_then_the_stale_image_is_removed() { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("slot.store"); + write_entry( + &path, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored"); + + write_entry(&path, (&stored_inputs(), &stored_output(), None), 8).expect("stored again"); + + assert!(!image_file(&path).exists()); + assert_eq!(read_delta_base(&path), None); +} + +#[test] +fn given_images_of_many_slots_when_storing_then_only_the_most_recently_used_two_are_kept() { + let test_cases = [ + ImagePruneTestCase { + description: "the two newest images are kept", + slots: 4, + read: &[], + expected_images: &["slot3.image", "slot4.image"], + }, + ImagePruneTestCase { + description: "a read image counts as used", + slots: 3, + read: &[1], + expected_images: &["slot1.image", "slot3.image"], + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + for slot in 0..test_case.slots { + let path = folder.path().join(format!("slot{slot}.store")); + write_entry( + &path, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored"); + let aged = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(slot + 1); + std::fs::File::options() + .write(true) + .open(image_file(&path)) + .and_then(|file| file.set_modified(aged)) + .expect("aged image"); + } + for slot in test_case.read { + assert!(read_delta_base(&folder.path().join(format!("slot{slot}.store"))).is_some()); + } + let newest = folder.path().join(format!("slot{}.store", test_case.slots)); + write_entry( + &newest, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored again"); + + let mut kept: Vec = std::fs::read_dir(folder.path()) + .expect("listed") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.ends_with(".image")) + .collect(); + kept.sort(); + + assert_eq!(kept, test_case.expected_images, "{}", test_case.description); + } +} + +#[test] +fn given_project_file_when_computing_delta_changes_then_only_content_changes_count() { + let test_cases = [ + ChangedFileTestCase { + description: "settled equal stamp is unchanged without reading", + stored_digest: None, + racy: false, + moved_stamp: false, + current_digest: None, + expected_changed: false, + }, + ChangedFileTestCase { + description: "moved stamp with equal bytes is unchanged", + stored_digest: Some("b3:one"), + racy: false, + moved_stamp: true, + current_digest: Some("b3:one"), + expected_changed: false, + }, + ChangedFileTestCase { + description: "moved stamp with new bytes is changed", + stored_digest: Some("b3:one"), + racy: false, + moved_stamp: true, + current_digest: Some("b3:two"), + expected_changed: true, + }, + ChangedFileTestCase { + description: "racy equal stamp with new bytes is changed", + stored_digest: Some("b3:one"), + racy: true, + moved_stamp: false, + current_digest: Some("b3:two"), + expected_changed: true, + }, + ChangedFileTestCase { + description: "moved stamp without a stored digest is changed", + stored_digest: None, + racy: false, + moved_stamp: true, + current_digest: Some("b3:one"), + expected_changed: true, + }, + ChangedFileTestCase { + description: "unreadable moved file is changed", + stored_digest: Some("b3:one"), + racy: false, + moved_stamp: true, + current_digest: None, + expected_changed: true, + }, + ]; + for test_case in test_cases { + let stored = vec![StoredProjectFile { + stamp: stamp("models/orders.sql", "f", None), + digest: test_case.stored_digest.map(str::to_owned), + racy: test_case.racy, + }]; + let mut current = stamp("models/orders.sql", "f", None); + if test_case.moved_stamp { + current.mtime_ns += 1; + } + let current = vec![current, stamp("models/customers.sql", "f", None)]; + let digests: HashMap = test_case + .current_digest + .map(|digest| HashMap::from([("models/orders.sql".to_owned(), digest.to_owned())])) + .unwrap_or_default(); + let unverified: HashSet = unverified_paths(&stored, ¤t); + + let changes = project_file_changes(&stored, ¤t, (&digests, &unverified)); + + let expected = ProjectFileChanges { + changed: if test_case.expected_changed { + vec!["models/orders.sql".to_owned()] + } else { + Vec::new() + }, + added: vec!["models/customers.sql".to_owned()], + removed: Vec::new(), + }; + assert_eq!(changes, expected, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs index ff85acfbe6..e7598675a7 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs @@ -69,7 +69,7 @@ fn given_stored_compile_when_reading_then_only_an_intact_slot_round_trips() { for test_case in test_cases { let folder = tempfile::tempdir().expect("store folder"); let path = folder.path().join("reuse/slot.store"); - write_entry(&path, &stored_inputs(), &stored_output(), 8).expect("stored compile"); + write_entry(&path, (&stored_inputs(), &stored_output(), None), 8).expect("stored compile"); (test_case.edit)(&path); let read = read_entry(&path); @@ -269,7 +269,7 @@ fn given_stored_compile_when_refreshing_inputs_then_only_the_slot_file_is_rewrit for test_case in test_cases { let folder = tempfile::tempdir().expect("store folder"); let path = folder.path().join("reuse/slot.store"); - write_entry(&path, &stored_inputs(), &stored_output(), 8).expect("stored compile"); + write_entry(&path, (&stored_inputs(), &stored_output(), None), 8).expect("stored compile"); let (name, bytes) = test_case.extra_section; add_slot_section(&path, name, bytes); let output_before = std::fs::read(output_file(&path)).expect("output bytes"); diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs index 9e2f58b920..edbb902334 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs @@ -59,3 +59,29 @@ pub(crate) struct LinkCycleTestCase { pub(crate) description: &'static str, pub(crate) expected_paths: &'static [&'static str], } + +/// A stored compile with an image, edited before the delta base is read back. +pub(crate) struct ImageReadTestCase { + pub(crate) description: &'static str, + pub(crate) edit: fn(&Path), + pub(crate) expected_read: bool, +} + +/// Slots stored oldest first, each with an image, then some read as delta bases. +pub(crate) struct ImagePruneTestCase { + pub(crate) description: &'static str, + pub(crate) slots: u64, + /// Slots read as delta bases before one more slot is stored. + pub(crate) read: &'static [u64], + pub(crate) expected_images: &'static [&'static str], +} + +/// A stored and a current project file and how the delta changed set sees them. +pub(crate) struct ChangedFileTestCase { + pub(crate) description: &'static str, + pub(crate) stored_digest: Option<&'static str>, + pub(crate) racy: bool, + pub(crate) moved_stamp: bool, + pub(crate) current_digest: Option<&'static str>, + pub(crate) expected_changed: bool, +} From 887b7117728a1ebd9a6239d7ba4501d79ad19262 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:44:20 +0100 Subject: [PATCH 02/32] feat(compile): let one compile's analysis and rules stand in for the whole project Model analysis sessions can keep their final relation facts and run in a given dependency order, the built-in rules request can be kept without its SQL text and evaluated again from cached findings only, and model and SQL test discovery can be limited to named files. A delta compile builds on these to analyse only the models that changed. --- .../_helpers/catalog_state.rs | 11 +++ .../analysis_session/_helpers/session.rs | 44 +++++++++--- .../src/assembly/analysis_session/models.rs | 29 ++++++++ .../src/assembly/analysis_session/tests.rs | 2 + .../analysis_session/tests/helpers.rs | 3 + .../tests/test_delta_session.rs | 67 +++++++++++++++++++ .../model_files/main/discover_model_files.rs | 6 ++ .../src/model_files/models.rs | 2 + .../src/model_files/tests/helpers.rs | 1 + .../src/sql_tests/_helpers/selection.rs | 3 +- .../sql_tests/main/discover_scenario_files.rs | 9 ++- .../sql_tests/main/discover_sql_test_files.rs | 2 +- .../src/sql_tests/models.rs | 2 + .../src/sql_tests/tests/helpers.rs | 1 + .../src/engine/_helpers/evaluation.rs | 7 ++ .../src/engine/_helpers/memo.rs | 58 ++++++++++++++-- .../src/engine/_helpers/rows.rs | 1 + .../src/engine/_helpers/speculation.rs | 7 +- crates/sqlbuild-rules/src/engine/tests.rs | 2 + .../src/engine/tests/helpers.rs | 1 + .../src/engine/tests/test_retained_rows.rs | 59 ++++++++++++++++ crates/sqlbuild-rules/src/models.rs | 15 +++++ 22 files changed, 312 insertions(+), 20 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs create mode 100644 crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs index 47310708ff..e604264fd8 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs @@ -115,6 +115,17 @@ impl SessionCatalog { self.schemas.get(name) } + /// Every closed shape the catalog holds, sorted by name. + pub(crate) fn schema_rows(&self) -> Shapes { + let mut rows: Shapes = self + .schemas + .iter() + .map(|(name, shape)| (name.clone(), shape.clone())) + .collect(); + rows.sort_by(|left, right| left.0.cmp(&right.0)); + rows + } + /// `(shared members, shared members re-analysed alone)` so far. pub(crate) fn sharing(&self) -> (usize, usize) { (self.shared_members, self.reanalysed_members) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 74e8b174c9..57d2485c17 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -33,7 +33,7 @@ use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_ use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, DeferredCleaning, FinishedSession, LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables, - SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, + RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::ProjectCatalog; @@ -49,13 +49,18 @@ impl AnalysisSession { .ok_or_else(|| native_failure(SESSION_CONTEXT, "two analysed models share a name"))?; let scheduled: Option>> = waves(&producers); let complete_shapes: ShapeTable = ShapeTable::from_shapes(&request.complete_schemas); - let dependency_ordered: bool = scheduled.is_some() - && request - .models - .iter() - .flat_map(|model| model.required_names.iter()) - .any(|name| complete_shapes.get(name).is_none_or(Vec::is_empty)); - let cyclic: bool = scheduled.is_none(); + let (dependency_ordered, cyclic): (bool, bool) = match request.forced_order { + Some(order) => order, + None => ( + scheduled.is_some() + && request + .models + .iter() + .flat_map(|model| model.required_names.iter()) + .any(|name| complete_shapes.get(name).is_none_or(Vec::is_empty)), + scheduled.is_none(), + ), + }; let waves: Vec> = scheduled.unwrap_or_default(); Ok(Self { catalog: SessionCatalog::new(catalog, &request.catalog_schemas), @@ -76,6 +81,7 @@ impl AnalysisSession { publications: Vec::new(), failures: Vec::new(), cache: None, + legacy_models: Vec::new(), request, }) } @@ -118,6 +124,16 @@ impl AnalysisSession { .into_iter() .collect::>() .ok_or("the session left a model unanalysed")?; + let relations: Option = + self.request.retain_relations.then(|| RelationTables { + types: owned_rows(&self.available_types), + nullability: owned_rows(&self.available_nullability), + complete: owned_rows(&self.complete_shapes), + schemas: self.catalog.schema_rows(), + dependency_ordered: self.dependency_ordered, + cyclic: self.cyclic, + legacy_models: self.legacy_models.clone(), + }); let (schema_additions, analysis_names) = self.catalog.into_changes(); let mut request: SessionRequest = self.request; for ((outcome, model), pending) in models @@ -155,6 +171,7 @@ impl AnalysisSession { pool, models: model_facts, dynamic_proofs, + relations, }; let outcome = SessionOutcome { models, @@ -254,7 +271,9 @@ impl AnalysisSession { let legacy: LegacyAnalysis = self.native_legacy(request, &result.cleaned_sql) .map_err(|reason| native_failure(&model_context(request), &reason))?; + let name: String = request.name.clone(); self.record_legacy_tables(*model, &mut tables); + self.legacy_models.push(name); legacy_outcome(legacy, lineage, diagnostics) } }; @@ -670,6 +689,15 @@ fn batch_member<'a>(model: &'a ModelRequest, schema: &'a Shapes) -> BatchMember< } /// Python's `{column.name: column.type or "UNKNOWN"}`. +/// A table's shapes in insertion order, owned. +fn owned_rows(table: &ShapeTable) -> Shapes { + table + .ordered() + .into_iter() + .map(|(name, shape)| (name.to_owned(), shape.clone())) + .collect() +} + pub(crate) fn shape_columns(columns: &[ColumnFact]) -> Pairs { dict_from_pairs(columns.iter().map(shape_column)) } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index c5a18ee5d3..2d5bb5c9a0 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -107,6 +107,8 @@ pub struct FinishedSession { pub(crate) models: HashMap, /// Each model's dynamic pivot proof from the session's own relation facts, by model name. pub(crate) dynamic_proofs: HashMap, + /// The relation facts the session ended with, when the request asked to keep them. + pub(crate) relations: Option, } /// The session's typed result for one compiled model, borrowed from the finished session. @@ -121,6 +123,12 @@ pub struct SessionModelAnalysis<'a> { } impl FinishedSession { + /// The relation facts the session ended with, when its request asked to keep them. + #[must_use] + pub fn relation_tables(&self) -> Option<&RelationTables> { + self.relations.as_ref() + } + /// The session's typed columns and dynamic proof for model `name`, without re-analysis. #[must_use] pub fn model_analysis(&self, name: &str) -> SessionModelAnalysis<'_> { @@ -173,6 +181,25 @@ pub struct SessionRequest { /// Python's `dynamic_families_by_table`, in dict order. pub dynamic_families_by_table: Vec<(String, Vec)>, pub models: Vec, + /// `(dependency ordered, cyclic)` to analyse with instead of deriving them from `models`: + /// a delta compile analyses a few models as the whole project's session did. + pub forced_order: Option<(bool, bool)>, + /// Keep the relation facts the session ends with, for the project image. + pub retain_relations: bool, +} + +/// The relation facts a finished session ends with, and how it ordered its analysis. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RelationTables { + pub types: Shapes, + pub nullability: Shapes, + pub complete: Shapes, + /// The binding catalog's closed shapes, sorted by name. + pub schemas: Shapes, + pub dependency_ordered: bool, + pub cyclic: bool, + /// Models whose analysis took the legacy path, which reads every relation's facts. + pub legacy_models: Vec, } /// One inferred output column; nullability is Python's `InferredNullability` value. @@ -323,6 +350,8 @@ pub struct AnalysisSession { pub(crate) publications: Shapes, pub(crate) failures: Vec, pub(crate) cache: Option, + /// Models whose analysis took the legacy path. + pub(crate) legacy_models: Vec, } /// Which of Python's column analyses one query outside the session takes. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs index b4f9db0702..95de9dfe5c 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs @@ -4,6 +4,8 @@ mod analysis_cache; mod analysis_session; #[path = "tests/test_contract_proof_json.rs"] mod contract_proof_json; +#[path = "tests/test_delta_session.rs"] +mod delta_session; #[path = "tests/test_expression_shapes.rs"] mod expression_shapes; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 16b5f60f76..f16b1eaebf 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -123,6 +123,8 @@ pub(crate) fn orders_request(models: Vec) -> SessionRequest { nullability_rules: Some(Vec::new()), nullability_callback: None, rich_type_inference: true, + forced_order: None, + retain_relations: false, column_types: raw_orders.clone(), column_nullability: shapes(&[( "raw_orders", @@ -384,6 +386,7 @@ pub(crate) fn finished_session(tables: PivotTables) -> FinishedSession { pool: catalog("duckdb", &Vec::new()).analysis_pool(), models: HashMap::new(), dynamic_proofs: HashMap::new(), + relations: None, } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs new file mode 100644 index 0000000000..6fa8b6dd5d --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs @@ -0,0 +1,67 @@ +use crate::assembly::analysis_session::models::{ModelRequest, RelationTables, SessionRequest}; +use crate::assembly::analysis_session::tests::helpers::{ + completed, model_requests, orders_request, started, +}; +use crate::assembly::analysis_session::tests::test_types::ModelSpec; + +const ORDER_MODELS: [ModelSpec; 3] = [ + ( + "stg_orders", + "SELECT order_id, amount * 2 AS doubled FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], + ), + ( + "orders_mart", + "SELECT doubled, order_id + 1 AS next_id FROM __ref(\"stg_orders\")", + &[], + &["stg_orders"], + ), + ( + "orders_star", + "SELECT * FROM __ref(\"stg_orders\")", + &[], + &["stg_orders"], + ), +]; + +/// `request` with `relations`' facts added for every relation it does not analyse. +fn injected(mut request: SessionRequest, relations: &RelationTables) -> SessionRequest { + let analysed: Vec = request + .models + .iter() + .map(|model| model.name.clone()) + .collect(); + for (table, facts) in [ + (&mut request.column_types, &relations.types), + (&mut request.column_nullability, &relations.nullability), + (&mut request.complete_schemas, &relations.complete), + ] { + let present: Vec = table.iter().map(|(name, _)| name.clone()).collect(); + table.extend( + facts + .iter() + .filter(|(name, _)| !analysed.contains(name) && !present.contains(name)) + .cloned(), + ); + } + request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); + request +} + +#[test] +fn given_whole_project_relations_when_analysing_one_model_then_it_matches_the_whole_session() { + let mut whole: SessionRequest = orders_request(model_requests(&ORDER_MODELS)); + whole.retain_relations = true; + let (_, whole_outcomes, finished) = completed(started(whole)); + let relations: RelationTables = finished.relation_tables().cloned().expect("relations kept"); + assert!(relations.types.iter().any(|(name, _)| name == "stg_orders")); + + for (index, spec) in ORDER_MODELS.iter().enumerate().skip(1) { + let one: Vec = model_requests(std::slice::from_ref(spec)); + let request: SessionRequest = injected(orders_request(one), &relations); + let (_, outcomes, _) = completed(started(request)); + + assert_eq!(outcomes, vec![whole_outcomes[index].clone()], "{}", spec.0); + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs b/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs index a365eddd22..aba0c8a20f 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs @@ -25,6 +25,12 @@ pub fn discover_model_files( Ok(paths .into_par_iter() .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) + .filter(|relative_path| { + options + .path_filter + .as_ref() + .is_none_or(|paths| paths.contains(relative_path)) + }) .map(|relative_path| DiscoveredFile { outcome: discover_one(root, tree, &relative_path, options), relative_path, diff --git a/crates/sqlbuild-discovery/src/model_files/models.rs b/crates/sqlbuild-discovery/src/model_files/models.rs index 3bbc25da51..e618f6aa7a 100644 --- a/crates/sqlbuild-discovery/src/model_files/models.rs +++ b/crates/sqlbuild-discovery/src/model_files/models.rs @@ -12,6 +12,8 @@ pub struct ModelFileOptions { pub removed_keys: Vec, pub extract_implicit_alias_columns: bool, pub extract_output_column_locations: bool, + /// Discover only these project-relative paths, when given. + pub path_filter: Option>>, } /// One parsed model file; values mirror the fields of Python's `DiscoveredSqlModelFile`. diff --git a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs index 6b8beef554..084d8f1953 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs @@ -14,6 +14,7 @@ const FILE_PATH: &str = "/project/models/orders.sql"; fn options() -> ModelFileOptions { ModelFileOptions { + path_filter: None, supported_keys: [ "audits", "columns", diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs index caf1f77e30..eb10a6a622 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs @@ -13,7 +13,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; pub(crate) fn discover_sql_files( root: &ProjectRoot, tree: &ProjectTree, - directory: &str, + (directory, path_filter): (&str, Option<&std::collections::BTreeSet>), parse: impl Fn(&str, String) -> Result + Sync, ) -> Result>, StageFailure> { let pool = discovery_pool().map_err(StageFailure::Internal)?; @@ -24,6 +24,7 @@ pub(crate) fn discover_sql_files( Ok(paths .into_par_iter() .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) + .filter(|relative_path| path_filter.is_none_or(|paths| paths.contains(relative_path))) .map(|relative_path| DiscoveredFile { outcome: read_and_parse(root, tree, &relative_path, &parse), relative_path, diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs index ccbf905907..3b175335b6 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs @@ -13,7 +13,10 @@ pub fn discover_scenario_files( tree: &ProjectTree, options: &SqlTestFileOptions, ) -> Result>, StageFailure> { - discover_sql_files(root, tree, SQL_SCENARIOS_ROOT, |file_path, contents| { - parse_scenario_file(file_path, contents, options) - }) + discover_sql_files( + root, + tree, + (SQL_SCENARIOS_ROOT, None), + |file_path, contents| parse_scenario_file(file_path, contents, options), + ) } diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs index 908475a3a0..a7286dff1a 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs @@ -16,7 +16,7 @@ pub fn discover_sql_test_files( discover_sql_files( root, tree, - &SQL_TESTS_ROOT.join("/"), + (&SQL_TESTS_ROOT.join("/"), options.path_filter.as_deref()), |file_path, contents| parse_sql_test_file(file_path, contents, options), ) } diff --git a/crates/sqlbuild-discovery/src/sql_tests/models.rs b/crates/sqlbuild-discovery/src/sql_tests/models.rs index 23a0099f72..6aa183b923 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/models.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/models.rs @@ -8,6 +8,8 @@ use sqlbuild_sqltext::compiler::models::AuthoredValue; pub struct SqlTestFileOptions { pub test_keys: Vec, pub scenario_keys: Vec, + /// Discover only these project-relative paths, when given. + pub path_filter: Option>>, } /// One `TEST(...)` block whose header parsed with only supported keys. diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs index 20ea382c79..b860bd6e41 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs @@ -13,6 +13,7 @@ fn keys(test_keys: &[&str]) -> Vec { fn options() -> SqlTestFileOptions { SqlTestFileOptions { + path_filter: None, test_keys: keys(&["name", "mode", "parameters", "cases"]), scenario_keys: keys(&["description", "tags"]), } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 383296000a..0355954257 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -60,6 +60,10 @@ struct PendingModelRule<'a> { identity: Option, } +/// The error a delta compile's rules request raises when a row without SQL needs evaluating. +pub(crate) const STRIPPED_ROW_MISSED: &str = + "delta compile unsupported: a model's rule findings are no longer cached"; + struct CompletedModelRule { index: usize, identity: Option, @@ -152,6 +156,9 @@ fn evaluate_native_models( identity, }); } + if pending_models.iter().any(|pending| pending.model.stripped) { + return Err(STRIPPED_ROW_MISSED.to_owned()); + } let project_dir = Path::new(&context.request.project_dir); let completed = evaluate_pending_models(pending_models, context)?; for completed_model in completed { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index b49f3ff516..7e1e010df4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -10,7 +10,7 @@ use crate::constants::{API_VERSION, NATIVE_ROWS_MEMO_VERSION}; use crate::engine::_helpers::cache::PendingBucketWrite; use crate::engine::_helpers::evaluation::{EvaluationControl, evaluate_request_with}; use crate::errors::RowsError; -use crate::models::{EvaluateRequest, Model, RowsEvaluation, RowsRequest}; +use crate::models::{EvaluateRequest, Model, RetainedRows, RowsEvaluation, RowsRequest}; /// A rows request whose models are digested, ready to evaluate once. #[derive(Debug)] @@ -20,6 +20,8 @@ pub(crate) struct PreparedRows { memo: Option<(String, PathBuf)>, /// The digest of every evaluation input the request carries, when it was asked for. pub(crate) key: Option, + /// The request kept for the project image, when it was asked for. + pub(crate) retained: Option, } /// What a speculative evaluation found, with every write it would have made still pending. @@ -41,7 +43,11 @@ pub(crate) fn evaluate_rows(rows: RowsRequest) -> Result Result { - let RowsRequest { mut request, memo } = rows; + let RowsRequest { + mut request, + memo, + retain, + } = rows; let models: Vec = std::mem::take(&mut request.models) .into_iter() .map(digested) @@ -55,13 +61,50 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result = retain.then(|| retained_rows(&request, &models)); request.models = models; - Ok(PreparedRows { request, memo, key }) + Ok(PreparedRows { + request, + memo, + key, + retained, + }) +} + +/// `request` with `models` whose SQL text is dropped, and each model's payload digest. +fn retained_rows(request: &EvaluateRequest, models: &[Model]) -> RetainedRows { + let mut kept: EvaluateRequest = EvaluateRequest { + models: Vec::new(), + ..request.clone() + }; + let mut digests: Vec = Vec::with_capacity(models.len()); + kept.models = models + .iter() + .map(|model| { + digests.push(model.payload_digest.clone().unwrap_or_default()); + Model { + query_sql: String::new(), + authored_sql: String::new(), + payload_digest: None, + stripped: true, + ..model.clone() + } + }) + .collect(); + RetainedRows { + request: kept, + digests, + } } /// Answer a prepared request from the memo or evaluate it, writing the cache and memo at once. pub(crate) fn evaluate_prepared(prepared: PreparedRows) -> Result { - let PreparedRows { request, memo, .. } = prepared; + let PreparedRows { + request, + memo, + retained, + .. + } = prepared; let reused: Option = memo .as_ref() .and_then(|(identity, path)| read_response(path, identity)); @@ -76,7 +119,9 @@ pub(crate) fn evaluate_prepared(prepared: PreparedRows) -> Result Result { + if model.payload_digest.is_some() { + return Ok(model); + } let mut hasher: blake3::Hasher = blake3::Hasher::new(); serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; model.payload_digest = Some(hasher.finalize().to_hex().to_string()); diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index 4058e6b6f2..fc694bf75b 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -93,6 +93,7 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result SpeculativeAnswer { - let prepared: PreparedRows = match prepare(rows, true) { + let mut prepared: PreparedRows = match prepare(rows, true) { Ok(prepared) => prepared, Err(error) => { drop(speculative); @@ -77,7 +77,10 @@ pub(crate) fn answer( }; match outcome { Some(outcome) => SpeculativeAnswer::Answered { - evaluation: commit(outcome), + evaluation: commit(outcome).map(|mut evaluation| { + evaluation.retained = prepared.retained.take(); + evaluation + }), used: true, }, None => SpeculativeAnswer::Answered { diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index ed6cc5ba16..5195d6dd8a 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -10,6 +10,8 @@ mod helpers; mod incremental_rules; #[path = "tests/test_query_facts.rs"] mod query_facts; +#[path = "tests/test_retained_rows.rs"] +mod retained_rows; #[path = "tests/test_scope_facts.rs"] mod scope_facts; #[path = "tests/test_speculative_rows.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index 93a3fee44b..89624c9562 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -531,6 +531,7 @@ pub(crate) fn speculative_rows_request( project_dir.join(SPECULATIVE_MEMO_FILE), ) }), + retain: false, }) } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs new file mode 100644 index 0000000000..a373ac7efd --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs @@ -0,0 +1,59 @@ +use tempfile::TempDir; + +use crate::engine::_helpers::evaluation::STRIPPED_ROW_MISSED; +use crate::engine::main::evaluate_rows::evaluate_rows; +use crate::engine::tests::helpers::{evaluation_summary, orders_models, speculative_rows_request}; +use crate::errors::RowsError; +use crate::models::{EvaluateRequest, RetainedRows, RowsRequest}; + +/// The retained request with each row's digest restored, as a delta compile rebuilds it. +fn restored(retained: &RetainedRows) -> EvaluateRequest { + let mut request: EvaluateRequest = retained.request.clone(); + for (model, digest) in request.models.iter_mut().zip(&retained.digests) { + model.payload_digest = Some(digest.clone()); + model.stripped = true; + } + request +} + +#[test] +fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_stand_in() +-> Result<(), String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let mut request: RowsRequest = + speculative_rows_request(project_dir.path(), true, &orders_models())?; + request.memo = None; + request.retain = true; + let first = evaluate_rows(request).map_err(|_| "first evaluation failed".to_owned())?; + let retained: RetainedRows = first.retained.clone().ok_or("nothing was retained")?; + assert_eq!(retained.digests.len(), retained.request.models.len()); + assert!( + retained + .request + .models + .iter() + .all(|model| model.query_sql.is_empty() && model.authored_sql.is_empty()) + ); + + let cached = evaluate_rows(RowsRequest { + request: restored(&retained), + memo: None, + retain: false, + }) + .map_err(|_| "cached evaluation failed".to_owned())?; + assert_eq!(evaluation_summary(&cached).0, evaluation_summary(&first).0); + + let uncached_dir = TempDir::new().map_err(|error| error.to_string())?; + let mut uncached: EvaluateRequest = restored(&retained); + uncached.project_dir = uncached_dir.path().to_string_lossy().into_owned(); + let missed = evaluate_rows(RowsRequest { + request: uncached, + memo: None, + retain: false, + }); + assert!( + matches!(&missed, Err(RowsError::Rules(message)) if message == STRIPPED_ROW_MISSED), + "a stripped row evaluated without its cached findings" + ); + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index c98860e8ff..c97836c7f2 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -571,6 +571,9 @@ pub struct Model { pub empty_input_only_test_count: u32, #[serde(skip)] pub payload_digest: Option, + /// Whether this row's SQL text was dropped, so only a cached result may stand for it. + #[serde(skip)] + pub stripped: bool, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -926,6 +929,16 @@ pub struct RowsRequest { pub request: EvaluateRequest, /// The compiled-code identity and memo path, when the rules cache is enabled. pub memo: Option<(String, std::path::PathBuf)>, + /// Keep the request, its models without SQL text, for the project image. + pub retain: bool, +} + +/// A request kept for the project image: its models' SQL text dropped, each model's digest kept. +#[derive(Clone, Debug, Default)] +pub struct RetainedRows { + pub request: EvaluateRequest, + /// Each model's payload digest, which keys its cached findings, in model order. + pub digests: Vec, } /// A built-in rules evaluation running speculatively on its own thread; dropping it cancels it. @@ -963,4 +976,6 @@ pub struct RowsEvaluation { pub built_in_ms: u64, #[serde(skip)] pub reused: bool, + #[serde(skip)] + pub retained: Option, } From 2fae7e50513549c72ee4e4a625de6181168e34b0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:32:10 +0100 Subject: [PATCH 03/32] perf(cache): read a delta compile's stored report only once it is needed A delta base now carries the stored inputs and image with the identity of the output they name, so the large stored report is read after the delta has run rather than before every attempt. --- .../src/project_reuse/_helpers/entry.rs | 19 ++++++++++++++----- .../src/project_reuse/main/read_delta_base.rs | 9 +++++++-- .../src/project_reuse/models.rs | 9 +++++++++ .../project_reuse/tests/test_project_image.rs | 7 +++++-- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs index fc4b0109c7..3ebb3ce730 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs @@ -12,7 +12,7 @@ use crate::project_reuse::constants::{ REUSE_MAX_STORED_IMAGES, REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX, }; use crate::project_reuse::models::{ - ImageSections, SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile, + DeltaBase, ImageSections, SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile, }; use crate::project_snapshot::models::PathStamp; use crate::store::errors::StoreDecodeError; @@ -115,12 +115,21 @@ pub(crate) fn read_entry(path: &Path) -> Option<(StoredInputs, StoredOutput)> { Some((slot.inputs, output)) } -/// The stored inputs, output and image of the slot, or `None` when any cannot be read. -pub(crate) fn read_delta_base(path: &Path) -> Option<(StoredInputs, StoredOutput, ImageSections)> { +/// The stored inputs and image of the slot, or `None` when either cannot be read. +pub(crate) fn read_delta_base(path: &Path) -> Option { let slot: StoredSlot = read_slot(path)?; let image: ImageSections = read_slot_image(path, &slot)?; - let output: StoredOutput = read_output(path, &slot)?; - Some((slot.inputs, output, image)) + Some(DeltaBase { + inputs: slot.inputs, + image, + output_identity: slot.output_identity, + }) +} + +/// The output of the stored compile `base` was read from, or `None` when it no longer is. +pub(crate) fn read_delta_output(path: &Path, base: &DeltaBase) -> Option { + let bytes: Vec = std::fs::read(output_path(path)).ok()?; + decoded_output(&bytes, &base.output_identity).ok() } /// The image the slot names, or `None` when it has none or it cannot be read. diff --git a/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs index c9c6d55d39..b3da76a610 100644 --- a/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs +++ b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs @@ -1,8 +1,13 @@ use std::path::Path; -use crate::project_reuse::models::{ImageSections, StoredInputs, StoredOutput}; +use crate::project_reuse::models::{DeltaBase, StoredOutput}; /// The stored compile at `store_path` with its image, the base a delta compile patches. -pub fn read_delta_base(store_path: &Path) -> Option<(StoredInputs, StoredOutput, ImageSections)> { +pub fn read_delta_base(store_path: &Path) -> Option { crate::project_reuse::_helpers::entry::read_delta_base(store_path) } + +/// The output of the stored compile `base` was read from, or `None` when it no longer is. +pub fn read_delta_output(store_path: &Path, base: &DeltaBase) -> Option { + crate::project_reuse::_helpers::entry::read_delta_output(store_path, base) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/models.rs b/crates/sqlbuild-cache/src/project_reuse/models.rs index 2a3c116066..96188c6cd4 100644 --- a/crates/sqlbuild-cache/src/project_reuse/models.rs +++ b/crates/sqlbuild-cache/src/project_reuse/models.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::PathBuf; +use crate::digest::types::ContentDigest; use crate::project_snapshot::models::{PathStamp, SnapshotRules}; /// Constants the Python host owns, passed once so both sides agree. @@ -72,6 +73,14 @@ pub struct StoredInputs { /// Named sections of one project image, in stored order. pub type ImageSections = Vec<(String, Vec)>; +/// The stored compile a delta compile patches: its inputs and image, and which output it names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeltaBase { + pub inputs: StoredInputs, + pub image: ImageSections, + pub output_identity: ContentDigest, +} + /// The project paths that differ from the stored compile's, each list sorted. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ProjectFileChanges { diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs index c78b1bf4ac..bbf15e173f 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; -use crate::project_reuse::_helpers::entry::{read_delta_base, write_entry}; +use crate::project_reuse::_helpers::entry::{read_delta_base, read_delta_output, write_entry}; use crate::project_reuse::_helpers::files::{project_file_changes, unverified_paths}; use crate::project_reuse::models::{ImageSections, ProjectFileChanges, StoredProjectFile}; use crate::project_reuse::tests::helpers::{stamp, stored_inputs, stored_output}; @@ -71,7 +71,10 @@ fn given_stored_compile_with_image_when_reading_delta_base_then_only_an_intact_i .expect("stored compile"); (test_case.edit)(&path); - let read = read_delta_base(&path); + let read = read_delta_base(&path).and_then(|base| { + let output = read_delta_output(&path, &base)?; + Some((base.inputs, output, base.image)) + }); assert_eq!( read, From 1fb6f8a16187fb433aaebcb27f1b68c13172fdba Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:32:10 +0100 Subject: [PATCH 04/32] feat(compile): compile only changed models and SQL tests against the stored compile On a reuse miss where only model or SQL test files changed, rerun the compile for those files with the rest of the project injected from the stored compile's image, splice their report items into the stored report and record the result as a normal reusable compile. Full compiles capture the image as they run. Any change the delta cannot reproduce exactly falls back to the full compile. SQLBUILD_DISABLE_DELTA_COMPILE=1 turns delta compiles off, and SQLBUILD_VERIFY_DELTA_COMPILE=1 runs each delta beside the full compile and exits 70 when their reports, exit codes or artifacts differ. --- .../analysis_session/model_analyses.rs | 9 + .../_helpers/analysis_session/session.rs | 5 + .../_helpers/compile_command/report.rs | 12 + .../_helpers/compile_command/reuse_attempt.rs | 102 ++- .../compile_command/reuse_recording.rs | 42 ++ .../_helpers/compile_command/target_writer.rs | 19 +- .../_helpers/compile_delta/context.rs | 276 +++++++ .../bindings/_helpers/compile_delta/driver.rs | 678 ++++++++++++++++++ .../_helpers/compile_delta/header_keys.rs | 156 ++++ .../bindings/_helpers/compile_delta/image.rs | 214 ++++++ .../_helpers/compile_delta/image_facts.rs | 49 ++ .../_helpers/compile_delta/lint_hooks.rs | 101 +++ .../bindings/_helpers/compile_delta/mod.rs | 10 + .../_helpers/compile_delta/report_hooks.rs | 278 +++++++ .../_helpers/compile_delta/rules_rows.rs | 87 +++ .../bindings/_helpers/compile_delta/splice.rs | 309 ++++++++ .../bindings/_helpers/compile_delta/verify.rs | 217 ++++++ .../_helpers/compile_driver/analysis.rs | 3 +- .../compile_driver/assembly_project.rs | 4 +- .../_helpers/compile_driver/command.rs | 193 +++-- .../_helpers/compile_outputs/json_report.rs | 3 + .../_helpers/compile_outputs/reuse.rs | 26 +- .../_helpers/discovery/model_records.rs | 17 + .../_helpers/discovery/project_files.rs | 1 + .../_helpers/discovery/sql_test_records.rs | 2 + .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/model_config/model_validator.rs | 5 +- .../_helpers/render/macro_call_memo.rs | 5 + .../_helpers/rules/custom_evaluation.rs | 50 +- .../src/bindings/_helpers/rules/rows.rs | 17 +- .../src/bindings/_helpers/rules/sql_rules.rs | 29 +- .../_helpers/scopes/scope_validation.rs | 15 +- .../_helpers/semantic_shapes/shapes.rs | 3 +- 33 files changed, 2857 insertions(+), 81 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_facts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 81960c08d6..2610876a8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -206,6 +206,15 @@ fn run_session<'py>( nullability.call_method1("setdefault", (&name, defaults))?; } let (additions, analysis_names, succeeded) = session.borrow_mut().finish(py)?; + if let Some(tables) = session + .borrow() + .finished_session() + .and_then(|finished| finished.relation_tables()) + { + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.relations = Some(tables.clone()); + }); + } record_catalog_changes(catalog, additions, &analysis_names, (&types, &nullability))?; record_cache(py, python, session.borrow().cache_stats())?; let positions: Bound<'py, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 6d7a78e232..83242dc1d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -275,6 +275,7 @@ pub(crate) fn start_model_analysis_session( ) -> PyResult { let py: Python<'_> = catalog.py(); let mut request: SessionRequest = session_request(request); + crate::bindings::_helpers::compile_delta::context::inject_relations(&mut request)?; let rule_failure: RuleFailure = RuleFailure::default(); request.nullability_callback = adapter_rules .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); @@ -282,6 +283,8 @@ pub(crate) fn start_model_analysis_session( let mut session: AnalysisSession = py .compiler_detach(|| start_analysis_session(request, catalog)) .map_err(|reason| internal_error(&reason))?; + let cache: Option<(PathBuf, String)> = + cache.filter(|_| crate::bindings::_helpers::compile_delta::context::injection().is_none()); let mut cache_path: Option = None; let mut cache_stats: Option = None; if let Some((path, environment)) = cache { @@ -368,6 +371,8 @@ fn session_request(request: RequestRow) -> SessionRequest { catalog_schemas, dynamic_families_by_table: families_by_table(dynamic_families_by_table), models: models.into_iter().map(model_request).collect(), + forced_order: None, + retain_relations: crate::bindings::_helpers::compile_delta::context::capturing(), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 929f554d35..486acbfff3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -296,6 +296,11 @@ fn shown_diagnostics( return Ok(Vec::new()); } let texts: Bound<'_, PyDict> = source_texts(&inputs.project)?; + crate::bindings::_helpers::compile_delta::report_hooks::complete_source_texts( + &inputs.project, + &inputs.diagnostics, + &texts, + )?; let warning_counts: Bound<'_, PyDict> = PyDict::new(py); for diagnostic in &inputs.diagnostics { if native_field(diagnostic, "severity")?.eq(WARNING_SEVERITY)? { @@ -358,6 +363,8 @@ pub(crate) fn format_compile_text<'py>( column_count: native_field(&model, "output_column_count")?.extract()?, }); } + let models: Vec = + crate::bindings::_helpers::compile_delta::report_hooks::text_model_lines(models); let lineage: LineageNote = if !inputs.lineage.is_none() && inputs.lineage_mode == RICH_LINEAGE { LineageNote::Rich } else if inputs.lineage.is_none() @@ -452,6 +459,11 @@ fn location_json<'py>(location: &Bound<'py, PyAny>) -> PyResult(diagnostic: &Bound<'py, PyAny>) -> PyResult> { + diagnostic_json(diagnostic) +} + fn diagnostic_json<'py>(diagnostic: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = diagnostic.py(); let payload: Bound<'py, PyDict> = dict( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 38fff9bfc4..9fc2c87329 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -30,6 +30,10 @@ use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::runtime_identity::{ invocation_identity, redirected_output_files, runtime_identity, }; +use crate::bindings::_helpers::compile_delta::driver::{ + self, DELTA_VERIFY_ENV_VAR, DeltaOutcome, delta_enabled, env_enabled, +}; +use crate::bindings::_helpers::compile_delta::{context, verify}; use crate::bindings::_helpers::compile_driver::command::run_compile_command; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; @@ -484,12 +488,98 @@ pub(crate) fn run_compile<'py>(request: &Bound<'py, PyAny>) -> PyResult = if missed && delta_enabled() { + compiled_with_delta(request, &attempt) + } else { + run_compile_command(py, request.clone(), Some(attempt)) + }; + context::reset(); + verify::disarm(); + code.and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py)) +} + +/// Compile the changed files when the delta can; otherwise, or under verification as well, +/// compile the whole project, capturing its image. +fn compiled_with_delta<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = request.py(); + let verifying: bool = env_enabled(DELTA_VERIFY_ENV_VAR); + let started: f64 = monotonic(py)?; + let delta: DeltaOutcome<'py> = match driver::run(request, attempt) { + Ok(delta) => delta, + Err(error) => DeltaOutcome::Fallback(format!("the delta compile failed: {error}")), + }; + context::reset(); + let project_dir: Bound<'py, PyAny> = native_field(attempt, "project_dir")?; + let cache: Bound<'py, PyAny> = cache_directory(&project_dir)?; + let compiled_root: PathBuf = project_dir + .call_method1("__truediv__", (TARGET_DIRECTORY_NAME,))? + .call_method1("__truediv__", (COMPILED_DIRECTORY_NAME,))? + .call_method0("absolute")? + .extract()?; + match delta { + DeltaOutcome::Finished(finished) if !verifying => { + let (files, models) = (finished.changed_files, finished.rerun_models); + let committing: std::time::Instant = std::time::Instant::now(); + let code: i64 = driver::commit(*finished, attempt)?; + driver::trace(py, "delta record and print", committing)?; + let seconds: f64 = elapsed_ms(py, started)? as f64 / MILLISECONDS_PER_SECOND; + print_stderr( + py, + &format!( + "Compiled {files} changed file(s) against the previous compile ({models} model(s), {seconds:.1}s)" + ) + .into_bound_py_any(py)?, + )?; + return Ok(code); + } + DeltaOutcome::Raised(error) if !verifying => return Err(error), + DeltaOutcome::Fallback(reason) => { + reuse_logger(py)?.call_method1("debug", ("Delta compile fell back: %s", &reason))?; + if verifying { + print_stderr( + py, + &format!("delta compile fell back: {reason}").into_bound_py_any(py)?, + )?; + } + context::start_capture(); + return run_compile_command(py, request.clone(), Some(attempt.clone())); + } + DeltaOutcome::Finished(finished) => verify::expect(verify::Expected { + stdout: Some(finished.stdout.clone()), + exit_code: Some(finished.phases.outcome.exit_code), + artifacts_written: finished.phases.outcome.artifacts_written, + raised: None, + compiled: verify::compiled_digests(&compiled_root), + compiled_root: compiled_root.to_string_lossy().into_owned(), + }), + DeltaOutcome::Raised(error) => verify::expect(verify::Expected { + raised: Some(error.to_string()), + compiled_root: compiled_root.to_string_lossy().into_owned(), + ..verify::Expected::default() + }), + } + context::start_capture(); + let full: PyResult = run_compile_command(py, request.clone(), Some(attempt.clone())); + let raised: Option = full.as_ref().err().map(ToString::to_string); + let differences: Vec = verify::differences(raised); + if differences.is_empty() { + print_stderr(py, &"delta compile verified".into_bound_py_any(py)?)?; + return full; + } + let cache_dir: PathBuf = cache.extract()?; + verify::write_mismatch(&cache_dir, &differences); + for difference in &differences { + print_stderr( + py, + &format!("delta compile mismatch: {difference}").into_bound_py_any(py)?, + )?; + } + full?; + Ok(verify::MISMATCH_EXIT_CODE) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 01eab6546c..e05e862fab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -81,6 +81,22 @@ impl CompileArtifactWrites { } } +impl CompileArtifactWrites { + /// Every recorded artifact's absolute path, with its digest when this compile wrote it. + pub(crate) fn written_digests(&self) -> HashMap> { + self.locked() + .iter() + .map(|(path, artifact)| { + let digest: Option = match artifact { + RecordedArtifact::Written(digest) => Some(digest.clone()), + RecordedArtifact::Kept { .. } => None, + }; + (path.clone(), digest) + }) + .collect() + } +} + #[pymethods] impl CompileArtifactWrites { /// Every recorded artifact by absolute path, as `(digest, size, mtime_ns)`. @@ -235,6 +251,13 @@ impl RecordedCompileOutput { fn locked(&self) -> std::sync::MutexGuard<'_, PrintedOutput> { self.output.lock().unwrap_or_else(PoisonError::into_inner) } + + /// Output already printed or recorded elsewhere: its stderr lines and its report. + pub(crate) fn recorded(stderr_lines: Vec>, stdout: Py) -> Self { + Self { + output: Mutex::new((stderr_lines, Some(stdout))), + } + } } /// `print(*values, file=sys.)`. @@ -480,6 +503,25 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { ("rules", reuse_rules(py, true)?.into_any()), ], )?; + if let Some(mut capture) = crate::bindings::_helpers::compile_delta::context::take_capture() { + capture.environment_names = native_field(&facts.input_reads, "environment_names")? + .try_iter()? + .map(|name| name?.extract::()) + .collect::>()?; + let json_output: bool = facts.json_output.is_truthy()?; + match crate::bindings::_helpers::compile_delta::image::ProjectImage::from_capture( + capture, + json_output, + ) { + Ok(image) => crate::bindings::_helpers::compile_delta::context::set_pending_image( + image.encode().ok(), + ), + Err(reason) => { + let _ = reuse_logger(py)? + .call_method1("debug", ("Compile reuse kept no project image: %s", reason))?; + } + } + } let (outcome, digested): (String, Bound<'_, PyAny>) = facts.native.call_method1("record", (payload,))?.extract()?; let _ = reuse_logger(py)?.call_method1( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 20a731b1df..25c5bdb758 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -467,6 +467,9 @@ fn plan_tests<'py>( count_stage(py, SQL_TEST_MODEL_CHAINS, 1)?; let chains: Bound<'py, PyAny> = sql_test_model_chain_names_by_key(project, &tests)?.into_any(); + crate::bindings::_helpers::compile_delta::report_hooks::record_test_chains( + &tests, &chains, + )?; Some((context, chains)) }; let tests: Vec> = tests.try_iter()?.collect::>()?; @@ -613,9 +616,16 @@ fn write_static_tests<'py>( } } } - record_compile_timing("cache_publication_ms", || { - write_cache(&native_field(project, "compile_cache_dir")?, &records) - })?; + let cache_dir: Bound<'py, PyAny> = native_field(project, "compile_cache_dir")?; + if crate::bindings::_helpers::compile_delta::context::injection().is_some() { + let mut kept: HashMap = read_cache(&cache_dir)?; + for (key, record) in records.drain(..) { + kept.insert(key, record); + } + records = kept.into_iter().collect(); + records.sort_by(|left, right| left.0.cmp(&right.0)); + } + record_compile_timing("cache_publication_ms", || write_cache(&cache_dir, &records))?; Ok((managed, diagnostics)) } @@ -627,7 +637,8 @@ pub(crate) fn write_static_target<'py>( prepared: Prepared<'_, 'py>, ) -> PyResult> { let py: Python<'py> = target_dir.py(); - let remove_stale_files: bool = prepared_target(target_dir)?; + let remove_stale_files: bool = prepared_target(target_dir)? + && crate::bindings::_helpers::compile_delta::context::injection().is_none(); let mut batch: ArtifactBatch = ArtifactBatch::new(remove_stale_files, prepared.recorder); let mut managed: Vec> = Vec::new(); let mut managed_models: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs new file mode 100644 index 0000000000..e86fdeb0ff --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -0,0 +1,276 @@ +//! What a full compile captures for its project image, and what a delta compile injects. +//! +//! Compiles run one at a time per process, but rules run on their own threads, so both the +//! capture and the injection live in process-wide slots rather than thread-local ones. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, SessionRequest}; +use sqlbuild_cache::project_reuse::models::ImageSections; +use sqlbuild_rules::models::RetainedRows; + +/// The facts a full compile leaves for its project image beyond its analysis record. +#[derive(Debug, Default)] +pub(crate) struct ImageCapture { + /// The relation facts the model analysis session ended with. + pub(crate) relations: Option, + /// The built-in rules request and its raw findings, before suppressions. + pub(crate) rules: Option, + /// Each custom-rule fact digest the custom rule cache compared, by fact key. + pub(crate) fact_digests: BTreeMap, String>, + /// Whether the JSON report fell back to the standard library's serializer. + pub(crate) stdlib_json: bool, + /// Each model's scope usage digest, by model file path. + pub(crate) usages: BTreeMap, + /// Each SQL test's compiled-artifact chain, by test key text. + pub(crate) test_chains: Vec, + /// Each project model's `(path, name, materialized)`, when custom rules ran. + pub(crate) model_subjects: Option)>>, + /// Each compiled model's `(name, relative path, upstream (kind, name) pairs)`. + pub(crate) models: Vec, + /// Each report diagnostic as its JSON item text, with what it names. + pub(crate) diagnostics: Vec, + /// The text report's model lines `(name, failed, column count)`, when it rendered one. + pub(crate) text_models: Option>, + /// The template environment names the compile read. + pub(crate) environment_names: Vec, + /// Each discovered model's header keys, by model file path. + pub(crate) header_keys: + BTreeMap, + /// Each model file's SQL lint cache key, in project model order. + pub(crate) lint_model_keys: Option>, + /// Each other SQL file's lint cache key, in walk order. + pub(crate) lint_file_keys: Option)>>, + /// The closed shape each model published for semantic checks, in publication order. + pub(crate) model_shapes: Option>, + /// Why this compile cannot be the base of a delta compile, when something ruled it out. + pub(crate) unsupported: Option, +} + +/// The built-in rules request a compile evaluated and its raw findings. +#[derive(Debug)] +pub(crate) struct CapturedRules { + pub(crate) retained: RetainedRows, +} + +/// One model's published `(column, type)` shape. +pub(crate) type ModelShape = (String, Vec<(String, String)>); + +/// One compiled model's identity and references. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct CapturedModel { + pub(crate) name: String, + pub(crate) relative_path: String, + pub(crate) upstream: Vec<(String, String)>, +} + +/// One report diagnostic: its JSON text and the resource and file it names. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct CapturedDiagnostic { + pub(crate) json: String, + pub(crate) resource_name: Option, + pub(crate) paths: Vec, +} + +/// One SQL test's file and the models its compiled artifact is keyed by. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct CapturedTest { + pub(crate) key: String, + pub(crate) relative_path: String, + pub(crate) chain: Vec, +} + +/// What a delta compile hands the stages it reruns for the changed models. +#[derive(Debug, Default)] +pub(crate) struct DeltaInjection { + /// Model files to discover, as project-relative POSIX paths. + pub(crate) model_paths: BTreeSet, + /// SQL test files to discover, as project-relative POSIX paths. + pub(crate) test_paths: BTreeSet, + /// The paths and names of every model the delta compile reruns. + pub(crate) rerun_names: BTreeSet, + /// The project directory the stored compile and the delta compile both read. + pub(crate) project_dir: std::path::PathBuf, + /// Every model and SQL test file of the whole project, as project-relative POSIX paths. + pub(crate) source_paths: BTreeSet, + /// Every model name of the whole project, which references resolve against. + pub(crate) model_names: Vec, + /// The whole project's relation facts, injected for relations the delta does not analyse. + pub(crate) relations: RelationTables, + /// The whole project's built-in rules request. + pub(crate) rules: Option, + /// The whole project's custom-rule fact digests. + pub(crate) fact_digests: BTreeMap, String>, + /// Each project model's `(path, name, materialized)`, the custom-rule subjects. + pub(crate) model_subjects: Option)>>, + /// The files that changed since the stored compile, as project-relative POSIX paths. + pub(crate) changed_paths: BTreeSet, + /// The whole project's SQL lint cache keys, as the image recorded them. + pub(crate) lint_model_keys: Option>, + pub(crate) lint_file_keys: Option)>>, + /// The closed shapes the whole project's models published. + pub(crate) model_shapes: Vec, + /// The whole project's text report model lines, with the rerun models' lines to replace. + pub(crate) text_models: Vec<(String, bool, i64)>, +} + +static CAPTURE: Mutex> = Mutex::new(None); +static INJECTION: Mutex>> = Mutex::new(None); + +fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Start capturing image facts for the compile about to run. +pub(crate) fn start_capture() { + *locked(&CAPTURE) = Some(ImageCapture::default()); +} + +/// Stop capturing and take what was captured. +pub(crate) fn take_capture() -> Option { + locked(&CAPTURE).take() +} + +/// Whether the running compile captures image facts. +pub(crate) fn capturing() -> bool { + locked(&CAPTURE).is_some() +} + +/// Record into the capture, when one is running. +pub(crate) fn capture(record: impl FnOnce(&mut ImageCapture)) { + if let Some(capture) = locked(&CAPTURE).as_mut() { + record(capture); + } +} + +/// Mark the running capture as unusable as a delta base. +pub(crate) fn capture_unsupported(reason: &str) { + capture(|capture| { + if capture.unsupported.is_none() { + capture.unsupported = Some(reason.to_owned()); + } + }); +} + +/// Run `body` with `injection` visible to the stages a delta compile reruns. +pub(crate) fn with_injection(injection: DeltaInjection, body: impl FnOnce() -> T) -> T { + *locked(&INJECTION) = Some(Arc::new(injection)); + let result: T = body(); + *locked(&INJECTION) = None; + result +} + +/// The running delta compile's injection, if any. +pub(crate) fn injection() -> Option> { + locked(&INJECTION).clone() +} + +/// Why the running delta compile cannot continue, once something ruled it out. +static UNSUPPORTED: Mutex> = Mutex::new(None); + +/// Rule out the running delta compile: returns the error that unwinds to its fallback. +pub(crate) fn delta_unsupported(reason: &str) -> pyo3::PyErr { + let mut slot = locked(&UNSUPPORTED); + if slot.is_none() { + *slot = Some(reason.to_owned()); + } + pyo3::exceptions::PyRuntimeError::new_err(format!("delta compile unsupported: {reason}")) +} + +/// Take the reason the delta compile was ruled out, if any. +pub(crate) fn take_delta_unsupported() -> Option { + locked(&UNSUPPORTED).take() +} + +/// The previous compile's digest of a custom-rule fact about a resource the delta did not rerun. +pub(crate) fn injected_fact_digest(key: &[String]) -> Option { + let injection: Arc = injection()?; + if key + .iter() + .skip(1) + .any(|part| injection.rerun_names.contains(part)) + { + return None; + } + injection.fact_digests.get(key).cloned() +} + +/// Give a delta compile's analysis session the whole project's facts for relations it does not +/// analyse, and the whole project's analysis order. +pub(crate) fn inject_relations(request: &mut SessionRequest) -> pyo3::PyResult<()> { + if !request.dynamic_families_by_table.is_empty() { + capture_unsupported("a model declares dynamic column families"); + } + let Some(injection) = injection() else { + return Ok(()); + }; + if !request.dynamic_families_by_table.is_empty() { + return Err(delta_unsupported( + "a model declares dynamic column families", + )); + } + let relations: &RelationTables = &injection.relations; + request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); + request.retain_relations = true; + let analysed: BTreeSet<&str> = request + .models + .iter() + .map(|model| model.name.as_str()) + .collect(); + for (table, injected) in [ + (&mut request.column_types, &relations.types), + (&mut request.column_nullability, &relations.nullability), + (&mut request.complete_schemas, &relations.complete), + ] { + let present: BTreeSet = table.iter().map(|(name, _)| name.clone()).collect(); + table.extend( + injected + .iter() + .filter(|(name, _)| !analysed.contains(name.as_str()) && !present.contains(name)) + .cloned(), + ); + } + Ok(()) +} + +/// What a delta compile's reuse record keeps of the stored compile it patched. +#[derive(Debug, Default)] +pub(crate) struct RecordMerge { + /// Each stored artifact's `(path, size, mtime_ns)`, kept unless the delta wrote it. + pub(crate) kept: Vec<(String, u64, i64)>, + /// The template environment names the stored compile read. + pub(crate) environment_names: Vec, +} + +static PENDING_IMAGE: Mutex> = Mutex::new(None); +static RECORD_MERGE: Mutex> = Mutex::new(None); + +/// Hand the next reuse record the image to keep beside it. +pub(crate) fn set_pending_image(image: Option) { + *locked(&PENDING_IMAGE) = image; +} + +/// The image the next reuse record keeps, taken. +pub(crate) fn take_pending_image() -> Option { + locked(&PENDING_IMAGE).take() +} + +/// Hand the next reuse record what it keeps of the patched compile. +pub(crate) fn set_record_merge(merge: Option) { + *locked(&RECORD_MERGE) = merge; +} + +/// What the next reuse record keeps of the patched compile, taken. +pub(crate) fn take_record_merge() -> Option { + locked(&RECORD_MERGE).take() +} + +/// Forget every capture, injection and pending record fact, after a compile ends either way. +pub(crate) fn reset() { + let _ = take_capture(); + *locked(&INJECTION) = None; + let _ = take_delta_unsupported(); + set_pending_image(None); + set_record_merge(None); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs new file mode 100644 index 0000000000..91dc768b7a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -0,0 +1,678 @@ +//! Compile only the changed models and SQL tests, patching the stored compile's image and +//! report; anything this cannot reproduce exactly falls back to a full compile. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyString, PyStringMethods}; +use sqlbuild_cache::project_reuse::main::read_delta_base::{read_delta_base, read_delta_output}; +use sqlbuild_cache::project_reuse::models::{ + DeltaBase, DeltaCandidate, ImageSections, ProjectFileChanges, +}; + +use crate::bindings::_helpers::compile_command::reuse_recording::RecordedCompileOutput; +use crate::bindings::_helpers::compile_delta::context::{ + CapturedDiagnostic, CapturedTest, DeltaInjection, ImageCapture, RecordMerge, start_capture, + take_capture, take_delta_unsupported, with_injection, +}; +use crate::bindings::_helpers::compile_delta::header_keys::{ + HeaderChange, custom_rule_reads, header_change, +}; +use crate::bindings::_helpers::compile_delta::image::{ImageRelations, ProjectImage}; +use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; +use crate::bindings::_helpers::compile_driver::command::{PhasesOutcome, unrecorded_compile}; +use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Set to `1` to always compile the whole project. +pub(crate) const DELTA_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_DELTA_COMPILE"; +/// Set to `1` to run each delta compile beside a full compile and fail on any difference. +pub(crate) const DELTA_VERIFY_ENV_VAR: &str = "SQLBUILD_VERIFY_DELTA_COMPILE"; +const ENABLED_VALUE: &str = "1"; +/// Rerunning more models than this, or this share of the project, compiles the whole project. +const MAX_RERUN_MODELS: usize = 500; +const MAX_RERUN_PERCENT: usize = 5; +/// How many times a changed relation shape may pull its dependents into the delta. +const MAX_ROUNDS: usize = 4; +const MODELS_ROOT: &str = "models/"; +const TESTS_ROOT: &str = "tests/"; +const SQL_SUFFIX: &str = ".sql"; +/// Under verification, the delta's own report is kept here for inspection. +const VERIFY_REPORT_DIRECTORY: &str = "target/cache/compiler-native-v1"; +const VERIFY_REPORT_FILE: &str = "delta-report.out"; + +/// Whether the environment turns `name` on. +pub(crate) fn env_enabled(name: &str) -> bool { + std::env::var(name).is_ok_and(|value| value == ENABLED_VALUE) +} + +/// Whether delta compiles, and so project images, are on for this process. +pub(crate) fn delta_enabled() -> bool { + !env_enabled(DELTA_DISABLE_ENV_VAR) +} + +/// A delta compile that finished, ready to be stored and printed or compared. +pub(crate) struct Finished<'py> { + pub(crate) request: Bound<'py, PyAny>, + pub(crate) stdout: String, + pub(crate) stderr_lines: Vec>, + pub(crate) phases: PhasesOutcome<'py>, + pub(crate) image: Option, + pub(crate) merge: RecordMerge, + pub(crate) changed_files: usize, + pub(crate) rerun_models: usize, +} + +/// What one delta attempt came to. +pub(crate) enum DeltaOutcome<'py> { + /// The delta cannot reproduce this compile; the reason is for diagnostics only. + Fallback(String), + /// The compile stopped on an error from a changed file, exactly as a full compile would. + Raised(PyErr), + Finished(Box>), +} + +fn fallback<'py>(reason: impl Into) -> PyResult> { + Ok(DeltaOutcome::Fallback(reason.into())) +} + +/// Whether the invocation asks for anything a delta compile does not reproduce. +fn ineligible(request: &Bound<'_, PyAny>) -> PyResult> { + let empty = |name: &str| -> PyResult { + let value: Bound<'_, PyAny> = native_field(request, name)?; + Ok(value.is_none() || !value.is_truthy()?) + }; + for (name, reason) in [ + ("select", "a selection was given"), + ("exclude", "an exclusion was given"), + ("dag_path", "a DAG artifact was requested"), + ("manifest", "a manifest was requested"), + ("defer_to", "deferral was requested"), + ("no_sql_validation", "SQL validation is off"), + ] { + if !empty(name)? { + return Ok(Some(reason)); + } + } + let lineage: Bound<'_, PyAny> = native_field(request, "lineage_mode")?; + if !lineage.is_none() && lineage.getattr("value")?.str()?.to_str()? != "fast" { + return Ok(Some("rich lineage was requested")); + } + let flags: Bound<'_, PyAny> = native_field(request, "profile_flags")?; + for name in [ + "skip_discovery_sql_analysis", + "skip_column_inference", + "skip_contracts", + "skip_write", + ] { + if native_field(&flags, name)?.is_truthy()? { + return Ok(Some("a profiling flag was given")); + } + } + Ok(None) +} + +/// A path the delta may recompile: an SQL file in a resource tree, not a declaration tree. +fn resource_sql(path: &str, root: &str) -> bool { + path.starts_with(root) + && path.ends_with(SQL_SUFFIX) + && !path.split('/').any(|part| part.starts_with('_')) +} + +/// The stored compile's inputs and image, read from the attempt's slot; its report is read +/// only once the delta has run. +struct Base { + store_path: PathBuf, + stored: DeltaBase, + image: ProjectImage, +} + +/// The project files the reuse check found changed, and the slot it checked. +fn changes_of( + attempt: &Bound<'_, PyAny>, +) -> PyResult> { + let native: Bound<'_, PyAny> = native_field(attempt, "native")?; + let Ok(native) = native.cast::() else { + return Ok(Err("the reuse check did not run".to_owned())); + }; + let (delta, store_path): (DeltaCandidate, PathBuf) = native + .get() + .with_attempt(|attempt| (attempt.delta.clone(), attempt.store_path.clone()))?; + Ok(match delta { + DeltaCandidate::Files(changes) => Ok((changes, store_path)), + DeltaCandidate::Unavailable(reason) => Err(reason.to_owned()), + }) +} + +fn base_of(store_path: PathBuf) -> Result { + let Some(stored) = read_delta_base(&store_path) else { + return Err("the stored compile has no project image".to_owned()); + }; + let Some(image) = ProjectImage::decode(&stored.image) else { + return Err("the project image cannot be read".to_owned()); + }; + Ok(Base { + store_path, + stored, + image, + }) +} + +/// The rerun set: models and SQL tests, with every path and name the delta recompiles. +#[derive(Clone, Debug, Default)] +struct Rerun { + changed: BTreeSet, + models: BTreeSet, + model_paths: BTreeSet, + test_paths: BTreeSet, + tests: BTreeSet, +} + +impl Rerun { + fn with_tests(mut self, image: &ProjectImage, changed_tests: &BTreeSet) -> Self { + self.test_paths = changed_tests.clone(); + let test_files: BTreeSet<&str> = image + .tests + .iter() + .map(|test| test.relative_path.as_str()) + .collect(); + for diagnostic in &image.diagnostics { + for path in &diagnostic.paths { + if test_files.contains(path.as_str()) { + self.test_paths.insert(path.clone()); + } + } + } + for test in &image.tests { + if test.chain.iter().any(|name| self.models.contains(name)) { + self.test_paths.insert(test.relative_path.clone()); + } + } + self.tests = image + .tests + .iter() + .filter(|test| self.test_paths.contains(&test.relative_path)) + .map(|test| test.key.clone()) + .collect(); + self + } + + fn names(&self) -> BTreeSet { + let mut names: BTreeSet = self.models.clone(); + names.extend(self.model_paths.iter().cloned()); + names.extend(self.tests.iter().cloned()); + names.extend(self.test_paths.iter().cloned()); + names + } + + fn owns(&self, diagnostic: &CapturedDiagnostic) -> bool { + diagnostic + .resource_name + .as_ref() + .is_some_and(|name| self.models.contains(name) || self.tests.contains(name)) + || diagnostic + .paths + .iter() + .any(|path| self.model_paths.contains(path) || self.test_paths.contains(path)) + } +} + +fn injection_for(base: &Base, rerun: &Rerun, project_dir: PathBuf) -> DeltaInjection { + let image: &ProjectImage = &base.image; + DeltaInjection { + project_dir, + model_paths: rerun.model_paths.clone(), + test_paths: rerun.test_paths.clone(), + rerun_names: rerun.names(), + source_paths: image + .models + .iter() + .map(|model| model.relative_path.clone()) + .chain(image.tests.iter().map(|test| test.relative_path.clone())) + .collect(), + model_names: image + .models + .iter() + .map(|model| model.name.clone()) + .collect(), + relations: (&image.relations).into(), + rules: image.rules.clone(), + fact_digests: image.fact_digests.iter().cloned().collect(), + model_subjects: image.model_subjects.clone(), + text_models: image.text_models.clone().unwrap_or_default(), + model_shapes: image.model_shapes.clone(), + changed_paths: rerun.changed.clone(), + lint_model_keys: image.lint_model_keys.clone(), + lint_file_keys: image.lint_file_keys.clone(), + } +} + +/// The relation facts of `name` that its dependents' analysis reads. +fn shape_of(relations: &ImageRelations, name: &str) -> [Option>; 3] { + let find = |table: &[(String, Vec<(String, String)>)]| { + table + .iter() + .find(|(relation, _)| relation == name) + .map(|(_, shape)| shape.clone()) + }; + [ + find(&relations.types), + find(&relations.nullability), + find(&relations.complete), + ] +} + +/// `base`'s rows with each of `fresh`'s rows replacing the one of its name, new names appended. +fn merged_rows( + base: &mut Vec<(String, Vec<(String, String)>)>, + fresh: Vec<(String, Vec<(String, String)>)>, +) { + let mut positions: HashMap = base + .iter() + .enumerate() + .map(|(index, (name, _))| (name.clone(), index)) + .collect(); + for (name, shape) in fresh { + match positions.get(&name) { + Some(index) => base[*index].1 = shape, + None => { + positions.insert(name.clone(), base.len()); + base.push((name, shape)); + } + } + } +} + +/// One restricted compile of `rerun`, with what it captured. +struct Round<'py> { + request: Bound<'py, PyAny>, + output: Bound<'py, PyAny>, + phases: PhasesOutcome<'py>, + capture: ImageCapture, +} + +enum RoundOutcome<'py> { + Ran(Box>), + Stop(DeltaOutcome<'py>), +} + +fn round<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, + (base, rerun): (&Base, &Rerun), +) -> PyResult> { + let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; + start_capture(); + let ran = with_injection(injection_for(base, rerun, project_dir), || { + unrecorded_compile(request, attempt) + }); + let unsupported: Option = take_delta_unsupported(); + let capture: Option = take_capture(); + if let Some(reason) = unsupported { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(reason))); + } + let (request, output, phases) = match ran { + Ok(ran) => ran, + Err(error) => { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(format!( + "the delta compile failed: {error}" + )))); + } + }; + let phases: PhasesOutcome<'py> = match phases { + Ok(phases) => phases, + Err(error) => { + let message: String = error.to_string(); + let owned: bool = rerun + .model_paths + .iter() + .chain(&rerun.test_paths) + .any(|path| message.contains(path.as_str())); + return Ok(RoundOutcome::Stop(if owned { + DeltaOutcome::Raised(error) + } else { + DeltaOutcome::Fallback(format!("the delta compile raised: {message}")) + })); + } + }; + let Some(capture) = capture else { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback( + "the delta compile captured nothing".to_owned(), + ))); + }; + if let Some(reason) = &capture.unsupported { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(reason.clone()))); + } + Ok(RoundOutcome::Ran(Box::new(Round { + request, + output, + phases, + capture, + }))) +} + +/// Compile the changed files against the stored compile, or say why that is not possible. +pub(crate) fn run<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, +) -> PyResult> { + if !delta_enabled() { + return fallback("delta compiles are off"); + } + if let Some(reason) = ineligible(request)? { + return fallback(reason); + } + let started: std::time::Instant = std::time::Instant::now(); + let (changes, store_path): (ProjectFileChanges, PathBuf) = match changes_of(attempt)? { + Ok(found) => found, + Err(reason) => return fallback(reason), + }; + if !changes.added.is_empty() || !changes.removed.is_empty() { + return fallback("files were added or removed"); + } + if changes.changed.is_empty() { + return fallback("no project file changed"); + } + if let Some(path) = changes + .changed + .iter() + .find(|path| !resource_sql(path, MODELS_ROOT) && !resource_sql(path, TESTS_ROOT)) + { + return fallback(format!("{path} is not a model or SQL test file")); + } + let base: Base = match base_of(store_path) { + Ok(base) => base, + Err(reason) => return fallback(reason), + }; + let json_output: bool = native_field(request, "json_output")?.is_truthy()?; + if base.image.json_output != json_output { + return fallback("the stored compile has the other output format"); + } + let model_by_path: HashMap<&str, &str> = base + .image + .models + .iter() + .map(|model| (model.relative_path.as_str(), model.name.as_str())) + .collect(); + let test_files: BTreeSet<&str> = base + .image + .tests + .iter() + .map(|test| test.relative_path.as_str()) + .collect(); + let mut rerun: Rerun = Rerun { + changed: changes.changed.iter().cloned().collect(), + ..Rerun::default() + }; + let mut changed_tests: BTreeSet = BTreeSet::new(); + for path in &changes.changed { + if resource_sql(path, MODELS_ROOT) + && let Some(name) = model_by_path.get(path.as_str()) + { + rerun.models.insert((*name).to_owned()); + rerun.model_paths.insert(path.clone()); + } else if resource_sql(path, TESTS_ROOT) && test_files.contains(path.as_str()) { + changed_tests.insert(path.clone()); + } else { + return fallback(format!("{path} is not a model or SQL test file")); + } + } + let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; + for path in &rerun.model_paths { + let name: &str = model_by_path + .get(path.as_str()) + .copied() + .unwrap_or_default(); + match header_change(&project_dir, path, base.image.header_keys.get(path)) { + HeaderChange::Unchanged | HeaderChange::Unparsed => {} + HeaderChange::Unsupported(reason) => return fallback(reason), + HeaderChange::Patched(keys) => { + if base.image.model_subjects.is_some() && keys.contains("materialized") { + return fallback("custom rules select models by their materialization"); + } + if let Some(reason) = + custom_rule_reads(&base.image.fact_digests, (path, name), &keys) + { + return fallback(reason); + } + } + } + } + let limit: usize = MAX_RERUN_MODELS.max(base.image.models.len() * MAX_RERUN_PERCENT / 100); + let mut last: Option> = None; + for _ in 0..MAX_ROUNDS { + if rerun.models.len() > limit { + return fallback("too many models would compile again"); + } + rerun = rerun.with_tests(&base.image, &changed_tests); + trace(request.py(), "delta base read", started)?; + let ran: Round<'py> = match round(request, attempt, (&base, &rerun))? { + RoundOutcome::Ran(ran) => *ran, + RoundOutcome::Stop(outcome) => return Ok(outcome), + }; + if rerun.models.is_empty() { + last = Some(ran); + break; + } + let Some(relations) = ran.capture.relations.clone() else { + return fallback("the delta compile analysed no models"); + }; + let relations: ImageRelations = relations.into(); + if relations.cyclic != base.image.relations.cyclic + || relations.dependency_ordered != base.image.relations.dependency_ordered + { + return fallback("the project's analysis order changed"); + } + for model in &ran.capture.models { + let stored = base + .image + .models + .iter() + .find(|stored| stored.name == model.name); + if stored != Some(model) { + return fallback(format!("{} changed its references", model.name)); + } + } + for path in &rerun.model_paths { + if ran.capture.usages.get(path) != base.image.usages.get(path) { + return fallback(format!("{path} changed the declarations it uses")); + } + } + let reshaped: BTreeSet = rerun + .models + .iter() + .filter(|name| shape_of(&relations, name) != shape_of(&base.image.relations, name)) + .cloned() + .collect(); + let dependents: BTreeSet = base.image.dependents(&reshaped); + let added: Vec = dependents + .into_iter() + .filter(|name| !rerun.models.contains(name)) + .collect(); + last = Some(ran); + if added.is_empty() { + break; + } + for name in added { + if let Some(model) = base.image.models.iter().find(|model| model.name == name) { + rerun.model_paths.insert(model.relative_path.clone()); + } + rerun.models.insert(name); + } + last = None; + } + let Some(ran) = last else { + return fallback("changed shapes reach too far"); + }; + trace(request.py(), "delta rounds", started)?; + let outcome = finished(ran, &base, &rerun, changes.changed.len()); + trace(request.py(), "delta report", started)?; + outcome +} + +/// Log how long the delta has run, for profiling. +pub(crate) fn trace(py: Python<'_>, phase: &str, started: std::time::Instant) -> PyResult<()> { + let milliseconds: u128 = started.elapsed().as_millis(); + py.import("logging")? + .getattr("getLogger")? + .call1(("sqlbuild.compile.delta",))? + .call_method1("debug", ("%s after %d ms", phase, milliseconds))?; + Ok(()) +} + +/// Every diagnostic not owned by the rerun set, which the delta must leave as it was. +fn unowned<'a>(rerun: &Rerun, diagnostics: &'a [CapturedDiagnostic]) -> Vec<&'a str> { + diagnostics + .iter() + .filter(|diagnostic| !rerun.owns(diagnostic)) + .map(|diagnostic| diagnostic.json.as_str()) + .collect() +} + +fn finished<'py>( + ran: Round<'py>, + base: &Base, + rerun: &Rerun, + changed_files: usize, +) -> PyResult> { + let py: Python<'py> = ran.request.py(); + if unowned(rerun, &base.image.diagnostics) != unowned(rerun, &ran.capture.diagnostics) { + return fallback("diagnostics outside the changed files differ"); + } + let recorded: Bound<'py, PyAny> = ran.output.getattr("stdout")?; + if recorded.is_none() { + return fallback("the delta compile recorded no report"); + } + let delta_stdout: String = recorded.extract()?; + let Some(stored_output) = read_delta_output(&base.store_path, &base.stored) else { + return fallback("the stored report changed"); + }; + if env_enabled(DELTA_VERIFY_ENV_VAR) { + let project_dir: PathBuf = native_field(&ran.request, "project_dir")?.extract()?; + let directory: PathBuf = project_dir.join(VERIFY_REPORT_DIRECTORY); + let _ = std::fs::create_dir_all(&directory); + let _ = std::fs::write(directory.join(VERIFY_REPORT_FILE), &delta_stdout); + } + let stdout: String = if base.image.json_output { + let scope: SpliceScope<'_> = SpliceScope { + models: &rerun.models, + tests: &rerun.tests, + }; + match spliced_report(&stored_output.stdout, &delta_stdout, &scope) { + Ok(stdout) => stdout, + Err(reason) => return fallback(reason), + } + } else { + delta_stdout + }; + let stderr_lines: Vec> = ran.output.getattr("stderr_lines")?.extract()?; + let image: Option = patched_image(&base.image, rerun, ran.capture).encode().ok(); + let merge: RecordMerge = RecordMerge { + kept: base + .stored + .inputs + .target_files + .iter() + .map(|stamp| (stamp.relative_path.clone(), stamp.size, stamp.mtime_ns)) + .collect(), + environment_names: base.image.environment_names.clone(), + }; + let _ = py; + Ok(DeltaOutcome::Finished(Box::new(Finished { + request: ran.request, + stdout, + stderr_lines, + phases: ran.phases, + image, + merge, + changed_files, + rerun_models: rerun.models.len(), + }))) +} + +/// The stored image with the rerun models' and tests' facts replaced by the delta's. +fn patched_image(base: &ProjectImage, rerun: &Rerun, capture: ImageCapture) -> ProjectImage { + let mut image: ProjectImage = base.clone(); + let fresh_models: HashMap = capture + .models + .into_iter() + .map(|model| (model.name.clone(), model)) + .collect(); + for model in &mut image.models { + if let Some(fresh) = fresh_models.get(&model.name) { + *model = fresh.clone(); + } + } + image.usages.extend(capture.usages); + let mut fresh_tests: HashMap = capture + .test_chains + .into_iter() + .map(|test| (test.key.clone(), test)) + .collect(); + for test in &mut image.tests { + if rerun.tests.contains(&test.key) + && let Some(fresh) = fresh_tests.remove(&test.key) + { + *test = fresh; + } + } + if let Some(relations) = capture.relations { + let fresh: ImageRelations = relations.into(); + merged_rows(&mut image.relations.types, fresh.types); + merged_rows(&mut image.relations.nullability, fresh.nullability); + merged_rows(&mut image.relations.complete, fresh.complete); + merged_rows(&mut image.relations.schemas, fresh.schemas); + let mut legacy: BTreeSet = image.relations.legacy_models.iter().cloned().collect(); + for name in &rerun.models { + legacy.remove(name); + } + legacy.extend(fresh.legacy_models); + image.relations.legacy_models = legacy.into_iter().collect(); + } + let mut digests: BTreeMap, String> = image.fact_digests.into_iter().collect(); + digests.extend(capture.fact_digests); + image.fact_digests = digests.into_iter().collect(); + image.diagnostics = capture.diagnostics; + if capture.text_models.is_some() { + image.text_models = capture.text_models; + } + image.rules = capture.rules.map(|rules| rules.retained); + if let Some(shapes) = capture.model_shapes { + image.model_shapes = shapes; + } + image.header_keys.extend(capture.header_keys); + image.lint_model_keys = capture.lint_model_keys; + image.lint_file_keys = capture.lint_file_keys; + image +} + +/// Store and print a finished delta compile; returns its exit code. +pub(crate) fn commit(finished: Finished<'_>, attempt: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = attempt.py(); + let Finished { + request, + stdout, + stderr_lines, + phases, + image, + merge, + .. + } = finished; + let report: &str = stdout.strip_suffix('\n').unwrap_or(&stdout); + let output: Bound<'_, PyAny> = Bound::new( + py, + RecordedCompileOutput::recorded( + stderr_lines, + PyString::new(py, &format!("{report}\n")).unbind(), + ), + )? + .into_any(); + crate::bindings::_helpers::compile_delta::context::set_pending_image(image); + crate::bindings::_helpers::compile_delta::context::set_record_merge(Some(merge)); + crate::bindings::_helpers::compile_driver::command::store_and_print( + (&request, attempt), + &output, + phases, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs new file mode 100644 index 0000000000..9c9d563f62 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs @@ -0,0 +1,156 @@ +//! Each model header's keys as digests, so a delta compile can tell which keys an edit changed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::bindings::_helpers::compile_delta::context::{capture, capturing}; +use crate::bindings::_helpers::discovery::model_records::model_file_options; + +/// One model's header keys with the digest of each authored value, sorted by key. +pub(crate) type HeaderKeys = Vec<(String, String)>; + +/// Header keys a delta compile recompiles the model for; any other changed key falls back. +const PATCHED_KEYS: [&str; 6] = [ + "description", + "columns", + "dynamic_columns", + "model_schema", + "audits", + "contract", +]; +/// `materialized` values a delta compile handles; others name materializations it cannot. +const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"]; +const MATERIALIZED_KEY: &str = "materialized"; + +fn header_keys(values: &[(String, AuthoredValue)]) -> HeaderKeys { + let mut keys: HeaderKeys = values + .iter() + .map(|(key, value)| { + let text: String = format!("{value:?}"); + (key.clone(), hex_digest(&bytes_digest(text.as_bytes()))) + }) + .collect(); + keys.sort(); + keys +} + +/// Record one discovered model's header keys for the image. +pub(crate) fn record_header_keys(relative_path: &str, values: &[(String, AuthoredValue)]) { + if !capturing() { + return; + } + let keys: HeaderKeys = header_keys(values); + let path: String = relative_path.replace('\\', "/"); + capture(|capture| { + capture.header_keys.insert(path, keys); + }); +} + +/// What an edit did to one model's header. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum HeaderChange { + /// The header's keys and values are unchanged. + Unchanged, + /// These keys changed, every one of which the delta recompiles. + Patched(BTreeSet), + /// Why the delta cannot recompile this header change. + Unsupported(String), + /// The file does not parse; the delta reports its error as a full compile would. + Unparsed, +} + +/// How the header of the model file at `path` changed against its recorded keys. +pub(crate) fn header_change( + project_dir: &Path, + relative_path: &str, + stored: Option<&HeaderKeys>, +) -> HeaderChange { + let Some(stored) = stored else { + return HeaderChange::Unsupported(format!("{relative_path} has no recorded header")); + }; + let Ok(contents) = std::fs::read_to_string(project_dir.join(relative_path)) else { + return HeaderChange::Unparsed; + }; + let file_path: String = project_dir + .join(relative_path) + .to_string_lossy() + .into_owned(); + let Ok(parsed) = parse_model_text(&file_path, contents, &model_file_options()) else { + return HeaderChange::Unparsed; + }; + let current: HeaderKeys = header_keys(&parsed.header_values); + let stored_map: BTreeMap<&str, &str> = stored + .iter() + .map(|(key, digest)| (key.as_str(), digest.as_str())) + .collect(); + let current_map: BTreeMap<&str, &str> = current + .iter() + .map(|(key, digest)| (key.as_str(), digest.as_str())) + .collect(); + let changed: BTreeSet = stored_map + .keys() + .chain(current_map.keys()) + .filter(|key| stored_map.get(*key) != current_map.get(*key)) + .map(|key| (*key).to_owned()) + .collect(); + if changed.is_empty() { + return HeaderChange::Unchanged; + } + for key in &changed { + if key == MATERIALIZED_KEY { + let value = parsed.header_values.iter().find(|(name, _)| name == key); + let supported: bool = matches!( + value, + Some((_, AuthoredValue::BareWord(word) | AuthoredValue::String(word))) + if PATCHED_MATERIALIZATIONS.contains(&word.as_str()) + ); + if !supported { + return HeaderChange::Unsupported(format!( + "{relative_path} changed its materialization beyond views and tables" + )); + } + } else if !PATCHED_KEYS.contains(&key.as_str()) { + return HeaderChange::Unsupported(format!( + "{relative_path} changed its {key} header key" + )); + } + } + HeaderChange::Patched(changed) +} + +/// Whether a custom rule reads a fact of this model that the changed header keys decide. +pub(crate) fn custom_rule_reads( + fact_digests: &[(Vec, String)], + model: (&str, &str), + changed: &BTreeSet, +) -> Option { + let (path, name) = model; + for (key, _) in fact_digests { + let Some((fact, arguments)) = key.split_first() else { + continue; + }; + if !arguments + .iter() + .any(|argument| argument == path || argument == name) + { + continue; + } + let reads: &[&str] = match fact.as_str() { + "contracts.enforced" | "contract" => &["contract"], + "contracts.grain" | "grain" => &["contract", "columns"], + "columns.declared" | "columns.names" => &["columns", "dynamic_columns"], + "materialized" => &["materialized"], + "tests.for_model" | "audits.for_model" => &["audits", "columns"], + _ => &[], + }; + if let Some(key) = reads.iter().find(|key| changed.contains(**key)) { + return Some(format!("a custom rule reads {name}'s {key}")); + } + } + None +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs new file mode 100644 index 0000000000..c5880d7036 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -0,0 +1,214 @@ +//! The project image a full or delta compile stores beside its reuse slot. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; +use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use sqlbuild_cache::project_reuse::models::ImageSections; +use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; + +use crate::bindings::_helpers::compile_delta::context::{ + CapturedDiagnostic, CapturedModel, CapturedTest, ImageCapture, ModelShape, +}; +use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; + +/// Changes whenever the image's meaning changes, so an older image is never read. +const IMAGE_FORMAT: &str = "sqlbuild-project-image-v1"; +const FACTS_SECTION: &str = "facts"; +const RULES_SECTION: &str = "rules"; + +/// Everything a delta compile reads of the compile it patches. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct ProjectImage { + pub(crate) format: String, + pub(crate) json_output: bool, + pub(crate) models: Vec, + pub(crate) usages: BTreeMap, + pub(crate) tests: Vec, + pub(crate) relations: ImageRelations, + pub(crate) fact_digests: Vec<(Vec, String)>, + pub(crate) model_subjects: Option)>>, + pub(crate) diagnostics: Vec, + pub(crate) text_models: Option>, + pub(crate) environment_names: Vec, + pub(crate) model_shapes: Vec, + pub(crate) lint_model_keys: Option>, + pub(crate) lint_file_keys: Option)>>, + pub(crate) header_keys: BTreeMap, + #[serde(skip)] + pub(crate) rules: Option, +} + +/// The analysis session's final relation facts, as stored. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct ImageRelations { + pub(crate) types: Shapes, + pub(crate) nullability: Shapes, + pub(crate) complete: Shapes, + pub(crate) schemas: Shapes, + pub(crate) dependency_ordered: bool, + pub(crate) cyclic: bool, + pub(crate) legacy_models: Vec, +} + +impl From for ImageRelations { + fn from(tables: RelationTables) -> Self { + Self { + types: tables.types, + nullability: tables.nullability, + complete: tables.complete, + schemas: tables.schemas, + dependency_ordered: tables.dependency_ordered, + cyclic: tables.cyclic, + legacy_models: tables.legacy_models, + } + } +} + +impl From<&ImageRelations> for RelationTables { + fn from(relations: &ImageRelations) -> Self { + Self { + types: relations.types.clone(), + nullability: relations.nullability.clone(), + complete: relations.complete.clone(), + schemas: relations.schemas.clone(), + dependency_ordered: relations.dependency_ordered, + cyclic: relations.cyclic, + legacy_models: relations.legacy_models.clone(), + } + } +} + +#[derive(Serialize, Deserialize)] +struct StoredRules { + request: serde_json::Value, + digests: Vec, +} + +/// Fields a request serializes but never reads back, by list; evaluation derives them again. +const DERIVED_FIELDS: [(&str, &str); 2] = [ + ("models", "empty_input_only_test_count"), + ("sql_tests", "empty_input_only"), +]; + +fn stored_request(request: &EvaluateRequest) -> Result { + let mut value: serde_json::Value = + serde_json::to_value(request).map_err(|error| error.to_string())?; + for (list, field) in DERIVED_FIELDS { + if let Some(items) = value + .get_mut(list) + .and_then(serde_json::Value::as_array_mut) + { + for item in items + .iter_mut() + .filter_map(serde_json::Value::as_object_mut) + { + item.remove(field); + } + } + } + Ok(value) +} + +impl ProjectImage { + /// The image of a finished compile's capture, or why it cannot be a delta base. + pub(crate) fn from_capture(capture: ImageCapture, json_output: bool) -> Result { + if let Some(reason) = capture.unsupported { + return Err(reason); + } + if capture.stdlib_json { + return Err("the report fell back to the standard library serializer".to_owned()); + } + let Some(relations) = capture.relations else { + return Err("no model analysis session ran".to_owned()); + }; + if !json_output && capture.text_models.is_none() { + return Err("no text report was rendered".to_owned()); + } + let Some(model_shapes) = capture.model_shapes else { + return Err("no semantic shapes were published".to_owned()); + }; + let mut environment_names: Vec = capture.environment_names; + environment_names.sort(); + environment_names.dedup(); + Ok(Self { + format: IMAGE_FORMAT.to_owned(), + json_output, + models: capture.models, + usages: capture.usages, + tests: capture.test_chains, + relations: relations.into(), + fact_digests: capture.fact_digests.into_iter().collect(), + model_subjects: capture.model_subjects, + diagnostics: capture.diagnostics, + text_models: capture.text_models, + environment_names, + model_shapes, + lint_model_keys: capture.lint_model_keys, + lint_file_keys: capture.lint_file_keys, + header_keys: capture.header_keys, + rules: capture.rules.map(|rules| rules.retained), + }) + } + + /// The image as the sections of its file. + pub(crate) fn encode(&self) -> Result { + let mut sections: ImageSections = vec![( + FACTS_SECTION.to_owned(), + serde_json::to_vec(self).map_err(|error| error.to_string())?, + )]; + if let Some(rules) = &self.rules { + let stored: StoredRules = StoredRules { + request: stored_request(&rules.request)?, + digests: rules.digests.clone(), + }; + sections.push(( + RULES_SECTION.to_owned(), + serde_json::to_vec(&stored).map_err(|error| error.to_string())?, + )); + } + Ok(sections) + } + + /// The image stored in `sections`, or `None` when it is unreadable or of another format. + pub(crate) fn decode(sections: &ImageSections) -> Option { + let section = |name: &str| { + sections + .iter() + .find(|(section, _)| section == name) + .map(|(_, bytes)| bytes.as_slice()) + }; + let mut image: Self = serde_json::from_slice(section(FACTS_SECTION)?).ok()?; + if image.format != IMAGE_FORMAT { + return None; + } + if let Some(bytes) = section(RULES_SECTION) { + let stored: StoredRules = serde_json::from_slice(bytes).ok()?; + let request: EvaluateRequest = serde_json::from_value(stored.request).ok()?; + if stored.digests.len() != request.models.len() { + return None; + } + image.rules = Some(RetainedRows { + request, + digests: stored.digests, + }); + } + Some(image) + } + + /// Every model whose reads include a relation of `names`, outside `names`. + pub(crate) fn dependents(&self, names: &BTreeSet) -> BTreeSet { + self.models + .iter() + .filter(|model| !names.contains(&model.name)) + .filter(|model| { + model + .upstream + .iter() + .any(|(kind, name)| kind == "model" && names.contains(name)) + }) + .map(|model| model.name.clone()) + .collect() + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_facts.rs new file mode 100644 index 0000000000..301c9528db --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_facts.rs @@ -0,0 +1,49 @@ +//! Facts a full compile's stages leave for its project image as they run. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyStringMethods; +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; + +use crate::bindings::_helpers::compile_delta::context::{capture, capture_unsupported}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Record the digest of one model's scope usages, which decide declaration placement. +pub(crate) fn record_model_usages( + model: &Bound<'_, PyAny>, + usages: (&[Bound<'_, PyAny>], &[Bound<'_, PyAny>]), +) -> PyResult<()> { + if let Err(error) = model_usages(model, usages) { + capture_unsupported(&format!( + "a model's scope usages could not be recorded: {error}" + )); + } + Ok(()) +} + +fn model_usages( + model: &Bound<'_, PyAny>, + (macro_usages, declaration_usages): (&[Bound<'_, PyAny>], &[Bound<'_, PyAny>]), +) -> PyResult<()> { + let Ok(path) = native_field(model, "model_file") + .and_then(|file| native_field(&file, "relative_path")) + .and_then(|path| Ok(path.str()?.to_str()?.replace('\\', "/"))) + else { + capture_unsupported("a model input has no file"); + return Ok(()); + }; + let mut text: String = String::new(); + for (group, usages) in [("macro", macro_usages), ("declaration", declaration_usages)] { + text.push_str(group); + text.push('\0'); + for usage in usages { + text.push_str(usage.repr()?.to_str()?); + text.push('\0'); + } + } + let digest: String = hex_digest(&bytes_digest(text.as_bytes())); + capture(|capture| { + capture.usages.insert(path, digest); + }); + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs new file mode 100644 index 0000000000..7962353b53 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs @@ -0,0 +1,101 @@ +//! The SQL lint keys a full compile leaves for its image, and a delta compile's use of them. + +use std::collections::{BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; + +use pyo3::PyResult; + +use crate::bindings::_helpers::compile_delta::context::{ + capture, capturing, delta_unsupported, injection, +}; +use crate::bindings::_helpers::rules::lint_pipeline::{LintFile, lint_sources}; + +/// Each model file's lint cache key: recorded for the image, and in a delta compile the whole +/// project's keys with the recompiled models' keys fresh. +pub(crate) fn model_identities(fresh: Vec<(String, String)>) -> PyResult> { + let Some(injection) = injection() else { + if capturing() { + let keys: Vec<(String, String)> = fresh.clone(); + capture(|capture| capture.lint_model_keys = Some(keys)); + } + return Ok(fresh); + }; + let Some(stored) = injection.lint_model_keys.as_ref() else { + return Err(delta_unsupported( + "the lint keys of the models were not recorded", + )); + }; + let mut fresh: HashMap = fresh.into_iter().collect(); + let whole: Vec<(String, String)> = stored + .iter() + .map(|(path, key)| match fresh.remove(path) { + Some(key) => (path.clone(), key), + None => (path.clone(), key.clone()), + }) + .collect(); + if !fresh.is_empty() { + return Err(delta_unsupported( + "a recompiled model has no recorded lint key", + )); + } + capture(|capture| capture.lint_model_keys = Some(whole.clone())); + Ok(whole) +} + +/// The changed files a delta compile lints, read from the project. +pub(crate) fn changed_files(project_dir: &Path) -> PyResult> { + let Some(injection) = injection() else { + return Ok(Vec::new()); + }; + let selected: BTreeSet = injection + .changed_paths + .iter() + .map(|path| { + let path: PathBuf = project_dir.join(path); + std::fs::canonicalize(&path).unwrap_or(path) + }) + .collect(); + lint_sources(project_dir, Some(&selected), None) +} + +/// Each non-model file's lint cache key, in walk order: recorded for the image, and in a delta +/// compile the whole project's keys with the changed files' keys fresh. +pub(crate) fn file_keys( + project_dir: &Path, + fresh: Vec<(PathBuf, String, Option)>, +) -> PyResult)>> { + let Some(injection) = injection() else { + if capturing() { + let keys: Vec<(String, Option)> = fresh + .iter() + .map(|(_, path, key)| (path.clone(), key.clone())) + .collect(); + capture(|capture| capture.lint_file_keys = Some(keys)); + } + return Ok(fresh); + }; + let Some(stored) = injection.lint_file_keys.as_ref() else { + return Err(delta_unsupported( + "the lint keys of the project files were not recorded", + )); + }; + let mut fresh: HashMap)> = + fresh.into_iter().map(|row| (row.1.clone(), row)).collect(); + let whole: Vec<(PathBuf, String, Option)> = stored + .iter() + .map(|(path, key)| { + fresh + .remove(path) + .unwrap_or_else(|| (project_dir.join(path), path.clone(), key.clone())) + }) + .collect(); + if !fresh.is_empty() { + return Err(delta_unsupported("a changed file has no recorded lint key")); + } + let keys: Vec<(String, Option)> = whole + .iter() + .map(|(_, path, key)| (path.clone(), key.clone())) + .collect(); + capture(|capture| capture.lint_file_keys = Some(keys)); + Ok(whole) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs new file mode 100644 index 0000000000..092677b283 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod context; +pub(crate) mod driver; +pub(crate) mod header_keys; +pub(crate) mod image; +pub(crate) mod image_facts; +pub(crate) mod lint_hooks; +pub(crate) mod report_hooks; +pub(crate) mod rules_rows; +pub(crate) mod splice; +pub(crate) mod verify; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs new file mode 100644 index 0000000000..a755e4c7f1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -0,0 +1,278 @@ +//! Where the compile's report and output stages leave image facts or take a delta's injection. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple, PyTupleMethods}; +use sqlbuild_core::cli_text::models::ModelLine; + +use crate::bindings::_helpers::compile_command::report::diagnostic_item; +use crate::bindings::_helpers::compile_delta::context::{ + CapturedDiagnostic, CapturedModel, CapturedTest, ModelShape, capture, capture_unsupported, + capturing, injection, +}; +use crate::bindings::_helpers::compile_outputs::json_report::plain_json; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// The text report's model lines: captured for the image, and in a delta compile the whole +/// project's lines with the rerun models' lines replaced. +pub(crate) fn text_model_lines(models: Vec) -> Vec { + let models: Vec = match injection() { + None => models, + Some(injection) => { + let mut fresh: HashMap = models + .into_iter() + .map(|line| (line.name.clone(), line)) + .collect(); + injection + .text_models + .iter() + .map(|(name, failed, column_count)| { + fresh.remove(name).unwrap_or_else(|| ModelLine { + name: name.clone(), + failed: *failed, + column_count: *column_count, + }) + }) + .collect() + } + }; + capture(|capture| { + capture.text_models = Some( + models + .iter() + .map(|line| (line.name.clone(), line.failed, line.column_count)) + .collect(), + ); + }); + models +} + +/// Record which models each SQL test's compiled artifact is keyed by. +fn record_test_chains_inner(tests: &Bound<'_, PyAny>, chains: &Bound<'_, PyAny>) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + let mut captured: Vec = Vec::new(); + for test in tests.try_iter()? { + let test: Bound<'_, PyAny> = test?; + let key: Bound<'_, PyAny> = native_field(&test, "key")?; + let chain: Vec = match chains.get_item(&key) { + Ok(chain) => chain.extract()?, + Err(_) => Vec::new(), + }; + captured.push(CapturedTest { + key: native_field(&test, "name")?.str()?.to_str()?.to_owned(), + relative_path: native_field(&native_field(&test, "test_file")?, "relative_path")? + .str()? + .to_str()? + .replace('\\', "/"), + chain, + }); + } + capture(|capture| capture.test_chains = captured); + Ok(()) +} + +/// Record the compiled models with their references, and the report's diagnostics. +fn record_compile_facts_inner( + graph: &Bound<'_, PyAny>, + diagnostics: &Bound<'_, PyTuple>, +) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + let native: Bound<'_, PyAny> = native_field(graph, "native")?; + let mut models: Vec = Vec::new(); + for model in native_field(&native_field(graph, "project")?, "models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let name: String = native_field(&model, "name")?.extract()?; + let mut upstream: Vec<(String, String)> = Vec::new(); + for item in native + .call_method1("upstream", (("model", name.as_str()),))? + .try_iter()? + { + upstream.push(item?.extract()?); + } + models.push(CapturedModel { + name, + relative_path: native_field(&model, "relative_path")? + .str()? + .to_str()? + .replace('\\', "/"), + upstream, + }); + } + let mut captured: Vec = Vec::new(); + for diagnostic in diagnostics.iter() { + captured.push(captured_diagnostic(&diagnostic)?); + } + let test_count: usize = native_field(&native_field(graph, "project")?, "sql_tests")?.len()?; + capture(|capture| { + capture.models = models; + capture.diagnostics = captured; + if capture.test_chains.len() != test_count { + capture + .unsupported + .get_or_insert_with(|| "the SQL tests' model chains were not recorded".to_owned()); + } + }); + Ok(()) +} + +/// One diagnostic as its report JSON, with the resource and paths it names. +pub(crate) fn captured_diagnostic(diagnostic: &Bound<'_, PyAny>) -> PyResult { + let item = diagnostic_item(diagnostic)?; + let json: String = plain_json(item.as_any())?; + let value: serde_json::Value = serde_json::from_str(&json).unwrap_or_default(); + let text = |value: Option<&serde_json::Value>| -> Option { + value.and_then(serde_json::Value::as_str).map(str::to_owned) + }; + let mut paths: Vec = Vec::new(); + paths.extend(text(value.get("path"))); + paths.extend(text( + value + .get("location") + .and_then(|location| location.get("path")), + )); + Ok(CapturedDiagnostic { + json, + resource_name: text(value.get("resource_name")), + paths: paths + .into_iter() + .map(|path| path.replace('\\', "/")) + .collect(), + }) +} + +/// In a delta compile, publish the shapes of every model it does not compile again first. +pub(crate) fn inject_model_shapes( + models: &[Bound<'_, PyAny>], + shapes: &Bound<'_, PyDict>, +) -> PyResult<()> { + let Some(injection) = injection() else { + return Ok(()); + }; + let mut rerun: HashSet = HashSet::with_capacity(models.len()); + for model in models { + rerun.insert(native_field(model, "name")?.extract()?); + } + let py = shapes.py(); + for (name, columns) in &injection.model_shapes { + if rerun.contains(name) { + continue; + } + let shape: Bound<'_, PyDict> = PyDict::new(py); + for (column, column_type) in columns { + shape.set_item(column, column_type)?; + } + shapes.set_item(name, shape)?; + } + Ok(()) +} + +/// Record the model shapes the first semantic shape pass published. +fn record_model_shapes_inner( + project: &Bound<'_, PyAny>, + shapes: &Bound<'_, PyDict>, +) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + let mut names: HashSet = HashSet::new(); + for model in native_field(project, "models")?.try_iter()? { + names.insert(native_field(&model?, "name")?.extract()?); + } + if let Some(injection) = injection() { + names.extend(injection.model_shapes.iter().map(|(name, _)| name.clone())); + } + let mut published: Vec = Vec::new(); + for (name, shape) in shapes.iter() { + let name: String = name.extract()?; + if !names.contains(&name) { + continue; + } + let shape: Bound<'_, PyDict> = shape.cast_into()?; + let mut columns: Vec<(String, String)> = Vec::with_capacity(shape.len()); + for (column, column_type) in shape.iter() { + columns.push((column.extract()?, column_type.extract()?)); + } + published.push((name, columns)); + } + capture(|capture| { + if capture.model_shapes.is_none() { + capture.model_shapes = Some(published); + } + }); + Ok(()) +} + +/// Run a capture hook; a failure only rules the compile out as a delta base. +fn guarded(hook: impl FnOnce() -> PyResult<()>) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + if let Err(error) = hook() { + capture_unsupported(&format!("an image fact could not be recorded: {error}")); + } + Ok(()) +} + +/// Record which models each SQL test's compiled artifact is keyed by. +pub(crate) fn record_test_chains( + tests: &Bound<'_, PyAny>, + chains: &Bound<'_, PyAny>, +) -> PyResult<()> { + guarded(|| record_test_chains_inner(tests, chains)) +} + +/// Record the compiled models with their references, and the report's diagnostics. +pub(crate) fn record_compile_facts( + graph: &Bound<'_, PyAny>, + diagnostics: &Bound<'_, PyTuple>, +) -> PyResult<()> { + guarded(|| record_compile_facts_inner(graph, diagnostics)) +} + +/// Record the model shapes the first semantic shape pass published. +pub(crate) fn record_model_shapes( + project: &Bound<'_, PyAny>, + shapes: &Bound<'_, PyDict>, +) -> PyResult<()> { + guarded(|| record_model_shapes_inner(project, shapes)) +} + +/// In a delta compile, the authored text of each model or SQL test file a diagnostic points +/// into but the delta did not compile, read from the project as the stored compile saw it. +pub(crate) fn complete_source_texts( + project: &Bound<'_, PyAny>, + diagnostics: &[Bound<'_, PyAny>], + texts: &Bound<'_, PyDict>, +) -> PyResult<()> { + let Some(injection) = injection() else { + return Ok(()); + }; + let _ = project; + let project_dir: &std::path::Path = &injection.project_dir; + let mut locations: Vec> = Vec::new(); + for diagnostic in diagnostics { + locations.push(native_field(diagnostic, "location")?); + for item in native_field(diagnostic, "related_locations")?.try_iter()? { + locations.push(native_field(&item?, "location")?); + } + } + for location in locations.into_iter().filter(|location| !location.is_none()) { + let path: Bound<'_, PyAny> = native_field(&location, "path")?; + if texts.contains(&path)? { + continue; + } + let relative: String = path.str()?.to_str()?.replace('\\', "/"); + if !injection.source_paths.contains(&relative) { + continue; + } + if let Ok(text) = std::fs::read_to_string(project_dir.join(&relative)) { + texts.set_item(&path, text)?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs new file mode 100644 index 0000000000..c746b7cd7f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs @@ -0,0 +1,87 @@ +//! A delta compile's built-in rules request: the previous compile's, with the rerun rows fresh. + +use std::collections::HashMap; + +use pyo3::PyResult; +use sqlbuild_rules::models::{EvaluateRequest, Model, RetainedRows, SqlTestFact}; + +use crate::bindings::_helpers::compile_delta::context::{delta_unsupported, injection}; + +/// `request` unchanged outside a delta compile; inside one, the whole project's request with +/// the rerun models' rows and the rerun SQL tests' facts taken from `request`. +pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult { + let Some(injection) = injection() else { + return Ok(request); + }; + let Some(base) = injection.rules.as_ref() else { + return Err(delta_unsupported( + "the previous compile did not run the built-in rules", + )); + }; + if !request.initial_findings.is_empty() || !base.request.initial_findings.is_empty() { + return Err(delta_unsupported( + "the rules request carries initial findings", + )); + } + let mut whole: EvaluateRequest = restored(base); + let mut fresh: HashMap = request + .models + .into_iter() + .map(|model| (model.name.clone(), model)) + .collect(); + for model in &mut whole.models { + if let Some(replacement) = fresh.remove(&model.name) { + *model = replacement; + } + } + if !fresh.is_empty() { + return Err(delta_unsupported( + "a rerun model is new to the rules request", + )); + } + whole.sql_tests = replaced_tests(&whole.sql_tests, request.sql_tests)?; + Ok(whole) +} + +/// The retained request with each row's payload digest restored. +fn restored(base: &RetainedRows) -> EvaluateRequest { + let mut request: EvaluateRequest = base.request.clone(); + for (model, digest) in request.models.iter_mut().zip(&base.digests) { + model.payload_digest = Some(digest.clone()); + model.stripped = true; + } + request +} + +/// `base` with the facts of every file `fresh` covers replaced, in place and in order. +fn replaced_tests(base: &[SqlTestFact], fresh: Vec) -> PyResult> { + let mut by_file: HashMap> = HashMap::new(); + let mut grouped: HashMap> = HashMap::new(); + for fact in fresh { + grouped + .entry(fact.source_path.clone()) + .or_default() + .push(fact); + } + let mut counts: HashMap<&str, usize> = HashMap::new(); + for fact in base { + *counts.entry(fact.source_path.as_str()).or_default() += 1; + } + for (path, facts) in grouped { + if counts.get(path.as_str()).copied().unwrap_or_default() != facts.len() { + return Err(delta_unsupported( + "a rerun SQL test file changed its test count", + )); + } + by_file.insert(path, facts.into_iter()); + } + Ok(base + .iter() + .map(|fact| { + by_file + .get_mut(&fact.source_path) + .and_then(Iterator::next) + .unwrap_or_else(|| fact.clone()) + }) + .collect()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs new file mode 100644 index 0000000000..82ea7a0d9c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs @@ -0,0 +1,309 @@ +//! Splice a delta compile's JSON report into the stored report it patches. + +use std::collections::{BTreeSet, HashMap}; + +use serde_json::Value; + +/// One JSON value's byte span in its document. +type Span = (usize, usize); + +/// The end of the JSON value starting at `start`, skipping strings and nested containers. +fn value_end(text: &[u8], start: usize) -> Option { + let mut depth: usize = 0; + let mut index: usize = start; + let mut in_string: bool = false; + while index < text.len() { + let byte: u8 = text[index]; + if in_string { + match byte { + b'\\' => index += 1, + b'"' => { + in_string = false; + if depth == 0 { + return Some(index + 1); + } + } + _ => {} + } + } else { + match byte { + b'"' => in_string = true, + b'{' | b'[' => depth += 1, + b'}' | b']' => { + if depth == 0 { + return Some(index); + } + depth -= 1; + if depth == 0 { + return Some(index + 1); + } + } + b',' | b'\n' if depth == 0 => return Some(index), + _ => {} + } + } + index += 1; + } + (depth == 0 && !in_string).then_some(index) +} + +fn skip_space(text: &[u8], mut index: usize) -> usize { + while index < text.len() && matches!(text[index], b' ' | b'\n' | b'\r' | b'\t') { + index += 1; + } + index +} + +/// The `(key, value span)` members of the object spanning `span`. +fn members(text: &str, span: Span) -> Option> { + let bytes: &[u8] = text.as_bytes(); + if bytes.get(span.0) != Some(&b'{') { + return None; + } + let mut found: Vec<(String, Span)> = Vec::new(); + let mut index: usize = skip_space(bytes, span.0 + 1); + while index < span.1 && bytes[index] != b'}' { + let key_end: usize = value_end(bytes, index)?; + let key: String = serde_json::from_str(&text[index..key_end]).ok()?; + index = skip_space(bytes, key_end); + if bytes.get(index) != Some(&b':') { + return None; + } + let start: usize = skip_space(bytes, index + 1); + let end: usize = value_end(bytes, start)?; + found.push((key, (start, end))); + index = skip_space(bytes, end); + if bytes.get(index) == Some(&b',') { + index = skip_space(bytes, index + 1); + } + } + Some(found) +} + +/// The element spans of the array spanning `span`. +fn elements(text: &str, span: Span) -> Option> { + let bytes: &[u8] = text.as_bytes(); + if bytes.get(span.0) != Some(&b'[') { + return None; + } + let mut found: Vec = Vec::new(); + let mut index: usize = skip_space(bytes, span.0 + 1); + while index < span.1 && bytes[index] != b']' { + let end: usize = value_end(bytes, index)?; + found.push((index, end)); + index = skip_space(bytes, end); + if bytes.get(index) == Some(&b',') { + index = skip_space(bytes, index + 1); + } + } + Some(found) +} + +fn text<'a>(document: &'a str, span: &Span) -> &'a str { + &document[span.0..span.1] +} + +fn whole(text: &str) -> Span { + let end: usize = text.trim_end().len(); + (skip_space(text.as_bytes(), 0), end) +} + +fn member<'a>(found: &'a [(String, Span)], key: &str) -> Option<&'a Span> { + found + .iter() + .find(|(name, _)| name == key) + .map(|(_, span)| span) +} + +/// What the splice replaces: the rerun models, their audits and the rerun SQL tests. +pub(crate) struct SpliceScope<'a> { + pub(crate) models: &'a BTreeSet, + pub(crate) tests: &'a BTreeSet, +} + +/// The string at `path` in the item spanning `span`: the first member of each key in turn. +/// +/// Reports keep each item's own keys before any nested object, so the first `"key": ` after +/// the previous key is that key's member; anything else is read as JSON. +fn item_field(text: &str, span: Span, path: &[&str]) -> Option { + let bytes: &[u8] = text.as_bytes(); + let mut start: usize = span.0; + for (index, key) in path.iter().enumerate() { + let needle: String = format!("\"{key}\": "); + let found: usize = text[start..span.1].find(&needle)?; + start += found + needle.len(); + if index + 1 < path.len() && bytes.get(start) != Some(&b'{') { + return None; + } + } + if bytes.get(start) != Some(&b'"') { + return slow_item_field(text, span, path); + } + let end: usize = value_end(bytes, start)?; + serde_json::from_str(&text[start..end]).ok() +} + +fn slow_item_field(text: &str, span: Span, path: &[&str]) -> Option { + let mut value: Value = serde_json::from_str(&text[span.0..span.1]).ok()?; + for key in path { + value = value.get(*key)?.clone(); + } + value.as_str().map(str::to_owned) +} + +/// The stored report with the delta's items for the rerun resources and the delta's +/// diagnostics, error counts and timings; `None` when the two reports do not line up. +pub(crate) fn spliced_report( + base: &str, + delta: &str, + scope: &SpliceScope<'_>, +) -> Result { + let base_members: Vec<(String, Span)> = + members(base, whole(base)).ok_or("the stored report cannot be read")?; + let delta_members: Vec<(String, Span)> = + members(delta, whole(delta)).ok_or("the delta report cannot be read")?; + let keys = + |found: &[(String, Span)]| found.iter().map(|(key, _)| key.clone()).collect::>(); + if keys(&base_members) != keys(&delta_members) { + return Err("the reports have different sections".to_owned()); + } + let mut output: String = String::with_capacity(base.len() + delta.len()); + let mut cursor: usize = 0; + for ((key, base_span), (_, delta_span)) in base_members.iter().zip(&delta_members) { + let replacement: String = match key.as_str() { + "has_errors" | "diagnostics" | "compile_timings" => text(delta, delta_span).to_owned(), + "summary" => spliced_summary(base, *base_span, delta, *delta_span) + .ok_or("the summaries do not line up")?, + "resources" => spliced_resources(base, *base_span, delta, *delta_span, scope) + .map_err(|kind| format!("the {kind} do not line up"))?, + _ => { + if text(base, base_span) != text(delta, delta_span) { + return Err(format!("the reports' {key} differ")); + } + continue; + } + }; + output.push_str(&base[cursor..base_span.0]); + output.push_str(&replacement); + cursor = base_span.1; + } + output.push_str(&base[cursor..]); + Ok(output) +} + +const DELTA_SUMMARY_KEYS: [&str; 2] = ["errors", "warnings"]; + +fn spliced_summary(base: &str, base_span: Span, delta: &str, delta_span: Span) -> Option { + let base_members: Vec<(String, Span)> = members(base, base_span)?; + let delta_members: Vec<(String, Span)> = members(delta, delta_span)?; + let mut output: String = String::new(); + let mut cursor: usize = base_span.0; + for (key, span) in &base_members { + if !DELTA_SUMMARY_KEYS.contains(&key.as_str()) { + continue; + } + let replacement: &Span = member(&delta_members, key)?; + output.push_str(&base[cursor..span.0]); + output.push_str(&delta[replacement.0..replacement.1]); + cursor = span.1; + } + output.push_str(&base[cursor..base_span.1]); + Some(output) +} + +fn spliced_resources( + base: &str, + base_span: Span, + delta: &str, + delta_span: Span, + scope: &SpliceScope<'_>, +) -> Result { + let resources: String = "resources".to_owned(); + let base_members: Vec<(String, Span)> = members(base, base_span).ok_or(resources.clone())?; + let delta_members: Vec<(String, Span)> = members(delta, delta_span).ok_or(resources)?; + let mut output: String = String::new(); + let mut cursor: usize = base_span.0; + for (key, span) in &base_members { + let delta_list: &Span = member(&delta_members, key).ok_or(key.clone())?; + let replacement: Option> = match key.as_str() { + "models" => spliced_items(base, *span, delta, *delta_list, |document, item| { + item_field(document, item, &["name"]).filter(|name| scope.models.contains(name)) + }), + "tests" => spliced_items(base, *span, delta, *delta_list, |document, item| { + item_field(document, item, &["name"]).filter(|name| scope.tests.contains(name)) + }), + "audits" => spliced_items(base, *span, delta, *delta_list, |document, item| { + let owner: Option = + item_field(document, item, &["attached_to", "resource_type"]); + let name: Option = item_field(document, item, &["attached_to", "name"]); + (owner.as_deref() == Some("model")) + .then_some(name) + .flatten() + .filter(|name| scope.models.contains(name)) + }), + _ => { + if base[span.0..span.1] != delta[delta_list.0..delta_list.1] { + return Err(key.clone()); + } + Some(None) + } + }; + let replacement: Option = replacement.ok_or(key.clone())?; + if let Some(replacement) = replacement { + output.push_str(&base[cursor..span.0]); + output.push_str(&replacement); + cursor = span.1; + } + } + output.push_str(&base[cursor..base_span.1]); + Ok(output) +} + +/// The base list with each item `owned` marks replaced, in order, by the delta's items `owned` +/// marks with the same owner; every other delta item must equal a base item. +fn spliced_items( + base: &str, + base_span: Span, + delta: &str, + delta_span: Span, + owned: impl Fn(&str, Span) -> Option, +) -> Option> { + let base_items: Vec = elements(base, base_span)?; + let delta_items: Vec = elements(delta, delta_span)?; + let mut fresh: HashMap> = HashMap::new(); + let mut unowned_delta: BTreeSet<&str> = BTreeSet::new(); + for item in &delta_items { + match owned(delta, *item) { + Some(owner) => fresh.entry(owner).or_default().push_back(*item), + None => { + unowned_delta.insert(&delta[item.0..item.1]); + } + } + } + let mut output: String = String::new(); + let mut cursor: usize = base_span.0; + let mut replaced: bool = false; + let mut unowned_base: BTreeSet<&str> = BTreeSet::new(); + for item in &base_items { + let Some(owner) = owned(base, *item) else { + unowned_base.insert(&base[item.0..item.1]); + continue; + }; + let replacement: Span = fresh.get_mut(&owner)?.pop_front()?; + output.push_str(&base[cursor..item.0]); + output.push_str(&delta[replacement.0..replacement.1]); + cursor = item.1; + replaced = true; + } + if fresh.values().any(|items| !items.is_empty()) { + return None; + } + if !unowned_delta.is_subset(&unowned_base) { + return None; + } + if !replaced { + return Some(None); + } + output.push_str(&base[cursor..base_span.1]); + Some(Some(output)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs new file mode 100644 index 0000000000..ad35ee8133 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs @@ -0,0 +1,217 @@ +//! The hidden differential check: run each delta compile, then the full compile it replaces, +//! and fail when their reports, exit codes or compiled artifacts differ. + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyStringMethods; +use sqlbuild_cache::digest::main::digest_files::digest_files; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_cache::project_reuse::constants::NATIVE_DIGEST_PREFIX; + +use crate::bindings::_helpers::compile_command::reuse_recording::CompileArtifactWrites; + +/// The exit code of a compile whose delta differed from its full compile. +pub(crate) const MISMATCH_EXIT_CODE: i64 = 70; +const MISMATCH_FILE: &str = "delta-mismatch.json"; + +/// What the delta compile produced, held until the full compile finishes. +#[derive(Debug, Default)] +pub(crate) struct Expected { + pub(crate) stdout: Option, + pub(crate) exit_code: Option, + pub(crate) raised: Option, + /// Every file under the compiled root after the delta, with its native digest. + pub(crate) compiled: BTreeMap, + pub(crate) compiled_root: String, + pub(crate) artifacts_written: bool, +} + +/// What the full compile produced. +#[derive(Debug, Default)] +struct Actual { + stdout: Option, + exit_code: Option, + artifacts_written: bool, + written: HashMap>, +} + +static EXPECTED: Mutex> = Mutex::new(None); +static ACTUAL: Mutex> = Mutex::new(None); + +fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn files_under(directory: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + match entry.file_type() { + Ok(kind) if kind.is_dir() => files_under(&path, found), + Ok(kind) if kind.is_file() => found.push(path), + _ => {} + } + } +} + +/// Every file under `root` with its native digest, keyed by absolute path text. +pub(crate) fn compiled_digests(root: &Path) -> BTreeMap { + let mut paths: Vec = Vec::new(); + files_under(root, &mut paths); + let digests = digest_files(&paths); + paths + .into_iter() + .zip(digests) + .filter_map(|(path, digest)| { + let digest = digest.ok()?; + Some(( + path.to_string_lossy().into_owned(), + format!("{NATIVE_DIGEST_PREFIX}{}", hex_digest(&digest)), + )) + }) + .collect() +} + +/// Hold the delta's result for comparison once the full compile finishes. +pub(crate) fn expect(expected: Expected) { + *locked(&EXPECTED) = Some(expected); + *locked(&ACTUAL) = None; +} + +/// Note the full compile's report, exit code and artifact writes, when a check is armed. +pub(crate) fn full_compile_finished( + output: &Bound<'_, PyAny>, + (exit_code, artifacts_written): (i64, bool), + artifact_writes: &Bound<'_, PyAny>, +) -> PyResult<()> { + if locked(&EXPECTED).is_none() { + return Ok(()); + } + let stdout: Bound<'_, PyAny> = output.getattr("stdout")?; + let written: HashMap> = artifact_writes + .cast::() + .map(|writes| writes.get().written_digests()) + .unwrap_or_default(); + *locked(&ACTUAL) = Some(Actual { + stdout: if stdout.is_none() { + None + } else { + Some(stdout.str()?.to_str()?.to_owned()) + }, + exit_code: Some(exit_code), + artifacts_written, + written, + }); + Ok(()) +} + +/// The report with its `compile_timings` value blanked. +fn without_timings(stdout: &str) -> String { + let Some(start) = stdout.find("\n \"compile_timings\": {") else { + return stdout.to_owned(); + }; + let Some(length) = stdout[start..].find("\n },\n") else { + return stdout.to_owned(); + }; + format!("{}{}", &stdout[..start], &stdout[start + length + 5..]) +} + +/// Compare the held delta result with the full compile's; every difference found. +pub(crate) fn differences(raised: Option) -> Vec { + let Some(expected) = locked(&EXPECTED).take() else { + return Vec::new(); + }; + let actual: Actual = locked(&ACTUAL).take().unwrap_or_default(); + let mut found: Vec = Vec::new(); + if expected.raised != raised { + found.push(format!( + "raised error differs: delta {:?}, full {:?}", + expected.raised, raised + )); + } + if expected.raised.is_some() || raised.is_some() { + return found; + } + if expected.exit_code != actual.exit_code { + found.push(format!( + "exit code differs: delta {:?}, full {:?}", + expected.exit_code, actual.exit_code + )); + } + let delta_stdout: Option = expected.stdout.as_deref().map(without_timings); + let full_stdout: Option = actual.stdout.as_deref().map(without_timings); + if delta_stdout != full_stdout { + let (delta, full) = ( + delta_stdout.unwrap_or_default(), + full_stdout.unwrap_or_default(), + ); + let common: usize = delta + .bytes() + .zip(full.bytes()) + .take_while(|(left, right)| left == right) + .count(); + let start: usize = common.saturating_sub(200); + let excerpt = |text: &str| -> String { + text.get(start..(common + 300).min(text.len())) + .unwrap_or_default() + .to_owned() + }; + found.push(format!( + "report differs at byte {common}:\n--- delta\n{}\n--- full\n{}", + excerpt(&delta), + excerpt(&full) + )); + } + if expected.artifacts_written != actual.artifacts_written { + found.push(format!( + "artifact writing differs: delta {}, full {}", + expected.artifacts_written, actual.artifacts_written + )); + } + if !actual.artifacts_written { + return found; + } + let prefix: &str = expected.compiled_root.as_str(); + let full_paths: BTreeMap<&str, &Option> = actual + .written + .iter() + .filter(|(path, _)| path.starts_with(prefix)) + .map(|(path, digest)| (path.as_str(), digest)) + .collect(); + for (path, digest) in &full_paths { + match (expected.compiled.get(*path), digest) { + (None, _) => found.push(format!("the delta left no artifact at {path}")), + (Some(left), Some(right)) if left != right => { + found.push(format!("artifact differs: {path}")); + } + _ => {} + } + } + for path in expected.compiled.keys() { + if !full_paths.contains_key(path.as_str()) { + found.push(format!( + "the delta left an artifact the full compile does not: {path}" + )); + } + } + found.truncate(50); + found +} + +/// Write the differences beside the compiler cache for inspection. +pub(crate) fn write_mismatch(cache_dir: &Path, differences: &[String]) { + let _ = std::fs::create_dir_all(cache_dir); + let text: String = serde_json::to_string_pretty(differences).unwrap_or_default(); + let _ = std::fs::write(cache_dir.join(MISMATCH_FILE), text); +} + +/// Forget any held result. +pub(crate) fn disarm() { + *locked(&EXPECTED) = None; + *locked(&ACTUAL) = None; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 23f91269da..50acead931 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -149,7 +149,8 @@ pub(crate) fn analyze<'py>( let options: AnalysisOptions<'py> = AnalysisOptions::read(py, options)?; let selection: &Bound<'py, PyAny> = &options.analysis_selection; let enabled: bool = !(native_field(selection, "select")?.is_truthy()? - || native_field(selection, "exclude")?.is_truthy()?); + || native_field(selection, "exclude")?.is_truthy()?) + && crate::bindings::_helpers::compile_delta::context::injection().is_none(); let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; let early_rules: EarlyRulesScope = EarlyRulesScope::new(); let analysis: PyResult> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 0e03181f2e..85444d1942 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -108,7 +108,9 @@ pub(crate) fn assembled_project<'py>( let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; let resources: Bound<'py, PyAny> = native_field(&native, "resources")?; let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; - report_scope_index_errors(&scope_index)?; + if crate::bindings::_helpers::compile_delta::context::injection().is_none() { + report_scope_index_errors(&scope_index)?; + } let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; let models: Bound<'py, PyTuple> = compiled_models(inputs, &context, (&native_field(&native, "models")?, names))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 5942689a4c..813f1473b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -57,11 +57,18 @@ fn retain_until_exit(value: &Bound<'_, PyAny>) -> PyResult<()> { } /// What one full compile reports for reuse recording. -struct CommandOutcome<'py> { - exit_code: i64, - compile_cache_enabled: bool, - artifacts_written: bool, - dag_artifact_path: Bound<'py, PyAny>, +pub(crate) struct CommandOutcome<'py> { + pub(crate) exit_code: i64, + pub(crate) compile_cache_enabled: bool, + pub(crate) artifacts_written: bool, + pub(crate) dag_artifact_path: Bound<'py, PyAny>, +} + +/// One compile's outcome and recorders, before it is stored and printed. +pub(crate) struct PhasesOutcome<'py> { + pub(crate) outcome: CommandOutcome<'py>, + pub(crate) input_reads: Bound<'py, PyAny>, + pub(crate) artifact_writes: Bound<'py, PyAny>, } /// The per-invocation recorders a compile runs inside. @@ -114,6 +121,45 @@ pub(crate) fn run_compile_command<'py>( exit_code } +/// The project-directory-resolved request, a fresh output recorder and the compile's phases, +/// run as `run_compile_command` runs them but neither stored nor printed. +pub(crate) fn unrecorded_compile<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, +) -> PyResult<( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + PyResult>, +)> { + let py: Python<'py> = request.py(); + let total_start: f64 = native_field(attempt, "started")?.extract()?; + let effective_request: Bound<'py, PyAny> = if native_field(request, "project_dir")?.is_none() { + replaced( + &request.clone(), + &[( + "project_dir", + imported(py, "pathlib", "Path")?.call_method0("cwd")?, + )], + )? + } else { + request.clone() + }; + let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); + let status: Status<'py> = Status(py.None().into_bound(py)); + let gc: Bound<'py, PyModule> = py.import("gc")?; + let was_enabled: bool = gc.call_method0("isenabled")?.is_truthy()?; + gc.call_method0("disable")?; + let phases: PyResult> = compile_phases( + (&effective_request, attempt), + (&output, &status), + total_start, + ); + if was_enabled { + gc.call_method0("enable")?; + } + Ok((effective_request, output, phases)) +} + /// `CompileReuseAttempt` of a compile that did not consult the reuse record. fn bypass_attempt<'py>( py: Python<'py>, @@ -166,55 +212,30 @@ fn recorded_compile<'py>( (output, status): (&Bound<'py, PyAny>, &Status<'py>), total_start: f64, ) -> PyResult { - let py: Python<'py> = request.py(); - let skip_write: bool = native_field(request, "profile_flags")? - .getattr("skip_write")? - .is_truthy()?; - let compiled: PyResult<(CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = - collect_compile_timings(|detailed_timings| { - within( - &Bound::new(py, record_compile_input_reads())?.into_any(), - |input_reads| { - let artifact_writes: Py = new_artifact_recorder(py)?; - let manager: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::PreparedCompileArtifacts)?, - &[ - ( - "enabled", - PyBool::new(py, !skip_write).to_owned().into_any(), - ), - ( - "artifact_writes", - artifact_writes.bind(py).clone().into_any(), - ), - ], - )?; - retain_until_exit(&manager)?; - within(&manager, |prepared_artifacts| { - let recorders: Recorders<'py> = Recorders { - detailed_timings: Rc::clone(detailed_timings), - input_reads: input_reads.clone(), - artifact_writes: artifact_writes.clone_ref(py), - prepared_artifacts: prepared_artifacts.clone(), - }; - let reuse_timings: Bound<'py, PyAny> = - compile_reuse_timings(attempt)?.into_any(); - let outcome: CommandOutcome<'py> = compiled_with_status( - (request, &recorders), - (status, output), - (total_start, &reuse_timings), - )?; - Ok(( - outcome, - recorders.input_reads, - recorders.artifact_writes.into_bound(py).into_any(), - )) - }) - }, - ) - }); + let phases: PyResult> = + compile_phases((request, attempt), (output, status), total_start); close_status(status)?; - let (outcome, input_reads, artifact_writes) = compiled?; + let phases: PhasesOutcome<'py> = phases?; + crate::bindings::_helpers::compile_delta::verify::full_compile_finished( + output, + (phases.outcome.exit_code, phases.outcome.artifacts_written), + &phases.artifact_writes, + )?; + store_and_print((request, attempt), output, phases) +} + +/// Store a finished compile for reuse, then print its recorded output. +pub(crate) fn store_and_print<'py>( + (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + output: &Bound<'py, PyAny>, + phases: PhasesOutcome<'py>, +) -> PyResult { + let py: Python<'py> = request.py(); + let PhasesOutcome { + outcome, + input_reads, + artifact_writes, + } = phases; let stored: PyResult<()> = write_reusable_compile( py, Some(keywords( @@ -248,6 +269,68 @@ fn recorded_compile<'py>( Ok(outcome.exit_code) } +/// Run the compile's phases inside its recorders, without storing or printing it. +pub(crate) fn compile_phases<'py>( + (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (output, status): (&Bound<'py, PyAny>, &Status<'py>), + total_start: f64, +) -> PyResult> { + let py: Python<'py> = request.py(); + let skip_write: bool = native_field(request, "profile_flags")? + .getattr("skip_write")? + .is_truthy()?; + let staging: bool = + !skip_write && crate::bindings::_helpers::compile_delta::context::injection().is_none(); + let (outcome, input_reads, artifact_writes): ( + CommandOutcome<'py>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = collect_compile_timings(|detailed_timings| { + within( + &Bound::new(py, record_compile_input_reads())?.into_any(), + |input_reads| { + let artifact_writes: Py = new_artifact_recorder(py)?; + let manager: Bound<'py, PyAny> = call_with( + ®istry::native(py, NativeAttr::PreparedCompileArtifacts)?, + &[ + ("enabled", PyBool::new(py, staging).to_owned().into_any()), + ( + "artifact_writes", + artifact_writes.bind(py).clone().into_any(), + ), + ], + )?; + retain_until_exit(&manager)?; + within(&manager, |prepared_artifacts| { + let recorders: Recorders<'py> = Recorders { + detailed_timings: Rc::clone(detailed_timings), + input_reads: input_reads.clone(), + artifact_writes: artifact_writes.clone_ref(py), + prepared_artifacts: prepared_artifacts.clone(), + }; + let reuse_timings: Bound<'py, PyAny> = + compile_reuse_timings(attempt)?.into_any(); + let outcome: CommandOutcome<'py> = compiled_with_status( + (request, &recorders), + (status, output), + (total_start, &reuse_timings), + )?; + Ok(( + outcome, + recorders.input_reads, + recorders.artifact_writes.into_bound(py).into_any(), + )) + }) + }, + ) + })?; + Ok(PhasesOutcome { + outcome, + input_reads, + artifact_writes, + }) +} + /// `_run_compile_with_status`: analyze, run the output phases and record the command output. fn compiled_with_status<'py>( (request, recorders): (&Bound<'py, PyAny>, &Recorders<'py>), @@ -309,6 +392,10 @@ fn compiled_with_status<'py>( .chain(withheld.try_iter()?) .collect::>>()?, )?; + crate::bindings::_helpers::compile_delta::report_hooks::record_compile_facts( + &native_field(&analysis, "graph")?, + &diagnostics, + )?; let mut exit_code: i64 = 0; for diagnostic in diagnostics.as_any().try_iter()? { if native_field(&diagnostic?, "is_error")?.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs index b1280c561e..96f3d65137 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -68,6 +68,9 @@ pub(crate) fn emit_json_report(report: &Bound<'_, PyAny>) -> PyResult { if let Some(error) = conversion.stdlib_error { return Err(error); } + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.stdlib_json = true; + }); emitted( &value, &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index ee68ca5a9e..517232a824 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -123,6 +123,14 @@ fn poisoned(_: T) -> pyo3::PyErr { PyValueError::new_err("compile reuse attempt is poisoned") } +impl NativeReuseAttempt { + /// Run `body` on the native attempt. + pub(crate) fn with_attempt(&self, body: impl FnOnce(&ReuseAttempt) -> T) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + Ok(body(&attempt)) + } +} + #[pymethods] impl NativeReuseAttempt { /// `"hit"`, `"miss"` or `"bypass"`. @@ -187,7 +195,7 @@ impl NativeReuseAttempt { fn record(&self, py: Python<'_>, record: CompileRecordPy) -> PyResult<(&'static str, usize)> { compiler_guard(|| { let rules: ReuseRules = record.rules.into(); - let compile: CompileRecord = CompileRecord { + let mut compile: CompileRecord = CompileRecord { stdout: record.stdout, stderr_lines: record.stderr_lines, exit_code: record.exit_code, @@ -207,7 +215,23 @@ impl NativeReuseAttempt { artifacts_written: record.artifacts_written, dag_path: record.dag_path, json_output: record.json_output, + image: crate::bindings::_helpers::compile_delta::context::take_pending_image(), }; + if let Some(merge) = + crate::bindings::_helpers::compile_delta::context::take_record_merge() + { + for (path, size, mtime_ns) in merge.kept { + compile + .artifacts + .entry(path) + .or_insert(RecordedArtifact::Kept { size, mtime_ns }); + } + compile + .template_environment_names + .extend(merge.environment_names); + compile.template_environment_names.sort(); + compile.template_environment_names.dedup(); + } let attempt = self.attempt.lock().map_err(poisoned)?; let result: RecordResult = py .detach(|| record_reuse(&attempt, &compile, &rules)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index 1515583bcb..b42b199fd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -69,6 +69,10 @@ fn model_items<'py>( for file in files { let raw: bool = tree.get().inner.raw_relative(&file.relative_path).is_some(); if !raw && let FileOutcome::Parsed(model) = file.outcome { + crate::bindings::_helpers::compile_delta::header_keys::record_header_keys( + &file.relative_path, + &model.header_values, + ); let file_path: String = root.display_path(&file.relative_path); let paths = DiscoveredPaths::Native(file.relative_path.clone(), file_path); let built = model_record(py, model, paths, options.extract_implicit_alias_columns)? @@ -102,6 +106,17 @@ pub(crate) struct ModelFileSelection<'py> { pub(crate) on_fault: Option>, } +/// The options model discovery parses headers with. +pub(crate) fn model_file_options() -> ModelFileOptions { + ModelFileOptions { + supported_keys: sorted_keys(&constants::SQL_MODEL_HEADER_KEYS), + removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), + extract_implicit_alias_columns: false, + extract_output_column_locations: false, + path_filter: None, + } +} + /// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. pub(crate) fn discovered_model_files<'py>( project_dir: &Bound<'py, PyAny>, @@ -115,6 +130,8 @@ pub(crate) fn discovered_model_files<'py>( removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), extract_implicit_alias_columns, extract_output_column_locations, + path_filter: crate::bindings::_helpers::compile_delta::context::injection() + .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), }; let items: Vec> = model_items(project_dir, &options)?; let faults = FileFaults { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index c7d5e780bf..0850dd66a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -105,6 +105,7 @@ fn parse_model_contents( removed_keys: request.removed_keys, extract_implicit_alias_columns: request.extract_implicit_alias_columns, extract_output_column_locations: request.extract_output_column_locations, + path_filter: None, }; let parsed = py .compiler_detach(|| Ok(parse_model_text(&request.file_path, contents, &options))) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index c5abe9872c..5ef3604e1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -57,6 +57,8 @@ fn options() -> PyResult { Ok(SqlTestFileOptions { test_keys: sorted_keys(&constants::SQL_TEST_HEADER_KEYS), scenario_keys: sorted_keys(&constants::SQL_SCENARIO_HEADER_KEYS), + path_filter: crate::bindings::_helpers::compile_delta::context::injection() + .map(|injection| std::sync::Arc::new(injection.test_paths.clone())), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 1ad13fd9e6..d9d2208ac8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -14,6 +14,7 @@ pub(crate) mod clone; pub(crate) mod command_runtime; pub(crate) mod compile_attachment; pub(crate) mod compile_command; +pub(crate) mod compile_delta; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compile_records; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index 8e23c3f973..68a0ba0b93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -58,9 +58,12 @@ impl NativeModelValidator { /// Model, seed, source, function and table function names discovered in `inputs`. fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { let model_stem = |file: &Bound<'_, PyAny>| stem(file, FILE_PATH_ATTRIBUTE); - let models: HashSet = each(inputs, "model_files", model_stem)? + let mut models: HashSet = each(inputs, "model_files", model_stem)? .into_iter() .collect(); + if let Some(injection) = crate::bindings::_helpers::compile_delta::context::injection() { + models.extend(injection.model_names.iter().cloned()); + } let mut seeds: HashSet = HashSet::new(); for schema in native_field(inputs, "schema_files")?.try_iter()? { seeds.extend(each(&schema?, "seed_entries", column_name)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs index 6e4cd932ca..a3501dd454 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -62,6 +62,11 @@ impl MacroCallMemo { path: PathBuf, environment: &str, ) -> PyResult<(Vec, usize)> { + if crate::bindings::_helpers::compile_delta::context::injection().is_some() { + return Err(pyo3::exceptions::PyRuntimeError::new_err( + "a delta compile leaves the macro call store to full compiles", + )); + } let store: NativeStore = open_store(py, &path, MACRO_CALL_STORE_KIND, environment)?; let opened = (store.metadata().to_vec(), store.loaded_entries()); self.inner.lock().map_err(poisoned)?.attach_store(store); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index f5169cf5c5..06199b4ab1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -97,7 +97,32 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( }; let project_dir: PathBuf = run.project_dir.extract()?; let mut digests: FactDigests<'py> = FactDigests::new(run.project, &project_dir)?; - let models: Vec<(String, String, Option)> = digests.replay.model_rows()?; + let models: Vec<(String, String, Option)> = + match crate::bindings::_helpers::compile_delta::context::injection() { + Some(injection) => { + let Some(mut rows) = injection.model_subjects.clone() else { + return Err( + crate::bindings::_helpers::compile_delta::context::delta_unsupported( + "the custom rules' subjects were not recorded", + ), + ); + }; + let fresh: Vec<(String, String, Option)> = digests.replay.model_rows()?; + for row in &mut rows { + if let Some(found) = fresh.iter().find(|candidate| candidate.0 == row.0) { + row.clone_from(found); + } + } + rows + } + None => { + let rows: Vec<(String, String, Option)> = digests.replay.model_rows()?; + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.model_subjects = Some(rows.clone()); + }); + rows + } + }; let mut model_paths: Vec = models.iter().map(|(path, _, _)| path.clone()).collect(); model_paths.sort(); let mut cache: CustomRuleCache = CustomRuleCache::new( @@ -112,6 +137,14 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( readset_digests.push(reproducible(digests.combined(readset))?); } let plan: HostPlan = cache.plan(readset_digests); + if !plan.is_empty() && crate::bindings::_helpers::compile_delta::context::injection().is_some() + { + return Err( + crate::bindings::_helpers::compile_delta::context::delta_unsupported( + "a custom rule must run again", + ), + ); + } if !plan.is_empty() { let rules: Vec<(String, HostRuleSubject)> = custom .iter() @@ -135,6 +168,15 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( let evaluations: Vec = evaluations(host_run, &mut digests, enabled)?; cache.record(py, evaluations)?; } + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.fact_digests.extend( + digests + .digests + .iter() + .filter(|(key, _)| !is_tree_fact(key)) + .map(|(key, digest)| (key.clone(), digest.clone())), + ); + }); let elapsed_ms: i64 = (started.elapsed().as_secs_f64() * 1000.0).round_ties_even() as i64; let cpu_ms: i64 = ((cpu_seconds(py)? - started_cpu) * 1000.0) as i64; outcome( @@ -232,6 +274,12 @@ impl<'py> FactDigests<'py> { if let Some(cached) = self.digests.get(key) { return Ok(cached.clone()); } + if let Some(injected) = + crate::bindings::_helpers::compile_delta::context::injected_fact_digest(key) + { + self.digests.insert(key.clone(), injected.clone()); + return Ok(injected); + } let digest: String = self.replay.digest(key)?; self.digests.insert(key.clone(), digest.clone()); Ok(digest) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index cef85ca350..dbeea55c9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -160,6 +160,7 @@ pub(crate) fn selectionless_rules_request( request: Mutex::new(Some(RowsRequest { request, memo: None, + retain: false, })), }) }) @@ -293,11 +294,17 @@ fn built_request( let model_rows: Vec = project_model_rows(facts.borrow().facts(), &names).map_err(value_error)?; request.models = built_models(&rows, &request, model_rows)?; + let request: EvaluateRequest = + crate::bindings::_helpers::compile_delta::rules_rows::whole_project_request(request)?; let memo: Option<(String, PathBuf)> = rows .memo .take() .map(|(identity, path)| (identity, PathBuf::from(path))); - Ok(RowsRequest { request, memo }) + Ok(RowsRequest { + request, + memo, + retain: crate::bindings::_helpers::compile_delta::context::capturing(), + }) }) } @@ -332,10 +339,16 @@ fn evaluated<'py>( .map_err(value_error)?, Some((early, interrupted)) => early.answer(py, request, interrupted)?, }; - let evaluation: RowsEvaluation = evaluation.map_err(|error| match error { + let mut evaluation: RowsEvaluation = evaluation.map_err(|error| match error { RowsError::Rules(message) => value_error(message), RowsError::Io(error) => PyOSError::new_err(error.to_string()), })?; + if let Some(retained) = evaluation.retained.take() { + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.rules = + Some(crate::bindings::_helpers::compile_delta::context::CapturedRules { retained }); + }); + } let classes = FindingClasses::new(py)?; let findings: Vec> = evaluation .faults diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index d3124c629b..a334bcd8cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -148,12 +148,15 @@ pub(crate) fn run_planned_sql_rules<'py>( }; let lint_key: String = settings_lint_key(&run.settings); let identities: Vec<(String, String)> = model_rule_identities(&run, &lint_key)?; - let project_files: Option> = - if !identities.is_empty() && run.prepared().is_none() && !run.model_selection { - Some(lint_sources(&run.project_dir, None, None)?) - } else { - None - }; + let project_files: Option> = if !identities.is_empty() + && run.prepared().is_none() + && !run.model_selection + && crate::bindings::_helpers::compile_delta::context::injection().is_none() + { + Some(lint_sources(&run.project_dir, None, None)?) + } else { + None + }; let started: Instant = Instant::now(); let (findings, hits, misses) = evaluated_sql_rules(&run, (&lint_key, identities), project_files)?; @@ -298,13 +301,14 @@ fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult = model_rule_keys(run.project.py(), lint_key, run.codes, &models); - Ok(models + let fresh: Vec<(String, String)> = models .into_iter() .map(|(path, ..)| { let key: String = keys[&path].clone(); (path, key) }) - .collect()) + .collect(); + crate::bindings::_helpers::compile_delta::lint_hooks::model_identities(fresh) } /// Canonical text of a hand-built model's dynamic column contract that SQL rule findings depend on. @@ -382,7 +386,11 @@ fn evaluated_sql_rules( for model in project_attribute(&run.project, "models")?.try_iter()? { model_paths.insert(project_dir.join(model_posix_path(&model?)?)); } + let delta = crate::bindings::_helpers::compile_delta::context::injection(); let files: Vec = match prepared_result(run, py) { + _ if delta.is_some() => { + crate::bindings::_helpers::compile_delta::lint_hooks::changed_files(project_dir)? + } Some(Ok(result)) => result.outcome.files.clone(), Some(Err(error)) if !error.is_instance_of::(py) => { return Err(error.clone_ref(py)); @@ -409,6 +417,11 @@ fn evaluated_sql_rules( ); file_keys.push((file.path.clone(), relative_path, key)); } + let file_keys: Vec<(PathBuf, String, Option)> = + crate::bindings::_helpers::compile_delta::lint_hooks::file_keys( + project_dir, + file_keys, + )?; let mut keyed: Vec<(String, String)> = Vec::new(); for (_, path, key) in &file_keys { if let Some(key) = key { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index f1672c0545..8338ebd9bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -355,10 +355,21 @@ pub(crate) fn scope_index_with_compile_usages<'py>( usages.extend(usage_records( model_schema_scope_usages(&schema_files, &resolver)?.as_any(), )?); + let digests: bool = crate::bindings::_helpers::compile_delta::context::capturing(); for model in native_field(inputs, "model_inputs")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - usages.extend(usage_records(&native_field(&model, "macro_usages")?)?); - usages.extend(usage_records(&native_field(&model, "declaration_usages")?)?); + let macro_usages: Vec> = + usage_records(&native_field(&model, "macro_usages")?)?; + let declaration_usages: Vec> = + usage_records(&native_field(&model, "declaration_usages")?)?; + if digests { + crate::bindings::_helpers::compile_delta::image_facts::record_model_usages( + &model, + (¯o_usages, &declaration_usages), + )?; + } + usages.extend(macro_usages); + usages.extend(declaration_usages); } for group in COMPILE_USAGE_GROUPS { for item in native_field(inputs, group)?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 6ea7f7db18..2973fa30de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -317,6 +317,7 @@ fn model_shapes<'py>( let mut pending: Vec> = native_field(project, "models")? .try_iter()? .collect::>()?; + crate::bindings::_helpers::compile_delta::report_hooks::inject_model_shapes(&pending, shapes)?; while !pending.is_empty() { let mut remaining: Vec> = Vec::new(); for model in &pending { @@ -362,7 +363,7 @@ fn model_shapes<'py>( } pending = remaining; } - Ok(()) + crate::bindings::_helpers::compile_delta::report_hooks::record_model_shapes(project, shapes) } fn all_known(names: &[String], shapes: &Bound<'_, PyDict>) -> PyResult { From 5324f54150203e5c98627c8dbf8f36e6bc3c1ff1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:57:56 +0100 Subject: [PATCH 05/32] perf(compile): decide delta fallbacks from a small image gate before decoding the image A delta compile now reads a gate section (format, output mode, stored errors, models, tests, header keys and custom-rule fact digests) to rule itself out before decoding the image's facts and rules, which it decodes in parallel. It moves the image's whole-project facts into the injection instead of cloning them each round, stops as soon as analysis or contracts report an error, and reruns the target model of a changed SQL test. Header digests and the stored rules request are cheaper to produce on every full compile. --- .../_helpers/compile_delta/context.rs | 27 +- .../bindings/_helpers/compile_delta/driver.rs | 310 +++++++++++------- .../_helpers/compile_delta/header_keys.rs | 61 +++- .../bindings/_helpers/compile_delta/image.rs | 297 +++++++++++++---- .../_helpers/compile_delta/report_hooks.rs | 1 + .../_helpers/compile_driver/analysis.rs | 5 + 6 files changed, 508 insertions(+), 193 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index e86fdeb0ff..1f055a6cfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -71,6 +71,9 @@ pub(crate) struct CapturedDiagnostic { pub(crate) json: String, pub(crate) resource_name: Option, pub(crate) paths: Vec, + /// Whether the diagnostic is an error. + #[serde(default)] + pub(crate) error: bool, } /// One SQL test's file and the models its compiled artifact is keyed by. @@ -154,11 +157,19 @@ pub(crate) fn capture_unsupported(reason: &str) { } /// Run `body` with `injection` visible to the stages a delta compile reruns. -pub(crate) fn with_injection(injection: DeltaInjection, body: impl FnOnce() -> T) -> T { +/// +/// Returns the injection back with the body's result, or `None` if something still holds it. +pub(crate) fn with_injection( + injection: DeltaInjection, + body: impl FnOnce() -> T, +) -> (T, Option) { *locked(&INJECTION) = Some(Arc::new(injection)); let result: T = body(); - *locked(&INJECTION) = None; - result + let injection: Option> = locked(&INJECTION).take(); + ( + result, + injection.and_then(|injection| Arc::try_unwrap(injection).ok()), + ) } /// The running delta compile's injection, if any. @@ -178,6 +189,16 @@ pub(crate) fn delta_unsupported(reason: &str) -> pyo3::PyErr { pyo3::exceptions::PyRuntimeError::new_err(format!("delta compile unsupported: {reason}")) } +/// In a delta compile that leaves errors to full compiles, stop once `failed` reports one. +pub(crate) fn stop_on_errors(failed: bool) -> pyo3::PyResult<()> { + let errors_on_delta_path: bool = + crate::bindings::_helpers::compile_delta::driver::ERRORS_ON_DELTA_PATH; + if failed && !errors_on_delta_path && injection().is_some() { + return Err(delta_unsupported("the edit reports errors")); + } + Ok(()) +} + /// Take the reason the delta compile was ruled out, if any. pub(crate) fn take_delta_unsupported() -> Option { locked(&UNSUPPORTED).take() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 91dc768b7a..ee7fb0a146 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyString, PyStringMethods}; +use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; use sqlbuild_cache::project_reuse::main::read_delta_base::{read_delta_base, read_delta_output}; use sqlbuild_cache::project_reuse::models::{ DeltaBase, DeltaCandidate, ImageSections, ProjectFileChanges, @@ -13,13 +14,13 @@ use sqlbuild_cache::project_reuse::models::{ use crate::bindings::_helpers::compile_command::reuse_recording::RecordedCompileOutput; use crate::bindings::_helpers::compile_delta::context::{ - CapturedDiagnostic, CapturedTest, DeltaInjection, ImageCapture, RecordMerge, start_capture, - take_capture, take_delta_unsupported, with_injection, + CapturedDiagnostic, CapturedModel, CapturedTest, DeltaInjection, ImageCapture, RecordMerge, + start_capture, take_capture, take_delta_unsupported, with_injection, }; use crate::bindings::_helpers::compile_delta::header_keys::{ HeaderChange, custom_rule_reads, header_change, }; -use crate::bindings::_helpers::compile_delta::image::{ImageRelations, ProjectImage}; +use crate::bindings::_helpers::compile_delta::image::{ImageGate, ImageRelations, ProjectImage}; use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; use crate::bindings::_helpers::compile_driver::command::{PhasesOutcome, unrecorded_compile}; use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; @@ -33,6 +34,8 @@ const ENABLED_VALUE: &str = "1"; /// Rerunning more models than this, or this share of the project, compiles the whole project. const MAX_RERUN_MODELS: usize = 500; const MAX_RERUN_PERCENT: usize = 5; +/// Whether edits that report or raise errors compile as deltas; until then they fall back. +pub(crate) const ERRORS_ON_DELTA_PATH: bool = false; /// How many times a changed relation shape may pull its dependents into the delta. const MAX_ROUNDS: usize = 4; const MODELS_ROOT: &str = "models/"; @@ -145,20 +148,6 @@ fn changes_of( }) } -fn base_of(store_path: PathBuf) -> Result { - let Some(stored) = read_delta_base(&store_path) else { - return Err("the stored compile has no project image".to_owned()); - }; - let Some(image) = ProjectImage::decode(&stored.image) else { - return Err("the project image cannot be read".to_owned()); - }; - Ok(Base { - store_path, - stored, - image, - }) -} - /// The rerun set: models and SQL tests, with every path and name the delta recompiles. #[derive(Clone, Debug, Default)] struct Rerun { @@ -218,13 +207,10 @@ impl Rerun { } } -fn injection_for(base: &Base, rerun: &Rerun, project_dir: PathBuf) -> DeltaInjection { - let image: &ProjectImage = &base.image; +/// The injection of the image's whole-project facts, moved out of `image` while the delta runs. +fn lent_injection(image: &mut ProjectImage, project_dir: PathBuf) -> DeltaInjection { DeltaInjection { project_dir, - model_paths: rerun.model_paths.clone(), - test_paths: rerun.test_paths.clone(), - rerun_names: rerun.names(), source_paths: image .models .iter() @@ -236,20 +222,42 @@ fn injection_for(base: &Base, rerun: &Rerun, project_dir: PathBuf) -> DeltaInjec .iter() .map(|model| model.name.clone()) .collect(), - relations: (&image.relations).into(), - rules: image.rules.clone(), - fact_digests: image.fact_digests.iter().cloned().collect(), - model_subjects: image.model_subjects.clone(), + relations: std::mem::take(&mut image.relations).into(), + rules: image.rules.take(), + fact_digests: std::mem::take(&mut image.fact_digests) + .into_iter() + .collect(), + model_subjects: image.model_subjects.take(), text_models: image.text_models.clone().unwrap_or_default(), - model_shapes: image.model_shapes.clone(), - changed_paths: rerun.changed.clone(), - lint_model_keys: image.lint_model_keys.clone(), - lint_file_keys: image.lint_file_keys.clone(), + model_shapes: std::mem::take(&mut image.model_shapes), + lint_model_keys: image.lint_model_keys.take(), + lint_file_keys: image.lint_file_keys.take(), + ..DeltaInjection::default() } } +/// Return the facts `lent_injection` moved out of `image`. +fn returned(image: &mut ProjectImage, injection: DeltaInjection) { + image.relations = injection.relations.into(); + image.rules = injection.rules; + image.fact_digests = injection.fact_digests.into_iter().collect(); + image.model_subjects = injection.model_subjects; + image.model_shapes = injection.model_shapes; + image.lint_model_keys = injection.lint_model_keys; + image.lint_file_keys = injection.lint_file_keys; +} + +/// `injection` set up to rerun `rerun`. +fn for_rerun(mut injection: DeltaInjection, rerun: &Rerun) -> DeltaInjection { + injection.model_paths = rerun.model_paths.clone(); + injection.test_paths = rerun.test_paths.clone(); + injection.rerun_names = rerun.names(); + injection.changed_paths = rerun.changed.clone(); + injection +} + /// The relation facts of `name` that its dependents' analysis reads. -fn shape_of(relations: &ImageRelations, name: &str) -> [Option>; 3] { +fn shape_of(relations: &RelationTables, name: &str) -> [Option>; 3] { let find = |table: &[(String, Vec<(String, String)>)]| { table .iter() @@ -300,13 +308,21 @@ enum RoundOutcome<'py> { fn round<'py>( request: &Bound<'py, PyAny>, attempt: &Bound<'py, PyAny>, - (base, rerun): (&Base, &Rerun), -) -> PyResult> { - let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; + (injection, rerun): (DeltaInjection, &Rerun), +) -> PyResult<(RoundOutcome<'py>, Option)> { start_capture(); - let ran = with_injection(injection_for(base, rerun, project_dir), || { - unrecorded_compile(request, attempt) - }); + let (ran, injection) = with_injection(injection, || unrecorded_compile(request, attempt)); + Ok((ran_round(ran, rerun)?, injection)) +} + +fn ran_round<'py>( + ran: PyResult<( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + PyResult>, + )>, + rerun: &Rerun, +) -> PyResult> { let unsupported: Option = take_delta_unsupported(); let capture: Option = take_capture(); if let Some(reason) = unsupported { @@ -329,7 +345,7 @@ fn round<'py>( .iter() .chain(&rerun.test_paths) .any(|path| message.contains(path.as_str())); - return Ok(RoundOutcome::Stop(if owned { + return Ok(RoundOutcome::Stop(if owned && ERRORS_ON_DELTA_PATH { DeltaOutcome::Raised(error) } else { DeltaOutcome::Fallback(format!("the delta compile raised: {message}")) @@ -381,112 +397,85 @@ pub(crate) fn run<'py>( { return fallback(format!("{path} is not a model or SQL test file")); } - let base: Base = match base_of(store_path) { - Ok(base) => base, - Err(reason) => return fallback(reason), + let Some(mut stored) = read_delta_base(&store_path) else { + return fallback("the stored compile has no project image"); + }; + let Some(gate) = ImageGate::decode(&stored.image) else { + return fallback("the project image cannot be read"); }; + if !ERRORS_ON_DELTA_PATH && gate.has_errors { + return fallback("the stored compile reported errors"); + } let json_output: bool = native_field(request, "json_output")?.is_truthy()?; - if base.image.json_output != json_output { + if gate.json_output != json_output { return fallback("the stored compile has the other output format"); } - let model_by_path: HashMap<&str, &str> = base - .image - .models - .iter() - .map(|model| (model.relative_path.as_str(), model.name.as_str())) - .collect(); - let test_files: BTreeSet<&str> = base - .image - .tests - .iter() - .map(|test| test.relative_path.as_str()) - .collect(); - let mut rerun: Rerun = Rerun { - changed: changes.changed.iter().cloned().collect(), - ..Rerun::default() - }; - let mut changed_tests: BTreeSet = BTreeSet::new(); - for path in &changes.changed { - if resource_sql(path, MODELS_ROOT) - && let Some(name) = model_by_path.get(path.as_str()) - { - rerun.models.insert((*name).to_owned()); - rerun.model_paths.insert(path.clone()); - } else if resource_sql(path, TESTS_ROOT) && test_files.contains(path.as_str()) { - changed_tests.insert(path.clone()); - } else { - return fallback(format!("{path} is not a model or SQL test file")); - } - } let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; - for path in &rerun.model_paths { - let name: &str = model_by_path - .get(path.as_str()) - .copied() - .unwrap_or_default(); - match header_change(&project_dir, path, base.image.header_keys.get(path)) { - HeaderChange::Unchanged | HeaderChange::Unparsed => {} - HeaderChange::Unsupported(reason) => return fallback(reason), - HeaderChange::Patched(keys) => { - if base.image.model_subjects.is_some() && keys.contains("materialized") { - return fallback("custom rules select models by their materialization"); - } - if let Some(reason) = - custom_rule_reads(&base.image.fact_digests, (path, name), &keys) - { - return fallback(reason); - } - } - } + let (mut rerun, changed_tests) = match first_rerun(&gate, &changes, &project_dir) { + Ok(found) => found, + Err(reason) => return fallback(reason), + }; + let limit: usize = MAX_RERUN_MODELS.max(gate.models.len() * MAX_RERUN_PERCENT / 100); + if rerun.models.len() > limit { + return fallback("too many models would compile again"); } - let limit: usize = MAX_RERUN_MODELS.max(base.image.models.len() * MAX_RERUN_PERCENT / 100); + let Some(mut image) = ProjectImage::decode(&stored.image, gate) else { + return fallback("the project image cannot be read"); + }; + stored.image = ImageSections::new(); + trace(request.py(), "delta base read", started)?; + let mut injection: DeltaInjection = lent_injection(&mut image, project_dir); let mut last: Option> = None; for _ in 0..MAX_ROUNDS { if rerun.models.len() > limit { return fallback("too many models would compile again"); } - rerun = rerun.with_tests(&base.image, &changed_tests); - trace(request.py(), "delta base read", started)?; - let ran: Round<'py> = match round(request, attempt, (&base, &rerun))? { + rerun = rerun.with_tests(&image, &changed_tests); + let (outcome, back) = round(request, attempt, (for_rerun(injection, &rerun), &rerun))?; + let ran: Round<'py> = match outcome { RoundOutcome::Ran(ran) => *ran, RoundOutcome::Stop(outcome) => return Ok(outcome), }; + let Some(back) = back else { + return fallback("the delta compile's injection is still in use"); + }; + injection = back; if rerun.models.is_empty() { last = Some(ran); break; } - let Some(relations) = ran.capture.relations.clone() else { + let Some(relations) = ran.capture.relations.as_ref() else { return fallback("the delta compile analysed no models"); }; - let relations: ImageRelations = relations.into(); - if relations.cyclic != base.image.relations.cyclic - || relations.dependency_ordered != base.image.relations.dependency_ordered + let stored_relations: &RelationTables = &injection.relations; + if relations.cyclic != stored_relations.cyclic + || relations.dependency_ordered != stored_relations.dependency_ordered { return fallback("the project's analysis order changed"); } + let stored_models: HashMap<&str, &CapturedModel> = image + .models + .iter() + .map(|model| (model.name.as_str(), model)) + .collect(); for model in &ran.capture.models { - let stored = base - .image - .models - .iter() - .find(|stored| stored.name == model.name); - if stored != Some(model) { + if stored_models.get(model.name.as_str()) != Some(&model) { return fallback(format!("{} changed its references", model.name)); } } for path in &rerun.model_paths { - if ran.capture.usages.get(path) != base.image.usages.get(path) { + if ran.capture.usages.get(path) != image.usages.get(path) { return fallback(format!("{path} changed the declarations it uses")); } } let reshaped: BTreeSet = rerun .models .iter() - .filter(|name| shape_of(&relations, name) != shape_of(&base.image.relations, name)) + .filter(|name| shape_of(relations, name) != shape_of(stored_relations, name)) .cloned() .collect(); - let dependents: BTreeSet = base.image.dependents(&reshaped); - let added: Vec = dependents + let added: Vec = image + .dependents(&reshaped) .into_iter() .filter(|name| !rerun.models.contains(name)) .collect(); @@ -495,22 +484,103 @@ pub(crate) fn run<'py>( break; } for name in added { - if let Some(model) = base.image.models.iter().find(|model| model.name == name) { + if let Some(model) = stored_models.get(name.as_str()) { rerun.model_paths.insert(model.relative_path.clone()); } rerun.models.insert(name); } last = None; } + returned(&mut image, injection); + let base: Base = Base { + store_path, + stored, + image, + }; let Some(ran) = last else { return fallback("changed shapes reach too far"); }; trace(request.py(), "delta rounds", started)?; - let outcome = finished(ran, &base, &rerun, changes.changed.len()); + let outcome = finished(ran, base, &rerun, changes.changed.len()); trace(request.py(), "delta report", started)?; outcome } +/// The models and SQL tests the changed files rerun, and the changed tests; or why the gate +/// rules a delta out. +fn first_rerun( + gate: &ImageGate, + changes: &ProjectFileChanges, + project_dir: &std::path::Path, +) -> Result<(Rerun, BTreeSet), String> { + let model_by_path: HashMap<&str, &str> = gate + .models + .iter() + .map(|model| (model.relative_path.as_str(), model.name.as_str())) + .collect(); + let test_files: BTreeSet<&str> = gate + .tests + .iter() + .map(|test| test.relative_path.as_str()) + .collect(); + let mut rerun: Rerun = Rerun { + changed: changes.changed.iter().cloned().collect(), + ..Rerun::default() + }; + let mut changed_tests: BTreeSet = BTreeSet::new(); + for path in &changes.changed { + if resource_sql(path, MODELS_ROOT) + && let Some(name) = model_by_path.get(path.as_str()) + { + rerun.models.insert((*name).to_owned()); + rerun.model_paths.insert(path.clone()); + } else if resource_sql(path, TESTS_ROOT) && test_files.contains(path.as_str()) { + changed_tests.insert(path.clone()); + } else { + return Err(format!("{path} is not a model or SQL test file")); + } + } + for path in &rerun.model_paths { + let name: &str = model_by_path + .get(path.as_str()) + .copied() + .unwrap_or_default(); + match header_change(project_dir, path, gate.header_keys.get(path)) { + HeaderChange::Unchanged => {} + HeaderChange::Unparsed if !ERRORS_ON_DELTA_PATH => { + return Err(format!("{path} does not parse")); + } + HeaderChange::Unparsed => {} + HeaderChange::Unsupported(reason) => return Err(reason), + HeaderChange::Patched(keys) => { + if gate.has_model_subjects && keys.contains("materialized") { + return Err("custom rules select models by their materialization".to_owned()); + } + if let Some(reason) = custom_rule_reads(&gate.fact_digests, (path, name), &keys) { + return Err(reason); + } + } + } + } + let path_by_model: HashMap<&str, &str> = model_by_path + .iter() + .map(|(path, name)| (*name, *path)) + .collect(); + for test in &gate.tests { + if !changed_tests.contains(&test.relative_path) { + continue; + } + for name in &test.chain { + let Some(path) = path_by_model.get(name.as_str()) else { + return Err(format!("{} tests a model the image lacks", test.key)); + }; + rerun.models.insert(name.clone()); + rerun.model_paths.insert((*path).to_owned()); + } + } + Ok((rerun, changed_tests)) +} + /// Log how long the delta has run, for profiling. pub(crate) fn trace(py: Python<'_>, phase: &str, started: std::time::Instant) -> PyResult<()> { let milliseconds: u128 = started.elapsed().as_millis(); @@ -532,11 +602,20 @@ fn unowned<'a>(rerun: &Rerun, diagnostics: &'a [CapturedDiagnostic]) -> Vec<&'a fn finished<'py>( ran: Round<'py>, - base: &Base, + base: Base, rerun: &Rerun, changed_files: usize, ) -> PyResult> { let py: Python<'py> = ran.request.py(); + if !ERRORS_ON_DELTA_PATH && ran.phases.outcome.exit_code != 0 { + let first: &str = ran + .capture + .diagnostics + .iter() + .find(|diagnostic| diagnostic.error) + .map_or("no error diagnostic", |diagnostic| diagnostic.json.as_str()); + return fallback(format!("the edit reports errors: {first}")); + } if unowned(rerun, &base.image.diagnostics) != unowned(rerun, &ran.capture.diagnostics) { return fallback("diagnostics outside the changed files differ"); } @@ -567,7 +646,7 @@ fn finished<'py>( delta_stdout }; let stderr_lines: Vec> = ran.output.getattr("stderr_lines")?.extract()?; - let image: Option = patched_image(&base.image, rerun, ran.capture).encode().ok(); + let environment_names: Vec = base.image.environment_names.clone(); let merge: RecordMerge = RecordMerge { kept: base .stored @@ -576,8 +655,9 @@ fn finished<'py>( .iter() .map(|stamp| (stamp.relative_path.clone(), stamp.size, stamp.mtime_ns)) .collect(), - environment_names: base.image.environment_names.clone(), + environment_names, }; + let image: Option = patched_image(base.image, rerun, ran.capture).encode().ok(); let _ = py; Ok(DeltaOutcome::Finished(Box::new(Finished { request: ran.request, @@ -592,8 +672,8 @@ fn finished<'py>( } /// The stored image with the rerun models' and tests' facts replaced by the delta's. -fn patched_image(base: &ProjectImage, rerun: &Rerun, capture: ImageCapture) -> ProjectImage { - let mut image: ProjectImage = base.clone(); +fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> ProjectImage { + let mut image: ProjectImage = base; let fresh_models: HashMap = capture .models .into_iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs index 9c9d563f62..57e7ab93c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs @@ -4,7 +4,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; -use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_sqltext::compiler::models::AuthoredValue; @@ -27,12 +26,68 @@ const PATCHED_KEYS: [&str; 6] = [ const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"]; const MATERIALIZED_KEY: &str = "materialized"; +/// Bytes of a digest kept per header value; a collision would need 2^64 work to find. +const KEPT_DIGEST_BYTES: usize = 16; +const HEX: &[u8; 16] = b"0123456789abcdef"; + +/// Append an unambiguous encoding of `value`: a tag byte, then length-prefixed parts. +fn encode_value(value: &AuthoredValue, out: &mut Vec) { + let text = |tag: u8, text: &str, out: &mut Vec| { + out.push(tag); + out.extend_from_slice(&(text.len() as u64).to_le_bytes()); + out.extend_from_slice(text.as_bytes()); + }; + let list = |tag: u8, items: &[AuthoredValue], out: &mut Vec| { + out.push(tag); + out.extend_from_slice(&(items.len() as u64).to_le_bytes()); + for item in items { + encode_value(item, out); + } + }; + let pairs = |tag: u8, items: &[(String, AuthoredValue)], out: &mut Vec| { + out.push(tag); + out.extend_from_slice(&(items.len() as u64).to_le_bytes()); + for (key, item) in items { + text(b'k', key, out); + encode_value(item, out); + } + }; + match value { + AuthoredValue::Null => out.push(b'n'), + AuthoredValue::Boolean(flag) => out.extend_from_slice(&[b'b', u8::from(*flag)]), + AuthoredValue::BareWord(word) => text(b'w', word, out), + AuthoredValue::String(string) => text(b's', string, out), + AuthoredValue::List(items) => list(b'l', items, out), + AuthoredValue::Map(items) => pairs(b'm', items, out), + AuthoredValue::Set(items) => list(b'e', items, out), + AuthoredValue::Tuple(items) => list(b't', items, out), + AuthoredValue::TypedConstant(items) => pairs(b'c', items, out), + AuthoredValue::InlineSqlHook(sql) => text(b'i', sql, out), + AuthoredValue::NamedSqlHook(name, items) => { + text(b'h', name, out); + pairs(b'a', items, out); + } + AuthoredValue::PythonHook(name, items) => { + text(b'p', name, out); + pairs(b'a', items, out); + } + } +} + fn header_keys(values: &[(String, AuthoredValue)]) -> HeaderKeys { + let mut buffer: Vec = Vec::new(); let mut keys: HeaderKeys = values .iter() .map(|(key, value)| { - let text: String = format!("{value:?}"); - (key.clone(), hex_digest(&bytes_digest(text.as_bytes()))) + buffer.clear(); + encode_value(value, &mut buffer); + let digest = bytes_digest(&buffer); + let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); + for byte in &digest[..KEPT_DIGEST_BYTES] { + hex.push(char::from(HEX[usize::from(byte >> 4)])); + hex.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + (key.clone(), hex) }) .collect(); keys.sort(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index c5880d7036..049adc3b3e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -14,12 +14,14 @@ use crate::bindings::_helpers::compile_delta::context::{ use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; /// Changes whenever the image's meaning changes, so an older image is never read. -const IMAGE_FORMAT: &str = "sqlbuild-project-image-v1"; +const IMAGE_FORMAT: &str = "sqlbuild-project-image-v3"; +const GATE_SECTION: &str = "gate"; const FACTS_SECTION: &str = "facts"; const RULES_SECTION: &str = "rules"; +const RULE_DIGESTS_SECTION: &str = "rule-digests"; /// Everything a delta compile reads of the compile it patches. -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default)] pub(crate) struct ProjectImage { pub(crate) format: String, pub(crate) json_output: bool, @@ -36,7 +38,6 @@ pub(crate) struct ProjectImage { pub(crate) lint_model_keys: Option>, pub(crate) lint_file_keys: Option)>>, pub(crate) header_keys: BTreeMap, - #[serde(skip)] pub(crate) rules: Option, } @@ -66,49 +67,143 @@ impl From for ImageRelations { } } -impl From<&ImageRelations> for RelationTables { - fn from(relations: &ImageRelations) -> Self { +impl From for RelationTables { + fn from(relations: ImageRelations) -> Self { Self { - types: relations.types.clone(), - nullability: relations.nullability.clone(), - complete: relations.complete.clone(), - schemas: relations.schemas.clone(), + types: relations.types, + nullability: relations.nullability, + complete: relations.complete, + schemas: relations.schemas, dependency_ordered: relations.dependency_ordered, cyclic: relations.cyclic, - legacy_models: relations.legacy_models.clone(), + legacy_models: relations.legacy_models, } } } +/// The prefix the keys of fields a rules request serializes but never reads back share; +/// evaluation derives them again. +const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; +const DERIVED_FIELDS: [&str; 2] = ["_test_count\":", "\":"]; + +/// `json` without the derived fields, each a bare number or boolean after its key. +/// +/// Compact JSON escapes every quote inside strings, so these keys only match as keys. +fn without_derived_fields(json: Vec) -> Vec { + let Ok(text) = std::str::from_utf8(&json) else { + return json; + }; + let mut kept: Vec = Vec::with_capacity(json.len()); + let mut copied: usize = 0; + for (start, _) in text.match_indices(DERIVED_FIELD_PREFIX) { + if start < copied { + continue; + } + let rest: &str = &text[start + DERIVED_FIELD_PREFIX.len()..]; + let Some(field) = DERIVED_FIELDS + .iter() + .find(|field| rest.starts_with(**field)) + else { + continue; + }; + let value: &str = &rest[field.len()..]; + let length: usize = value.find([',', '}']).unwrap_or(value.len()); + kept.extend_from_slice(&json[copied..start]); + copied = start + DERIVED_FIELD_PREFIX.len() + field.len() + length; + } + kept.extend_from_slice(&json[copied..]); + kept +} + +/// Rows of `table` that differ from the same relation's `types` row; `None` where equal. +fn differing(table: &Shapes, types: &Shapes) -> Vec<(String, Option>)> { + let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types + .iter() + .map(|(name, shape)| (name.as_str(), shape)) + .collect(); + table + .iter() + .map(|(name, shape)| { + let same: bool = types + .get(name.as_str()) + .is_some_and(|typed| *typed == shape); + (name.clone(), (!same).then(|| shape.clone())) + }) + .collect() +} + +/// `differing`'s rows restored against `types`. +fn restored(rows: Vec<(String, Option>)>, types: &Shapes) -> Option { + let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types + .iter() + .map(|(name, shape)| (name.as_str(), shape)) + .collect(); + rows.into_iter() + .map(|(name, shape)| { + let shape: Vec<(String, String)> = match shape { + Some(shape) => shape, + None => (*types.get(name.as_str())?).clone(), + }; + Some((name, shape)) + }) + .collect() +} + +/// A shape table as stored: `None` where a relation's row equals its `types` row. +type StoredShapes = Vec<(String, Option>)>; + +/// What a delta compile reads first to decide, cheaply, whether it can patch the image. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct ImageGate { + pub(crate) format: String, + pub(crate) json_output: bool, + /// Whether the stored compile reported any error diagnostic. + pub(crate) has_errors: bool, + /// Whether custom rules recorded which models they select. + pub(crate) has_model_subjects: bool, + pub(crate) models: Vec, + pub(crate) tests: Vec, + pub(crate) fact_digests: Vec<(Vec, String)>, + pub(crate) header_keys: BTreeMap, +} + +/// The rest of the image's facts as stored; shapes equal to a relation's types are stored once. #[derive(Serialize, Deserialize)] -struct StoredRules { - request: serde_json::Value, - digests: Vec, +struct StoredFacts { + usages: BTreeMap, + types: Shapes, + nullability: Shapes, + complete: StoredShapes, + schemas: StoredShapes, + dependency_ordered: bool, + cyclic: bool, + legacy_models: Vec, + model_subjects: Option)>>, + diagnostics: Vec, + text_models: Option>, + environment_names: Vec, + model_shapes: StoredShapes, + lint_model_keys: Option>, + lint_file_keys: Option)>>, } -/// Fields a request serializes but never reads back, by list; evaluation derives them again. -const DERIVED_FIELDS: [(&str, &str); 2] = [ - ("models", "empty_input_only_test_count"), - ("sql_tests", "empty_input_only"), -]; - -fn stored_request(request: &EvaluateRequest) -> Result { - let mut value: serde_json::Value = - serde_json::to_value(request).map_err(|error| error.to_string())?; - for (list, field) in DERIVED_FIELDS { - if let Some(items) = value - .get_mut(list) - .and_then(serde_json::Value::as_array_mut) - { - for item in items - .iter_mut() - .filter_map(serde_json::Value::as_object_mut) - { - item.remove(field); - } - } +fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { + sections + .iter() + .find(|(section, _)| section == name) + .map(|(_, bytes)| bytes.as_slice()) +} + +fn json(value: &T) -> Result, String> { + serde_json::to_vec(value).map_err(|error| error.to_string()) +} + +impl ImageGate { + /// The gate stored in `sections`, or `None` when it is unreadable or of another format. + pub(crate) fn decode(sections: &ImageSections) -> Option { + let gate: Self = serde_json::from_slice(section(sections, GATE_SECTION)?).ok()?; + (gate.format == IMAGE_FORMAT).then_some(gate) } - Ok(value) } impl ProjectImage { @@ -153,48 +248,106 @@ impl ProjectImage { } /// The image as the sections of its file. - pub(crate) fn encode(&self) -> Result { - let mut sections: ImageSections = vec![( - FACTS_SECTION.to_owned(), - serde_json::to_vec(self).map_err(|error| error.to_string())?, - )]; - if let Some(rules) = &self.rules { - let stored: StoredRules = StoredRules { - request: stored_request(&rules.request)?, - digests: rules.digests.clone(), - }; - sections.push(( - RULES_SECTION.to_owned(), - serde_json::to_vec(&stored).map_err(|error| error.to_string())?, - )); + pub(crate) fn encode(self) -> Result { + let gate: ImageGate = ImageGate { + format: self.format, + json_output: self.json_output, + has_errors: self.diagnostics.iter().any(|diagnostic| diagnostic.error), + has_model_subjects: self.model_subjects.is_some(), + models: self.models, + tests: self.tests, + fact_digests: self.fact_digests, + header_keys: self.header_keys, + }; + let relations: ImageRelations = self.relations; + let types: &Shapes = &relations.types; + let facts: StoredFacts = StoredFacts { + usages: self.usages, + complete: differing(&relations.complete, types), + schemas: differing(&relations.schemas, types), + model_shapes: differing(&self.model_shapes, types), + types: relations.types, + nullability: relations.nullability, + dependency_ordered: relations.dependency_ordered, + cyclic: relations.cyclic, + legacy_models: relations.legacy_models, + model_subjects: self.model_subjects, + diagnostics: self.diagnostics, + text_models: self.text_models, + environment_names: self.environment_names, + lint_model_keys: self.lint_model_keys, + lint_file_keys: self.lint_file_keys, + }; + let rules: Option = self.rules; + let ((gate, facts), rules) = rayon::join( + || (json(&gate), json(&facts)), + || { + rules + .map(|rules| -> Result<(Vec, Vec), String> { + Ok(( + without_derived_fields(json(&rules.request)?), + json(&rules.digests)?, + )) + }) + .transpose() + }, + ); + let mut sections: ImageSections = vec![ + (GATE_SECTION.to_owned(), gate?), + (FACTS_SECTION.to_owned(), facts?), + ]; + if let Some((request, digests)) = rules? { + sections.push((RULES_SECTION.to_owned(), request)); + sections.push((RULE_DIGESTS_SECTION.to_owned(), digests)); } Ok(sections) } - /// The image stored in `sections`, or `None` when it is unreadable or of another format. - pub(crate) fn decode(sections: &ImageSections) -> Option { - let section = |name: &str| { - sections - .iter() - .find(|(section, _)| section == name) - .map(|(_, bytes)| bytes.as_slice()) + /// The image stored in `sections` behind `gate`, or `None` when it is unreadable. + pub(crate) fn decode(sections: &ImageSections, gate: ImageGate) -> Option { + let (facts, rules) = rayon::join( + || serde_json::from_slice::(section(sections, FACTS_SECTION)?).ok(), + || -> Option> { + let Some(bytes) = section(sections, RULES_SECTION) else { + return Some(None); + }; + let request: EvaluateRequest = serde_json::from_slice(bytes).ok()?; + let digests: Vec = + serde_json::from_slice(section(sections, RULE_DIGESTS_SECTION)?).ok()?; + (digests.len() == request.models.len()) + .then_some(Some(RetainedRows { request, digests })) + }, + ); + let facts: StoredFacts = facts?; + let types: Shapes = facts.types; + let relations: ImageRelations = ImageRelations { + complete: restored(facts.complete, &types)?, + schemas: restored(facts.schemas, &types)?, + nullability: facts.nullability, + dependency_ordered: facts.dependency_ordered, + cyclic: facts.cyclic, + legacy_models: facts.legacy_models, + types: Shapes::new(), }; - let mut image: Self = serde_json::from_slice(section(FACTS_SECTION)?).ok()?; - if image.format != IMAGE_FORMAT { - return None; - } - if let Some(bytes) = section(RULES_SECTION) { - let stored: StoredRules = serde_json::from_slice(bytes).ok()?; - let request: EvaluateRequest = serde_json::from_value(stored.request).ok()?; - if stored.digests.len() != request.models.len() { - return None; - } - image.rules = Some(RetainedRows { - request, - digests: stored.digests, - }); - } - Some(image) + let model_shapes: Shapes = restored(facts.model_shapes, &types)?; + Some(Self { + format: gate.format, + json_output: gate.json_output, + models: gate.models, + usages: facts.usages, + tests: gate.tests, + relations: ImageRelations { types, ..relations }, + fact_digests: gate.fact_digests, + model_subjects: facts.model_subjects, + diagnostics: facts.diagnostics, + text_models: facts.text_models, + environment_names: facts.environment_names, + model_shapes, + lint_model_keys: facts.lint_model_keys, + lint_file_keys: facts.lint_file_keys, + header_keys: gate.header_keys, + rules: rules?, + }) } /// Every model whose reads include a relation of `names`, outside `names`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs index a755e4c7f1..75a1d507e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -142,6 +142,7 @@ pub(crate) fn captured_diagnostic(diagnostic: &Bound<'_, PyAny>) -> PyResult( driver.graph((&discovered, &adapter), &lint_dialect)?; let project: Bound<'py, PyAny> = native_field(&graph, "project")?; let graph_failed: bool = any_error(&native_field(&project, "diagnostics")?)?; + crate::bindings::_helpers::compile_delta::context::stop_on_errors(graph_failed)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { if let Some(early_rules) = early_rules { @@ -219,6 +220,10 @@ fn analyzed<'py>( } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, &dialect)?; + crate::bindings::_helpers::compile_delta::context::stop_on_errors(any_error(&native_field( + &contracts.result, + "diagnostics", + )?)?)?; let selected_keys: Bound<'py, PyAny> = select_project_graph(&graph, &driver.select, &driver.exclude)?.into_any(); let core_diagnostics: Bound<'py, PyTuple> = PyTuple::new( From ced2ecf0fbc88f81db8495379a96a28a6c37e4fc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 23:21:26 +0100 Subject: [PATCH 06/32] perf(compile): stop copying the rules request and walking every lint file in delta compiles The rules engine now retains a compile's request as JSON, encoded in place without cloning its models, so full compiles pay less for their image and the image stores those bytes as they are. A delta compile hands its decoded request to the rules round instead of cloning it, lints its changed files without walking and canonicalising every SQL file, encodes the patched image while it splices the report, reads the stored report only in JSON mode, and skips the macro call store's project fingerprint. --- .../_helpers/compile_delta/context.rs | 6 +- .../bindings/_helpers/compile_delta/driver.rs | 69 +++++++++----- .../bindings/_helpers/compile_delta/image.rs | 94 +++++++++---------- .../_helpers/compile_delta/lint_hooks.rs | 32 ++++--- .../_helpers/compile_delta/rules_rows.rs | 18 ++-- .../_helpers/render/macro_call_memo.rs | 5 + .../src/engine/_helpers/memo.rs | 85 +++++++++++++---- .../src/engine/tests/test_retained_rows.rs | 19 ++-- crates/sqlbuild-rules/src/models.rs | 7 +- src/sqlbuild/_native.pyi | 1 + .../macro_bridge/classes/macro_bridge.py | 2 +- 11 files changed, 211 insertions(+), 127 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 1f055a6cfa..3ddf02ee47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -10,6 +10,8 @@ use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, Sess use sqlbuild_cache::project_reuse::models::ImageSections; use sqlbuild_rules::models::RetainedRows; +use crate::bindings::_helpers::compile_delta::image::BaseRules; + /// The facts a full compile leaves for its project image beyond its analysis record. #[derive(Debug, Default)] pub(crate) struct ImageCapture { @@ -101,8 +103,8 @@ pub(crate) struct DeltaInjection { pub(crate) model_names: Vec, /// The whole project's relation facts, injected for relations the delta does not analyse. pub(crate) relations: RelationTables, - /// The whole project's built-in rules request. - pub(crate) rules: Option, + /// The whole project's built-in rules request, taken by the round that evaluates it. + pub(crate) rules: Mutex>, /// The whole project's custom-rule fact digests. pub(crate) fact_digests: BTreeMap, String>, /// Each project model's `(path, name, materialized)`, the custom-rule subjects. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index ee7fb0a146..8c2237c11a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -20,7 +20,9 @@ use crate::bindings::_helpers::compile_delta::context::{ use crate::bindings::_helpers::compile_delta::header_keys::{ HeaderChange, custom_rule_reads, header_change, }; -use crate::bindings::_helpers::compile_delta::image::{ImageGate, ImageRelations, ProjectImage}; +use crate::bindings::_helpers::compile_delta::image::{ + BaseRules, ImageGate, ImageRelations, ImageRules, ProjectImage, +}; use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; use crate::bindings::_helpers::compile_driver::command::{PhasesOutcome, unrecorded_compile}; use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; @@ -223,7 +225,10 @@ fn lent_injection(image: &mut ProjectImage, project_dir: PathBuf) -> DeltaInject .map(|model| model.name.clone()) .collect(), relations: std::mem::take(&mut image.relations).into(), - rules: image.rules.take(), + rules: std::sync::Mutex::new(match image.rules.take() { + Some(ImageRules::Decoded(rules)) => Some(rules), + Some(ImageRules::Retained(_)) | None => None, + }), fact_digests: std::mem::take(&mut image.fact_digests) .into_iter() .collect(), @@ -239,7 +244,6 @@ fn lent_injection(image: &mut ProjectImage, project_dir: PathBuf) -> DeltaInject /// Return the facts `lent_injection` moved out of `image`. fn returned(image: &mut ProjectImage, injection: DeltaInjection) { image.relations = injection.relations.into(); - image.rules = injection.rules; image.fact_digests = injection.fact_digests.into_iter().collect(); image.model_subjects = injection.model_subjects; image.model_shapes = injection.model_shapes; @@ -479,10 +483,19 @@ pub(crate) fn run<'py>( .into_iter() .filter(|name| !rerun.models.contains(name)) .collect(); - last = Some(ran); if added.is_empty() { + last = Some(ran); break; } + let rules: Option = ran + .capture + .rules + .as_ref() + .and_then(|rules| BaseRules::decoded(&rules.retained)); + let Some(rules) = rules else { + return fallback("the delta compile retained no rules request"); + }; + injection.rules = std::sync::Mutex::new(Some(rules)); for name in added { if let Some(model) = stored_models.get(name.as_str()) { rerun.model_paths.insert(model.relative_path.clone()); @@ -624,29 +637,13 @@ fn finished<'py>( return fallback("the delta compile recorded no report"); } let delta_stdout: String = recorded.extract()?; - let Some(stored_output) = read_delta_output(&base.store_path, &base.stored) else { - return fallback("the stored report changed"); - }; if env_enabled(DELTA_VERIFY_ENV_VAR) { let project_dir: PathBuf = native_field(&ran.request, "project_dir")?.extract()?; let directory: PathBuf = project_dir.join(VERIFY_REPORT_DIRECTORY); let _ = std::fs::create_dir_all(&directory); let _ = std::fs::write(directory.join(VERIFY_REPORT_FILE), &delta_stdout); } - let stdout: String = if base.image.json_output { - let scope: SpliceScope<'_> = SpliceScope { - models: &rerun.models, - tests: &rerun.tests, - }; - match spliced_report(&stored_output.stdout, &delta_stdout, &scope) { - Ok(stdout) => stdout, - Err(reason) => return fallback(reason), - } - } else { - delta_stdout - }; let stderr_lines: Vec> = ran.output.getattr("stderr_lines")?.extract()?; - let environment_names: Vec = base.image.environment_names.clone(); let merge: RecordMerge = RecordMerge { kept: base .stored @@ -655,9 +652,33 @@ fn finished<'py>( .iter() .map(|stamp| (stamp.relative_path.clone(), stamp.size, stamp.mtime_ns)) .collect(), - environment_names, + environment_names: base.image.environment_names.clone(), + }; + let json_output: bool = base.image.json_output; + let (store_path, stored, image) = (base.store_path, base.stored, base.image); + let capture: ImageCapture = ran.capture; + let (image, stdout): (Option, Result) = py.detach(|| { + rayon::join( + || patched_image(image, rerun, capture).encode().ok(), + || { + if !json_output { + return Ok(delta_stdout); + } + let Some(stored_output) = read_delta_output(&store_path, &stored) else { + return Err("the stored report changed".to_owned()); + }; + let scope: SpliceScope<'_> = SpliceScope { + models: &rerun.models, + tests: &rerun.tests, + }; + spliced_report(&stored_output.stdout, &delta_stdout, &scope) + }, + ) + }); + let stdout: String = match stdout { + Ok(stdout) => stdout, + Err(reason) => return fallback(reason), }; - let image: Option = patched_image(base.image, rerun, ran.capture).encode().ok(); let _ = py; Ok(DeltaOutcome::Finished(Box::new(Finished { request: ran.request, @@ -717,7 +738,9 @@ fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> Pr if capture.text_models.is_some() { image.text_models = capture.text_models; } - image.rules = capture.rules.map(|rules| rules.retained); + image.rules = capture + .rules + .map(|rules| ImageRules::Retained(rules.retained)); if let Some(shapes) = capture.model_shapes { image.model_shapes = shapes; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 049adc3b3e..1396f3ec22 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -38,7 +38,33 @@ pub(crate) struct ProjectImage { pub(crate) lint_model_keys: Option>, pub(crate) lint_file_keys: Option)>>, pub(crate) header_keys: BTreeMap, - pub(crate) rules: Option, + pub(crate) rules: Option, +} + +/// A stored compile's built-in rules request, as decoded for a delta compile. +#[derive(Clone, Debug)] +pub(crate) struct BaseRules { + pub(crate) request: EvaluateRequest, + /// Each model's payload digest, which keys its cached findings, in model order. + pub(crate) digests: Vec, +} + +impl BaseRules { + /// The rows a compile retained, decoded. + pub(crate) fn decoded(retained: &RetainedRows) -> Option { + let request: EvaluateRequest = serde_json::from_slice(&retained.request_json).ok()?; + (request.models.len() == retained.digests.len()).then(|| Self { + request, + digests: retained.digests.clone(), + }) + } +} + +/// An image's rules request: decoded from a stored image, or as a compile just retained it. +#[derive(Clone, Debug)] +pub(crate) enum ImageRules { + Decoded(BaseRules), + Retained(RetainedRows), } /// The analysis session's final relation facts, as stored. @@ -81,40 +107,6 @@ impl From for RelationTables { } } -/// The prefix the keys of fields a rules request serializes but never reads back share; -/// evaluation derives them again. -const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; -const DERIVED_FIELDS: [&str; 2] = ["_test_count\":", "\":"]; - -/// `json` without the derived fields, each a bare number or boolean after its key. -/// -/// Compact JSON escapes every quote inside strings, so these keys only match as keys. -fn without_derived_fields(json: Vec) -> Vec { - let Ok(text) = std::str::from_utf8(&json) else { - return json; - }; - let mut kept: Vec = Vec::with_capacity(json.len()); - let mut copied: usize = 0; - for (start, _) in text.match_indices(DERIVED_FIELD_PREFIX) { - if start < copied { - continue; - } - let rest: &str = &text[start + DERIVED_FIELD_PREFIX.len()..]; - let Some(field) = DERIVED_FIELDS - .iter() - .find(|field| rest.starts_with(**field)) - else { - continue; - }; - let value: &str = &rest[field.len()..]; - let length: usize = value.find([',', '}']).unwrap_or(value.len()); - kept.extend_from_slice(&json[copied..start]); - copied = start + DERIVED_FIELD_PREFIX.len() + field.len() + length; - } - kept.extend_from_slice(&json[copied..]); - kept -} - /// Rows of `table` that differ from the same relation's `types` row; `None` where equal. fn differing(table: &Shapes, types: &Shapes) -> Vec<(String, Option>)> { let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types @@ -243,7 +235,9 @@ impl ProjectImage { lint_model_keys: capture.lint_model_keys, lint_file_keys: capture.lint_file_keys, header_keys: capture.header_keys, - rules: capture.rules.map(|rules| rules.retained), + rules: capture + .rules + .map(|rules| ImageRules::Retained(rules.retained)), }) } @@ -278,20 +272,16 @@ impl ProjectImage { lint_model_keys: self.lint_model_keys, lint_file_keys: self.lint_file_keys, }; - let rules: Option = self.rules; - let ((gate, facts), rules) = rayon::join( - || (json(&gate), json(&facts)), - || { - rules - .map(|rules| -> Result<(Vec, Vec), String> { - Ok(( - without_derived_fields(json(&rules.request)?), - json(&rules.digests)?, - )) - }) - .transpose() - }, - ); + let rules: Option = match self.rules { + Some(ImageRules::Retained(rules)) => Some(rules), + Some(ImageRules::Decoded(_)) | None => None, + }; + let (gate, facts) = rayon::join(|| json(&gate), || json(&facts)); + let rules = rules + .map(|rules| -> Result<(Vec, Vec), String> { + Ok((rules.request_json, json(&rules.digests)?)) + }) + .transpose(); let mut sections: ImageSections = vec![ (GATE_SECTION.to_owned(), gate?), (FACTS_SECTION.to_owned(), facts?), @@ -307,7 +297,7 @@ impl ProjectImage { pub(crate) fn decode(sections: &ImageSections, gate: ImageGate) -> Option { let (facts, rules) = rayon::join( || serde_json::from_slice::(section(sections, FACTS_SECTION)?).ok(), - || -> Option> { + || -> Option> { let Some(bytes) = section(sections, RULES_SECTION) else { return Some(None); }; @@ -315,7 +305,7 @@ impl ProjectImage { let digests: Vec = serde_json::from_slice(section(sections, RULE_DIGESTS_SECTION)?).ok()?; (digests.len() == request.models.len()) - .then_some(Some(RetainedRows { request, digests })) + .then_some(Some(ImageRules::Decoded(BaseRules { request, digests }))) }, ); let facts: StoredFacts = facts?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs index 7962353b53..08c0c46d78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs @@ -1,6 +1,6 @@ //! The SQL lint keys a full compile leaves for its image, and a delta compile's use of them. -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use pyo3::PyResult; @@ -8,7 +8,9 @@ use pyo3::PyResult; use crate::bindings::_helpers::compile_delta::context::{ capture, capturing, delta_unsupported, injection, }; -use crate::bindings::_helpers::rules::lint_pipeline::{LintFile, lint_sources}; +use sqlbuild_core::format_command::constants::LINT_DIRECTORY_NAMES; + +use crate::bindings::_helpers::rules::lint_pipeline::LintFile; /// Each model file's lint cache key: recorded for the image, and in a delta compile the whole /// project's keys with the recompiled models' keys fresh. @@ -42,20 +44,26 @@ pub(crate) fn model_identities(fresh: Vec<(String, String)>) -> PyResult PyResult> { let Some(injection) = injection() else { return Ok(Vec::new()); }; - let selected: BTreeSet = injection - .changed_paths - .iter() - .map(|path| { - let path: PathBuf = project_dir.join(path); - std::fs::canonicalize(&path).unwrap_or(path) - }) - .collect(); - lint_sources(project_dir, Some(&selected), None) + let mut files: Vec = Vec::new(); + for path in &injection.changed_paths { + let linted: bool = path + .split_once('/') + .is_some_and(|(root, _)| LINT_DIRECTORY_NAMES.contains(&root)); + if !linted { + continue; + } + let path: PathBuf = project_dir.join(path); + let contents: String = std::fs::read_to_string(&path) + .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.to_string()))?; + files.push(LintFile { path, contents }); + } + Ok(files) } /// Each non-model file's lint cache key, in walk order: recorded for the image, and in a delta diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs index c746b7cd7f..26787e6bd7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs @@ -3,9 +3,10 @@ use std::collections::HashMap; use pyo3::PyResult; -use sqlbuild_rules::models::{EvaluateRequest, Model, RetainedRows, SqlTestFact}; +use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact}; use crate::bindings::_helpers::compile_delta::context::{delta_unsupported, injection}; +use crate::bindings::_helpers::compile_delta::image::BaseRules; /// `request` unchanged outside a delta compile; inside one, the whole project's request with /// the rerun models' rows and the rerun SQL tests' facts taken from `request`. @@ -13,7 +14,12 @@ pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult = injection + .rules + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let Some(base) = taken else { return Err(delta_unsupported( "the previous compile did not run the built-in rules", )); @@ -44,10 +50,10 @@ pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult EvaluateRequest { - let mut request: EvaluateRequest = base.request.clone(); - for (model, digest) in request.models.iter_mut().zip(&base.digests) { - model.payload_digest = Some(digest.clone()); +fn restored(base: BaseRules) -> EvaluateRequest { + let mut request: EvaluateRequest = base.request; + for (model, digest) in request.models.iter_mut().zip(base.digests) { + model.payload_digest = Some(digest); model.stripped = true; } request diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs index a3501dd454..33c0d0fcb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -81,6 +81,11 @@ impl MacroCallMemo { Ok(()) } + /// Whether this compile may use the call store; a delta compile leaves it to full compiles. + fn store_allowed(&self) -> bool { + crate::bindings::_helpers::compile_delta::context::injection().is_none() + } + /// Whether this compile recorded calls the attached store does not hold yet. fn store_needs_save(&self) -> PyResult { Ok(self diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index 7e1e010df4..dd17474675 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -61,7 +61,12 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result = retain.then(|| retained_rows(&request, &models)); + let mut models: Vec = models; + let retained: Option = if retain { + retained_rows(&mut request, &mut models) + } else { + None + }; request.models = models; Ok(PreparedRows { request, @@ -71,30 +76,72 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result RetainedRows { - let mut kept: EvaluateRequest = EvaluateRequest { - models: Vec::new(), - ..request.clone() - }; - let mut digests: Vec = Vec::with_capacity(models.len()); - kept.models = models +/// `request` with `models` as JSON, each model's SQL text and digest dropped, and each model's +/// payload digest; the models are put back as they were. `None` when it cannot be encoded. +fn retained_rows(request: &mut EvaluateRequest, models: &mut Vec) -> Option { + let digests: Vec = models .iter() + .map(|model| model.payload_digest.clone().unwrap_or_default()) + .collect(); + let kept: Vec<(String, String, Option, bool)> = models + .iter_mut() .map(|model| { - digests.push(model.payload_digest.clone().unwrap_or_default()); - Model { - query_sql: String::new(), - authored_sql: String::new(), - payload_digest: None, - stripped: true, - ..model.clone() - } + ( + std::mem::take(&mut model.query_sql), + std::mem::take(&mut model.authored_sql), + model.payload_digest.take(), + std::mem::replace(&mut model.stripped, true), + ) }) .collect(); - RetainedRows { - request: kept, + request.models = std::mem::take(models); + let json: Option> = serde_json::to_vec(&*request).ok(); + *models = std::mem::take(&mut request.models); + for (model, (query_sql, authored_sql, payload_digest, stripped)) in models.iter_mut().zip(kept) + { + model.query_sql = query_sql; + model.authored_sql = authored_sql; + model.payload_digest = payload_digest; + model.stripped = stripped; + } + Some(RetainedRows { + request_json: without_derived_fields(json?), digests, + }) +} + +/// The prefix the keys of fields a request serializes but never reads back share; evaluation +/// derives them again. +const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; +const DERIVED_FIELDS: [&str; 2] = ["_test_count\":", "\":"]; + +/// `json` without the derived fields, each a bare number or boolean after its key. +/// +/// Compact JSON escapes every quote inside strings, so these keys only match as keys. +fn without_derived_fields(json: Vec) -> Vec { + let Ok(text) = std::str::from_utf8(&json) else { + return json; + }; + let mut kept: Vec = Vec::with_capacity(json.len()); + let mut copied: usize = 0; + for (start, _) in text.match_indices(DERIVED_FIELD_PREFIX) { + if start < copied { + continue; + } + let rest: &str = &text[start + DERIVED_FIELD_PREFIX.len()..]; + let Some(field) = DERIVED_FIELDS + .iter() + .find(|field| rest.starts_with(**field)) + else { + continue; + }; + let value: &str = &rest[field.len()..]; + let length: usize = value.find([',', '}']).unwrap_or(value.len()); + kept.extend_from_slice(&json[copied..start]); + copied = start + DERIVED_FIELD_PREFIX.len() + field.len() + length; } + kept.extend_from_slice(&json[copied..]); + kept } /// Answer a prepared request from the memo or evaluate it, writing the cache and memo at once. diff --git a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs index a373ac7efd..aaae4165fa 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs @@ -7,13 +7,14 @@ use crate::errors::RowsError; use crate::models::{EvaluateRequest, RetainedRows, RowsRequest}; /// The retained request with each row's digest restored, as a delta compile rebuilds it. -fn restored(retained: &RetainedRows) -> EvaluateRequest { - let mut request: EvaluateRequest = retained.request.clone(); +fn restored(retained: &RetainedRows) -> Result { + let mut request: EvaluateRequest = serde_json::from_slice(&retained.request_json) + .map_err(|error| format!("the retained request does not decode: {error}"))?; for (model, digest) in request.models.iter_mut().zip(&retained.digests) { model.payload_digest = Some(digest.clone()); model.stripped = true; } - request + Ok(request) } #[test] @@ -26,17 +27,17 @@ fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_sta request.retain = true; let first = evaluate_rows(request).map_err(|_| "first evaluation failed".to_owned())?; let retained: RetainedRows = first.retained.clone().ok_or("nothing was retained")?; - assert_eq!(retained.digests.len(), retained.request.models.len()); + let kept: EvaluateRequest = serde_json::from_slice(&retained.request_json) + .map_err(|error| format!("the retained request does not decode: {error}"))?; + assert_eq!(retained.digests.len(), kept.models.len()); assert!( - retained - .request - .models + kept.models .iter() .all(|model| model.query_sql.is_empty() && model.authored_sql.is_empty()) ); let cached = evaluate_rows(RowsRequest { - request: restored(&retained), + request: restored(&retained)?, memo: None, retain: false, }) @@ -44,7 +45,7 @@ fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_sta assert_eq!(evaluation_summary(&cached).0, evaluation_summary(&first).0); let uncached_dir = TempDir::new().map_err(|error| error.to_string())?; - let mut uncached: EvaluateRequest = restored(&retained); + let mut uncached: EvaluateRequest = restored(&retained)?; uncached.project_dir = uncached_dir.path().to_string_lossy().into_owned(); let missed = evaluate_rows(RowsRequest { request: uncached, diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index c97836c7f2..c8899aee30 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -933,10 +933,11 @@ pub struct RowsRequest { pub retain: bool, } -/// A request kept for the project image: its models' SQL text dropped, each model's digest kept. -#[derive(Clone, Debug, Default)] +/// A request kept for the project image, as the JSON an `EvaluateRequest` reads back: its +/// models' SQL text and digests dropped and each model marked stripped. +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct RetainedRows { - pub request: EvaluateRequest, + pub request_json: Vec, /// Each model's payload digest, which keys its cached findings, in model order. pub digests: Vec, } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bddfa169bb..c7bd5f8fe3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5292,6 +5292,7 @@ class MacroCallMemo: def stats(self) -> tuple[int, int, int]: ... def attach_store(self, path: str, environment: str) -> tuple[bytes, int]: ... def discard_store(self) -> None: ... + def store_allowed(self) -> bool: ... def store_needs_save(self) -> bool: ... def store_stats(self) -> tuple[int, int]: ... def save_store(self, path: str, metadata: bytes) -> int | None: ... diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py index 262903156b..6ff6923335 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -43,7 +43,7 @@ def __init__(self) -> None: def attach_store(self, *, cache_dir: Path, project_dir: Path, model_paths: list[str]) -> None: """Reuse call results stored by earlier compiles of the same code and project files.""" - if self._store_path is not None: + if self._store_path is not None or not self.memo.store_allowed(): return path: Path = cache_dir / MACRO_CALL_STORE_FILE_NAME metadata: bytes From 562cdceb4080bbc3069ed956578a592e4a408fbf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 23:49:46 +0100 Subject: [PATCH 07/32] chore(compile): log the files a delta compile considers changed --- .../bindings/_helpers/compile_delta/driver.rs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 8c2237c11a..4c81ab4fed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -388,6 +388,35 @@ pub(crate) fn run<'py>( Ok(found) => found, Err(reason) => return fallback(reason), }; + request + .py() + .import("logging")? + .getattr("getLogger")? + .call1(("sqlbuild.compile.delta",))? + .call_method1( + "debug", + ( + "Delta compile candidates: changed %s, added %s, removed %s", + changes + .changed + .iter() + .take(5) + .cloned() + .collect::>(), + changes + .added + .iter() + .take(5) + .cloned() + .collect::>(), + changes + .removed + .iter() + .take(5) + .cloned() + .collect::>(), + ), + )?; if !changes.added.is_empty() || !changes.removed.is_empty() { return fallback("files were added or removed"); } From 874e7939d809c5438d0e8ce112cd628797133999 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:08:43 +0100 Subject: [PATCH 08/32] feat(compile): compile edits that report or raise errors as deltas Edits that introduce or fix error diagnostics, and changed files that stop the compile with an error, now take the delta path; the verifier compares them like any other delta. --- .../src/bindings/_helpers/compile_delta/driver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 4c81ab4fed..a778eeed13 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -37,7 +37,7 @@ const ENABLED_VALUE: &str = "1"; const MAX_RERUN_MODELS: usize = 500; const MAX_RERUN_PERCENT: usize = 5; /// Whether edits that report or raise errors compile as deltas; until then they fall back. -pub(crate) const ERRORS_ON_DELTA_PATH: bool = false; +pub(crate) const ERRORS_ON_DELTA_PATH: bool = true; /// How many times a changed relation shape may pull its dependents into the delta. const MAX_ROUNDS: usize = 4; const MODELS_ROOT: &str = "models/"; From 4718eb88257a6bb7b67b3775f50b1b775c289d34 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:22:07 +0100 Subject: [PATCH 09/32] perf(compile): store the project image in the native record format The image's gate and facts are now length-prefixed native records with a string table instead of JSON, and the rules engine assembles its retained request from the row and request JSON it already serializes for its digests, so a full compile pays about 1.7% for its image instead of 3.4%. --- .../bindings/_helpers/compile_delta/image.rs | 250 ++++++++++++++++-- .../_helpers/compile_delta/image_codec.rs | 150 +++++++++++ .../bindings/_helpers/compile_delta/mod.rs | 1 + .../src/engine/_helpers/memo.rs | 169 +++++++++--- 4 files changed, 508 insertions(+), 62 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 1396f3ec22..04da1be82b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet}; -use serde::{Deserialize, Serialize}; use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_cache::project_reuse::models::ImageSections; @@ -12,9 +11,10 @@ use crate::bindings::_helpers::compile_delta::context::{ CapturedDiagnostic, CapturedModel, CapturedTest, ImageCapture, ModelShape, }; use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; +use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; /// Changes whenever the image's meaning changes, so an older image is never read. -const IMAGE_FORMAT: &str = "sqlbuild-project-image-v3"; +const IMAGE_FORMAT: &str = "sqlbuild-project-image-v4"; const GATE_SECTION: &str = "gate"; const FACTS_SECTION: &str = "facts"; const RULES_SECTION: &str = "rules"; @@ -68,7 +68,7 @@ pub(crate) enum ImageRules { } /// The analysis session's final relation facts, as stored. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct ImageRelations { pub(crate) types: Shapes, pub(crate) nullability: Shapes, @@ -145,7 +145,7 @@ fn restored(rows: Vec<(String, Option>)>, types: &Shapes) type StoredShapes = Vec<(String, Option>)>; /// What a delta compile reads first to decide, cheaply, whether it can patch the image. -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default)] pub(crate) struct ImageGate { pub(crate) format: String, pub(crate) json_output: bool, @@ -160,7 +160,6 @@ pub(crate) struct ImageGate { } /// The rest of the image's facts as stored; shapes equal to a relation's types are stored once. -#[derive(Serialize, Deserialize)] struct StoredFacts { usages: BTreeMap, types: Shapes, @@ -186,18 +185,235 @@ fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { .map(|(_, bytes)| bytes.as_slice()) } -fn json(value: &T) -> Result, String> { - serde_json::to_vec(value).map_err(|error| error.to_string()) +/// The diagnostics and report lines an image keeps as JSON, being small and varied. +type StoredMisc = ( + Vec, + Option>, + Vec, +); + +fn put_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a Shapes) { + encoder.count(shapes.len()); + for (name, shape) in shapes { + encoder.string(name); + encoder.pairs(shape); + } +} + +fn put_stored_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a StoredShapes) { + encoder.count(shapes.len()); + for (name, shape) in shapes { + encoder.string(name); + encoder.flag(shape.is_some()); + if let Some(shape) = shape { + encoder.pairs(shape); + } + } +} + +fn shapes(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| Some((decoder.string()?, decoder.pairs()?))) +} + +fn stored_shapes(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| { + let name: String = decoder.string()?; + let shape: Option> = if decoder.flag()? { + Some(decoder.pairs()?) + } else { + None + }; + Some((name, shape)) + }) } impl ImageGate { + fn encode(&self) -> Vec { + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.string(&self.format); + encoder.flag(self.json_output); + encoder.flag(self.has_errors); + encoder.flag(self.has_model_subjects); + encoder.count(self.models.len()); + for model in &self.models { + encoder.string(&model.name); + encoder.string(&model.relative_path); + encoder.pairs(&model.upstream); + } + encoder.count(self.tests.len()); + for test in &self.tests { + encoder.string(&test.key); + encoder.string(&test.relative_path); + encoder.strings(test.chain.iter().map(String::as_str)); + } + encoder.count(self.fact_digests.len()); + for (key, digest) in &self.fact_digests { + encoder.strings(key.iter().map(String::as_str)); + encoder.string(digest); + } + encoder.count(self.header_keys.len()); + for (path, keys) in &self.header_keys { + encoder.string(path); + encoder.pairs(keys); + } + encoder.finish() + } + /// The gate stored in `sections`, or `None` when it is unreadable or of another format. pub(crate) fn decode(sections: &ImageSections) -> Option { - let gate: Self = serde_json::from_slice(section(sections, GATE_SECTION)?).ok()?; - (gate.format == IMAGE_FORMAT).then_some(gate) + let mut decoder: Decoder<'_> = Decoder::new(section(sections, GATE_SECTION)?)?; + let format: String = decoder.string()?; + if format != IMAGE_FORMAT { + return None; + } + let gate: Self = Self { + format, + json_output: decoder.flag()?, + has_errors: decoder.flag()?, + has_model_subjects: decoder.flag()?, + models: decoder.list(|decoder| { + Some(CapturedModel { + name: decoder.string()?, + relative_path: decoder.string()?, + upstream: decoder.pairs()?, + }) + })?, + tests: decoder.list(|decoder| { + Some(CapturedTest { + key: decoder.string()?, + relative_path: decoder.string()?, + chain: decoder.strings()?, + }) + })?, + fact_digests: decoder.list(|decoder| Some((decoder.strings()?, decoder.string()?)))?, + header_keys: decoder + .list(|decoder| Some((decoder.string()?, decoder.pairs()?)))? + .into_iter() + .collect(), + }; + decoder.finish()?; + Some(gate) } } +impl StoredFacts { + fn encode(&self) -> Result, String> { + let misc: Vec = serde_json::to_vec(&( + &self.diagnostics, + &self.text_models, + &self.environment_names, + )) + .map_err(|error| error.to_string())?; + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.count(self.usages.len()); + for (path, digest) in &self.usages { + encoder.string(path); + encoder.string(digest); + } + put_shapes(&mut encoder, &self.types); + put_shapes(&mut encoder, &self.nullability); + put_stored_shapes(&mut encoder, &self.complete); + put_stored_shapes(&mut encoder, &self.schemas); + put_stored_shapes(&mut encoder, &self.model_shapes); + encoder.flag(self.dependency_ordered); + encoder.flag(self.cyclic); + encoder.strings(self.legacy_models.iter().map(String::as_str)); + encoder.flag(self.model_subjects.is_some()); + if let Some(subjects) = &self.model_subjects { + encoder.count(subjects.len()); + for (path, name, materialized) in subjects { + encoder.string(path); + encoder.string(name); + encoder.optional_string(materialized.as_deref()); + } + } + encoder.flag(self.lint_model_keys.is_some()); + if let Some(keys) = &self.lint_model_keys { + encoder.pairs(keys); + } + encoder.flag(self.lint_file_keys.is_some()); + if let Some(keys) = &self.lint_file_keys { + encoder.count(keys.len()); + for (path, key) in keys { + encoder.string(path); + encoder.optional_string(key.as_deref()); + } + } + encoder.bytes(&misc); + Ok(encoder.finish()) + } + + fn decode(section: &[u8]) -> Option { + let mut decoder: Decoder<'_> = Decoder::new(section)?; + let usages: BTreeMap = decoder + .list(|decoder| Some((decoder.string()?, decoder.string()?)))? + .into_iter() + .collect(); + let types: Shapes = shapes(&mut decoder)?; + let nullability: Shapes = shapes(&mut decoder)?; + let complete: StoredShapes = stored_shapes(&mut decoder)?; + let schemas: StoredShapes = stored_shapes(&mut decoder)?; + let model_shapes: StoredShapes = stored_shapes(&mut decoder)?; + let dependency_ordered: bool = decoder.flag()?; + let cyclic: bool = decoder.flag()?; + let legacy_models: Vec = decoder.strings()?; + let model_subjects = if decoder.flag()? { + Some(decoder.list(|decoder| { + Some(( + decoder.string()?, + decoder.string()?, + decoder.optional_string()?, + )) + })?) + } else { + None + }; + let lint_model_keys = if decoder.flag()? { + Some(decoder.pairs()?) + } else { + None + }; + let lint_file_keys = if decoder.flag()? { + Some(decoder.list(|decoder| Some((decoder.string()?, decoder.optional_string()?)))?) + } else { + None + }; + let (diagnostics, text_models, environment_names): StoredMisc = + serde_json::from_slice(decoder.bytes()?).ok()?; + decoder.finish()?; + Some(Self { + usages, + types, + nullability, + complete, + schemas, + dependency_ordered, + cyclic, + legacy_models, + model_subjects, + diagnostics, + text_models, + environment_names, + model_shapes, + lint_model_keys, + lint_file_keys, + }) + } +} + +fn encoded_strings(values: &[String]) -> Vec { + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.strings(values.iter().map(String::as_str)); + encoder.finish() +} + +fn decoded_strings(section: &[u8]) -> Option> { + let mut decoder: Decoder<'_> = Decoder::new(section)?; + let values: Vec = decoder.strings()?; + decoder.finish()?; + Some(values) +} + impl ProjectImage { /// The image of a finished compile's capture, or why it cannot be a delta base. pub(crate) fn from_capture(capture: ImageCapture, json_output: bool) -> Result { @@ -276,17 +492,13 @@ impl ProjectImage { Some(ImageRules::Retained(rules)) => Some(rules), Some(ImageRules::Decoded(_)) | None => None, }; - let (gate, facts) = rayon::join(|| json(&gate), || json(&facts)); - let rules = rules - .map(|rules| -> Result<(Vec, Vec), String> { - Ok((rules.request_json, json(&rules.digests)?)) - }) - .transpose(); + let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); + let rules = rules.map(|rules| (rules.request_json, encoded_strings(&rules.digests))); let mut sections: ImageSections = vec![ - (GATE_SECTION.to_owned(), gate?), + (GATE_SECTION.to_owned(), gate), (FACTS_SECTION.to_owned(), facts?), ]; - if let Some((request, digests)) = rules? { + if let Some((request, digests)) = rules { sections.push((RULES_SECTION.to_owned(), request)); sections.push((RULE_DIGESTS_SECTION.to_owned(), digests)); } @@ -296,14 +508,14 @@ impl ProjectImage { /// The image stored in `sections` behind `gate`, or `None` when it is unreadable. pub(crate) fn decode(sections: &ImageSections, gate: ImageGate) -> Option { let (facts, rules) = rayon::join( - || serde_json::from_slice::(section(sections, FACTS_SECTION)?).ok(), + || StoredFacts::decode(section(sections, FACTS_SECTION)?), || -> Option> { let Some(bytes) = section(sections, RULES_SECTION) else { return Some(None); }; let request: EvaluateRequest = serde_json::from_slice(bytes).ok()?; let digests: Vec = - serde_json::from_slice(section(sections, RULE_DIGESTS_SECTION)?).ok()?; + decoded_strings(section(sections, RULE_DIGESTS_SECTION)?)?; (digests.len() == request.models.len()) .then_some(Some(ImageRules::Decoded(BaseRules { request, digests }))) }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs new file mode 100644 index 0000000000..386af395a6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs @@ -0,0 +1,150 @@ +//! The project image's binary sections: the native store's length-prefixed records, with each +//! distinct string written once and referred to by its index. + +use std::collections::HashMap; + +use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; + +/// The most items a list reserves room for before reading them, against corrupt counts. +const MAX_RESERVED_ITEMS: usize = 1 << 16; + +/// Writes one section: its values refer to a table of the distinct strings they hold. +#[derive(Default)] +pub(crate) struct Encoder<'a> { + ids: HashMap<&'a str, u64>, + strings: Vec<&'a str>, + body: RecordWriter, +} + +impl<'a> Encoder<'a> { + pub(crate) fn string(&mut self, value: &'a str) { + let next: u64 = self.strings.len() as u64; + let id: u64 = *self.ids.entry(value).or_insert(next); + if id == next { + self.strings.push(value); + } + self.body.put_u64(id); + } + + pub(crate) fn optional_string(&mut self, value: Option<&'a str>) { + self.flag(value.is_some()); + if let Some(value) = value { + self.string(value); + } + } + + pub(crate) fn count(&mut self, count: usize) { + self.body.put_u64(count as u64); + } + + pub(crate) fn flag(&mut self, value: bool) { + self.body.put_u64(u64::from(value)); + } + + pub(crate) fn bytes(&mut self, value: &[u8]) { + self.body.put_bytes(value); + } + + pub(crate) fn strings(&mut self, values: impl ExactSizeIterator) { + self.count(values.len()); + for value in values { + self.string(value); + } + } + + pub(crate) fn pairs(&mut self, pairs: &'a [(String, String)]) { + self.count(pairs.len()); + for (left, right) in pairs { + self.string(left); + self.string(right); + } + } + + /// The section: the string table, then the values. + pub(crate) fn finish(self) -> Vec { + let mut section: RecordWriter = RecordWriter::default(); + section.put_u64(self.strings.len() as u64); + for value in &self.strings { + section.put_str(value); + } + section.put_bytes(&self.body.into_bytes()); + section.into_bytes() + } +} + +/// Reads a section an [`Encoder`] wrote; any malformed value fails the whole read. +pub(crate) struct Decoder<'a> { + strings: Vec, + body: RecordReader<'a>, +} + +impl<'a> Decoder<'a> { + pub(crate) fn new(section: &'a [u8]) -> Option { + let mut reader: RecordReader<'a> = RecordReader::new(section); + let count: usize = usize::try_from(reader.u64().ok()?).ok()?; + let mut strings: Vec = Vec::with_capacity(count.min(section.len())); + for _ in 0..count { + strings.push(reader.string().ok()?); + } + let body: &'a [u8] = reader.bytes().ok()?; + reader.finish().ok()?; + Some(Self { + strings, + body: RecordReader::new(body), + }) + } + + pub(crate) fn string(&mut self) -> Option { + let id: usize = usize::try_from(self.body.u64().ok()?).ok()?; + self.strings.get(id).cloned() + } + + pub(crate) fn optional_string(&mut self) -> Option> { + Some(if self.flag()? { + Some(self.string()?) + } else { + None + }) + } + + pub(crate) fn count(&mut self) -> Option { + usize::try_from(self.body.u64().ok()?).ok() + } + + pub(crate) fn flag(&mut self) -> Option { + match self.body.u64().ok()? { + 0 => Some(false), + 1 => Some(true), + _ => None, + } + } + + pub(crate) fn bytes(&mut self) -> Option<&'a [u8]> { + self.body.bytes().ok() + } + + pub(crate) fn list( + &mut self, + mut item: impl FnMut(&mut Self) -> Option, + ) -> Option> { + let count: usize = self.count()?; + let mut items: Vec = Vec::with_capacity(count.min(MAX_RESERVED_ITEMS)); + for _ in 0..count { + items.push(item(self)?); + } + Some(items) + } + + pub(crate) fn strings(&mut self) -> Option> { + self.list(Self::string) + } + + pub(crate) fn pairs(&mut self) -> Option> { + self.list(|decoder| Some((decoder.string()?, decoder.string()?))) + } + + /// Whether every value was read. + pub(crate) fn finish(self) -> Option<()> { + self.body.finish().ok() + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs index 092677b283..1740d34376 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod context; pub(crate) mod driver; pub(crate) mod header_keys; pub(crate) mod image; +pub(crate) mod image_codec; pub(crate) mod image_facts; pub(crate) mod lint_hooks; pub(crate) mod report_hooks; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index dd17474675..8ffaa0b75f 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -48,12 +48,22 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result = std::mem::take(&mut request.models) + let digested_rows: Vec<(Model, Option>)> = std::mem::take(&mut request.models) .into_iter() - .map(digested) + .map(|model| digested(model, retain)) .collect::>()?; + let mut row_json: Vec> = Vec::new(); + let mut models: Vec = Vec::with_capacity(digested_rows.len()); + for (model, json) in digested_rows { + models.push(model); + row_json.extend(json); + } + let mut request_json: Option> = None; let memo: Option<(String, PathBuf)> = match memo { - Some((code_identity, path)) => Some((identity(&request, &models, &code_identity)?, path)), + Some((code_identity, path)) => { + let kept: Option<&mut Option>> = retain.then_some(&mut request_json); + Some((identity(&request, &models, &code_identity, kept)?, path)) + } None => None, }; let key: Option = if keyed { @@ -61,9 +71,8 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result = models; let retained: Option = if retain { - retained_rows(&mut request, &mut models) + retained_rows(&request, &models, (request_json, row_json)) } else { None }; @@ -76,40 +85,90 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result) -> Option { - let digests: Vec = models - .iter() - .map(|model| model.payload_digest.clone().unwrap_or_default()) - .collect(); - let kept: Vec<(String, String, Option, bool)> = models - .iter_mut() - .map(|model| { - ( - std::mem::take(&mut model.query_sql), - std::mem::take(&mut model.authored_sql), - model.payload_digest.take(), - std::mem::replace(&mut model.stripped, true), - ) - }) - .collect(); - request.models = std::mem::take(models); - let json: Option> = serde_json::to_vec(&*request).ok(); - *models = std::mem::take(&mut request.models); - for (model, (query_sql, authored_sql, payload_digest, stripped)) in models.iter_mut().zip(kept) - { - model.query_sql = query_sql; - model.authored_sql = authored_sql; - model.payload_digest = payload_digest; - model.stripped = stripped; +const NO_SQL_KEYS: &str = "a retained model row has no SQL keys"; +/// Where the model-free request's JSON holds its empty model list, which the rows fill. +const EMPTY_MODELS: &[u8] = b",\"models\":[],\"sql_tests\":"; +const MODELS_KEY: &[u8] = b",\"models\":["; + +/// `request` as JSON with each model's row as `prepare` serialized it for its digest, the SQL +/// text dropped; and each model's payload digest. `None` when it cannot be assembled. +fn retained_rows( + request: &EvaluateRequest, + models: &[Model], + (request_json, row_json): (Option>, Vec>), +) -> Option { + if row_json.len() != models.len() { + return None; } + let request_json: Vec = match request_json { + Some(json) => json, + None => serde_json::to_vec(request).ok()?, + }; + let mut found = request_json + .windows(EMPTY_MODELS.len()) + .enumerate() + .filter(|(_, window)| *window == EMPTY_MODELS) + .map(|(index, _)| index); + let (Some(at), None) = (found.next(), found.next()) else { + return None; + }; + let rows: usize = row_json.iter().map(Vec::len).sum::() + row_json.len(); + let mut json: Vec = Vec::with_capacity(request_json.len() + rows); + json.extend_from_slice(&request_json[..at]); + json.extend_from_slice(MODELS_KEY); + for (index, row) in row_json.iter().enumerate() { + if index > 0 { + json.push(b','); + } + json.extend_from_slice(row); + } + json.push(b']'); + json.extend_from_slice(&request_json[at + MODELS_KEY.len() + 1..]); Some(RetainedRows { - request_json: without_derived_fields(json?), - digests, + request_json: without_derived_fields(json), + digests: models + .iter() + .map(|model| model.payload_digest.clone().unwrap_or_default()) + .collect(), }) } +/// A model row's JSON with its `query_sql` and `authored_sql` text emptied. +/// +/// They are the first string-valued keys after `name` and `relative_path`, and compact JSON +/// escapes every quote inside strings, so the first match of each key is that key. +fn without_sql(json: &[u8]) -> Option> { + let mut kept: Vec = Vec::with_capacity(json.len()); + let mut copied: usize = 0; + for key in [&b"\"query_sql\":\""[..], &b"\"authored_sql\":\""[..]] { + let start: usize = copied + find(&json[copied..], key)? + key.len(); + let end: usize = start + string_end(&json[start..])?; + kept.extend_from_slice(&json[copied..start]); + copied = end; + } + kept.extend_from_slice(&json[copied..]); + Some(kept) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +/// The offset of the quote closing the JSON string whose text `bytes` starts with. +fn string_end(bytes: &[u8]) -> Option { + let mut index: usize = 0; + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + b'"' => return Some(index), + _ => index += 1, + } + } + None +} + /// The prefix the keys of fields a request serializes but never reads back share; evaluation /// derives them again. const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; @@ -246,35 +305,59 @@ fn speculation_key( } None => { hasher.update(b"unmemoized\0"); - hasher.update(identity(request, models, "")?.as_bytes()); + hasher.update(identity(request, models, "", None)?.as_bytes()); } } Ok(hasher.finalize().to_hex().to_string()) } -/// `model` with the BLAKE3 digest of its exact facts, which keys its cached findings. -fn digested(mut model: Model) -> Result { +/// `model` with the BLAKE3 digest of its exact facts, which keys its cached findings; with +/// `retain`, also its row as JSON without its SQL text. +fn digested(mut model: Model, retain: bool) -> Result<(Model, Option>), RowsError> { if model.payload_digest.is_some() { - return Ok(model); + let json: Option> = if retain { + let json: Vec = serde_json::to_vec(&model).map_err(rules_error)?; + Some(without_sql(&json).ok_or_else(|| RowsError::Rules(NO_SQL_KEYS.to_owned()))?) + } else { + None + }; + return Ok((model, json)); } - let mut hasher: blake3::Hasher = blake3::Hasher::new(); - serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; - model.payload_digest = Some(hasher.finalize().to_hex().to_string()); - Ok(model) + if !retain { + let mut hasher: blake3::Hasher = blake3::Hasher::new(); + serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; + model.payload_digest = Some(hasher.finalize().to_hex().to_string()); + return Ok((model, None)); + } + let json: Vec = serde_json::to_vec(&model).map_err(rules_error)?; + model.payload_digest = Some(blake3::hash(&json).to_hex().to_string()); + let stripped: Vec = + without_sql(&json).ok_or_else(|| RowsError::Rules(NO_SQL_KEYS.to_owned()))?; + Ok((model, Some(stripped))) } /// The memo version, build, model-free request and every model's digest, in order. +/// +/// With `kept`, the model-free request's JSON is kept there as well. fn identity( request: &EvaluateRequest, models: &[Model], code_identity: &str, + kept: Option<&mut Option>>, ) -> Result { let mut hasher: blake3::Hasher = blake3::Hasher::new(); for part in [NATIVE_ROWS_MEMO_VERSION, code_identity] { hasher.update(part.as_bytes()); hasher.update(b"\0"); } - serde_json::to_writer(&mut hasher, request).map_err(rules_error)?; + match kept { + Some(kept) => { + let json: Vec = serde_json::to_vec(request).map_err(rules_error)?; + hasher.update(&json); + *kept = Some(json); + } + None => serde_json::to_writer(&mut hasher, request).map_err(rules_error)?, + } hasher.update(b"\0"); hasher.update(&(models.len() as u64).to_le_bytes()); for model in models { From 767cd53cfd26efaed062781f615d1913380cf94d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:33:15 +0100 Subject: [PATCH 10/32] perf(compile): discover only the sources a delta compile's models read When no stored diagnostic concerns a file outside models and SQL tests, a delta compile builds records only for the sources its rerun models read and splices their report items into the stored source list. --- .../_helpers/compile_delta/context.rs | 2 + .../bindings/_helpers/compile_delta/driver.rs | 37 ++++++++++++++++++- .../bindings/_helpers/compile_delta/splice.rs | 15 ++++++-- .../_helpers/discovery/yaml_entries.rs | 9 +++++ 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 3ddf02ee47..686a6e62d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -111,6 +111,8 @@ pub(crate) struct DeltaInjection { pub(crate) model_subjects: Option)>>, /// The files that changed since the stored compile, as project-relative POSIX paths. pub(crate) changed_paths: BTreeSet, + /// The only sources discovery keeps, those the rerun models read; `None` keeps every one. + pub(crate) source_names: Option>, /// The whole project's SQL lint cache keys, as the image recorded them. pub(crate) lint_model_keys: Option>, pub(crate) lint_file_keys: Option)>>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index a778eeed13..d46e834f1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -41,6 +41,8 @@ pub(crate) const ERRORS_ON_DELTA_PATH: bool = true; /// How many times a changed relation shape may pull its dependents into the delta. const MAX_ROUNDS: usize = 4; const MODELS_ROOT: &str = "models/"; +/// The upstream kind of a source a model reads. +const SOURCE_KIND: &str = "source"; const TESTS_ROOT: &str = "tests/"; const SQL_SUFFIX: &str = ".sql"; /// Under verification, the delta's own report is kept here for inspection. @@ -154,6 +156,8 @@ fn changes_of( #[derive(Clone, Debug, Default)] struct Rerun { changed: BTreeSet, + /// The sources the rerun models read, the only ones discovery keeps; `None` keeps all. + sources: Option>, models: BTreeSet, model_paths: BTreeSet, test_paths: BTreeSet, @@ -252,11 +256,16 @@ fn returned(image: &mut ProjectImage, injection: DeltaInjection) { } /// `injection` set up to rerun `rerun`. -fn for_rerun(mut injection: DeltaInjection, rerun: &Rerun) -> DeltaInjection { +fn for_rerun( + mut injection: DeltaInjection, + rerun: &Rerun, + sources_read: Option>, +) -> DeltaInjection { injection.model_paths = rerun.model_paths.clone(); injection.test_paths = rerun.test_paths.clone(); injection.rerun_names = rerun.names(); injection.changed_paths = rerun.changed.clone(); + injection.source_names = sources_read; injection } @@ -464,7 +473,20 @@ pub(crate) fn run<'py>( return fallback("too many models would compile again"); } rerun = rerun.with_tests(&image, &changed_tests); - let (outcome, back) = round(request, attempt, (for_rerun(injection, &rerun), &rerun))?; + let sources_read: Option> = + (!image.diagnostics.iter().any(outside_resources)).then(|| { + image + .models + .iter() + .filter(|model| rerun.models.contains(&model.name)) + .flat_map(|model| &model.upstream) + .filter(|(kind, _)| kind == SOURCE_KIND) + .map(|(_, name)| name.clone()) + .collect() + }); + rerun.sources = sources_read.clone(); + let injection_for_round: DeltaInjection = for_rerun(injection, &rerun, sources_read); + let (outcome, back) = round(request, attempt, (injection_for_round, &rerun))?; let ran: Round<'py> = match outcome { RoundOutcome::Ran(ran) => *ran, RoundOutcome::Stop(outcome) => return Ok(outcome), @@ -623,6 +645,16 @@ fn first_rerun( Ok((rerun, changed_tests)) } +/// Whether a stored diagnostic concerns a file other than a model or SQL test, which a delta +/// compile then reads in full. +fn outside_resources(diagnostic: &CapturedDiagnostic) -> bool { + diagnostic + .paths + .iter() + .any(|path| !path.starts_with(MODELS_ROOT) && !path.starts_with(TESTS_ROOT)) + || diagnostic.paths.is_empty() && diagnostic.resource_name.is_none() +} + /// Log how long the delta has run, for profiling. pub(crate) fn trace(py: Python<'_>, phase: &str, started: std::time::Instant) -> PyResult<()> { let milliseconds: u128 = started.elapsed().as_millis(); @@ -699,6 +731,7 @@ fn finished<'py>( let scope: SpliceScope<'_> = SpliceScope { models: &rerun.models, tests: &rerun.tests, + sources: rerun.sources.as_ref(), }; spliced_report(&stored_output.stdout, &delta_stdout, &scope) }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs index 82ea7a0d9c..29c9f6e5af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs @@ -119,6 +119,8 @@ fn member<'a>(found: &'a [(String, Span)], key: &str) -> Option<&'a Span> { pub(crate) struct SpliceScope<'a> { pub(crate) models: &'a BTreeSet, pub(crate) tests: &'a BTreeSet, + /// The sources the delta discovered, when it discovered only some; `None` when all. + pub(crate) sources: Option<&'a BTreeSet>, } /// The string at `path` in the item spanning `span`: the first member of each key in turn. @@ -225,14 +227,19 @@ fn spliced_resources( let mut cursor: usize = base_span.0; for (key, span) in &base_members { let delta_list: &Span = member(&delta_members, key).ok_or(key.clone())?; - let replacement: Option> = match key.as_str() { - "models" => spliced_items(base, *span, delta, *delta_list, |document, item| { + let replacement: Option> = match (key.as_str(), scope.sources) { + ("sources", Some(sources)) => { + spliced_items(base, *span, delta, *delta_list, |document, item| { + item_field(document, item, &["name"]).filter(|name| sources.contains(name)) + }) + } + ("models", _) => spliced_items(base, *span, delta, *delta_list, |document, item| { item_field(document, item, &["name"]).filter(|name| scope.models.contains(name)) }), - "tests" => spliced_items(base, *span, delta, *delta_list, |document, item| { + ("tests", _) => spliced_items(base, *span, delta, *delta_list, |document, item| { item_field(document, item, &["name"]).filter(|name| scope.tests.contains(name)) }), - "audits" => spliced_items(base, *span, delta, *delta_list, |document, item| { + ("audits", _) => spliced_items(base, *span, delta, *delta_list, |document, item| { let owner: Option = item_field(document, item, &["attached_to", "resource_type"]); let name: Option = item_field(document, item, &["attached_to", "name"]); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index b2b19a4c73..3a121e2b00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -348,8 +348,17 @@ pub(crate) fn parse_sources_document<'py>( .map_err(|error| { parse_error(py, error, SOURCE_PARSE_ERROR).unwrap_or_else(|error| error) })?; + let referenced: Option> = + crate::bindings::_helpers::compile_delta::context::injection() + .and_then(|injection| injection.source_names.clone()); let mut built: Vec> = Vec::with_capacity(sources.len()); for source in &sources { + if let Some(referenced) = &referenced + && let Ok(name) = source.name.0.extract::() + && !referenced.contains(&name) + { + continue; + } built.push(source_entry(py, &classes, source)?); } PyTuple::new(py, built) From fa61615ef066db76469af52329367f674eaff630 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:54:08 +0100 Subject: [PATCH 11/32] perf(compile): cut whole-project work from delta compiles and SQL-test rules - Index scope resources once instead of scanning them for every SQL test. - Summarise each SQL-test fixture once per evaluation and keep the summaries beside the bulk rules cache, so unchanged fixtures are not parsed again. - Drop unread source entries from loaded source YAML before building them. - Skip the rules response memo in delta compiles, whose request never repeats. - Record a delta compile's kept artifacts with the stamps the reuse check verified instead of walking the compiled tree again. - Scan string bytes a word at a time when splicing the stored report. --- .../src/project_reuse/_helpers/target.rs | 24 +- .../src/project_reuse/models.rs | 3 + .../_helpers/compile_delta/context.rs | 4 +- .../bindings/_helpers/compile_delta/driver.rs | 8 +- .../_helpers/compile_delta/report_hooks.rs | 50 ++++ .../bindings/_helpers/compile_delta/splice.rs | 44 +++- .../_helpers/compile_outputs/reuse.rs | 11 +- .../_helpers/discovery/yaml_records.rs | 11 +- .../src/bindings/_helpers/rules/rows.rs | 2 + .../src/engine/_helpers/cache.rs | 12 + .../src/engine/_helpers/evaluation.rs | 19 +- .../src/rules/_helpers/sql_test_rules.rs | 233 ++++++++++++++++-- 12 files changed, 376 insertions(+), 45 deletions(-) diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs index edfecaebac..141682526a 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs @@ -1,6 +1,6 @@ //! Stat identity of the artifacts a compile leaves in target/. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use crate::project_reuse::_helpers::files::file_digest; @@ -96,6 +96,23 @@ fn matches_recorded( } } +/// Each expected artifact's stamp: the vouched one for a kept artifact, read for a written one. +fn vouched_stamps(expected: &BTreeMap<&str, &RecordedArtifact>, vouched: &[PathStamp]) -> Stamps { + let vouched: HashMap<&str, &PathStamp> = vouched + .iter() + .map(|stamp| (stamp.relative_path.as_str(), stamp)) + .collect(); + expected + .iter() + .filter_map(|(path, artifact)| match artifact { + RecordedArtifact::Kept { .. } => vouched + .get(path) + .map(|stamp| ((*path).to_owned(), (*stamp).clone())), + RecordedArtifact::Written { .. } => keyed_stamp(path), + }) + .collect() +} + /// Stamp this compile's artifacts, or `None` when any changed since it wrote them. pub(crate) fn verified_target_files( project_dir: &Path, @@ -113,7 +130,10 @@ pub(crate) fn verified_target_files( .map(|(path, artifact)| (path.as_str(), artifact)) .collect(); let paths: Vec<&str> = expected.keys().copied().collect(); - let stamps: Stamps = target_stamps(project_dir, &paths, written, rules); + let stamps: Stamps = match &record.vouched_target_stamps { + Some(vouched) => vouched_stamps(&expected, vouched), + None => target_stamps(project_dir, &paths, written, rules), + }; let keys_match: bool = stamps.len() == expected.len() && stamps .keys() diff --git a/crates/sqlbuild-cache/src/project_reuse/models.rs b/crates/sqlbuild-cache/src/project_reuse/models.rs index 96188c6cd4..e32a91d068 100644 --- a/crates/sqlbuild-cache/src/project_reuse/models.rs +++ b/crates/sqlbuild-cache/src/project_reuse/models.rs @@ -170,6 +170,9 @@ pub struct CompileRecord { pub json_output: bool, /// The project image to keep beside the stored compile, when one was built. pub image: Option, + /// Stamps the caller verified for artifacts it kept; with them the compiled tree is not + /// walked again and only written artifacts are stamped. + pub vouched_target_stamps: Option>, } /// What one compile left at one artifact path. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 686a6e62d1..9e50a9ea63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -262,8 +262,8 @@ pub(crate) fn inject_relations(request: &mut SessionRequest) -> pyo3::PyResult<( /// What a delta compile's reuse record keeps of the stored compile it patched. #[derive(Debug, Default)] pub(crate) struct RecordMerge { - /// Each stored artifact's `(path, size, mtime_ns)`, kept unless the delta wrote it. - pub(crate) kept: Vec<(String, u64, i64)>, + /// Each stored artifact's stamp, verified by the reuse check, kept unless the delta wrote it. + pub(crate) kept: Vec, /// The template environment names the stored compile read. pub(crate) environment_names: Vec, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index d46e834f1b..759b259e5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -706,13 +706,7 @@ fn finished<'py>( } let stderr_lines: Vec> = ran.output.getattr("stderr_lines")?.extract()?; let merge: RecordMerge = RecordMerge { - kept: base - .stored - .inputs - .target_files - .iter() - .map(|stamp| (stamp.relative_path.clone(), stamp.size, stamp.mtime_ns)) - .collect(), + kept: base.stored.inputs.target_files.clone(), environment_names: base.image.environment_names.clone(), }; let json_output: bool = base.image.json_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs index 75a1d507e4..eb0f2cd61f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -277,3 +277,53 @@ pub(crate) fn complete_source_texts( } Ok(()) } + +/// In a delta compile that reads only some sources, drop every other source entry from loaded +/// source YAML documents before they are built. +pub(crate) fn retain_read_sources( + outcomes: &mut [sqlbuild_discovery::yaml_files::models::YamlFileOutcome], +) { + use sqlbuild_config::models::ConfigValue; + use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; + let Some(injection) = injection() else { + return; + }; + let Some(read) = injection.source_names.as_ref() else { + return; + }; + fn text(value: &ConfigValue) -> Option<&str> { + match value { + ConfigValue::String(text) => Some(text.as_str()), + _ => None, + } + } + for outcome in outcomes { + let YamlFileOutcome::Loaded { + value: ConfigValue::Map(entries), + .. + } = outcome + else { + continue; + }; + for (key, value) in entries { + if text(key) != Some(SOURCES_KEY) { + continue; + } + let ConfigValue::List(sources) = value else { + continue; + }; + sources.retain(|source| { + let name: Option<&str> = source.as_map().and_then(|entries| { + entries + .iter() + .find(|(key, _)| text(key) == Some(NAME_KEY)) + .and_then(|(_, name)| text(name)) + }); + name.is_none_or(|name| read.contains(name)) + }); + } + } +} + +const SOURCES_KEY: &str = "sources"; +const NAME_KEY: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs index 29c9f6e5af..894796f19f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs @@ -7,6 +7,35 @@ use serde_json::Value; /// One JSON value's byte span in its document. type Span = (usize, usize); +/// Bytes per word `string_stop` checks at once. +const WORD: usize = 8; +const LOW_BITS: u64 = 0x0101_0101_0101_0101; +const HIGH_BITS: u64 = 0x8080_8080_8080_8080; + +/// Whether any byte of `word` equals `byte`. +fn has_byte(word: u64, byte: u8) -> bool { + let matched: u64 = word ^ (LOW_BITS * u64::from(byte)); + matched.wrapping_sub(LOW_BITS) & !matched & HIGH_BITS != 0 +} + +/// The first quote or backslash at or after `index`, or the text's end; words without either +/// are skipped whole. +fn string_stop(text: &[u8], mut index: usize) -> usize { + while index + WORD <= text.len() { + let mut word: [u8; WORD] = [0; WORD]; + word.copy_from_slice(&text[index..index + WORD]); + let word: u64 = u64::from_le_bytes(word); + if has_byte(word, b'"') || has_byte(word, b'\\') { + break; + } + index += WORD; + } + while index < text.len() && !matches!(text[index], b'"' | b'\\') { + index += 1; + } + index +} + /// The end of the JSON value starting at `start`, skipping strings and nested containers. fn value_end(text: &[u8], start: usize) -> Option { let mut depth: usize = 0; @@ -15,6 +44,10 @@ fn value_end(text: &[u8], start: usize) -> Option { while index < text.len() { let byte: u8 = text[index]; if in_string { + index = string_stop(text, index); + let Some(&byte) = text.get(index) else { + break; + }; match byte { b'\\' => index += 1, b'"' => { @@ -278,22 +311,23 @@ fn spliced_items( let base_items: Vec = elements(base, base_span)?; let delta_items: Vec = elements(delta, delta_span)?; let mut fresh: HashMap> = HashMap::new(); - let mut unowned_delta: BTreeSet<&str> = BTreeSet::new(); + let mut unmatched_delta: std::collections::HashSet<&str> = std::collections::HashSet::new(); for item in &delta_items { match owned(delta, *item) { Some(owner) => fresh.entry(owner).or_default().push_back(*item), None => { - unowned_delta.insert(&delta[item.0..item.1]); + unmatched_delta.insert(&delta[item.0..item.1]); } } } let mut output: String = String::new(); let mut cursor: usize = base_span.0; let mut replaced: bool = false; - let mut unowned_base: BTreeSet<&str> = BTreeSet::new(); for item in &base_items { let Some(owner) = owned(base, *item) else { - unowned_base.insert(&base[item.0..item.1]); + if !unmatched_delta.is_empty() { + unmatched_delta.remove(&base[item.0..item.1]); + } continue; }; let replacement: Span = fresh.get_mut(&owner)?.pop_front()?; @@ -305,7 +339,7 @@ fn spliced_items( if fresh.values().any(|items| !items.is_empty()) { return None; } - if !unowned_delta.is_subset(&unowned_base) { + if !unmatched_delta.is_empty() { return None; } if !replaced { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index 517232a824..7983364ffd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -216,16 +216,21 @@ impl NativeReuseAttempt { dag_path: record.dag_path, json_output: record.json_output, image: crate::bindings::_helpers::compile_delta::context::take_pending_image(), + vouched_target_stamps: None, }; if let Some(merge) = crate::bindings::_helpers::compile_delta::context::take_record_merge() { - for (path, size, mtime_ns) in merge.kept { + for stamp in &merge.kept { compile .artifacts - .entry(path) - .or_insert(RecordedArtifact::Kept { size, mtime_ns }); + .entry(stamp.relative_path.clone()) + .or_insert(RecordedArtifact::Kept { + size: stamp.size, + mtime_ns: stamp.mtime_ns, + }); } + compile.vouched_target_stamps = Some(merge.kept); compile .template_environment_names .extend(merge.environment_names); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index 6349270505..fc5063fb32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -166,9 +166,14 @@ fn load_files<'py>( let failure_kind: FailureKind = kind.failure_kind(); py.detach(|| load_yaml(&root, inner, &named, failure_kind)) }; - let mut outcomes = outcomes - .map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))? - .into_iter(); + let mut outcomes: Vec = + outcomes.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; + if matches!(kind, YamlKind::Source) { + crate::bindings::_helpers::compile_delta::report_hooks::retain_read_sources( + &mut outcomes, + ); + } + let mut outcomes = outcomes.into_iter(); for path in batch { let relative: Bound<'py, PyAny> = path_object(py, &tree.get().inner, path.clone())?.into_bound(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index dbeea55c9e..94c3848ffd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -296,9 +296,11 @@ fn built_request( request.models = built_models(&rows, &request, model_rows)?; let request: EvaluateRequest = crate::bindings::_helpers::compile_delta::rules_rows::whole_project_request(request)?; + let delta: bool = crate::bindings::_helpers::compile_delta::context::injection().is_some(); let memo: Option<(String, PathBuf)> = rows .memo .take() + .filter(|_| !delta) .map(|(identity, path)| (identity, PathBuf::from(path))); Ok(RowsRequest { request, diff --git a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs index 6430d93cea..670c724fc7 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs @@ -15,6 +15,18 @@ pub(crate) struct RuleCacheEntry { pub faults: Vec, } +/// Where the SQL-test fixture summaries sit, beside the bulk native rules cache. +pub(crate) fn fixture_summaries_path(project_dir: &Path, rules_cache_path: &str) -> PathBuf { + let native: PathBuf = if rules_cache_path.is_empty() { + project_dir.join("target/rules-cache/bulk/native.json") + } else { + PathBuf::from(rules_cache_path) + }; + native.with_file_name(FIXTURE_SUMMARIES_FILE) +} + +const FIXTURE_SUMMARIES_FILE: &str = "sql-test-fixtures.json"; + pub(crate) struct Cache { native_rules_path: PathBuf, } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 0355954257..871155c23d 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -22,7 +22,7 @@ use rayon::iter::IndexedParallelIterator; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; @@ -334,6 +334,23 @@ pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result, +) -> Result { + let store: Option = request.config.cache.enabled.then(|| { + crate::engine::_helpers::cache::fixture_summaries_path( + Path::new(&request.project_dir), + &request.rules_cache_path, + ) + }); + crate::rules::_helpers::sql_test_rules::with_fixture_memo(store.as_deref(), || { + let evaluated: Result = evaluated_request(request, control); + let keep: bool = evaluated.is_ok() && !control.cancelled(); + (evaluated, keep) + }) +} + +fn evaluated_request( + request: EvaluateRequest, + control: &EvaluationControl<'_>, ) -> Result { if request.version != API_VERSION { return Err(format!( diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs index 678022978a..9b2cce8828 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs @@ -10,9 +10,14 @@ use crate::rules::_helpers::evaluation::path_fault; use crate::rules::models::ProjectEvaluationRequest; use polyglot_sql::Expression; use polyglot_sql::expressions::{Literal, Select}; +use serde::{Deserialize, Serialize}; use sqlbuild_analysis::compiler::main::sql_test_fixture_facts::fixture_facts; +use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; use std::path::Path; +use std::rc::Rc; + +use crate::constants::NATIVE_BUILD_IDENTITY; const UNIT_ROOT: &str = "tests/unit"; const SCENARIO_ROOT: &str = "tests/scenarios"; @@ -23,6 +28,11 @@ use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_PATH; pub(crate) fn evaluate_project( evaluation: ProjectEvaluationRequest<'_>, ) -> Result, String> { + let resources: &[ScopeResource] = &evaluation.request.scope_index.resources; + with_scope_lookup(resources, || evaluated_project(evaluation)) +} + +fn evaluated_project(evaluation: ProjectEvaluationRequest<'_>) -> Result, String> { let mut faults: Vec = Vec::new(); let mut tests: Vec<&SqlTestFact> = evaluation.request.sql_tests.iter().collect(); tests.sort_by(|left, right| { @@ -56,7 +66,13 @@ pub(crate) fn evaluate_project( continue; } if let Err(reason) = fixture_eligibility.entry(&cte.sql).or_insert_with(|| { - fixture_query(&cte.sql, &evaluation.request.dialect).map(|_| ()) + match fixture_summary(&cte.sql, &evaluation.request.dialect) + .error + .clone() + { + Some(error) => Err(error), + None => Ok(()), + } }) { let mut fault = path_fault( rule, @@ -381,17 +397,57 @@ fn allowed_subjects(evaluation: &ProjectEvaluationRequest<'_>, test: &SqlTestFac Vec::new() } +/// Each `(is a model, name)`'s first scope resource index, for the resources it was built from. +type ScopeLookup = (usize, HashMap<(bool, String), usize>); + +thread_local! { + static SCOPE_LOOKUP: RefCell> = const { RefCell::new(None) }; +} + +/// Run `body` with the scope resources indexed by kind and name on this thread. +fn with_scope_lookup(resources: &[ScopeResource], body: impl FnOnce() -> T) -> T { + let mut index: HashMap<(bool, String), usize> = HashMap::new(); + for (position, item) in resources.iter().enumerate() { + let key: Option = match item.kind { + ResourceKind::Model => Some(true), + ResourceKind::Function => Some(false), + _ => None, + }; + if let Some(model) = key { + index.entry((model, item.name.clone())).or_insert(position); + } + } + let outer = SCOPE_LOOKUP.with(|slot| slot.borrow_mut().replace((resources.len(), index))); + let result: T = body(); + SCOPE_LOOKUP.with(|slot| *slot.borrow_mut() = outer); + result +} + fn scope_resource<'a>( evaluation: &'a ProjectEvaluationRequest<'_>, kind: ResourceKind, name: &str, ) -> Option<&'a ScopeResource> { - evaluation - .request - .scope_index - .resources - .iter() - .find(|item| same_resource_kind(&item.kind, &kind) && item.name == name) + let resources: &'a [ScopeResource] = &evaluation.request.scope_index.resources; + let model: Option = match kind { + ResourceKind::Model => Some(true), + ResourceKind::Function => Some(false), + _ => None, + }; + let indexed: Option> = SCOPE_LOOKUP.with(|slot| { + let slot = slot.borrow(); + let (count, index) = slot.as_ref()?; + if *count != resources.len() { + return None; + } + Some(model.and_then(|model| index.get(&(model, name.to_owned())).copied())) + }); + match indexed { + Some(position) => position.map(|position| &resources[position]), + None => resources + .iter() + .find(|item| same_resource_kind(&item.kind, &kind) && item.name == name), + } } fn same_resource_kind(left: &ResourceKind, right: &ResourceKind) -> bool { @@ -690,21 +746,146 @@ fn empty_relation(cte: &SqlTestCteFact, dialect: &str) -> bool { if fixture_facts(&cte.name, &cte.sql).empty_fixture_marker { return true; } - let Ok(fixture) = fixture_query(&cte.sql, dialect) else { - return false; - }; + fixture_summary(&cte.sql, dialect).empty +} + +/// Whether a parsed fixture selects no rows. +fn selects_no_rows(fixture: &ParsedFixture) -> bool { let Expression::Select(select) = &fixture.statement else { return false; }; select.with.is_none() && (limited_to_zero_rows(select) - || (filtered_to_zero_rows(select, &fixture) && !fixture.order_by_calls_function())) + || (filtered_to_zero_rows(select, fixture) && !fixture.order_by_calls_function())) +} + +/// What the SQL-test rules read of one parsed fixture. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct FixtureSummary { + /// Why the fixture does not parse, when it does not. + error: Option, + /// Whether it selects no rows. + empty: bool, + /// The model a bare row-existence assertion reads, when the fixture is one. + row_existence_target: Option, +} + +/// Fixture summaries by digest of `(dialect, SQL)`, shared by the rules of one evaluation. +#[derive(Default)] +struct FixtureMemo { + summaries: HashMap>, + /// The digests this evaluation read, which are the ones saved again. + used: std::collections::HashSet, + /// Whether any summary was computed rather than loaded. + computed: bool, +} + +thread_local! { + static FIXTURE_MEMO: RefCell> = const { RefCell::new(None) }; +} + +/// The stored fixture summaries, kept only for one native build. +#[derive(Default, Serialize, Deserialize)] +struct StoredSummaries { + build: String, + summaries: BTreeMap, +} + +/// Run `body` with each fixture summarised at most once on this thread, starting from the +/// summaries stored at `store` and saving them again when `body` says its result stands. +pub(crate) fn with_fixture_memo(store: Option<&Path>, body: impl FnOnce() -> (T, bool)) -> T { + let loaded: BTreeMap = store + .and_then(|path| std::fs::read(path).ok()) + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .filter(|stored| stored.build == NATIVE_BUILD_IDENTITY) + .map(|stored| stored.summaries) + .unwrap_or_default(); + let loaded_count: usize = loaded.len(); + let memo: FixtureMemo = FixtureMemo { + summaries: loaded + .into_iter() + .map(|(key, summary)| (key, Rc::new(summary))) + .collect(), + used: std::collections::HashSet::new(), + computed: false, + }; + let outer: Option = FIXTURE_MEMO.with(|slot| slot.borrow_mut().replace(memo)); + let (result, write): (T, bool) = body(); + let memo: Option = FIXTURE_MEMO.with(|slot| { + let mut slot = slot.borrow_mut(); + let memo = slot.take(); + *slot = outer; + memo + }); + if let (Some(path), Some(memo), true) = (store, memo, write) + && !memo.used.is_empty() + && (memo.computed || memo.used.len() != loaded_count) + { + let FixtureMemo { + summaries, used, .. + } = memo; + let stored: StoredSummaries = StoredSummaries { + build: NATIVE_BUILD_IDENTITY.to_owned(), + summaries: summaries + .into_iter() + .filter(|(key, _)| used.contains(key)) + .map(|(key, summary)| (key, summary.as_ref().clone())) + .collect(), + }; + let _ = write_summaries(path, &stored); + } + result } -fn fixture_query(sql: &str, dialect: &str) -> Result { - model_shape(sql, dialect) +fn write_summaries(path: &Path, stored: &StoredSummaries) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary = path.with_extension(format!("tmp-{}", std::process::id())); + std::fs::write(&temporary, serde_json::to_vec(stored)?)?; + std::fs::rename(&temporary, path) +} + +fn summarised(sql: &str, dialect: &str) -> FixtureSummary { + let parsed: Result = model_shape(sql, dialect) .and_then(|shape| parse_fixture(shape, dialect)) - .map_err(|error| format!("could not parse SQL-test fixture: {error}")) + .map_err(|error| format!("could not parse SQL-test fixture: {error}")); + match parsed { + Err(error) => FixtureSummary { + error: Some(error), + empty: false, + row_existence_target: None, + }, + Ok(fixture) => FixtureSummary { + error: None, + empty: selects_no_rows(&fixture), + row_existence_target: row_existence_target(&fixture), + }, + } +} + +fn fixture_summary(sql: &str, dialect: &str) -> Rc { + FIXTURE_MEMO.with(|memo| { + let mut memo = memo.borrow_mut(); + let Some(memo) = memo.as_mut() else { + return Rc::new(summarised(sql, dialect)); + }; + let mut hasher: blake3::Hasher = blake3::Hasher::new(); + hasher.update(dialect.as_bytes()); + hasher.update(b"\0"); + hasher.update(sql.as_bytes()); + let key: String = hasher.finalize().to_hex().to_string(); + if let Some(summary) = memo.summaries.get(&key) { + let summary: Rc = Rc::clone(summary); + memo.used.insert(key); + return summary; + } + let summary: Rc = Rc::new(summarised(sql, dialect)); + memo.summaries.insert(key.clone(), Rc::clone(&summary)); + memo.used.insert(key); + memo.computed = true; + summary + }) } fn limited_to_zero_rows(select: &Select) -> bool { @@ -755,18 +936,23 @@ fn numeric_literal(expression: &Expression) -> Option { } fn bare_row_existence(sql: &str, targets: &[String], dialect: &str) -> bool { - let Ok(fixture) = fixture_query(sql, dialect) else { - return false; - }; + fixture_summary(sql, dialect) + .row_existence_target + .as_ref() + .is_some_and(|target| targets.contains(target)) +} + +/// The model a fixture reads when it is a bare existence check of one model's rows. +fn row_existence_target(fixture: &ParsedFixture) -> Option { let Expression::Select(select) = &fixture.statement else { - return false; + return None; }; let [source] = select .from .as_ref() .map_or(&[][..], |from| from.expressions.as_slice()) else { - return false; + return None; }; let unfiltered = select.where_clause.is_none() && select.prewhere.is_none() @@ -775,16 +961,19 @@ fn bare_row_existence(sql: &str, targets: &[String], dialect: &str) -> bool { && select.connect.is_none() && select.lateral_views.is_empty() && select.group_by.is_none(); - select.with.is_none() + let bare: bool = select.with.is_none() && unfiltered && select.joins.is_empty() - && reference_target(source, &fixture).is_some_and(|target| targets.contains(&target)) && !fixture.projection_calls_function() && !fixture.order_by_calls_function() && select.limit.is_none() && select.offset.is_none() && select.fetch.is_none() - && fixture.query_count() == 1 + && fixture.query_count() == 1; + if !bare { + return None; + } + reference_target(source, fixture) } fn reference_target(source: &Expression, fixture: &ParsedFixture) -> Option { From 4cda946c40c3d59442d1da27e4bcc6614b5ee816 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:02:01 +0100 Subject: [PATCH 12/32] perf(compile): keep loaded source YAML for later delta compiles A delta compile stores each source YAML document it loads, keyed by the digest of its text, in a native store beside the reuse slots, and decodes it instead of parsing it when the text is unchanged. --- .../_helpers/compile_delta/context.rs | 2 + .../bindings/_helpers/compile_delta/driver.rs | 12 +- .../bindings/_helpers/compile_delta/mod.rs | 1 + .../_helpers/compile_delta/yaml_cache.rs | 266 ++++++++++++++++++ .../_helpers/discovery/yaml_records.rs | 21 +- 5 files changed, 299 insertions(+), 3 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 9e50a9ea63..878d3c6798 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -111,6 +111,8 @@ pub(crate) struct DeltaInjection { pub(crate) model_subjects: Option)>>, /// The files that changed since the stored compile, as project-relative POSIX paths. pub(crate) changed_paths: BTreeSet, + /// Where loaded YAML documents are kept for later delta compiles. + pub(crate) yaml_store: Option, /// The only sources discovery keeps, those the rerun models read; `None` keeps every one. pub(crate) source_names: Option>, /// The whole project's SQL lint cache keys, as the image recorded them. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 759b259e5a..5837e96017 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -214,8 +214,16 @@ impl Rerun { } /// The injection of the image's whole-project facts, moved out of `image` while the delta runs. -fn lent_injection(image: &mut ProjectImage, project_dir: PathBuf) -> DeltaInjection { +fn lent_injection( + image: &mut ProjectImage, + project_dir: PathBuf, + store_path: &std::path::Path, +) -> DeltaInjection { DeltaInjection { + yaml_store: store_path + .parent() + .and_then(std::path::Path::parent) + .map(crate::bindings::_helpers::compile_delta::yaml_cache::yaml_store_path), project_dir, source_paths: image .models @@ -466,7 +474,7 @@ pub(crate) fn run<'py>( }; stored.image = ImageSections::new(); trace(request.py(), "delta base read", started)?; - let mut injection: DeltaInjection = lent_injection(&mut image, project_dir); + let mut injection: DeltaInjection = lent_injection(&mut image, project_dir, &store_path); let mut last: Option> = None; for _ in 0..MAX_ROUNDS { if rerun.models.len() > limit { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs index 1740d34376..1e2e6a90b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs @@ -9,3 +9,4 @@ pub(crate) mod report_hooks; pub(crate) mod rules_rows; pub(crate) mod splice; pub(crate) mod verify; +pub(crate) mod yaml_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs new file mode 100644 index 0000000000..70da741d8a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs @@ -0,0 +1,266 @@ +//! Loaded YAML documents a delta compile keeps by content digest, so the unchanged YAML it +//! reads again is decoded rather than parsed. + +use std::path::PathBuf; + +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_cache::digest::types::ContentDigest; +use sqlbuild_cache::store::models::NativeStore; +use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use sqlbuild_core::text::main::decode_text::decode_text; +use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; + +use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; + +/// The native store kind holding loaded YAML documents. +pub(crate) const YAML_STORE_KIND: &str = "yaml-documents"; +/// The store file beside the compile reuse slots. +pub(crate) const YAML_STORE_FILE_NAME: &str = "yaml-documents.bin"; +/// Bumped whenever the stored encoding changes; older stores open empty. +pub(crate) const YAML_STORE_ENVIRONMENT: &str = "yaml-documents-v1"; + +/// Where a delta compile keeps loaded YAML documents. +pub(crate) fn yaml_store_path(cache_dir: &std::path::Path) -> PathBuf { + cache_dir.join(YAML_STORE_FILE_NAME) +} + +/// The store key of a document's text. +pub(crate) fn document_key(contents: &str) -> ContentDigest { + bytes_digest(contents.as_bytes()) +} + +const NULL: usize = 0; +const BOOL: usize = 1; +const INTEGER: usize = 2; +const BIG_INTEGER: usize = 3; +const FLOAT: usize = 4; +const STRING: usize = 5; +const DATE: usize = 6; +const DATE_TIME: usize = 7; +const TIME: usize = 8; +const LIST: usize = 9; +const MAP: usize = 10; +const NO_OFFSET: usize = 0; +const OFFSET: usize = 1; + +fn put_date(encoder: &mut Encoder<'_>, date: &ConfigDate) { + encoder.count(usize::from(date.year)); + encoder.count(usize::from(date.month)); + encoder.count(usize::from(date.day)); +} + +fn put_time(encoder: &mut Encoder<'_>, time: &ConfigTime) { + encoder.count(usize::from(time.hour)); + encoder.count(usize::from(time.minute)); + encoder.count(usize::from(time.second)); + encoder.bytes(&time.microsecond.to_le_bytes()); +} + +fn put_value<'a>(encoder: &mut Encoder<'a>, value: &'a ConfigValue) { + match value { + ConfigValue::Null => encoder.count(NULL), + ConfigValue::Bool(flag) => { + encoder.count(BOOL); + encoder.flag(*flag); + } + ConfigValue::Integer(number) => { + encoder.count(INTEGER); + encoder.bytes(&number.to_le_bytes()); + } + ConfigValue::BigInteger(digits) => { + encoder.count(BIG_INTEGER); + encoder.string(digits); + } + ConfigValue::Float(number) => { + encoder.count(FLOAT); + encoder.bytes(&number.to_bits().to_le_bytes()); + } + ConfigValue::String(text) => { + encoder.count(STRING); + encoder.string(text); + } + ConfigValue::Date(date) => { + encoder.count(DATE); + put_date(encoder, date); + } + ConfigValue::DateTime(moment) => { + encoder.count(DATE_TIME); + put_date(encoder, &moment.date); + put_time(encoder, &moment.time); + match moment.utc_offset_seconds { + Some(offset) => { + encoder.count(OFFSET); + encoder.bytes(&offset.to_le_bytes()); + } + None => encoder.count(NO_OFFSET), + } + } + ConfigValue::Time(time) => { + encoder.count(TIME); + put_time(encoder, time); + } + ConfigValue::List(items) => { + encoder.count(LIST); + encoder.count(items.len()); + for item in items { + put_value(encoder, item); + } + } + ConfigValue::Map(entries) => { + encoder.count(MAP); + encoder.count(entries.len()); + for (key, item) in entries { + put_value(encoder, key); + put_value(encoder, item); + } + } + } +} + +/// A loaded document as stored. +pub(crate) fn encoded_document(value: &ConfigValue) -> Vec { + let mut encoder: Encoder<'_> = Encoder::default(); + put_value(&mut encoder, value); + encoder.finish() +} + +fn fixed(decoder: &mut Decoder<'_>) -> Option<[u8; N]> { + decoder.bytes()?.try_into().ok() +} + +fn small>(decoder: &mut Decoder<'_>) -> Option { + T::try_from(decoder.count()?).ok() +} + +fn date(decoder: &mut Decoder<'_>) -> Option { + Some(ConfigDate { + year: small(decoder)?, + month: small(decoder)?, + day: small(decoder)?, + }) +} + +fn time(decoder: &mut Decoder<'_>) -> Option { + Some(ConfigTime { + hour: small(decoder)?, + minute: small(decoder)?, + second: small(decoder)?, + microsecond: u32::from_le_bytes(fixed(decoder)?), + }) +} + +fn value(decoder: &mut Decoder<'_>) -> Option { + Some(match decoder.count()? { + NULL => ConfigValue::Null, + BOOL => ConfigValue::Bool(decoder.flag()?), + INTEGER => ConfigValue::Integer(i64::from_le_bytes(fixed(decoder)?)), + BIG_INTEGER => ConfigValue::BigInteger(decoder.string()?), + FLOAT => ConfigValue::Float(f64::from_bits(u64::from_le_bytes(fixed(decoder)?))), + STRING => ConfigValue::String(decoder.string()?), + DATE => ConfigValue::Date(date(decoder)?), + DATE_TIME => ConfigValue::DateTime(ConfigDateTime { + date: date(decoder)?, + time: time(decoder)?, + utc_offset_seconds: match decoder.count()? { + NO_OFFSET => None, + OFFSET => Some(i32::from_le_bytes(fixed(decoder)?)), + _ => return None, + }, + }), + TIME => ConfigValue::Time(time(decoder)?), + LIST => ConfigValue::List(decoder.list(value)?), + MAP => ConfigValue::Map(decoder.list(|decoder| Some((value(decoder)?, value(decoder)?)))?), + _ => return None, + }) +} + +/// A stored document, or `None` when it cannot be decoded. +pub(crate) fn decoded_document(stored: &[u8]) -> Option { + let mut decoder: Decoder<'_> = Decoder::new(stored)?; + let document: ConfigValue = value(&mut decoder)?; + decoder.finish()?; + Some(document) +} + +/// The store a delta compile reads and fills, opened beside its reuse slots. +pub(crate) struct DocumentStore { + pub(crate) store: NativeStore, + pub(crate) path: PathBuf, +} + +/// In a delta compile, the document store beside its reuse slots; `None` otherwise or when it +/// cannot be opened, which only means parsing again. +pub(crate) fn open_documents(py: pyo3::Python<'_>) -> Option { + let path: PathBuf = crate::bindings::_helpers::compile_delta::context::injection()? + .yaml_store + .clone()?; + let store: NativeStore = crate::bindings::_helpers::cache::native_store::open_store( + py, + &path, + YAML_STORE_KIND, + YAML_STORE_ENVIRONMENT, + ) + .ok()?; + Some(DocumentStore { store, path }) +} + +/// Save the documents this compile loaded; failing to only means parsing them again. +pub(crate) fn save_documents(py: pyo3::Python<'_>, documents: &DocumentStore) { + let _ = crate::bindings::_helpers::cache::native_store::save_store( + py, + &documents.store, + &documents.path, + &[], + ); +} + +/// Each file's loaded document: decoded from `documents` when its text is stored, otherwise +/// loaded by `load` and stored. +pub(crate) fn load_with( + documents: &mut DocumentStore, + paths: &[String], + absolute: impl Fn(&str) -> PathBuf, + load: impl FnOnce(&[String]) -> Result, E>, +) -> Result, E> { + let mut outcomes: Vec> = Vec::with_capacity(paths.len()); + let mut missed: Vec = Vec::new(); + for path in paths { + let stored: Option = std::fs::read(absolute(path)) + .ok() + .and_then(|bytes| decode_text(&bytes).ok()) + .and_then(|contents| { + let value: ConfigValue = + decoded_document(documents.store.get(&document_key(&contents))?)?; + Some(YamlFileOutcome::Loaded { contents, value }) + }); + if stored.is_none() { + missed.push(path.clone()); + } + outcomes.push(stored); + } + let mut loaded = if missed.is_empty() { + Vec::new() + } else { + load(&missed)? + } + .into_iter(); + let mut complete: Vec = Vec::with_capacity(paths.len()); + for outcome in outcomes { + let outcome: YamlFileOutcome = match outcome { + Some(outcome) => outcome, + None => { + let Some(outcome) = loaded.next() else { + break; + }; + if let YamlFileOutcome::Loaded { contents, value } = &outcome { + documents + .store + .put(document_key(contents), encoded_document(value)); + } + outcome + } + }; + complete.push(outcome); + } + Ok(complete) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index fc5063fb32..1e48ff6bb5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -155,6 +155,12 @@ fn load_files<'py>( let py: Python<'py> = project_dir.py(); let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; let root: ProjectRoot = project_root(project_dir)?; + let mut documents: Option = + if matches!(kind, YamlKind::Source) { + crate::bindings::_helpers::compile_delta::yaml_cache::open_documents(py) + } else { + None + }; for batch in batches(&tree.get().inner, paths, YAML_BATCH_BYTES) { let named: Vec = batch .iter() @@ -164,7 +170,17 @@ fn load_files<'py>( let outcomes: Result, StageFailure> = { let inner = &tree.get().inner; let failure_kind: FailureKind = kind.failure_kind(); - py.detach(|| load_yaml(&root, inner, &named, failure_kind)) + match documents.as_mut() { + Some(documents) => py.detach(|| { + crate::bindings::_helpers::compile_delta::yaml_cache::load_with( + documents, + &named, + |path| inner.absolute(path), + |missed| load_yaml(&root, inner, missed, failure_kind), + ) + }), + None => py.detach(|| load_yaml(&root, inner, &named, failure_kind)), + } }; let mut outcomes: Vec = outcomes.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; @@ -195,6 +211,9 @@ fn load_files<'py>( on_file((relative_path, file_path), built)?; } } + if let Some(documents) = &documents { + crate::bindings::_helpers::compile_delta::yaml_cache::save_documents(py, documents); + } Ok(()) } From c5bbfa4390c16d43dd95c81234fa235ebcec910c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:08:33 +0100 Subject: [PATCH 13/32] perf(compile): overlap a delta compile's slow reads and writes Decode the stored rules request on its own thread while the delta compile discovers and analyses, and write a stored compile's output and image at once. --- .../src/project_reuse/_helpers/entry.rs | 36 ++++++---- .../_helpers/compile_delta/context.rs | 4 +- .../bindings/_helpers/compile_delta/driver.rs | 8 +-- .../bindings/_helpers/compile_delta/image.rs | 69 +++++++++++++------ .../_helpers/compile_delta/rules_rows.rs | 5 +- 5 files changed, 78 insertions(+), 44 deletions(-) diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs index 3ebb3ce730..a61b875cf2 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs @@ -143,23 +143,29 @@ pub(crate) fn write_entry( (inputs, output, image): (&StoredInputs, &StoredOutput, Option<&ImageSections>), max_entries: usize, ) -> std::io::Result<()> { - let meta: Vec = encode_output(output); - let (encoded, identity) = encode_sections( - OUTPUT_ROLE, - &[ - (OUTPUT_META_SECTION, &meta), - (STDOUT_SECTION, output.stdout.as_bytes()), - ], - ); - write_atomically(&output_path(path), &encoded)?; - drop(encoded); - let image_identity: Option = match image { - Some(sections) => Some(write_image(path, sections)?), - None => { - let _ = std::fs::remove_file(image_path(path)); - None + let write_output = || -> std::io::Result { + let meta: Vec = encode_output(output); + let (encoded, identity) = encode_sections( + OUTPUT_ROLE, + &[ + (OUTPUT_META_SECTION, &meta), + (STDOUT_SECTION, output.stdout.as_bytes()), + ], + ); + write_atomically(&output_path(path), &encoded)?; + Ok(identity) + }; + let write_named_image = || -> std::io::Result> { + match image { + Some(sections) => Ok(Some(write_image(path, sections)?)), + None => { + let _ = std::fs::remove_file(image_path(path)); + Ok(None) + } } }; + let (identity, image_identity) = rayon::join(write_output, write_named_image); + let (identity, image_identity) = (identity?, image_identity?); let encoded_inputs: Vec = encode_inputs(inputs); let mut sections: Vec<(&str, &[u8])> = vec![ (INPUTS_SECTION, &encoded_inputs), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 878d3c6798..4271f356a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -10,7 +10,7 @@ use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, Sess use sqlbuild_cache::project_reuse::models::ImageSections; use sqlbuild_rules::models::RetainedRows; -use crate::bindings::_helpers::compile_delta::image::BaseRules; +use crate::bindings::_helpers::compile_delta::image::PendingRules; /// The facts a full compile leaves for its project image beyond its analysis record. #[derive(Debug, Default)] @@ -104,7 +104,7 @@ pub(crate) struct DeltaInjection { /// The whole project's relation facts, injected for relations the delta does not analyse. pub(crate) relations: RelationTables, /// The whole project's built-in rules request, taken by the round that evaluates it. - pub(crate) rules: Mutex>, + pub(crate) rules: Mutex>, /// The whole project's custom-rule fact digests. pub(crate) fact_digests: BTreeMap, String>, /// Each project model's `(path, name, materialized)`, the custom-rule subjects. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 5837e96017..80f97ceffc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -21,7 +21,7 @@ use crate::bindings::_helpers::compile_delta::header_keys::{ HeaderChange, custom_rule_reads, header_change, }; use crate::bindings::_helpers::compile_delta::image::{ - BaseRules, ImageGate, ImageRelations, ImageRules, ProjectImage, + BaseRules, ImageGate, ImageRelations, ImageRules, PendingRules, ProjectImage, }; use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; use crate::bindings::_helpers::compile_driver::command::{PhasesOutcome, unrecorded_compile}; @@ -238,7 +238,7 @@ fn lent_injection( .collect(), relations: std::mem::take(&mut image.relations).into(), rules: std::sync::Mutex::new(match image.rules.take() { - Some(ImageRules::Decoded(rules)) => Some(rules), + Some(ImageRules::Decoding(rules)) => Some(rules), Some(ImageRules::Retained(_)) | None => None, }), fact_digests: std::mem::take(&mut image.fact_digests) @@ -469,7 +469,7 @@ pub(crate) fn run<'py>( if rerun.models.len() > limit { return fallback("too many models would compile again"); } - let Some(mut image) = ProjectImage::decode(&stored.image, gate) else { + let Some(mut image) = ProjectImage::decode(&mut stored.image, gate) else { return fallback("the project image cannot be read"); }; stored.image = ImageSections::new(); @@ -554,7 +554,7 @@ pub(crate) fn run<'py>( let Some(rules) = rules else { return fallback("the delta compile retained no rules request"); }; - injection.rules = std::sync::Mutex::new(Some(rules)); + injection.rules = std::sync::Mutex::new(Some(PendingRules::Ready(rules))); for name in added { if let Some(model) = stored_models.get(name.as_str()) { rerun.model_paths.insert(model.relative_path.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 04da1be82b..881941db79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -21,7 +21,7 @@ const RULES_SECTION: &str = "rules"; const RULE_DIGESTS_SECTION: &str = "rule-digests"; /// Everything a delta compile reads of the compile it patches. -#[derive(Clone, Debug, Default)] +#[derive(Debug, Default)] pub(crate) struct ProjectImage { pub(crate) format: String, pub(crate) json_output: bool, @@ -60,13 +60,37 @@ impl BaseRules { } } -/// An image's rules request: decoded from a stored image, or as a compile just retained it. -#[derive(Clone, Debug)] +/// An image's rules request: decoding from a stored image, or as a compile just retained it. +#[derive(Debug)] pub(crate) enum ImageRules { - Decoded(BaseRules), + Decoding(PendingRules), Retained(RetainedRows), } +/// A stored rules request decoding on its own thread while the delta compile starts, or +/// decoded already. +#[derive(Debug)] +pub(crate) enum PendingRules { + Decoding(std::thread::JoinHandle>), + Ready(BaseRules), +} + +impl PendingRules { + /// The decoded request, or `None` when it could not be decoded. + pub(crate) fn resolved(self) -> Option { + match self { + Self::Decoding(handle) => handle.join().ok().flatten(), + Self::Ready(rules) => Some(rules), + } + } +} + +fn decoded_rules(request: &[u8], digests: &[u8]) -> Option { + let request: EvaluateRequest = serde_json::from_slice(request).ok()?; + let digests: Vec = decoded_strings(digests)?; + (digests.len() == request.models.len()).then_some(BaseRules { request, digests }) +} + /// The analysis session's final relation facts, as stored. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct ImageRelations { @@ -490,7 +514,7 @@ impl ProjectImage { }; let rules: Option = match self.rules { Some(ImageRules::Retained(rules)) => Some(rules), - Some(ImageRules::Decoded(_)) | None => None, + Some(ImageRules::Decoding(_)) | None => None, }; let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); let rules = rules.map(|rules| (rules.request_json, encoded_strings(&rules.digests))); @@ -506,21 +530,24 @@ impl ProjectImage { } /// The image stored in `sections` behind `gate`, or `None` when it is unreadable. - pub(crate) fn decode(sections: &ImageSections, gate: ImageGate) -> Option { - let (facts, rules) = rayon::join( - || StoredFacts::decode(section(sections, FACTS_SECTION)?), - || -> Option> { - let Some(bytes) = section(sections, RULES_SECTION) else { - return Some(None); - }; - let request: EvaluateRequest = serde_json::from_slice(bytes).ok()?; - let digests: Vec = - decoded_strings(section(sections, RULE_DIGESTS_SECTION)?)?; - (digests.len() == request.models.len()) - .then_some(Some(ImageRules::Decoded(BaseRules { request, digests }))) - }, - ); - let facts: StoredFacts = facts?; + /// + /// The rules request, the largest section, is taken out of `sections` and decodes on its + /// own thread. + pub(crate) fn decode(sections: &mut ImageSections, gate: ImageGate) -> Option { + let mut taken = |name: &str| { + sections + .iter_mut() + .find(|(section, _)| section == name) + .map(|(_, bytes)| std::mem::take(bytes)) + }; + let rules: Option = match (taken(RULES_SECTION), taken(RULE_DIGESTS_SECTION)) { + (Some(request), Some(digests)) => Some(ImageRules::Decoding(PendingRules::Decoding( + std::thread::spawn(move || decoded_rules(&request, &digests)), + ))), + (None, None) => None, + _ => return None, + }; + let facts: StoredFacts = StoredFacts::decode(section(sections, FACTS_SECTION)?)?; let types: Shapes = facts.types; let relations: ImageRelations = ImageRelations { complete: restored(facts.complete, &types)?, @@ -548,7 +575,7 @@ impl ProjectImage { lint_model_keys: facts.lint_model_keys, lint_file_keys: facts.lint_file_keys, header_keys: gate.header_keys, - rules: rules?, + rules, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs index 26787e6bd7..579f4c1e07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs @@ -6,7 +6,7 @@ use pyo3::PyResult; use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact}; use crate::bindings::_helpers::compile_delta::context::{delta_unsupported, injection}; -use crate::bindings::_helpers::compile_delta::image::BaseRules; +use crate::bindings::_helpers::compile_delta::image::{BaseRules, PendingRules}; /// `request` unchanged outside a delta compile; inside one, the whole project's request with /// the rerun models' rows and the rerun SQL tests' facts taken from `request`. @@ -18,7 +18,8 @@ pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult Date: Mon, 19 Oct 2026 02:14:56 +0100 Subject: [PATCH 14/32] fix(compile): keep every source a delta compile's functions, audits and tests read A delta compile that discovers only some sources now also keeps those that functions and audits read and those its rerun SQL tests read, falls back early when a model edit changes which sources it names, and falls back when anything still reports an unknown source. --- .../_helpers/compile_delta/context.rs | 27 +++++++ .../bindings/_helpers/compile_delta/driver.rs | 33 +++++++++ .../_helpers/compile_delta/header_keys.rs | 73 ++++++++++++++++--- .../bindings/_helpers/compile_delta/image.rs | 29 +++++++- .../_helpers/compiled_project/audit_loop.rs | 7 ++ .../compiled_project/function_loop.rs | 10 +++ .../compiled_project/scenario_loop.rs | 7 ++ .../compiled_project/sql_test_loop.rs | 2 + .../_helpers/discovery/model_records.rs | 1 + 9 files changed, 178 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 4271f356a4..349421e885 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -15,6 +15,10 @@ use crate::bindings::_helpers::compile_delta::image::PendingRules; /// The facts a full compile leaves for its project image beyond its analysis record. #[derive(Debug, Default)] pub(crate) struct ImageCapture { + /// The sources functions and audits read, which a delta compile always keeps. + pub(crate) pinned_sources: BTreeSet, + /// The sources each SQL test file reads, which a delta compile keeps when it reruns it. + pub(crate) test_sources: BTreeMap>, /// The relation facts the model analysis session ended with. pub(crate) relations: Option, /// The built-in rules request and its raw findings, before suppressions. @@ -153,6 +157,29 @@ pub(crate) fn capture(record: impl FnOnce(&mut ImageCapture)) { } } +/// Record sources a resource other than a model reads, when a capture is running. +pub(crate) fn record_source_reads<'a>(names: impl IntoIterator) { + capture(|capture| { + capture + .pinned_sources + .extend(names.into_iter().map(str::to_owned)); + }); +} + +/// Record sources an SQL test file reads, when a capture is running. +pub(crate) fn record_test_source_reads<'a>( + relative_path: &str, + names: impl IntoIterator, +) { + capture(|capture| { + capture + .test_sources + .entry(relative_path.replace('\\', "/")) + .or_default() + .extend(names.into_iter().map(str::to_owned)); + }); +} + /// Mark the running capture as unusable as a delta base. pub(crate) fn capture_unsupported(reason: &str) { capture(|capture| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 80f97ceffc..750d2620eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -41,6 +41,8 @@ pub(crate) const ERRORS_ON_DELTA_PATH: bool = true; /// How many times a changed relation shape may pull its dependents into the delta. const MAX_ROUNDS: usize = 4; const MODELS_ROOT: &str = "models/"; +/// What every error for a reference to an undiscovered source says. +const UNKNOWN_SOURCE: &str = "unknown source"; /// The upstream kind of a source a model reads. const SOURCE_KIND: &str = "source"; const TESTS_ROOT: &str = "tests/"; @@ -366,6 +368,11 @@ fn ran_round<'py>( .iter() .chain(&rerun.test_paths) .any(|path| message.contains(path.as_str())); + if rerun.sources.is_some() && message.contains(UNKNOWN_SOURCE) { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(format!( + "the delta compile read a source it did not discover: {message}" + )))); + } return Ok(RoundOutcome::Stop(if owned && ERRORS_ON_DELTA_PATH { DeltaOutcome::Raised(error) } else { @@ -490,6 +497,15 @@ pub(crate) fn run<'py>( .flat_map(|model| &model.upstream) .filter(|(kind, _)| kind == SOURCE_KIND) .map(|(_, name)| name.clone()) + .chain(image.pinned_sources.iter().cloned()) + .chain( + rerun + .test_paths + .iter() + .filter_map(|path| image.test_sources.get(path)) + .flatten() + .cloned(), + ) .collect() }); rerun.sources = sources_read.clone(); @@ -698,6 +714,15 @@ fn finished<'py>( .map_or("no error diagnostic", |diagnostic| diagnostic.json.as_str()); return fallback(format!("the edit reports errors: {first}")); } + if rerun.sources.is_some() + && ran + .capture + .diagnostics + .iter() + .any(|diagnostic| diagnostic.json.contains(UNKNOWN_SOURCE)) + { + return fallback("the delta compile read a source it did not discover"); + } if unowned(rerun, &base.image.diagnostics) != unowned(rerun, &ran.capture.diagnostics) { return fallback("diagnostics outside the changed files differ"); } @@ -809,6 +834,14 @@ fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> Pr image.model_shapes = shapes; } image.header_keys.extend(capture.header_keys); + let mut pinned: BTreeSet = std::mem::take(&mut image.pinned_sources) + .into_iter() + .collect(); + pinned.extend(capture.pinned_sources); + image.pinned_sources = pinned.into_iter().collect(); + for (path, names) in capture.test_sources { + image.test_sources.insert(path, names.into_iter().collect()); + } image.lint_model_keys = capture.lint_model_keys; image.lint_file_keys = capture.lint_file_keys; image diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs index 57e7ab93c7..0885aaf061 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs @@ -6,6 +6,9 @@ use std::path::Path; use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_sqltext::compiler::models::AuthoredValue; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::compile_delta::context::{capture, capturing}; use crate::bindings::_helpers::discovery::model_records::model_file_options; @@ -74,32 +77,77 @@ fn encode_value(value: &AuthoredValue, out: &mut Vec) { } } -fn header_keys(values: &[(String, AuthoredValue)]) -> HeaderKeys { +/// The pseudo header key holding a digest of the sources a model's authored query names. +const REFERENCES_KEY: &str = "\u{0}sources"; +const SOURCE_KIND: &str = "source"; + +/// A digest of the source calls `query_sql` names, scanned with plain `--` comments. A delta +/// compile discovers only the sources its models read before, so an edit changing it falls back. +fn references_digest(query_sql: &str) -> String { + let syntax: LexicalSyntax = LexicalSyntax { + line_comment_prefixes: vec![LINE_COMMENT.to_owned()], + ..LexicalSyntax::default() + }; + let mut buffer: Vec = Vec::new(); + match extract_sql_references(query_sql, &syntax) { + ReferenceExtraction::Extracted(scan) => { + for reference in scan + .references + .iter() + .filter(|reference| reference.kind == SOURCE_KIND) + { + for part in [ + reference.kind, + &reference.name, + reference.package.as_deref().unwrap_or(""), + ] { + buffer.extend_from_slice(&(part.len() as u64).to_le_bytes()); + buffer.extend_from_slice(part.as_bytes()); + } + } + } + ReferenceExtraction::Failed(_) => buffer.push(0xff), + } + short_hex(&bytes_digest(&buffer)) +} + +const LINE_COMMENT: &str = "--"; + +fn short_hex(digest: &[u8]) -> String { + let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); + for byte in &digest[..KEPT_DIGEST_BYTES] { + hex.push(char::from(HEX[usize::from(byte >> 4)])); + hex.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + hex +} + +fn header_keys(values: &[(String, AuthoredValue)], query_sql: &str) -> HeaderKeys { let mut buffer: Vec = Vec::new(); let mut keys: HeaderKeys = values .iter() .map(|(key, value)| { buffer.clear(); encode_value(value, &mut buffer); - let digest = bytes_digest(&buffer); - let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); - for byte in &digest[..KEPT_DIGEST_BYTES] { - hex.push(char::from(HEX[usize::from(byte >> 4)])); - hex.push(char::from(HEX[usize::from(byte & 0x0f)])); - } + let hex: String = short_hex(&bytes_digest(&buffer)); (key.clone(), hex) }) .collect(); + keys.push((REFERENCES_KEY.to_owned(), references_digest(query_sql))); keys.sort(); keys } /// Record one discovered model's header keys for the image. -pub(crate) fn record_header_keys(relative_path: &str, values: &[(String, AuthoredValue)]) { +pub(crate) fn record_header_keys( + relative_path: &str, + values: &[(String, AuthoredValue)], + query_sql: &str, +) { if !capturing() { return; } - let keys: HeaderKeys = header_keys(values); + let keys: HeaderKeys = header_keys(values, query_sql); let path: String = relative_path.replace('\\', "/"); capture(|capture| { capture.header_keys.insert(path, keys); @@ -138,7 +186,7 @@ pub(crate) fn header_change( let Ok(parsed) = parse_model_text(&file_path, contents, &model_file_options()) else { return HeaderChange::Unparsed; }; - let current: HeaderKeys = header_keys(&parsed.header_values); + let current: HeaderKeys = header_keys(&parsed.header_values, &parsed.query_sql); let stored_map: BTreeMap<&str, &str> = stored .iter() .map(|(key, digest)| (key.as_str(), digest.as_str())) @@ -156,6 +204,11 @@ pub(crate) fn header_change( if changed.is_empty() { return HeaderChange::Unchanged; } + if changed.contains(REFERENCES_KEY) { + return HeaderChange::Unsupported(format!( + "{relative_path} changed which sources it reads" + )); + } for key in &changed { if key == MATERIALIZED_KEY { let value = parsed.header_values.iter().find(|(name, _)| name == key); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 881941db79..54153c248a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; /// Changes whenever the image's meaning changes, so an older image is never read. -const IMAGE_FORMAT: &str = "sqlbuild-project-image-v4"; +const IMAGE_FORMAT: &str = "sqlbuild-project-image-v5"; const GATE_SECTION: &str = "gate"; const FACTS_SECTION: &str = "facts"; const RULES_SECTION: &str = "rules"; @@ -34,6 +34,8 @@ pub(crate) struct ProjectImage { pub(crate) diagnostics: Vec, pub(crate) text_models: Option>, pub(crate) environment_names: Vec, + pub(crate) pinned_sources: Vec, + pub(crate) test_sources: BTreeMap>, pub(crate) model_shapes: Vec, pub(crate) lint_model_keys: Option>, pub(crate) lint_file_keys: Option)>>, @@ -197,6 +199,8 @@ struct StoredFacts { diagnostics: Vec, text_models: Option>, environment_names: Vec, + pinned_sources: Vec, + test_sources: BTreeMap>, model_shapes: StoredShapes, lint_model_keys: Option>, lint_file_keys: Option)>>, @@ -363,6 +367,12 @@ impl StoredFacts { encoder.optional_string(key.as_deref()); } } + encoder.strings(self.pinned_sources.iter().map(String::as_str)); + encoder.count(self.test_sources.len()); + for (path, names) in &self.test_sources { + encoder.string(path); + encoder.strings(names.iter().map(String::as_str)); + } encoder.bytes(&misc); Ok(encoder.finish()) } @@ -402,6 +412,11 @@ impl StoredFacts { } else { None }; + let pinned_sources: Vec = decoder.strings()?; + let test_sources: BTreeMap> = decoder + .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? + .into_iter() + .collect(); let (diagnostics, text_models, environment_names): StoredMisc = serde_json::from_slice(decoder.bytes()?).ok()?; decoder.finish()?; @@ -418,6 +433,8 @@ impl StoredFacts { diagnostics, text_models, environment_names, + pinned_sources, + test_sources, model_shapes, lint_model_keys, lint_file_keys, @@ -471,6 +488,12 @@ impl ProjectImage { diagnostics: capture.diagnostics, text_models: capture.text_models, environment_names, + pinned_sources: capture.pinned_sources.into_iter().collect(), + test_sources: capture + .test_sources + .into_iter() + .map(|(path, names)| (path, names.into_iter().collect())) + .collect(), model_shapes, lint_model_keys: capture.lint_model_keys, lint_file_keys: capture.lint_file_keys, @@ -509,6 +532,8 @@ impl ProjectImage { diagnostics: self.diagnostics, text_models: self.text_models, environment_names: self.environment_names, + pinned_sources: self.pinned_sources, + test_sources: self.test_sources, lint_model_keys: self.lint_model_keys, lint_file_keys: self.lint_file_keys, }; @@ -571,6 +596,8 @@ impl ProjectImage { diagnostics: facts.diagnostics, text_models: facts.text_models, environment_names: facts.environment_names, + pinned_sources: facts.pinned_sources, + test_sources: facts.test_sources, model_shapes, lint_model_keys: facts.lint_model_keys, lint_file_keys: facts.lint_file_keys, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs index 20869e5c8f..f748dffb8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -1,5 +1,6 @@ //! `build_project_audit_inputs`: singular and attached generic audits attached natively. +use crate::bindings::_helpers::compile_delta::context::record_source_reads; use std::collections::{BTreeMap, HashMap}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; @@ -573,6 +574,12 @@ impl<'py> AuditScope<'_, 'py> { .downcast()? .clone(); let known: KnownResources<'_> = validator.get().known_resources(); + record_source_reads( + calls + .iter() + .filter(|(kind, _)| *kind == SOURCE_TARGET) + .map(|(_, name)| *name), + ); match audit_reference_error(relative_path, calls, &known) { Some(message) => Err(compile_error(self.context.classes, &message, None)), None => Ok(()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs index fe97660e20..ea653300d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs @@ -1,5 +1,6 @@ //! `build_sql_function_inputs`: every SQL and Python function attached natively. +use crate::bindings::_helpers::compile_delta::context::record_source_reads; use std::collections::{HashMap, HashSet}; use pyo3::exceptions::PyValueError; @@ -51,6 +52,9 @@ const TABLE_RETURNS: &str = "TABLE"; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; +/// The reference kind of a source. +const SOURCE_KIND: &str = "source"; + /// A function's paths, stem, discovered file, header values, body and SQL file contents. pub(crate) type FunctionRow = ( String, @@ -518,6 +522,12 @@ fn function_references( .iter() .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) .collect(); + record_source_reads( + calls + .iter() + .filter(|(kind, _)| *kind == SOURCE_KIND) + .map(|(_, name)| *name), + ); if let Some(message) = function_reference_error(relative_path, &calls, &known) { return Err(compile_error(context.classes, &message, None)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs index 2771b1321c..11af682292 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs @@ -1,5 +1,6 @@ //! `build_scenario_inputs`: SQL scenarios expanded, classified and checked natively. +use crate::bindings::_helpers::compile_delta::context::record_test_source_reads; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_helper_scope::scenario_helper_scope; @@ -154,6 +155,12 @@ impl<'py> TestScope<'_, 'py> { sources: source_names(&references), }); } + record_test_source_reads( + origin.1, + sources + .iter() + .flat_map(|cte| cte.sources.iter().map(String::as_str)), + ); if let Some(message) = scenario_source_violation(origin.1, &sources, &self.catalog.sources) { return Err(compile_error(self.context.classes, &message, None)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs index c181628576..07563842eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs @@ -1,5 +1,6 @@ //! `build_test_inputs`: SQL test blocks expanded, classified and checked natively. +use crate::bindings::_helpers::compile_delta::context::record_test_source_reads; use std::collections::{HashMap, HashSet}; use std::convert::Infallible; @@ -748,6 +749,7 @@ impl<'py> TestScope<'_, 'py> { .into_iter() .filter(|name| !mock_models.contains(name)) .collect(); + record_test_source_reads(relative_path, mock_sources.iter().map(String::as_str)); if let Some(message) = unknown_test_target( relative_path, &self.catalog, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index b42b199fd8..1d293297e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -72,6 +72,7 @@ fn model_items<'py>( crate::bindings::_helpers::compile_delta::header_keys::record_header_keys( &file.relative_path, &model.header_values, + &model.query_sql, ); let file_path: String = root.display_path(&file.relative_path); let paths = DiscoveredPaths::Native(file.relative_path.clone(), file_path); From ce2f8d65a5e57d3c7cf376eb7c61590fe176b366 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:31:41 +0100 Subject: [PATCH 15/32] feat(compile): add the delta_compile setting and fix recording multi-round deltas - [settings] delta_compile (default true, overridable locally) stops compiles capturing a project image, so no delta compile follows. - Record a delta compile's kept artifacts with the reuse check's stamps only where they still match; a multi-round delta rewrites some of them, and the stale stamp made recording drop the stored compile. - Fall back before compiling when an edited model's authored references change, which a delta compile would otherwise only find after its first round. - Unit tests for the report splice, the project image round trip and damaged images, and the stored YAML documents. --- .../src/project_reuse/_helpers/target.rs | 12 +- .../src/project/_helpers/sections.rs | 12 +- .../sqlbuild-config/src/project/constants.rs | 4 +- .../_helpers/compile_delta/context.rs | 5 + .../_helpers/compile_delta/header_keys.rs | 19 +- .../bindings/_helpers/compile_delta/image.rs | 2 +- .../bindings/_helpers/compile_delta/mod.rs | 3 + .../bindings/_helpers/compile_delta/tests.rs | 8 + .../compile_delta/tests/test_image.rs | 234 ++++++++++++++++++ .../compile_delta/tests/test_splice.rs | 83 +++++++ .../compile_delta/tests/test_types.rs | 27 ++ .../compile_delta/tests/test_yaml_cache.rs | 93 +++++++ .../_helpers/compile_driver/discovery.rs | 14 ++ .../_helpers/contract_records/constants.rs | 1 + src/sqlbuild/_native.pyi | 2 + 15 files changed, 498 insertions(+), 21 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_splice.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_yaml_cache.rs diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs index 141682526a..e259fd0b41 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs @@ -96,7 +96,8 @@ fn matches_recorded( } } -/// Each expected artifact's stamp: the vouched one for a kept artifact, read for a written one. +/// Each expected artifact's stamp: the vouched one for an artifact kept as it was vouched for, +/// read for any other. fn vouched_stamps(expected: &BTreeMap<&str, &RecordedArtifact>, vouched: &[PathStamp]) -> Stamps { let vouched: HashMap<&str, &PathStamp> = vouched .iter() @@ -105,9 +106,12 @@ fn vouched_stamps(expected: &BTreeMap<&str, &RecordedArtifact>, vouched: &[PathS expected .iter() .filter_map(|(path, artifact)| match artifact { - RecordedArtifact::Kept { .. } => vouched - .get(path) - .map(|stamp| ((*path).to_owned(), (*stamp).clone())), + RecordedArtifact::Kept { size, mtime_ns } => match vouched.get(path) { + Some(stamp) if stamp.size == *size && stamp.mtime_ns == *mtime_ns => { + Some(((*path).to_owned(), (*stamp).clone())) + } + _ => keyed_stamp(path), + }, RecordedArtifact::Written { .. } => keyed_stamp(path), }) .collect() diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index ecb135feaf..021b10dbc3 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -21,8 +21,8 @@ use crate::project::constants::{ USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ - DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, - TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, + DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, + SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; @@ -242,6 +242,14 @@ pub(crate) fn load_settings(payload: Option<&ConfigValue>, file: ConfigFile) -> UNIT_TEST_MAX_REAL_MODELS_KEY, Built::Value(real_model_limit(mapping)?), ), + ( + DELTA_COMPILE_KEY, + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + DELTA_COMPILE_KEY, + true, + )?)), + ), ]; let mut fields = fields; for (key, seconds_key, default) in [ diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index dc1c783554..d225cbba22 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -40,9 +40,10 @@ pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; pub(crate) const UNIT_TEST_MAX_REAL_MODELS_KEY: &str = "unit_test_max_real_models"; pub(crate) const DEFAULT_UNIT_TEST_MAX_REAL_MODELS: i64 = 10; pub(crate) const TEST_TIMEOUT_KEY: &str = "test_timeout"; +pub(crate) const DELTA_COMPILE_KEY: &str = "delta_compile"; pub(crate) const SCENARIO_TIMEOUT_KEY: &str = "scenario_timeout"; pub(crate) const DEFAULT_TEST_TIMEOUT: &str = "30s"; -pub(crate) const SETTINGS_KEYS: [&str; 14] = [ +pub(crate) const SETTINGS_KEYS: [&str; 15] = [ "sql_analysis", "require_sql_analysis", "query_change_tracking", @@ -57,6 +58,7 @@ pub(crate) const SETTINGS_KEYS: [&str; 14] = [ "unit_test_max_real_models", "test_timeout", "scenario_timeout", + "delta_compile", ]; pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "append_cursor_inclusive", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 349421e885..74dd2bd62e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -140,6 +140,11 @@ pub(crate) fn start_capture() { *locked(&CAPTURE) = Some(ImageCapture::default()); } +/// Stop capturing without keeping anything, for a project that turns delta compiles off. +pub(crate) fn stop_capture() { + *locked(&CAPTURE) = None; +} + /// Stop capturing and take what was captured. pub(crate) fn take_capture() -> Option { locked(&CAPTURE).take() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs index 0885aaf061..d2e8646c72 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs @@ -77,12 +77,11 @@ fn encode_value(value: &AuthoredValue, out: &mut Vec) { } } -/// The pseudo header key holding a digest of the sources a model's authored query names. -const REFERENCES_KEY: &str = "\u{0}sources"; -const SOURCE_KIND: &str = "source"; +/// The pseudo header key holding a digest of the references a model's authored query names. +const REFERENCES_KEY: &str = "\u{0}references"; -/// A digest of the source calls `query_sql` names, scanned with plain `--` comments. A delta -/// compile discovers only the sources its models read before, so an edit changing it falls back. +/// A digest of the reference calls `query_sql` names, scanned with plain `--` comments. An edit +/// that changes them changes what the model reads, which a delta compile does not patch. fn references_digest(query_sql: &str) -> String { let syntax: LexicalSyntax = LexicalSyntax { line_comment_prefixes: vec![LINE_COMMENT.to_owned()], @@ -91,11 +90,7 @@ fn references_digest(query_sql: &str) -> String { let mut buffer: Vec = Vec::new(); match extract_sql_references(query_sql, &syntax) { ReferenceExtraction::Extracted(scan) => { - for reference in scan - .references - .iter() - .filter(|reference| reference.kind == SOURCE_KIND) - { + for reference in &scan.references { for part in [ reference.kind, &reference.name, @@ -205,9 +200,7 @@ pub(crate) fn header_change( return HeaderChange::Unchanged; } if changed.contains(REFERENCES_KEY) { - return HeaderChange::Unsupported(format!( - "{relative_path} changed which sources it reads" - )); + return HeaderChange::Unsupported(format!("{relative_path} changed what it references")); } for key in &changed { if key == MATERIALIZED_KEY { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 54153c248a..8220874967 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; /// Changes whenever the image's meaning changes, so an older image is never read. -const IMAGE_FORMAT: &str = "sqlbuild-project-image-v5"; +pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v5"; const GATE_SECTION: &str = "gate"; const FACTS_SECTION: &str = "facts"; const RULES_SECTION: &str = "rules"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs index 1e2e6a90b7..5380d7f686 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs @@ -10,3 +10,6 @@ pub(crate) mod rules_rows; pub(crate) mod splice; pub(crate) mod verify; pub(crate) mod yaml_cache; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests.rs new file mode 100644 index 0000000000..f7ab89f124 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_image.rs"] +mod image; +#[path = "tests/test_splice.rs"] +mod splice; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_yaml_cache.rs"] +mod yaml_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs new file mode 100644 index 0000000000..7dcaf46192 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs @@ -0,0 +1,234 @@ +use std::collections::BTreeMap; + +use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; + +use crate::bindings::_helpers::compile_delta::context::{ + CapturedDiagnostic, CapturedModel, CapturedTest, +}; +use crate::bindings::_helpers::compile_delta::image::{ + BaseRules, IMAGE_FORMAT, ImageGate, ImageRelations, ImageRules, PendingRules, ProjectImage, +}; +use crate::bindings::_helpers::compile_delta::tests::test_types::DamagedImageTestCase; + +fn pairs(values: &[(&str, &str)]) -> Vec<(String, String)> { + values + .iter() + .map(|(left, right)| ((*left).to_owned(), (*right).to_owned())) + .collect() +} + +fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// An image holding at least one value of every kind it stores. +fn image() -> ProjectImage { + let order_types: Vec<(String, String)> = pairs(&[("order_id", "BIGINT"), ("amount", "DOUBLE")]); + let customer_types: Vec<(String, String)> = pairs(&[("customer_id", "BIGINT")]); + let request: Vec = serde_json::to_vec(&EvaluateRequest::default()).expect("request"); + ProjectImage { + format: String::new(), + json_output: true, + models: vec![ + CapturedModel { + name: "orders".to_owned(), + relative_path: "models/orders.sql".to_owned(), + upstream: pairs(&[("source", "raw_orders"), ("model", "customers")]), + }, + CapturedModel { + name: "customers".to_owned(), + relative_path: "models/customers.sql".to_owned(), + upstream: Vec::new(), + }, + ], + usages: BTreeMap::from([("models/orders.sql".to_owned(), "digest-1".to_owned())]), + tests: vec![CapturedTest { + key: "orders_test".to_owned(), + relative_path: "tests/unit/orders.sql".to_owned(), + chain: strings(&["orders", "customers"]), + }], + relations: ImageRelations { + types: vec![ + ("orders".to_owned(), order_types.clone()), + ("customers".to_owned(), customer_types.clone()), + ], + nullability: vec![("orders".to_owned(), pairs(&[("order_id", "never")]))], + complete: vec![("orders".to_owned(), order_types.clone())], + schemas: vec![("customers".to_owned(), pairs(&[("customer_id", "INTEGER")]))], + dependency_ordered: true, + cyclic: false, + legacy_models: strings(&["customers"]), + }, + fact_digests: vec![(strings(&["model", "orders"]), "fact-digest".to_owned())], + model_subjects: Some(vec![ + ( + "models/orders.sql".to_owned(), + "orders".to_owned(), + Some("table".to_owned()), + ), + ( + "models/customers.sql".to_owned(), + "customers".to_owned(), + None, + ), + ]), + diagnostics: vec![CapturedDiagnostic { + json: "{\"code\": \"W1\"}".to_owned(), + resource_name: Some("orders".to_owned()), + paths: strings(&["models/orders.sql"]), + error: false, + }], + text_models: Some(vec![("orders".to_owned(), true, 3)]), + environment_names: strings(&["SQLBUILD_TARGET"]), + pinned_sources: strings(&["raw_customers"]), + test_sources: BTreeMap::from([( + "tests/unit/orders.sql".to_owned(), + strings(&["raw_orders"]), + )]), + model_shapes: vec![ + ("orders".to_owned(), order_types), + ("customers".to_owned(), pairs(&[("customer_id", "VARCHAR")])), + ], + lint_model_keys: Some(pairs(&[("models/orders.sql", "lint-key")])), + lint_file_keys: Some(vec![ + ("models/orders.sql".to_owned(), Some("file-key".to_owned())), + ("models/customers.sql".to_owned(), None), + ]), + header_keys: BTreeMap::from([( + "models/orders.sql".to_owned(), + pairs(&[("description", "0123")]), + )]), + rules: Some(ImageRules::Retained(RetainedRows { + request_json: request, + digests: Vec::new(), + })), + } +} + +/// Every stored fact of `image` but its rules, in a comparable form. +fn facts(image: &ProjectImage) -> String { + format!( + "{:?}", + ( + ( + &image.json_output, + &image.models, + &image.usages, + &image.tests, + &image.relations, + &image.fact_digests, + &image.model_subjects, + ), + ( + &image.diagnostics, + &image.text_models, + &image.environment_names, + &image.pinned_sources, + &image.test_sources, + &image.model_shapes, + &image.lint_model_keys, + &image.lint_file_keys, + &image.header_keys, + ), + ) + ) +} + +fn current_format() -> String { + IMAGE_FORMAT.to_owned() +} + +fn encoded() -> Vec<(String, Vec)> { + let mut stored: ProjectImage = image(); + stored.format = current_format(); + stored.encode().expect("encoded") +} + +#[test] +fn given_project_image_when_encoded_and_decoded_then_every_fact_and_the_rules_survive() { + let mut sections = encoded(); + let gate: ImageGate = ImageGate::decode(§ions).expect("gate"); + assert!(gate.json_output); + assert!(!gate.has_errors); + assert!(gate.has_model_subjects); + let decoded: ProjectImage = ProjectImage::decode(&mut sections, gate).expect("image"); + let mut expected: ProjectImage = image(); + expected.format = current_format(); + assert_eq!(decoded.format, expected.format); + assert_eq!(facts(&decoded), facts(&expected)); + let Some(ImageRules::Decoding(rules)) = decoded.rules else { + panic!("the rules request was not decoding"); + }; + let rules: BaseRules = PendingRules::resolved(rules).expect("rules"); + assert!(rules.request.models.is_empty()); + assert!(rules.digests.is_empty()); +} + +#[test] +fn given_damaged_image_when_decoded_then_it_is_never_read() { + let test_cases = [ + DamagedImageTestCase { + description: "intact", + damage: |_| {}, + expected_gate: true, + expected_image: true, + }, + DamagedImageTestCase { + description: "truncated facts", + damage: |sections| { + let facts = sections + .iter_mut() + .find(|(name, _)| name == "facts") + .expect("facts"); + facts.1.truncate(facts.1.len() / 2); + }, + expected_gate: true, + expected_image: false, + }, + DamagedImageTestCase { + description: "rules request without its digests", + damage: |sections| sections.retain(|(name, _)| name != "rule-digests"), + expected_gate: true, + expected_image: false, + }, + DamagedImageTestCase { + description: "gate of another format", + damage: |sections| { + let mut other: ProjectImage = ProjectImage::default(); + other.format = "sqlbuild-project-image-v0".to_owned(); + let gate = other + .encode() + .expect("encoded") + .into_iter() + .find(|(name, _)| name == "gate") + .expect("gate"); + for section in sections.iter_mut() { + if section.0 == "gate" { + section.1 = gate.1.clone(); + } + } + }, + expected_gate: false, + expected_image: false, + }, + ]; + for test_case in test_cases { + let mut sections = encoded(); + (test_case.damage)(&mut sections); + let gate: Option = ImageGate::decode(§ions); + assert_eq!( + gate.is_some(), + test_case.expected_gate, + "{}", + test_case.description + ); + let decoded: Option = + gate.and_then(|gate| ProjectImage::decode(&mut sections, gate)); + assert_eq!( + decoded.is_some(), + test_case.expected_image, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_splice.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_splice.rs new file mode 100644 index 0000000000..667f913b50 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_splice.rs @@ -0,0 +1,83 @@ +use std::collections::BTreeSet; + +use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; +use crate::bindings::_helpers::compile_delta::tests::test_types::SpliceTestCase; + +const BASE: &str = r#"{"has_errors": false, "summary": {"models": 2, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 1}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "not_null", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "compiled"}]}, "diagnostics": [], "compile_timings": {"total_ms": 9}}"#; + +fn names(values: &[&str]) -> BTreeSet { + values.iter().map(|value| (*value).to_owned()).collect() +} + +#[test] +fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_sections_change() { + let test_cases = [ + SpliceTestCase { + description: "a rerun model, its audit and the delta's diagnostics replace the stored ones", + base: BASE, + delta: r#"{"has_errors": true, "summary": {"models": 1, "errors": 1, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 3}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "unique", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": []}, "diagnostics": [{"code": "E1"}], "compile_timings": {"total_ms": 1}}"#, + models: &["orders"], + tests: &[], + sources: None, + expected: Some( + r#"{"has_errors": true, "summary": {"models": 2, "errors": 1, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 3}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "unique", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "compiled"}]}, "diagnostics": [{"code": "E1"}], "compile_timings": {"total_ms": 1}}"#, + ), + }, + SpliceTestCase { + description: "a rerun SQL test and the only discovered source replace the stored ones", + base: BASE, + delta: r#"{"has_errors": false, "summary": {"models": 0, "errors": 0, "warnings": 0}, "resources": {"models": [], "sources": [{"name": "raw_customers", "column_count": 5}], "audits": [], "tests": [{"name": "orders_test", "status": "changed"}]}, "diagnostics": [], "compile_timings": {"total_ms": 1}}"#, + models: &[], + tests: &["orders_test"], + sources: Some(&["raw_customers"]), + expected: Some( + r#"{"has_errors": false, "summary": {"models": 2, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 1}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 5}], "audits": [{"name": "not_null", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "changed"}]}, "diagnostics": [], "compile_timings": {"total_ms": 1}}"#, + ), + }, + SpliceTestCase { + description: "a delta item for a resource outside the rerun set must already be stored", + base: BASE, + delta: r#"{"has_errors": false, "summary": {"models": 1, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "customers", "columns": 9}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [], "tests": []}, "diagnostics": [], "compile_timings": {"total_ms": 1}}"#, + models: &["orders"], + tests: &[], + sources: None, + expected: None, + }, + SpliceTestCase { + description: "a whole-project section that differs cannot be spliced", + base: r#"{"has_errors": false, "target": "dev", "summary": {"errors": 0, "warnings": 0}, "resources": {}, "diagnostics": [], "compile_timings": {}}"#, + delta: r#"{"has_errors": false, "target": "prod", "summary": {"errors": 0, "warnings": 0}, "resources": {}, "diagnostics": [], "compile_timings": {}}"#, + models: &[], + tests: &[], + sources: None, + expected: None, + }, + SpliceTestCase { + description: "reports with different sections cannot be spliced", + base: r#"{"has_errors": false, "resources": {}}"#, + delta: r#"{"has_errors": false}"#, + models: &[], + tests: &[], + sources: None, + expected: None, + }, + ]; + for test_case in test_cases { + let models: BTreeSet = names(test_case.models); + let tests: BTreeSet = names(test_case.tests); + let sources: Option> = test_case.sources.map(names); + let scope: SpliceScope<'_> = SpliceScope { + models: &models, + tests: &tests, + sources: sources.as_ref(), + }; + let spliced: Result = + spliced_report(test_case.base, test_case.delta, &scope); + assert_eq!( + spliced.ok().as_deref(), + test_case.expected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_types.rs new file mode 100644 index 0000000000..73216caed5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_types.rs @@ -0,0 +1,27 @@ +use sqlbuild_config::models::ConfigValue; + +/// A stored report, a delta report and what splicing the delta into the stored one gives. +pub(crate) struct SpliceTestCase { + pub(crate) description: &'static str, + pub(crate) base: &'static str, + pub(crate) delta: &'static str, + pub(crate) models: &'static [&'static str], + pub(crate) tests: &'static [&'static str], + pub(crate) sources: Option<&'static [&'static str]>, + /// The spliced report, or `None` when the reports must not line up. + pub(crate) expected: Option<&'static str>, +} + +/// A loaded YAML document kept by the delta compile's document store. +pub(crate) struct DocumentTestCase { + pub(crate) description: &'static str, + pub(crate) document: ConfigValue, +} + +/// Stored image bytes damaged before they are read back. +pub(crate) struct DamagedImageTestCase { + pub(crate) description: &'static str, + pub(crate) damage: fn(&mut Vec<(String, Vec)>), + pub(crate) expected_gate: bool, + pub(crate) expected_image: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_yaml_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_yaml_cache.rs new file mode 100644 index 0000000000..56ae88c523 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_yaml_cache.rs @@ -0,0 +1,93 @@ +use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; + +use crate::bindings::_helpers::compile_delta::tests::test_types::DocumentTestCase; +use crate::bindings::_helpers::compile_delta::yaml_cache::{decoded_document, encoded_document}; + +fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) +} + +#[test] +fn given_loaded_document_when_stored_and_read_back_then_it_is_unchanged() { + let date: ConfigDate = ConfigDate { + year: 2026, + month: 2, + day: 28, + }; + let time: ConfigTime = ConfigTime { + hour: 23, + minute: 59, + second: 58, + microsecond: 123_456, + }; + let test_cases = [ + DocumentTestCase { + description: "empty document", + document: ConfigValue::Null, + }, + DocumentTestCase { + description: "sources document with repeated strings", + document: ConfigValue::Map(vec![( + text("sources"), + ConfigValue::List(vec![ + ConfigValue::Map(vec![ + (text("name"), text("orders")), + (text("schema"), text("raw")), + ]), + ConfigValue::Map(vec![ + (text("name"), text("customers")), + (text("schema"), text("raw")), + ]), + ]), + )]), + }, + DocumentTestCase { + description: "every scalar kind", + document: ConfigValue::List(vec![ + ConfigValue::Bool(true), + ConfigValue::Bool(false), + ConfigValue::Integer(-42), + ConfigValue::BigInteger("123456789012345678901234567890".to_owned()), + ConfigValue::Float(-0.5), + ConfigValue::Date(date), + ConfigValue::Time(time), + ConfigValue::DateTime(ConfigDateTime { + date, + time, + utc_offset_seconds: Some(-3600), + }), + ConfigValue::DateTime(ConfigDateTime { + date, + time, + utc_offset_seconds: None, + }), + text(""), + text("unicode ✓"), + ]), + }, + DocumentTestCase { + description: "mapping with non-string keys", + document: ConfigValue::Map(vec![ + (ConfigValue::Integer(1), text("one")), + (ConfigValue::Null, ConfigValue::List(Vec::new())), + ]), + }, + ]; + for test_case in test_cases { + let stored: Vec = encoded_document(&test_case.document); + assert_eq!( + decoded_document(&stored), + Some(test_case.document.clone()), + "{}", + test_case.description + ); + for length in 0..stored.len() { + assert_ne!( + decoded_document(&stored[..length]), + Some(test_case.document.clone()), + "{}: truncated to {length} bytes", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index d747ec27e6..6cd405d255 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -49,6 +49,8 @@ use crate::bindings::_helpers::records::ids::RecordId; use pyo3::IntoPyObjectExt; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; +/// The setting that turns delta compiles, and the project image they read, off. +const DELTA_COMPILE_SETTING_KEY: &str = "delta_compile"; /// Declaration file collections counted into the declaration-parse lifecycle, in field order. const DECLARATION_COLLECTIONS: [&str; 14] = [ @@ -236,6 +238,18 @@ fn assembled_inputs<'py>( local_config: &local_config, })?; reject_unsupported_python_paths(project_dir)?; + let delta_settings: &Bound<'py, PyAny> = + if native_field(&local_config, "setting_overrides")?.contains(DELTA_COMPILE_SETTING_KEY)? { + &local_config + } else { + &project_config + }; + if !native_field(delta_settings, "settings")? + .getattr(DELTA_COMPILE_SETTING_KEY)? + .is_truthy()? + { + crate::bindings::_helpers::compile_delta::context::stop_capture(); + } let sql_analysis_override: &Bound<'py, PyAny> = request.sql_analysis_enabled_override.bind(py); let sql_analysis_enabled: Bound<'py, PyAny> = if !sql_analysis_override.is_none() { sql_analysis_override.clone() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 159055abea..8df9e95178 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -212,6 +212,7 @@ const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("test_timeout_seconds", FieldDefault::Int(30)), FieldSpec::new("scenario_timeout", FieldDefault::None), FieldSpec::new("scenario_timeout_seconds", FieldDefault::None), + FieldSpec::new("delta_compile", FieldDefault::True), ], }; const R_SCOPES_CONFIG: RecordSpec = RecordSpec { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c7bd5f8fe3..370bf8a534 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -13756,6 +13756,7 @@ class SettingsConfig(NativeRecord): test_timeout_seconds: int | None scenario_timeout: str | None scenario_timeout_seconds: int | None + delta_compile: bool def __init__( self, sql_analysis: bool = ..., @@ -13774,6 +13775,7 @@ class SettingsConfig(NativeRecord): test_timeout_seconds: int | None = ..., scenario_timeout: str | None = ..., scenario_timeout_seconds: int | None = ..., + delta_compile: bool = ..., ) -> None: ... @property def sql_validation(self) -> bool: ... From a5fdc0942be7702a610bf6b37f01a25ab2ed1652 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:31:42 +0100 Subject: [PATCH 16/32] docs(compile): describe compiling only the changed files after an edit --- website/src/content/docs/docs/cli/compile.mdx | 35 +++++++++++++++---- .../docs/concepts/project-configuration.mdx | 2 ++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 978f0493ee..425c0428ad 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -120,11 +120,31 @@ prints `Recording compile for reuse...` and a completion line to stderr before t ### Compiling after an edit -Any change runs the whole compile again: every model, test, and audit is rendered again, and -contracts and semantic validation run for the whole project. No render is carried over from the -previous compile; rendering runs natively and does not run unchanged macro calls again. Every -model's SQL references are scanned again too; no reference scan is kept between compiles. A few -caches under `target/cache/` shorten the rest: +When only model SQL files and SQL test files changed since the stored compile, `sqb compile` +compiles just those files against it. It renders, analyses and checks the changed models and SQL +tests, plus any model whose inputs' columns, types or nullability the edit changed, and takes +everything else from a project image stored beside the stored compile. It prints one line to +stderr, for example `Compiled 1 changed file(s) against the previous compile (1 model(s), 2.1s)`, +then the same report and exit code a full compile would produce, including any errors the edit +introduces or fixes. Project-wide rules still run over the whole project. + +The compile runs in full instead, decided before any compile work starts, when: + +- A file was added, removed or renamed, or any file other than a model or SQL test changed, such + as `sqlbuild_project.toml`, a source or schema YAML file, a macro, a seed or a function. +- An edited model changes what it references, or changes a header setting other than its + `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch + between `view` and `table` materialization. +- A custom rule reads something the edit changed, such as a model's contract or materialization. +- The stored compile used the other output format (`--json` or text), or has no project image. + +A few edits are found to need a full compile only part-way through, for example when a macro's +arguments change which models or sources it reads, or when the edit changes the shape of so many +models that a full compile is cheaper. + +Otherwise, an edit runs the whole compile again, and a few caches under `target/cache/` shorten +it: every model, test, and audit is rendered again, and contracts and semantic validation run for +the whole project. Every model's SQL references are scanned again too. - An unchanged model takes its column analysis from `model-analyses.bin`. Entries are keyed by the model's query and settings, the inferred columns, types and nullability of every relation @@ -137,8 +157,9 @@ caches under `target/cache/` shorten the rest: - Macro call results are reused as described in [Macros must be deterministic](/docs/concepts/macros#macros-must-be-deterministic). -An edit therefore costs a full render of the project plus the analysis, test scans, and rules of -what changed. The output is byte-identical to `sqb compile --no-cache`. +Either way, the output is byte-identical to `sqb compile --no-cache`. To turn compiling only the +changed files off, set `delta_compile = false` under `[settings]` or set +`SQLBUILD_DISABLE_DELTA_COMPILE=1`; compiles then no longer store a project image. Known limits: diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 19e4b3324f..d2badeca2e 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -629,6 +629,7 @@ default_audit_run_scope = "delta_and_final" unit_test_max_real_models = 10 test_timeout = "30s" scenario_timeout = "15m" +delta_compile = true ``` | Field | Default | Description | @@ -646,6 +647,7 @@ scenario_timeout = "15m" | `unit_test_max_real_models` | `10` | Most real (unmocked) models one SQL unit test may execute; a larger test fails compilation. See [Unit test and scenario limits](#unit-test-and-scenario-limits). | | `test_timeout` | `30s` | Time limit for each SQL unit test, or `"disabled"` | | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | +| `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](/docs/cli/compile#compiling-after-an-edit). | ### Unit test and scenario limits From dda82c5b583fc4b96f488f09a7c18bb68b03d13e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:54:27 +0100 Subject: [PATCH 17/32] test(compile): verify seeded edit series of delta compiles end to end - A seeded series of edits to a 300-model DuckDB project, in JSON and text, checks every delta against a full compile and every expected fallback. - Cover replaying a delta, multi-round deltas followed by another edit, and the delta_compile setting and environment opt-outs. - Fall back before compiling when the stored compile reports non-rule problems in files the delta would not compile again, which it cannot reproduce, and log which diagnostic differs when one still does. - Export the delta compile environment variable names. --- .../bindings/_helpers/compile_delta/driver.rs | 37 ++- .../_helpers/exported_constants/constants.rs | 5 + src/sqlbuild/_native.pyi | 2 + .../cli/commands/main/compile/_test_types.py | 27 ++ .../cli/commands/main/compile/helpers.py | 42 +++ .../main/compile/test_compile_delta.py | 287 ++++++++++++++++++ 6 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 750d2620eb..3ba7bd0335 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -41,6 +41,8 @@ pub(crate) const ERRORS_ON_DELTA_PATH: bool = true; /// How many times a changed relation shape may pull its dependents into the delta. const MAX_ROUNDS: usize = 4; const MODELS_ROOT: &str = "models/"; +/// How a rules diagnostic's report JSON names its phase; rules run over the whole project. +const RULE_PHASE: &str = "\"phase\": \"rule\""; /// What every error for a reference to an undiscovered source says. const UNKNOWN_SOURCE: &str = "unknown source"; /// The upstream kind of a source a model reads. @@ -488,6 +490,13 @@ pub(crate) fn run<'py>( return fallback("too many models would compile again"); } rerun = rerun.with_tests(&image, &changed_tests); + if image + .diagnostics + .iter() + .any(|diagnostic| kept_only_by_full_compiles(&rerun, diagnostic)) + { + return fallback("the stored compile reports problems in files the edit leaves alone"); + } let sources_read: Option> = (!image.diagnostics.iter().any(outside_resources)).then(|| { image @@ -669,6 +678,17 @@ fn first_rerun( Ok((rerun, changed_tests)) } +/// Whether a stored diagnostic concerns a model or SQL test the delta does not compile again +/// and comes from a phase other than rules, which only a full compile reports again. +fn kept_only_by_full_compiles(rerun: &Rerun, diagnostic: &CapturedDiagnostic) -> bool { + !rerun.owns(diagnostic) + && !diagnostic.json.contains(RULE_PHASE) + && diagnostic + .paths + .iter() + .any(|path| path.starts_with(MODELS_ROOT) || path.starts_with(TESTS_ROOT)) +} + /// Whether a stored diagnostic concerns a file other than a model or SQL test, which a delta /// compile then reads in full. fn outside_resources(diagnostic: &CapturedDiagnostic) -> bool { @@ -723,8 +743,21 @@ fn finished<'py>( { return fallback("the delta compile read a source it did not discover"); } - if unowned(rerun, &base.image.diagnostics) != unowned(rerun, &ran.capture.diagnostics) { - return fallback("diagnostics outside the changed files differ"); + let (stored, fresh) = ( + unowned(rerun, &base.image.diagnostics), + unowned(rerun, &ran.capture.diagnostics), + ); + if stored != fresh { + let only = |left: &[&str], right: &[&str]| { + left.iter() + .find(|item| !right.contains(item)) + .map_or(String::new(), |item| (*item).to_owned()) + }; + return fallback(format!( + "diagnostics outside the changed files differ: stored {} / delta {}", + only(&stored, &fresh), + only(&fresh, &stored) + )); } let recorded: Bound<'py, PyAny> = ran.output.getattr("stdout")?; if recorded.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index c02be409fd..044bc2611c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -41,6 +41,9 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::{ use crate::bindings::_helpers::compile_command::constants::{ REUSE_DISABLE_ENV_VAR, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, }; +use crate::bindings::_helpers::compile_delta::driver::{ + DELTA_DISABLE_ENV_VAR, DELTA_VERIFY_ENV_VAR, +}; use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; use crate::bindings::_helpers::native_host::constants::CUSTOM_HOST_LAUNCH_MODULE; use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; @@ -69,6 +72,8 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("STAGED_PROMOTION_MODE", STAGED_PROMOTION_MODE), ("STAGE_CAPTURE_DIR_ENV_VAR", CAPTURE_DIR_ENV_VAR), ("COMPILE_REUSE_DISABLE_ENV_VAR", REUSE_DISABLE_ENV_VAR), + ("DELTA_COMPILE_DISABLE_ENV_VAR", DELTA_DISABLE_ENV_VAR), + ("DELTA_COMPILE_VERIFY_ENV_VAR", DELTA_VERIFY_ENV_VAR), ("SQLBUILD_QUERY_TAG_APP", QUERY_TAG_APP), ("COMMAND_OUTPUT_RECORD_TYPE", COMMAND_OUTPUT_RECORD_TYPE), ( diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 370bf8a534..bc9c83bb49 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -176,6 +176,8 @@ TABLE_PROMOTION_MODE_SETTING_KEY: str STAGED_PROMOTION_MODE: str STAGE_CAPTURE_DIR_ENV_VAR: str COMPILE_REUSE_DISABLE_ENV_VAR: str +DELTA_COMPILE_DISABLE_ENV_VAR: str +DELTA_COMPILE_VERIFY_ENV_VAR: str SQLBUILD_QUERY_TAG_APP: str STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 0a68348ecf..2a24618817 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1103,3 +1103,30 @@ class EarlyBuiltinRulesEditTestCase: expected_diagnostics: tuple[tuple[str, str, int, int], ...] expected_early_outcomes: tuple[tuple[int, int], ...] expected_rule_cache_outcomes: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class DeltaCompileEdit: + """One edit a seeded delta compile series can make, and whether it may compile as a delta.""" + + kind: str + pattern: str + replacement: str + on_test_file: bool = False + # A fallback reason fragment when the edit must compile in full; `None` when it is a delta. + fallback_reason: str | None = None + + +@dataclass(frozen=True) +class DeltaCompileSeriesTestCase: + description: str + seed: int + json_output: bool + steps: int + + +@dataclass(frozen=True) +class DeltaCompileOptOutTestCase: + description: str + settings: str + env: dict[str, str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index b852152b73..1f08108c46 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5095,3 +5095,45 @@ def json_diagnostic_codes(stdout: str) -> tuple[str, ...]: document: dict[str, Any] = json.loads(stdout or "{}") return tuple(str(diagnostic["code"]) for diagnostic in document.get("diagnostics", ())) + + +DELTA_COMPILE_LINE: str = "against the previous compile" +DELTA_VERIFIED_LINE: str = "delta compile verified" +DELTA_FALLBACK_PREFIX: str = "delta compile fell back: " +DELTA_MISMATCH_PREFIX: str = "delta compile mismatch: " + + +class DeltaCompileRun(NamedTuple): + """One fresh-process compile with compile reuse and, optionally, the delta verifier.""" + + returncode: int + stdout: str + stderr: str + + @property + def verified(self) -> bool: + return DELTA_VERIFIED_LINE in self.stderr + + @property + def mismatches(self) -> list[str]: + return [line for line in self.stderr.splitlines() if line.startswith(DELTA_MISMATCH_PREFIX)] + + @property + def fallback_reason(self) -> str | None: + for line in self.stderr.splitlines(): + if line.startswith(DELTA_FALLBACK_PREFIX): + return line.removeprefix(DELTA_FALLBACK_PREFIX) + return None + + +def run_delta_compile( + *, project_dir: Path, json_output: bool, env: dict[str, str] | None = None +) -> DeltaCompileRun: + """Compile in a fresh process with compile reuse enabled and the given extra environment.""" + + result: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, + args=("compile", "--json") if json_output else ("compile",), + env={REUSE_DISABLE_ENV_VAR: "0", **(env or {})}, + ) + return DeltaCompileRun(returncode=result.returncode, stdout=result.stdout, stderr=result.stderr) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py new file mode 100644 index 0000000000..50cc23d903 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -0,0 +1,287 @@ +"""Delta compiles must print exactly what a full compile of the same edit prints.""" + +from __future__ import annotations + +import random +import re +from pathlib import Path + +import pytest + +from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from sqlbuild._native import DELTA_COMPILE_DISABLE_ENV_VAR, DELTA_COMPILE_VERIFY_ENV_VAR +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + DeltaCompileEdit, + DeltaCompileOptOutTestCase, + DeltaCompileSeriesTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + COMPILE_REUSE_HIT_LINE, + DELTA_COMPILE_LINE, + DeltaCompileRun, + replace_project_text, + run_delta_compile, +) + +_MODEL_COUNT: int = 300 +_VERIFY_ENV: dict[str, str] = {DELTA_COMPILE_VERIFY_ENV_VAR: "1"} +_MISMATCH_EXIT_CODE: int = 70 +_REVERT_PROBABILITY: float = 0.5 +_MINIMUM_DELTAS: int = 3 +# Edits that compile as deltas are drawn more often than those that must compile in full. +_DELTA_WEIGHT: int = 3 +_FALLBACK_WEIGHT: int = 1 +_EDIT_ATTEMPTS: int = 50 +_PROJECT_CONFIG: str = "sqlbuild_project.toml" +# Problems a full compile reported in files a delta does not compile again keep it full. +_STALE_PROBLEMS: str = "reports problems in files the edit leaves alone" +_REUSE_ROOT: str = "target/cache/compiler-native-v1/project-reuse-v3" + +_EDITS: tuple[DeltaCompileEdit, ...] = ( + DeltaCompileEdit("body_constant", r"COALESCE\(b\.amount, \d+\)", "COALESCE(b.amount, {value})"), + DeltaCompileEdit("body_comment", r"\Z", "\n-- reviewed {value}\n"), + DeltaCompileEdit( + "description", r'MODEL \(description "[^"]*"', 'MODEL (description "Edited {value}."' + ), + DeltaCompileEdit( + "column_description", + r'description "Order quantity measure"', + 'description "Edited measure {value}"', + ), + DeltaCompileEdit("type_error", r"quantity_000 \(type DOUBLE", "quantity_000 (type VARCHAR"), + DeltaCompileEdit("parse_error", r"projected AS \(SELECT", "projected AS (SELEC"), + DeltaCompileEdit( + "test_expectation", + r"(__expected__\w+ AS \(SELECT CAST\()\d+", + r"\g<1>{value}", + on_test_file=True, + ), + DeltaCompileEdit( + "materialization", + r"materialized table", + "materialized view", + fallback_reason="custom rules select models by their materialization", + ), + DeltaCompileEdit( + "contract", + r"contract enforced, ", + "", + fallback_reason="a custom rule reads", + ), + DeltaCompileEdit( + "unknown_reference", + r'__ref\("([^"]+)"\)', + r'__ref("\1_missing")', + fallback_reason="changed what it references", + ), +) +_CONFIG_EDIT: DeltaCompileEdit = DeltaCompileEdit( + "project_config", + r"\Z", + "\n# reviewed {value}\n", + fallback_reason="sqlbuild_project.toml is not a model or SQL test file", +) + + +def write_delta_compile_project(*, project_dir: Path, model_count: int) -> None: + """Write the dense synthetic project on DuckDB: sources, seeds, functions, macros, SQL tests + and custom rules.""" + + write_dense_compile_project(project_dir=project_dir, model_count=model_count) + replace_project_text( + project_dir, _PROJECT_CONFIG, 'adapter = "snowflake"', 'adapter = "duckdb"' + ) + + +def _project_files(project_dir: Path, on_test_file: bool) -> list[Path]: + root: Path = project_dir / ("tests" if on_test_file else "models") + return sorted(path for path in root.rglob("*.sql") if "/_" not in path.as_posix()) + + +def _applied(path: Path, edit: DeltaCompileEdit, rng: random.Random) -> bool: + text: str = path.read_text(encoding="utf-8") + replacement: str = edit.replacement.replace("{value}", str(rng.randint(1, 10**6))) + edited, count = re.subn(edit.pattern, replacement, text, count=1) + if count == 0 or edited == text: + return False + path.write_text(edited, encoding="utf-8") + return True + + +def _edited_file(project_dir: Path, edit: DeltaCompileEdit, rng: random.Random) -> tuple[Path, str]: + """Apply `edit` to a randomly chosen file it changes; return the file and its old text.""" + + if edit is _CONFIG_EDIT: + candidates: list[Path] = [project_dir / _PROJECT_CONFIG] + else: + candidates = _project_files(project_dir, edit.on_test_file) + for _ in range(_EDIT_ATTEMPTS): + path: Path = rng.choice(candidates) + original: str = path.read_text(encoding="utf-8") + if _applied(path, edit, rng): + return path, original + raise AssertionError(f"no file takes the {edit.kind} edit") + + +def _assert_outcome( + run: DeltaCompileRun, edit: DeltaCompileEdit, step: str, other_files_edited: bool +) -> bool: + """Assert `run` compiled `edit` as it must; return whether it compiled as a delta.""" + + assert run.returncode != _MISMATCH_EXIT_CODE, f"{step}: {run.mismatches}" + assert run.mismatches == [], step + if COMPILE_REUSE_HIT_LINE in run.stderr: + # A revert to the state of a compile that stored nothing, such as one that raised, + # replays the compile stored before it. + assert "revert" in step, step + return False + reason: str | None = run.fallback_reason + if other_files_edited and reason is not None and _STALE_PROBLEMS in reason: + return False + if edit.fallback_reason is None: + assert run.verified, f"{step}: fell back: {reason}\n{run.stderr[-2000:]}" + return True + assert reason is not None and edit.fallback_reason in reason, f"{step}: {reason}" + return False + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileSeriesTestCase(description="json_report", seed=11, json_output=True, steps=12), + DeltaCompileSeriesTestCase(description="text_report", seed=12, json_output=False, steps=12), + DeltaCompileSeriesTestCase( + description="long_json_series", seed=13, json_output=True, steps=20 + ), + ], + ids=lambda case: case.description, +) +def test_given_seeded_edit_series_when_compiling_each_edit_then_deltas_match_full_compiles( + test_case: DeltaCompileSeriesTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) + rng: random.Random = random.Random(test_case.seed) + baseline: DeltaCompileRun = run_delta_compile( + project_dir=project_dir, json_output=test_case.json_output, env=_VERIFY_ENV + ) + assert baseline.returncode == 0, baseline.stderr[-2000:] + pending: list[tuple[Path, str, DeltaCompileEdit]] = [] + edits: tuple[DeltaCompileEdit, ...] = (*_EDITS, _CONFIG_EDIT) + weights: list[int] = [ + _FALLBACK_WEIGHT if edit.fallback_reason else _DELTA_WEIGHT for edit in edits + ] + failing: bool = False + deltas: int = 0 + for index in range(test_case.steps): + if pending and (failing or rng.random() < _REVERT_PROBABILITY): + path, original, edit = pending.pop() + path.write_text(original, encoding="utf-8") + step: str = f"step {index}: revert {edit.kind} in {path.name}" + else: + edit = rng.choices(edits, weights=weights)[0] + path, original = _edited_file(project_dir, edit, rng) + pending.append((path, original, edit)) + step = f"step {index}: {edit.kind} in {path.name}" + run: DeltaCompileRun = run_delta_compile( + project_dir=project_dir, json_output=test_case.json_output, env=_VERIFY_ENV + ) + other_files_edited: bool = any(other != path for other, _, _ in pending) + deltas += _assert_outcome(run, edit, step, other_files_edited) + failing = run.returncode != 0 + assert deltas >= _MINIMUM_DELTAS + + +def test_given_unchanged_project_when_compiling_after_a_delta_then_the_delta_is_replayed( + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) + full: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) + edited: Path = _project_files(project_dir, on_test_file=False)[7] + rng: random.Random = random.Random(5) + assert _applied(edited, _EDITS[0], rng) + delta: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) + replayed: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) + + assert full.returncode == delta.returncode == replayed.returncode == 0 + assert DELTA_COMPILE_LINE in delta.stderr + assert DELTA_COMPILE_LINE not in replayed.stderr + assert COMPILE_REUSE_HIT_LINE in replayed.stderr + assert replayed.stdout.count('"compile_timings"') == 1 + assert _without_timings(replayed.stdout) == _without_timings(delta.stdout) + + +def test_given_edit_changing_downstream_shapes_when_compiling_the_next_edit_then_both_are_deltas( + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) + models: list[Path] = _project_files(project_dir, on_test_file=False) + reshaped: Path = next( + path for path in models if "quantity_000 (type DOUBLE" in path.read_text(encoding="utf-8") + ) + baseline: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) + reshaped.write_text( + reshaped.read_text(encoding="utf-8") + .replace("quantity_000 (type DOUBLE", "quantity_000 (type FLOAT", 1) + .replace("AS DOUBLE) AS quantity_000", "AS FLOAT) AS quantity_000"), + encoding="utf-8", + ) + shape_edit: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) + assert _applied(models[11], _EDITS[0], random.Random(3)) + next_edit: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) + full: DeltaCompileRun = run_delta_compile( + project_dir=project_dir, json_output=True, env={DELTA_COMPILE_DISABLE_ENV_VAR: "1"} + ) + + assert baseline.returncode == 0, baseline.stderr[-2000:] + assert re.search(r"\(([2-9]|\d\d+) model\(s\)", shape_edit.stderr), shape_edit.stderr[-2000:] + assert DELTA_COMPILE_LINE in next_edit.stderr, next_edit.stderr[-2000:] + assert DELTA_COMPILE_LINE not in full.stderr + assert _without_timings(next_edit.stdout) == _without_timings(full.stdout) + + +def _without_timings(report: str) -> str: + return re.sub(r'"compile_timings": \{[^}]*\}', "", report) + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileOptOutTestCase( + description="project_setting", settings="\n[settings]\ndelta_compile = false\n", env={} + ), + DeltaCompileOptOutTestCase( + description="environment", settings="", env={DELTA_COMPILE_DISABLE_ENV_VAR: "1"} + ), + ], + ids=lambda case: case.description, +) +def test_given_delta_compiles_turned_off_when_compiling_an_edit_then_it_compiles_in_full( + test_case: DeltaCompileOptOutTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) + config: Path = project_dir / _PROJECT_CONFIG + config.write_text(config.read_text(encoding="utf-8") + test_case.settings, encoding="utf-8") + first: DeltaCompileRun = run_delta_compile( + project_dir=project_dir, json_output=False, env=test_case.env + ) + replace_project_text( + project_dir, + _project_files(project_dir, on_test_file=False)[3].relative_to(project_dir).as_posix(), + "COALESCE(b.amount, ", + "COALESCE(b.amount, 1 + ", + ) + edited: DeltaCompileRun = run_delta_compile( + project_dir=project_dir, json_output=False, env=test_case.env + ) + + assert first.returncode == edited.returncode == 0, edited.stderr[-2000:] + assert DELTA_COMPILE_LINE not in edited.stderr + assert sorted((project_dir / _REUSE_ROOT).glob("*.image")) == [] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From fbbf8f59637c15c882e14f77cd99663584c6445a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:54:37 +0100 Subject: [PATCH 18/32] docs(compile): note that problems in untouched files keep compiles full --- website/src/content/docs/docs/cli/compile.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 425c0428ad..0ded0472b2 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -136,6 +136,8 @@ The compile runs in full instead, decided before any compile work starts, when: `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch between `view` and `table` materialization. - A custom rule reads something the edit changed, such as a model's contract or materialization. +- The stored compile reported errors or warnings other than rule findings in model or SQL test + files the edit did not change. - The stored compile used the other output format (`--json` or text), or has no project image. A few edits are found to need a full compile only part-way through, for example when a macro's From fc2d29b1e3ce0d03ba6601940b91d6dac3d522ad Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:54:49 +0100 Subject: [PATCH 19/32] docs(compile): regenerate the skill reference pages for delta compiles --- .../sqlbuild/references/docs/cli/compile.md | 37 +++++++++++++++---- .../docs/concepts/project-configuration.md | 2 + 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 7fec47b9f4..bfffb7f32d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -129,11 +129,33 @@ prints `Recording compile for reuse...` and a completion line to stderr before t ### Compiling after an edit -Any change runs the whole compile again: every model, test, and audit is rendered again, and -contracts and semantic validation run for the whole project. No render is carried over from the -previous compile; rendering runs natively and does not run unchanged macro calls again. Every -model's SQL references are scanned again too; no reference scan is kept between compiles. A few -caches under `target/cache/` shorten the rest: +When only model SQL files and SQL test files changed since the stored compile, `sqb compile` +compiles just those files against it. It renders, analyses and checks the changed models and SQL +tests, plus any model whose inputs' columns, types or nullability the edit changed, and takes +everything else from a project image stored beside the stored compile. It prints one line to +stderr, for example `Compiled 1 changed file(s) against the previous compile (1 model(s), 2.1s)`, +then the same report and exit code a full compile would produce, including any errors the edit +introduces or fixes. Project-wide rules still run over the whole project. + +The compile runs in full instead, decided before any compile work starts, when: + +- A file was added, removed or renamed, or any file other than a model or SQL test changed, such + as `sqlbuild_project.toml`, a source or schema YAML file, a macro, a seed or a function. +- An edited model changes what it references, or changes a header setting other than its + `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch + between `view` and `table` materialization. +- A custom rule reads something the edit changed, such as a model's contract or materialization. +- The stored compile reported errors or warnings other than rule findings in model or SQL test + files the edit did not change. +- The stored compile used the other output format (`--json` or text), or has no project image. + +A few edits are found to need a full compile only part-way through, for example when a macro's +arguments change which models or sources it reads, or when the edit changes the shape of so many +models that a full compile is cheaper. + +Otherwise, an edit runs the whole compile again, and a few caches under `target/cache/` shorten +it: every model, test, and audit is rendered again, and contracts and semantic validation run for +the whole project. Every model's SQL references are scanned again too. - An unchanged model takes its column analysis from `model-analyses.bin`. Entries are keyed by the model's query and settings, the inferred columns, types and nullability of every relation @@ -146,8 +168,9 @@ caches under `target/cache/` shorten the rest: - Macro call results are reused as described in [Macros must be deterministic](../concepts/macros.md#macros-must-be-deterministic). -An edit therefore costs a full render of the project plus the analysis, test scans, and rules of -what changed. The output is byte-identical to `sqb compile --no-cache`. +Either way, the output is byte-identical to `sqb compile --no-cache`. To turn compiling only the +changed files off, set `delta_compile = false` under `[settings]` or set +`SQLBUILD_DISABLE_DELTA_COMPILE=1`; compiles then no longer store a project image. Known limits: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index b47766fc67..b4a40fe677 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -652,6 +652,7 @@ default_audit_run_scope = "delta_and_final" unit_test_max_real_models = 10 test_timeout = "30s" scenario_timeout = "15m" +delta_compile = true ``` | Field | Default | Description | @@ -669,6 +670,7 @@ scenario_timeout = "15m" | `unit_test_max_real_models` | `10` | Most real (unmocked) models one SQL unit test may execute; a larger test fails compilation. See [Unit test and scenario limits](#unit-test-and-scenario-limits). | | `test_timeout` | `30s` | Time limit for each SQL unit test, or `"disabled"` | | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | +| `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](../cli/compile.md#compiling-after-an-edit). | ### Unit test and scenario limits From 2b6ea49ba6d7acd44d6882233401b3211bc811f8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 03:52:46 +0100 Subject: [PATCH 20/32] style(compile): satisfy clippy in the project image --- .../src/bindings/_helpers/compile_delta/driver.rs | 2 +- .../src/bindings/_helpers/compile_delta/image.rs | 8 ++++---- .../bindings/_helpers/compile_delta/tests/test_image.rs | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 3ba7bd0335..45bf68145f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -579,7 +579,7 @@ pub(crate) fn run<'py>( let Some(rules) = rules else { return fallback("the delta compile retained no rules request"); }; - injection.rules = std::sync::Mutex::new(Some(PendingRules::Ready(rules))); + injection.rules = std::sync::Mutex::new(Some(PendingRules::Ready(Box::new(rules)))); for name in added { if let Some(model) = stored_models.get(name.as_str()) { rerun.model_paths.insert(model.relative_path.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 8220874967..cd6cdc1889 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -74,7 +74,7 @@ pub(crate) enum ImageRules { #[derive(Debug)] pub(crate) enum PendingRules { Decoding(std::thread::JoinHandle>), - Ready(BaseRules), + Ready(Box), } impl PendingRules { @@ -82,7 +82,7 @@ impl PendingRules { pub(crate) fn resolved(self) -> Option { match self { Self::Decoding(handle) => handle.join().ok().flatten(), - Self::Ready(rules) => Some(rules), + Self::Ready(rules) => Some(*rules), } } } @@ -134,7 +134,7 @@ impl From for RelationTables { } /// Rows of `table` that differ from the same relation's `types` row; `None` where equal. -fn differing(table: &Shapes, types: &Shapes) -> Vec<(String, Option>)> { +fn differing(table: &Shapes, types: &Shapes) -> StoredShapes { let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types .iter() .map(|(name, shape)| (name.as_str(), shape)) @@ -151,7 +151,7 @@ fn differing(table: &Shapes, types: &Shapes) -> Vec<(String, Option>)>, types: &Shapes) -> Option { +fn restored(rows: StoredShapes, types: &Shapes) -> Option { let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types .iter() .map(|(name, shape)| (name.as_str(), shape)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs index 7dcaf46192..03efd9fb17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs @@ -194,8 +194,10 @@ fn given_damaged_image_when_decoded_then_it_is_never_read() { DamagedImageTestCase { description: "gate of another format", damage: |sections| { - let mut other: ProjectImage = ProjectImage::default(); - other.format = "sqlbuild-project-image-v0".to_owned(); + let other: ProjectImage = ProjectImage { + format: "sqlbuild-project-image-v0".to_owned(), + ..ProjectImage::default() + }; let gate = other .encode() .expect("encoded") From f80fbd32e0bf183124fbb377ac33eb612b1329b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:15:40 +0100 Subject: [PATCH 21/32] refactor(compile): bring the cache, rules and analysis delta code in line with workspace policy - Serialize retained rules rows without their SQL text directly instead of scanning each row's JSON for the SQL keys, which a full compile paid for. - Single-line doc comments, constants ahead of behavior, explicit handling of unreadable stores, the image section alias in types.rs, one entry function per module, and table-driven tests with helpers in helpers.rs. --- .../analysis_session/_helpers/session.rs | 2 +- .../src/assembly/analysis_session/models.rs | 3 +- .../analysis_session/tests/helpers.rs | 28 ++++ .../tests/test_delta_session.rs | 55 ++++---- .../analysis_session/tests/test_types.rs | 7 + .../src/project_reuse/_helpers/attempt.rs | 12 +- .../src/project_reuse/_helpers/entry.rs | 13 +- .../src/project_reuse/_helpers/files.rs | 5 +- .../src/project_reuse/_helpers/image.rs | 17 ++- .../src/project_reuse/_helpers/target.rs | 3 +- .../src/project_reuse/main/mod.rs | 1 + .../src/project_reuse/main/read_delta_base.rs | 7 +- .../project_reuse/main/read_delta_output.rs | 8 ++ .../sqlbuild-cache/src/project_reuse/mod.rs | 1 + .../src/project_reuse/models.rs | 7 +- .../src/project_reuse/tests/helpers.rs | 16 ++- .../project_reuse/tests/test_project_image.rs | 113 ++++++++++------- .../src/project_reuse/tests/test_types.rs | 11 +- .../sqlbuild-cache/src/project_reuse/types.rs | 4 + .../_helpers/compile_delta/context.rs | 2 +- .../bindings/_helpers/compile_delta/driver.rs | 8 +- .../bindings/_helpers/compile_delta/image.rs | 2 +- .../src/engine/_helpers/cache.rs | 4 +- .../src/engine/_helpers/evaluation.rs | 5 +- .../src/engine/_helpers/memo.rs | 97 ++++---------- .../src/engine/tests/helpers.rs | 14 +- .../src/engine/tests/test_retained_rows.rs | 120 +++++++++++------- .../src/engine/tests/test_types.rs | 8 ++ crates/sqlbuild-rules/src/models.rs | 3 +- .../src/rules/_helpers/sql_test_rules.rs | 18 ++- 30 files changed, 342 insertions(+), 252 deletions(-) create mode 100644 crates/sqlbuild-cache/src/project_reuse/main/read_delta_output.rs create mode 100644 crates/sqlbuild-cache/src/project_reuse/types.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 57d2485c17..66f3c1196b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -688,7 +688,6 @@ fn batch_member<'a>(model: &'a ModelRequest, schema: &'a Shapes) -> BatchMember< } } -/// Python's `{column.name: column.type or "UNKNOWN"}`. /// A table's shapes in insertion order, owned. fn owned_rows(table: &ShapeTable) -> Shapes { table @@ -698,6 +697,7 @@ fn owned_rows(table: &ShapeTable) -> Shapes { .collect() } +/// Python's `{column.name: column.type or "UNKNOWN"}`. pub(crate) fn shape_columns(columns: &[ColumnFact]) -> Pairs { dict_from_pairs(columns.iter().map(shape_column)) } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 2d5bb5c9a0..3a625a8170 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -181,8 +181,7 @@ pub struct SessionRequest { /// Python's `dynamic_families_by_table`, in dict order. pub dynamic_families_by_table: Vec<(String, Vec)>, pub models: Vec, - /// `(dependency ordered, cyclic)` to analyse with instead of deriving them from `models`: - /// a delta compile analyses a few models as the whole project's session did. + /// The whole project's `(dependency ordered, cyclic)`, for a session of only some models. pub forced_order: Option<(bool, bool)>, /// Keep the relation facts the session ends with, for the project image. pub retain_relations: bool, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index f16b1eaebf..28908a0405 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -1,3 +1,4 @@ +use crate::assembly::analysis_session::models::RelationTables; use std::collections::HashMap; use crate::assembly::analysis_session::_helpers::cte_facts::{ @@ -657,3 +658,30 @@ pub(crate) fn column_view(finished: &ModelColumns) -> ColumnView<'_> { finished.proven_star, ) } + +/// `request` with `relations`' facts added for every relation it does not analyse. +pub(crate) fn injected_relations( + mut request: SessionRequest, + relations: &RelationTables, +) -> SessionRequest { + let analysed: Vec = request + .models + .iter() + .map(|model| model.name.clone()) + .collect(); + for (table, facts) in [ + (&mut request.column_types, &relations.types), + (&mut request.column_nullability, &relations.nullability), + (&mut request.complete_schemas, &relations.complete), + ] { + let present: Vec = table.iter().map(|(name, _)| name.clone()).collect(); + table.extend( + facts + .iter() + .filter(|(name, _)| !analysed.contains(name) && !present.contains(name)) + .cloned(), + ); + } + request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); + request +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs index 6fa8b6dd5d..2588501a63 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs @@ -1,8 +1,8 @@ use crate::assembly::analysis_session::models::{ModelRequest, RelationTables, SessionRequest}; use crate::assembly::analysis_session::tests::helpers::{ - completed, model_requests, orders_request, started, + completed, injected_relations, model_requests, orders_request, started, }; -use crate::assembly::analysis_session::tests::test_types::ModelSpec; +use crate::assembly::analysis_session::tests::test_types::{DeltaSessionTestCase, ModelSpec}; const ORDER_MODELS: [ModelSpec; 3] = [ ( @@ -25,43 +25,36 @@ const ORDER_MODELS: [ModelSpec; 3] = [ ), ]; -/// `request` with `relations`' facts added for every relation it does not analyse. -fn injected(mut request: SessionRequest, relations: &RelationTables) -> SessionRequest { - let analysed: Vec = request - .models - .iter() - .map(|model| model.name.clone()) - .collect(); - for (table, facts) in [ - (&mut request.column_types, &relations.types), - (&mut request.column_nullability, &relations.nullability), - (&mut request.complete_schemas, &relations.complete), - ] { - let present: Vec = table.iter().map(|(name, _)| name.clone()).collect(); - table.extend( - facts - .iter() - .filter(|(name, _)| !analysed.contains(name) && !present.contains(name)) - .cloned(), - ); - } - request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); - request -} - #[test] fn given_whole_project_relations_when_analysing_one_model_then_it_matches_the_whole_session() { + let test_cases = [ + DeltaSessionTestCase { + description: "a model reading another model's columns", + model_index: 1, + expected_whole_session_outcome: true, + }, + DeltaSessionTestCase { + description: "a model selecting every column of another model", + model_index: 2, + expected_whole_session_outcome: true, + }, + ]; let mut whole: SessionRequest = orders_request(model_requests(&ORDER_MODELS)); whole.retain_relations = true; let (_, whole_outcomes, finished) = completed(started(whole)); let relations: RelationTables = finished.relation_tables().cloned().expect("relations kept"); - assert!(relations.types.iter().any(|(name, _)| name == "stg_orders")); - - for (index, spec) in ORDER_MODELS.iter().enumerate().skip(1) { + for test_case in test_cases { + let spec: &ModelSpec = &ORDER_MODELS[test_case.model_index]; let one: Vec = model_requests(std::slice::from_ref(spec)); - let request: SessionRequest = injected(orders_request(one), &relations); + let request: SessionRequest = injected_relations(orders_request(one), &relations); + let (_, outcomes, _) = completed(started(request)); - assert_eq!(outcomes, vec![whole_outcomes[index].clone()], "{}", spec.0); + assert_eq!( + outcomes == vec![whole_outcomes[test_case.model_index].clone()], + test_case.expected_whole_session_outcome, + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index 21e234622c..c74ecd0b9e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -201,3 +201,10 @@ pub(crate) struct ContractProofJsonTestCase { pub(crate) proof: ContractProof, pub(crate) expected_json: &'static str, } + +/// One model of a session analysed alone against the whole session's relation facts. +pub(crate) struct DeltaSessionTestCase { + pub(crate) description: &'static str, + pub(crate) model_index: usize, + pub(crate) expected_whole_session_outcome: bool, +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs index 94a4c79ece..0ceec82d72 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs @@ -27,6 +27,12 @@ use crate::project_snapshot::main::snapshot_project_files::snapshot_project_file use crate::project_snapshot::main::text_path::text_path; use crate::project_snapshot::models::PathStamp; +const NO_STORED_COMPILE: &str = "no stored compile"; +const REUSE_BYPASSED: &str = "compile reuse is off"; +const OUTPUT_UNREADABLE: &str = "the stored output cannot be replayed"; +const IDENTITY_CHANGED: &str = "the invocation, runtime, environment or settings changed"; +const TARGET_CHANGED: &str = "the compiled target changed"; + fn now_ns() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -35,12 +41,6 @@ fn now_ns() -> i64 { }) } -const NO_STORED_COMPILE: &str = "no stored compile"; -const REUSE_BYPASSED: &str = "compile reuse is off"; -const OUTPUT_UNREADABLE: &str = "the stored output cannot be replayed"; -const IDENTITY_CHANGED: &str = "the invocation, runtime, environment or settings changed"; -const TARGET_CHANGED: &str = "the compiled target changed"; - fn bypassed(check: &ReuseCheck<'_>) -> bool { check.bypass_requested || check.rules.disabling_variables.iter().any(|(name, value)| { diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs index a61b875cf2..c042aea9ae 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs @@ -12,8 +12,9 @@ use crate::project_reuse::constants::{ REUSE_MAX_STORED_IMAGES, REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX, }; use crate::project_reuse::models::{ - DeltaBase, ImageSections, SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile, + DeltaBase, SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile, }; +use crate::project_reuse::types::ImageSections; use crate::project_snapshot::models::PathStamp; use crate::store::errors::StoreDecodeError; use crate::store::models::{RecordReader, RecordWriter}; @@ -128,8 +129,14 @@ pub(crate) fn read_delta_base(path: &Path) -> Option { /// The output of the stored compile `base` was read from, or `None` when it no longer is. pub(crate) fn read_delta_output(path: &Path, base: &DeltaBase) -> Option { - let bytes: Vec = std::fs::read(output_path(path)).ok()?; - decoded_output(&bytes, &base.output_identity).ok() + let bytes: Vec = match std::fs::read(output_path(path)) { + Ok(bytes) => bytes, + Err(_unreadable) => return None, + }; + let Ok(output) = decoded_output(&bytes, &base.output_identity) else { + return None; + }; + Some(output) } /// The image the slot names, or `None` when it has none or it cannot be read. diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs index 85b34391d4..ce48822f3e 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs @@ -203,10 +203,7 @@ pub(crate) fn pending_digest_bytes( .sum() } -/// The project paths whose content, kind or presence differs from the stored compile's. -/// -/// A file whose stamp still matches is unchanged; every file in `unverified` is compared by -/// content, and one without a usable digest on either side counts as changed. +/// Paths whose content, kind or presence differs; files in `unverified` compare by digest. pub(crate) fn project_file_changes( stored: &[StoredProjectFile], current: &[PathStamp], diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs index dbb64d9079..b4b5cb273a 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs @@ -5,12 +5,12 @@ use std::time::SystemTime; use crate::digest::types::ContentDigest; use crate::project_reuse::_helpers::sections::{ - SectionFile, decode_sections, encode_sections, write_atomically, + decode_sections, encode_sections, write_atomically, }; use crate::project_reuse::constants::{ REUSE_IMAGE_SUFFIX, REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX, }; -use crate::project_reuse::models::ImageSections; +use crate::project_reuse::types::ImageSections; const IMAGE_ROLE: &str = "project-reuse-image"; @@ -30,13 +30,16 @@ pub(crate) fn write_image(path: &Path, sections: &ImageSections) -> std::io::Res Ok(identity) } -/// The image the slot names, or `None` when it is missing, damaged or another compile's. -/// -/// Reading marks the image as used, so pruning keeps the images of recently used slots. +/// The slot's image, marked as used, or `None` when it is missing, damaged or another compile's. pub(crate) fn read_image(path: &Path, identity: &ContentDigest) -> Option { let file_path: PathBuf = image_path(path); - let bytes: Vec = std::fs::read(&file_path).ok()?; - let file: SectionFile<'_> = decode_sections(&bytes, IMAGE_ROLE).ok()?; + let bytes: Vec = match std::fs::read(&file_path) { + Ok(bytes) => bytes, + Err(_unreadable) => return None, + }; + let Ok(file) = decode_sections(&bytes, IMAGE_ROLE) else { + return None; + }; if file.identity != *identity { return None; } diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs index e259fd0b41..649a4737ae 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs @@ -96,8 +96,7 @@ fn matches_recorded( } } -/// Each expected artifact's stamp: the vouched one for an artifact kept as it was vouched for, -/// read for any other. +/// Each expected artifact's stamp: vouched when kept as vouched for, otherwise read. fn vouched_stamps(expected: &BTreeMap<&str, &RecordedArtifact>, vouched: &[PathStamp]) -> Stamps { let vouched: HashMap<&str, &PathStamp> = vouched .iter() diff --git a/crates/sqlbuild-cache/src/project_reuse/main/mod.rs b/crates/sqlbuild-cache/src/project_reuse/main/mod.rs index f45ee7b384..ba6988e321 100644 --- a/crates/sqlbuild-cache/src/project_reuse/main/mod.rs +++ b/crates/sqlbuild-cache/src/project_reuse/main/mod.rs @@ -1,5 +1,6 @@ pub mod check_reuse; pub mod pending_digest_bytes; pub mod read_delta_base; +pub mod read_delta_output; pub mod record_reuse; pub mod replay_stdout; diff --git a/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs index b3da76a610..f4347e59f8 100644 --- a/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs +++ b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs @@ -1,13 +1,8 @@ use std::path::Path; -use crate::project_reuse::models::{DeltaBase, StoredOutput}; +use crate::project_reuse::models::DeltaBase; /// The stored compile at `store_path` with its image, the base a delta compile patches. pub fn read_delta_base(store_path: &Path) -> Option { crate::project_reuse::_helpers::entry::read_delta_base(store_path) } - -/// The output of the stored compile `base` was read from, or `None` when it no longer is. -pub fn read_delta_output(store_path: &Path, base: &DeltaBase) -> Option { - crate::project_reuse::_helpers::entry::read_delta_output(store_path, base) -} diff --git a/crates/sqlbuild-cache/src/project_reuse/main/read_delta_output.rs b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_output.rs new file mode 100644 index 0000000000..ebbb734700 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_output.rs @@ -0,0 +1,8 @@ +use std::path::Path; + +use crate::project_reuse::models::{DeltaBase, StoredOutput}; + +/// The output of the stored compile `base` was read from, or `None` when it no longer is. +pub fn read_delta_output(store_path: &Path, base: &DeltaBase) -> Option { + crate::project_reuse::_helpers::entry::read_delta_output(store_path, base) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/mod.rs b/crates/sqlbuild-cache/src/project_reuse/mod.rs index 2a66cc14e3..76910b98c4 100644 --- a/crates/sqlbuild-cache/src/project_reuse/mod.rs +++ b/crates/sqlbuild-cache/src/project_reuse/mod.rs @@ -5,6 +5,7 @@ pub mod constants; pub mod errors; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-cache/src/project_reuse/models.rs b/crates/sqlbuild-cache/src/project_reuse/models.rs index e32a91d068..893ec924f6 100644 --- a/crates/sqlbuild-cache/src/project_reuse/models.rs +++ b/crates/sqlbuild-cache/src/project_reuse/models.rs @@ -1,5 +1,6 @@ //! The stored compile, the facts it is checked against, and one invocation's reuse state. +use crate::project_reuse::types::ImageSections; use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::PathBuf; @@ -70,9 +71,6 @@ pub struct StoredInputs { pub settings_digest: String, } -/// Named sections of one project image, in stored order. -pub type ImageSections = Vec<(String, Vec)>; - /// The stored compile a delta compile patches: its inputs and image, and which output it names. #[derive(Clone, Debug, PartialEq, Eq)] pub struct DeltaBase { @@ -170,8 +168,7 @@ pub struct CompileRecord { pub json_output: bool, /// The project image to keep beside the stored compile, when one was built. pub image: Option, - /// Stamps the caller verified for artifacts it kept; with them the compiled tree is not - /// walked again and only written artifacts are stamped. + /// Verified stamps of kept artifacts, which spare recording a walk of the compiled tree. pub vouched_target_stamps: Option>, } diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs b/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs index 09ff216d01..4359a23b90 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs @@ -1,9 +1,10 @@ use std::collections::BTreeMap; -use std::path::Path; +use std::path::{Path, PathBuf}; use crate::project_reuse::_helpers::entry::write_entry; use crate::project_reuse::_helpers::sections::{decode_sections, encode_sections}; use crate::project_reuse::models::{SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile}; +use crate::project_reuse::types::ImageSections; use crate::project_snapshot::models::PathStamp; pub(crate) fn stamp(relative_path: &str, kind: &'static str, link: Option<&str>) -> PathStamp { @@ -169,3 +170,16 @@ pub(crate) fn damage(path: &Path) { bytes[last] ^= 0xff; std::fs::write(path, bytes).expect("damaged slot"); } + +/// A small project image of two sections. +pub(crate) fn image() -> ImageSections { + vec![ + ("facts".to_owned(), b"{\"models\": [\"orders\"]}".to_vec()), + ("rules".to_owned(), b"{}".to_vec()), + ] +} + +/// The image file beside the slot at `path`. +pub(crate) fn image_file(path: &Path) -> PathBuf { + path.with_extension("image") +} diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs index bbf15e173f..a3bd567c9c 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs @@ -1,24 +1,15 @@ use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; use crate::project_reuse::_helpers::entry::{read_delta_base, read_delta_output, write_entry}; use crate::project_reuse::_helpers::files::{project_file_changes, unverified_paths}; -use crate::project_reuse::models::{ImageSections, ProjectFileChanges, StoredProjectFile}; -use crate::project_reuse::tests::helpers::{stamp, stored_inputs, stored_output}; +use crate::project_reuse::models::{ProjectFileChanges, StoredProjectFile}; +use crate::project_reuse::tests::helpers::{ + image, image_file, stamp, stored_inputs, stored_output, +}; use crate::project_reuse::tests::test_types::{ - ChangedFileTestCase, ImagePruneTestCase, ImageReadTestCase, + ChangedFileTestCase, ImagePruneTestCase, ImageReadTestCase, ImageRestoreTestCase, }; - -fn image() -> ImageSections { - vec![ - ("facts".to_owned(), b"{\"models\": [\"orders\"]}".to_vec()), - ("rules".to_owned(), b"{}".to_vec()), - ] -} - -fn image_file(path: &Path) -> PathBuf { - path.with_extension("image") -} +use crate::project_reuse::types::ImageSections; #[test] fn given_stored_compile_with_image_when_reading_delta_base_then_only_an_intact_image_is_read() { @@ -88,20 +79,50 @@ fn given_stored_compile_with_image_when_reading_delta_base_then_only_an_intact_i } #[test] -fn given_stored_compile_without_image_when_storing_again_then_the_stale_image_is_removed() { - let folder = tempfile::tempdir().expect("store folder"); - let path = folder.path().join("slot.store"); - write_entry( - &path, - (&stored_inputs(), &stored_output(), Some(&image())), - 8, - ) - .expect("stored"); +fn given_stored_compile_with_image_when_storing_again_then_only_a_new_image_is_kept() { + let test_cases = [ + ImageRestoreTestCase { + description: "storing without an image removes the stale one", + stored_again: None, + expected_image_kept: false, + }, + ImageRestoreTestCase { + description: "storing with an image keeps it", + stored_again: Some(image), + expected_image_kept: true, + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("slot.store"); + write_entry( + &path, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored"); + let again: Option = test_case.stored_again.map(|image| image()); - write_entry(&path, (&stored_inputs(), &stored_output(), None), 8).expect("stored again"); + write_entry( + &path, + (&stored_inputs(), &stored_output(), again.as_ref()), + 8, + ) + .expect("stored again"); - assert!(!image_file(&path).exists()); - assert_eq!(read_delta_base(&path), None); + assert_eq!( + image_file(&path).exists(), + test_case.expected_image_kept, + "{}", + test_case.description + ); + assert_eq!( + read_delta_base(&path).is_some(), + test_case.expected_image_kept, + "{}", + test_case.description + ); + } } #[test] @@ -167,49 +188,49 @@ fn given_project_file_when_computing_delta_changes_then_only_content_changes_cou description: "settled equal stamp is unchanged without reading", stored_digest: None, racy: false, - moved_stamp: false, + mtime_shift_ns: 0, current_digest: None, - expected_changed: false, + expected_changed: &[], }, ChangedFileTestCase { description: "moved stamp with equal bytes is unchanged", stored_digest: Some("b3:one"), racy: false, - moved_stamp: true, + mtime_shift_ns: 1, current_digest: Some("b3:one"), - expected_changed: false, + expected_changed: &[], }, ChangedFileTestCase { description: "moved stamp with new bytes is changed", stored_digest: Some("b3:one"), racy: false, - moved_stamp: true, + mtime_shift_ns: 1, current_digest: Some("b3:two"), - expected_changed: true, + expected_changed: &["models/orders.sql"], }, ChangedFileTestCase { description: "racy equal stamp with new bytes is changed", stored_digest: Some("b3:one"), racy: true, - moved_stamp: false, + mtime_shift_ns: 0, current_digest: Some("b3:two"), - expected_changed: true, + expected_changed: &["models/orders.sql"], }, ChangedFileTestCase { description: "moved stamp without a stored digest is changed", stored_digest: None, racy: false, - moved_stamp: true, + mtime_shift_ns: 1, current_digest: Some("b3:one"), - expected_changed: true, + expected_changed: &["models/orders.sql"], }, ChangedFileTestCase { description: "unreadable moved file is changed", stored_digest: Some("b3:one"), racy: false, - moved_stamp: true, + mtime_shift_ns: 1, current_digest: None, - expected_changed: true, + expected_changed: &["models/orders.sql"], }, ]; for test_case in test_cases { @@ -219,9 +240,7 @@ fn given_project_file_when_computing_delta_changes_then_only_content_changes_cou racy: test_case.racy, }]; let mut current = stamp("models/orders.sql", "f", None); - if test_case.moved_stamp { - current.mtime_ns += 1; - } + current.mtime_ns += test_case.mtime_shift_ns; let current = vec![current, stamp("models/customers.sql", "f", None)]; let digests: HashMap = test_case .current_digest @@ -232,11 +251,11 @@ fn given_project_file_when_computing_delta_changes_then_only_content_changes_cou let changes = project_file_changes(&stored, ¤t, (&digests, &unverified)); let expected = ProjectFileChanges { - changed: if test_case.expected_changed { - vec!["models/orders.sql".to_owned()] - } else { - Vec::new() - }, + changed: test_case + .expected_changed + .iter() + .map(|path| (*path).to_owned()) + .collect(), added: vec!["models/customers.sql".to_owned()], removed: Vec::new(), }; diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs index edbb902334..6c2416836d 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs @@ -81,7 +81,14 @@ pub(crate) struct ChangedFileTestCase { pub(crate) description: &'static str, pub(crate) stored_digest: Option<&'static str>, pub(crate) racy: bool, - pub(crate) moved_stamp: bool, + pub(crate) mtime_shift_ns: i64, pub(crate) current_digest: Option<&'static str>, - pub(crate) expected_changed: bool, + pub(crate) expected_changed: &'static [&'static str], +} + +/// A stored compile with an image, stored again with or without one. +pub(crate) struct ImageRestoreTestCase { + pub(crate) description: &'static str, + pub(crate) stored_again: Option crate::project_reuse::types::ImageSections>, + pub(crate) expected_image_kept: bool, } diff --git a/crates/sqlbuild-cache/src/project_reuse/types.rs b/crates/sqlbuild-cache/src/project_reuse/types.rs new file mode 100644 index 0000000000..476b4b4da0 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/types.rs @@ -0,0 +1,4 @@ +//! Type aliases of the project reuse store. + +/// Named sections of one project image, in stored order. +pub type ImageSections = Vec<(String, Vec)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 74dd2bd62e..69eb3824a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -7,7 +7,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, SessionRequest}; -use sqlbuild_cache::project_reuse::models::ImageSections; +use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_rules::models::RetainedRows; use crate::bindings::_helpers::compile_delta::image::PendingRules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 45bf68145f..a50482eaa6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -7,10 +7,10 @@ use std::path::PathBuf; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyString, PyStringMethods}; use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; -use sqlbuild_cache::project_reuse::main::read_delta_base::{read_delta_base, read_delta_output}; -use sqlbuild_cache::project_reuse::models::{ - DeltaBase, DeltaCandidate, ImageSections, ProjectFileChanges, -}; +use sqlbuild_cache::project_reuse::main::read_delta_base::read_delta_base; +use sqlbuild_cache::project_reuse::main::read_delta_output::read_delta_output; +use sqlbuild_cache::project_reuse::models::{DeltaBase, DeltaCandidate, ProjectFileChanges}; +use sqlbuild_cache::project_reuse::types::ImageSections; use crate::bindings::_helpers::compile_command::reuse_recording::RecordedCompileOutput; use crate::bindings::_helpers::compile_delta::context::{ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index cd6cdc1889..fa8ad5c931 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; -use sqlbuild_cache::project_reuse::models::ImageSections; +use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::compile_delta::context::{ diff --git a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs index 670c724fc7..5bf79b7bc4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs @@ -15,6 +15,8 @@ pub(crate) struct RuleCacheEntry { pub faults: Vec, } +const FIXTURE_SUMMARIES_FILE: &str = "sql-test-fixtures.json"; + /// Where the SQL-test fixture summaries sit, beside the bulk native rules cache. pub(crate) fn fixture_summaries_path(project_dir: &Path, rules_cache_path: &str) -> PathBuf { let native: PathBuf = if rules_cache_path.is_empty() { @@ -25,8 +27,6 @@ pub(crate) fn fixture_summaries_path(project_dir: &Path, rules_cache_path: &str) native.with_file_name(FIXTURE_SUMMARIES_FILE) } -const FIXTURE_SUMMARIES_FILE: &str = "sql-test-fixtures.json"; - pub(crate) struct Cache { native_rules_path: PathBuf, } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 871155c23d..f0f4e3c2ed 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -343,7 +343,10 @@ pub(crate) fn evaluate_request_with( }); crate::rules::_helpers::sql_test_rules::with_fixture_memo(store.as_deref(), || { let evaluated: Result = evaluated_request(request, control); - let keep: bool = evaluated.is_ok() && !control.cancelled(); + let keep: bool = match &evaluated { + Ok(_) => !control.cancelled(), + Err(_failed) => false, + }; (evaluated, keep) }) } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index 8ffaa0b75f..8c8805ba36 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -12,6 +12,14 @@ use crate::engine::_helpers::evaluation::{EvaluationControl, evaluate_request_wi use crate::errors::RowsError; use crate::models::{EvaluateRequest, Model, RetainedRows, RowsEvaluation, RowsRequest}; +/// Where the model-free request's JSON holds its empty model list, which the rows fill. +const EMPTY_MODELS: &[u8] = b",\"models\":[],\"sql_tests\":"; +const MODELS_KEY: &[u8] = b",\"models\":["; + +/// The shared key prefix of fields a request serializes but evaluation derives again. +const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; +const DERIVED_FIELDS: [&str; 2] = ["_test_count\":", "\":"]; + /// A rows request whose models are digested, ready to evaluate once. #[derive(Debug)] pub(crate) struct PreparedRows { @@ -85,13 +93,7 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result = match request_json { Some(json) => json, - None => serde_json::to_vec(request).ok()?, + None => match serde_json::to_vec(request) { + Ok(json) => json, + Err(_unencodable) => return None, + }, }; let mut found = request_json .windows(EMPTY_MODELS.len()) @@ -133,50 +138,7 @@ fn retained_rows( }) } -/// A model row's JSON with its `query_sql` and `authored_sql` text emptied. -/// -/// They are the first string-valued keys after `name` and `relative_path`, and compact JSON -/// escapes every quote inside strings, so the first match of each key is that key. -fn without_sql(json: &[u8]) -> Option> { - let mut kept: Vec = Vec::with_capacity(json.len()); - let mut copied: usize = 0; - for key in [&b"\"query_sql\":\""[..], &b"\"authored_sql\":\""[..]] { - let start: usize = copied + find(&json[copied..], key)? + key.len(); - let end: usize = start + string_end(&json[start..])?; - kept.extend_from_slice(&json[copied..start]); - copied = end; - } - kept.extend_from_slice(&json[copied..]); - Some(kept) -} - -fn find(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - -/// The offset of the quote closing the JSON string whose text `bytes` starts with. -fn string_end(bytes: &[u8]) -> Option { - let mut index: usize = 0; - while index < bytes.len() { - match bytes[index] { - b'\\' => index += 2, - b'"' => return Some(index), - _ => index += 1, - } - } - None -} - -/// The prefix the keys of fields a request serializes but never reads back share; evaluation -/// derives them again. -const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; -const DERIVED_FIELDS: [&str; 2] = ["_test_count\":", "\":"]; - -/// `json` without the derived fields, each a bare number or boolean after its key. -/// -/// Compact JSON escapes every quote inside strings, so these keys only match as keys. +/// `json` without the derived fields; compact JSON escapes quotes, so keys only match as keys. fn without_derived_fields(json: Vec) -> Vec { let Ok(text) = std::str::from_utf8(&json) else { return json; @@ -311,34 +273,25 @@ fn speculation_key( Ok(hasher.finalize().to_hex().to_string()) } -/// `model` with the BLAKE3 digest of its exact facts, which keys its cached findings; with -/// `retain`, also its row as JSON without its SQL text. +/// `model` with the BLAKE3 digest of its exact facts and, with `retain`, its JSON without SQL. fn digested(mut model: Model, retain: bool) -> Result<(Model, Option>), RowsError> { - if model.payload_digest.is_some() { - let json: Option> = if retain { - let json: Vec = serde_json::to_vec(&model).map_err(rules_error)?; - Some(without_sql(&json).ok_or_else(|| RowsError::Rules(NO_SQL_KEYS.to_owned()))?) - } else { - None - }; - return Ok((model, json)); - } - if !retain { + if model.payload_digest.is_none() { let mut hasher: blake3::Hasher = blake3::Hasher::new(); serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; model.payload_digest = Some(hasher.finalize().to_hex().to_string()); + } + if !retain { return Ok((model, None)); } - let json: Vec = serde_json::to_vec(&model).map_err(rules_error)?; - model.payload_digest = Some(blake3::hash(&json).to_hex().to_string()); - let stripped: Vec = - without_sql(&json).ok_or_else(|| RowsError::Rules(NO_SQL_KEYS.to_owned()))?; - Ok((model, Some(stripped))) + let query_sql: String = std::mem::take(&mut model.query_sql); + let authored_sql: String = std::mem::take(&mut model.authored_sql); + let stripped: Result, serde_json::Error> = serde_json::to_vec(&model); + model.query_sql = query_sql; + model.authored_sql = authored_sql; + Ok((model, Some(stripped.map_err(rules_error)?))) } -/// The memo version, build, model-free request and every model's digest, in order. -/// -/// With `kept`, the model-free request's JSON is kept there as well. +/// The memo version, build, request and model digests, keeping the request's JSON in `kept`. fn identity( request: &EvaluateRequest, models: &[Model], diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index 89624c9562..f9482f9c95 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -9,7 +9,8 @@ use std::sync::atomic::AtomicBool; use crate::engine::main::answer_speculative_rows::answer_speculative_rows; use crate::engine::main::discard_speculative_rows::discard_speculative_rows; use crate::models::{ - EvaluateRequest, Fault, RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows, + EvaluateRequest, Fault, RetainedRows, RowsEvaluation, RowsRequest, SpeculativeAnswer, + SpeculativeRows, }; use crate::engine::_helpers::evaluation::evaluate_json; @@ -638,3 +639,14 @@ pub(crate) fn query_facts_faults( }) .collect()) } + +/// The retained request with each row's digest restored, as a delta compile rebuilds it. +pub(crate) fn restored_rows(retained: &RetainedRows) -> Result { + let mut request: EvaluateRequest = serde_json::from_slice(&retained.request_json) + .map_err(|error| format!("the retained request does not decode: {error}"))?; + for (model, digest) in request.models.iter_mut().zip(&retained.digests) { + model.payload_digest = Some(digest.clone()); + model.stripped = true; + } + Ok(request) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs index aaae4165fa..e720d8fa80 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs @@ -2,59 +2,85 @@ use tempfile::TempDir; use crate::engine::_helpers::evaluation::STRIPPED_ROW_MISSED; use crate::engine::main::evaluate_rows::evaluate_rows; -use crate::engine::tests::helpers::{evaluation_summary, orders_models, speculative_rows_request}; +use crate::engine::tests::helpers::{ + evaluation_summary, orders_models, restored_rows, speculative_rows_request, +}; +use crate::engine::tests::test_types::RetainedRowsTestCase; use crate::errors::RowsError; -use crate::models::{EvaluateRequest, RetainedRows, RowsRequest}; - -/// The retained request with each row's digest restored, as a delta compile rebuilds it. -fn restored(retained: &RetainedRows) -> Result { - let mut request: EvaluateRequest = serde_json::from_slice(&retained.request_json) - .map_err(|error| format!("the retained request does not decode: {error}"))?; - for (model, digest) in request.models.iter_mut().zip(&retained.digests) { - model.payload_digest = Some(digest.clone()); - model.stripped = true; - } - Ok(request) -} +use crate::models::{EvaluateRequest, Fault, RetainedRows, RowsEvaluation, RowsRequest}; #[test] fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_stand_in() -> Result<(), String> { - let project_dir = TempDir::new().map_err(|error| error.to_string())?; - let mut request: RowsRequest = - speculative_rows_request(project_dir.path(), true, &orders_models())?; - request.memo = None; - request.retain = true; - let first = evaluate_rows(request).map_err(|_| "first evaluation failed".to_owned())?; - let retained: RetainedRows = first.retained.clone().ok_or("nothing was retained")?; - let kept: EvaluateRequest = serde_json::from_slice(&retained.request_json) - .map_err(|error| format!("the retained request does not decode: {error}"))?; - assert_eq!(retained.digests.len(), kept.models.len()); - assert!( - kept.models - .iter() - .all(|model| model.query_sql.is_empty() && model.authored_sql.is_empty()) - ); + let test_cases = [ + RetainedRowsTestCase { + description: "the findings cached by the retaining evaluation stand in", + fresh_cache: false, + expected_same_findings: true, + expected_missed: None, + }, + RetainedRowsTestCase { + description: "without cached findings a stripped row is refused", + fresh_cache: true, + expected_same_findings: false, + expected_missed: Some(STRIPPED_ROW_MISSED), + }, + ]; + for test_case in test_cases { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let mut request: RowsRequest = + speculative_rows_request(project_dir.path(), true, &orders_models())?; + request.memo = None; + request.retain = true; + let first = evaluate_rows(request).map_err(|_| "first evaluation failed".to_owned())?; + let retained: RetainedRows = first.retained.clone().ok_or("nothing was retained")?; + let kept: EvaluateRequest = serde_json::from_slice(&retained.request_json) + .map_err(|error| format!("the retained request does not decode: {error}"))?; + let other_dir = TempDir::new().map_err(|error| error.to_string())?; + let mut stripped: EvaluateRequest = restored_rows(&retained)?; + stripped.project_dir = [project_dir.path(), other_dir.path()] + [usize::from(test_case.fresh_cache)] + .to_string_lossy() + .into_owned(); - let cached = evaluate_rows(RowsRequest { - request: restored(&retained)?, - memo: None, - retain: false, - }) - .map_err(|_| "cached evaluation failed".to_owned())?; - assert_eq!(evaluation_summary(&cached).0, evaluation_summary(&first).0); + let evaluated: Result = evaluate_rows(RowsRequest { + request: stripped, + memo: None, + retain: false, + }); - let uncached_dir = TempDir::new().map_err(|error| error.to_string())?; - let mut uncached: EvaluateRequest = restored(&retained)?; - uncached.project_dir = uncached_dir.path().to_string_lossy().into_owned(); - let missed = evaluate_rows(RowsRequest { - request: uncached, - memo: None, - retain: false, - }); - assert!( - matches!(&missed, Err(RowsError::Rules(message)) if message == STRIPPED_ROW_MISSED), - "a stripped row evaluated without its cached findings" - ); + assert_eq!( + retained.digests.len(), + kept.models.len(), + "{}", + test_case.description + ); + assert!( + kept.models + .iter() + .all(|model| model.query_sql.is_empty() && model.authored_sql.is_empty()), + "{}", + test_case.description + ); + let findings: Option> = match &evaluated { + Ok(evaluation) => Some(evaluation_summary(evaluation).0), + Err(_refused) => None, + }; + assert_eq!( + findings == Some(evaluation_summary(&first).0), + test_case.expected_same_findings, + "{}", + test_case.description + ); + let missed: Option<&str> = match &evaluated { + Err(RowsError::Rules(message)) => Some(message.as_str()), + _ => None, + }; + assert_eq!( + missed, test_case.expected_missed, + "{}", + test_case.description + ); + } Ok(()) } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index a2501f8c01..7753d9c716 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -282,3 +282,11 @@ pub(crate) struct QueryFactsTestCase { pub(crate) columns: Value, pub(crate) expected_faults: &'static [(&'static str, u64, u64, &'static str)], } + +/// A retained request evaluated again without SQL, with or without the cached findings. +pub(crate) struct RetainedRowsTestCase { + pub(crate) description: &'static str, + pub(crate) fresh_cache: bool, + pub(crate) expected_same_findings: bool, + pub(crate) expected_missed: Option<&'static str>, +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index c8899aee30..9e0ab232b9 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -933,8 +933,7 @@ pub struct RowsRequest { pub retain: bool, } -/// A request kept for the project image, as the JSON an `EvaluateRequest` reads back: its -/// models' SQL text and digests dropped and each model marked stripped. +/// A request kept for the project image as `EvaluateRequest` JSON without its models' SQL text. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct RetainedRows { pub request_json: Vec, diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs index 9b2cce8828..71c4365e9a 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs @@ -784,6 +784,18 @@ thread_local! { static FIXTURE_MEMO: RefCell> = const { RefCell::new(None) }; } +/// The fixture summaries stored at `path`; missing or unreadable ones are summarised again. +fn stored_summaries(path: &Path) -> Option { + let bytes: Vec = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(_unreadable) => return None, + }; + match serde_json::from_slice::(&bytes) { + Ok(stored) => Some(stored), + Err(_damaged) => None, + } +} + /// The stored fixture summaries, kept only for one native build. #[derive(Default, Serialize, Deserialize)] struct StoredSummaries { @@ -791,12 +803,10 @@ struct StoredSummaries { summaries: BTreeMap, } -/// Run `body` with each fixture summarised at most once on this thread, starting from the -/// summaries stored at `store` and saving them again when `body` says its result stands. +/// Run `body` summarising each fixture once, from and, when `body` says so, back to `store`. pub(crate) fn with_fixture_memo(store: Option<&Path>, body: impl FnOnce() -> (T, bool)) -> T { let loaded: BTreeMap = store - .and_then(|path| std::fs::read(path).ok()) - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .and_then(stored_summaries) .filter(|stored| stored.build == NATIVE_BUILD_IDENTITY) .map(|stored| stored.summaries) .unwrap_or_default(); From 9e71f5d4fd77fa591cdfc65db44ee65a73157579 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:58:03 +0100 Subject: [PATCH 22/32] perf(compile): check a delta's references against the stored upstream instead of digesting every model's - Full compiles no longer scan every model's authored references for the image; a delta compile scans only the changed models and falls back when one names a model, source or seed its stored compile did not read. - Find the retained request's model list at its first match and key the image string table with the fast hasher. --- Cargo.lock | 1 + crates/sqlbuild-python/Cargo.toml | 1 + .../bindings/_helpers/compile_delta/driver.rs | 11 ++- .../_helpers/compile_delta/header_keys.rs | 72 +++++++++---------- .../bindings/_helpers/compile_delta/image.rs | 2 +- .../_helpers/compile_delta/image_codec.rs | 5 +- .../_helpers/discovery/model_records.rs | 1 - .../src/engine/_helpers/memo.rs | 11 +-- 8 files changed, 53 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37ac81b26c..de1498a7da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1244,6 +1244,7 @@ dependencies = [ "mimalloc", "pyo3", "rayon", + "rustc-hash", "serde", "serde_json", "sqlbuild-analysis", diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 077bf8b416..8741c66d01 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -20,6 +20,7 @@ fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true rayon.workspace = true +rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index a50482eaa6..406fc7a80e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -637,12 +637,21 @@ fn first_rerun( return Err(format!("{path} is not a model or SQL test file")); } } + let upstream_by_path: HashMap<&str, &[(String, String)]> = gate + .models + .iter() + .map(|model| (model.relative_path.as_str(), model.upstream.as_slice())) + .collect(); for path in &rerun.model_paths { let name: &str = model_by_path .get(path.as_str()) .copied() .unwrap_or_default(); - match header_change(project_dir, path, gate.header_keys.get(path)) { + let upstream: &[(String, String)] = upstream_by_path + .get(path.as_str()) + .copied() + .unwrap_or_default(); + match header_change(project_dir, path, (gate.header_keys.get(path), upstream)) { HeaderChange::Unchanged => {} HeaderChange::Unparsed if !ERRORS_ON_DELTA_PATH => { return Err(format!("{path} does not parse")); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs index d2e8646c72..d8e658ae4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs @@ -28,6 +28,15 @@ const PATCHED_KEYS: [&str; 6] = [ /// `materialized` values a delta compile handles; others name materializations it cannot. const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"]; const MATERIALIZED_KEY: &str = "materialized"; +/// Plain `--` comments, which every adapter's SQL reads as comments. +const LINE_COMMENT: &str = "--"; +/// Reference call kinds and the upstream kinds they resolve to. +const REF_CALL: &str = "ref"; +const SOURCE_CALL: &str = "source"; +const SEED_CALL: &str = "seed"; +const MODEL_KIND: &str = "model"; +const SOURCE_KIND: &str = "source"; +const SEED_KIND: &str = "seed"; /// Bytes of a digest kept per header value; a collision would need 2^64 work to find. const KEPT_DIGEST_BYTES: usize = 16; @@ -77,37 +86,31 @@ fn encode_value(value: &AuthoredValue, out: &mut Vec) { } } -/// The pseudo header key holding a digest of the references a model's authored query names. -const REFERENCES_KEY: &str = "\u{0}references"; - -/// A digest of the reference calls `query_sql` names, scanned with plain `--` comments. An edit -/// that changes them changes what the model reads, which a delta compile does not patch. -fn references_digest(query_sql: &str) -> String { +/// Whether every model, source and seed `query_sql` names was among the stored `upstream`. +fn references_read(query_sql: &str, upstream: &[(String, String)]) -> bool { let syntax: LexicalSyntax = LexicalSyntax { line_comment_prefixes: vec![LINE_COMMENT.to_owned()], ..LexicalSyntax::default() }; - let mut buffer: Vec = Vec::new(); - match extract_sql_references(query_sql, &syntax) { - ReferenceExtraction::Extracted(scan) => { - for reference in &scan.references { - for part in [ - reference.kind, - &reference.name, - reference.package.as_deref().unwrap_or(""), - ] { - buffer.extend_from_slice(&(part.len() as u64).to_le_bytes()); - buffer.extend_from_slice(part.as_bytes()); - } - } + let ReferenceExtraction::Extracted(scan) = extract_sql_references(query_sql, &syntax) else { + return false; + }; + scan.references.iter().all(|reference| { + let kind: Option<&str> = match reference.kind { + REF_CALL => Some(MODEL_KIND), + SOURCE_CALL => Some(SOURCE_KIND), + SEED_CALL => Some(SEED_KIND), + _ => None, + }; + match (kind, &reference.package) { + (Some(kind), None) => upstream + .iter() + .any(|(stored_kind, name)| stored_kind == kind && *name == reference.name), + _ => true, } - ReferenceExtraction::Failed(_) => buffer.push(0xff), - } - short_hex(&bytes_digest(&buffer)) + }) } -const LINE_COMMENT: &str = "--"; - fn short_hex(digest: &[u8]) -> String { let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); for byte in &digest[..KEPT_DIGEST_BYTES] { @@ -117,7 +120,7 @@ fn short_hex(digest: &[u8]) -> String { hex } -fn header_keys(values: &[(String, AuthoredValue)], query_sql: &str) -> HeaderKeys { +fn header_keys(values: &[(String, AuthoredValue)]) -> HeaderKeys { let mut buffer: Vec = Vec::new(); let mut keys: HeaderKeys = values .iter() @@ -128,21 +131,16 @@ fn header_keys(values: &[(String, AuthoredValue)], query_sql: &str) -> HeaderKey (key.clone(), hex) }) .collect(); - keys.push((REFERENCES_KEY.to_owned(), references_digest(query_sql))); keys.sort(); keys } /// Record one discovered model's header keys for the image. -pub(crate) fn record_header_keys( - relative_path: &str, - values: &[(String, AuthoredValue)], - query_sql: &str, -) { +pub(crate) fn record_header_keys(relative_path: &str, values: &[(String, AuthoredValue)]) { if !capturing() { return; } - let keys: HeaderKeys = header_keys(values, query_sql); + let keys: HeaderKeys = header_keys(values); let path: String = relative_path.replace('\\', "/"); capture(|capture| { capture.header_keys.insert(path, keys); @@ -166,7 +164,7 @@ pub(crate) enum HeaderChange { pub(crate) fn header_change( project_dir: &Path, relative_path: &str, - stored: Option<&HeaderKeys>, + (stored, upstream): (Option<&HeaderKeys>, &[(String, String)]), ) -> HeaderChange { let Some(stored) = stored else { return HeaderChange::Unsupported(format!("{relative_path} has no recorded header")); @@ -181,7 +179,10 @@ pub(crate) fn header_change( let Ok(parsed) = parse_model_text(&file_path, contents, &model_file_options()) else { return HeaderChange::Unparsed; }; - let current: HeaderKeys = header_keys(&parsed.header_values, &parsed.query_sql); + if !references_read(&parsed.query_sql, upstream) { + return HeaderChange::Unsupported(format!("{relative_path} changed what it references")); + } + let current: HeaderKeys = header_keys(&parsed.header_values); let stored_map: BTreeMap<&str, &str> = stored .iter() .map(|(key, digest)| (key.as_str(), digest.as_str())) @@ -199,9 +200,6 @@ pub(crate) fn header_change( if changed.is_empty() { return HeaderChange::Unchanged; } - if changed.contains(REFERENCES_KEY) { - return HeaderChange::Unsupported(format!("{relative_path} changed what it references")); - } for key in &changed { if key == MATERIALIZED_KEY { let value = parsed.header_values.iter().find(|(name, _)| name == key); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index fa8ad5c931..769edf4ade 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; /// Changes whenever the image's meaning changes, so an older image is never read. -pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v5"; +pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v6"; const GATE_SECTION: &str = "gate"; const FACTS_SECTION: &str = "facts"; const RULES_SECTION: &str = "rules"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs index 386af395a6..f2c98be98d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs @@ -1,8 +1,7 @@ //! The project image's binary sections: the native store's length-prefixed records, with each //! distinct string written once and referred to by its index. -use std::collections::HashMap; - +use rustc_hash::FxHashMap; use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; /// The most items a list reserves room for before reading them, against corrupt counts. @@ -11,7 +10,7 @@ const MAX_RESERVED_ITEMS: usize = 1 << 16; /// Writes one section: its values refer to a table of the distinct strings they hold. #[derive(Default)] pub(crate) struct Encoder<'a> { - ids: HashMap<&'a str, u64>, + ids: FxHashMap<&'a str, u64>, strings: Vec<&'a str>, body: RecordWriter, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index 1d293297e3..b42b199fd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -72,7 +72,6 @@ fn model_items<'py>( crate::bindings::_helpers::compile_delta::header_keys::record_header_keys( &file.relative_path, &model.header_values, - &model.query_sql, ); let file_path: String = root.display_path(&file.relative_path); let paths = DiscoveredPaths::Native(file.relative_path.clone(), file_path); diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index 8c8805ba36..ed6eaece1a 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -12,7 +12,7 @@ use crate::engine::_helpers::evaluation::{EvaluationControl, evaluate_request_wi use crate::errors::RowsError; use crate::models::{EvaluateRequest, Model, RetainedRows, RowsEvaluation, RowsRequest}; -/// Where the model-free request's JSON holds its empty model list, which the rows fill. +/// Where the model-free request's JSON holds its empty model list; strings escape their quotes. const EMPTY_MODELS: &[u8] = b",\"models\":[],\"sql_tests\":"; const MODELS_KEY: &[u8] = b",\"models\":["; @@ -109,14 +109,9 @@ fn retained_rows( Err(_unencodable) => return None, }, }; - let mut found = request_json + let at: usize = request_json .windows(EMPTY_MODELS.len()) - .enumerate() - .filter(|(_, window)| *window == EMPTY_MODELS) - .map(|(index, _)| index); - let (Some(at), None) = (found.next(), found.next()) else { - return None; - }; + .position(|window| window == EMPTY_MODELS)?; let rows: usize = row_json.iter().map(Vec::len).sum::() + row_json.len(); let mut json: Vec = Vec::with_capacity(request_json.len() + rows); json.extend_from_slice(&request_json[..at]); From 19ca61bdbf5fead99bf0897ec59a6f57d49206f3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:01:01 +0100 Subject: [PATCH 23/32] refactor(compile): bring the delta compile in line with workspace policy - Split the delta driver into preparing, running rounds and finishing; pass owned values instead of mutable references; read recorded output and graph edges through their native types instead of Python attributes; log through one compile reuse debug helper. - Keep only text, not Python strings or the request, in finished deltas, and keep a compile's phases as a tuple as before. - Handle unreadable stores, images and JSON explicitly. - Move the delta compile unit tests to bindings/tests with shared helpers and table-driven cases, and the seeded E2E edit series into scripts/delta_compile_series. --- .../project_reuse/tests/test_project_image.rs | 26 +- .../_helpers/analysis_session/session.rs | 6 +- .../_helpers/compile_command/report.rs | 2 +- .../_helpers/compile_command/reuse_attempt.rs | 44 +- .../compile_command/reuse_recording.rs | 48 +- .../_helpers/compile_delta/context.rs | 36 +- .../bindings/_helpers/compile_delta/driver.rs | 502 +++++++++++------- .../_helpers/compile_delta/header_keys.rs | 103 ++-- .../bindings/_helpers/compile_delta/image.rs | 79 ++- .../_helpers/compile_delta/image_codec.rs | 38 +- .../_helpers/compile_delta/lint_hooks.rs | 9 +- .../bindings/_helpers/compile_delta/mod.rs | 3 - .../_helpers/compile_delta/report_hooks.rs | 33 +- .../_helpers/compile_delta/rules_rows.rs | 3 +- .../bindings/_helpers/compile_delta/splice.rs | 78 +-- .../bindings/_helpers/compile_delta/verify.rs | 39 +- .../_helpers/compile_delta/yaml_cache.rs | 372 +++++++------ .../_helpers/compile_driver/command.rs | 105 ++-- .../_helpers/compile_driver/discovery.rs | 8 +- .../_helpers/discovery/yaml_records.rs | 3 +- .../bindings/_helpers/graph/project_graph.rs | 2 +- .../src/bindings/_helpers/rules/sql_rules.rs | 38 +- crates/sqlbuild-python/src/bindings/mod.rs | 3 + .../{_helpers/compile_delta => }/tests.rs | 2 + .../tests/test_image.rs => tests/helpers.rs} | 175 +++--- .../src/bindings/tests/test_image.rs | 111 ++++ .../compile_delta => }/tests/test_splice.rs | 19 +- .../compile_delta => }/tests/test_types.rs | 18 +- .../tests/test_yaml_cache.rs | 36 +- .../src/engine/tests/test_retained_rows.rs | 18 +- .../src/rules/_helpers/sql_test_rules.rs | 8 +- scripts/delta_compile_series/__init__.py | 1 + .../delta_compile_series/_helpers/__init__.py | 1 + .../delta_compile_series/_helpers/edits.py | 55 ++ .../delta_compile_series/_helpers/outcomes.py | 43 ++ .../delta_compile_series/_helpers/sequence.py | 52 ++ scripts/delta_compile_series/constants.py | 71 +++ scripts/delta_compile_series/exceptions.py | 5 + scripts/delta_compile_series/main/__init__.py | 1 + .../main/run_delta_compile_sequence.py | 41 ++ .../main/run_delta_compile_series.py | 58 ++ .../main/write_delta_compile_project.py | 19 + scripts/delta_compile_series/models.py | 46 ++ scripts/delta_compile_series/types.py | 21 + .../cli/commands/main/compile/_test_types.py | 26 +- .../cli/commands/main/compile/helpers.py | 50 +- .../main/compile/test_compile_delta.py | 299 ++++------- 47 files changed, 1673 insertions(+), 1083 deletions(-) rename crates/sqlbuild-python/src/bindings/{_helpers/compile_delta => }/tests.rs (81%) rename crates/sqlbuild-python/src/bindings/{_helpers/compile_delta/tests/test_image.rs => tests/helpers.rs} (54%) create mode 100644 crates/sqlbuild-python/src/bindings/tests/test_image.rs rename crates/sqlbuild-python/src/bindings/{_helpers/compile_delta => }/tests/test_splice.rs (93%) rename crates/sqlbuild-python/src/bindings/{_helpers/compile_delta => }/tests/test_types.rs (56%) rename crates/sqlbuild-python/src/bindings/{_helpers/compile_delta => }/tests/test_yaml_cache.rs (77%) create mode 100644 scripts/delta_compile_series/__init__.py create mode 100644 scripts/delta_compile_series/_helpers/__init__.py create mode 100644 scripts/delta_compile_series/_helpers/edits.py create mode 100644 scripts/delta_compile_series/_helpers/outcomes.py create mode 100644 scripts/delta_compile_series/_helpers/sequence.py create mode 100644 scripts/delta_compile_series/constants.py create mode 100644 scripts/delta_compile_series/exceptions.py create mode 100644 scripts/delta_compile_series/main/__init__.py create mode 100644 scripts/delta_compile_series/main/run_delta_compile_sequence.py create mode 100644 scripts/delta_compile_series/main/run_delta_compile_series.py create mode 100644 scripts/delta_compile_series/main/write_delta_compile_project.py create mode 100644 scripts/delta_compile_series/models.py create mode 100644 scripts/delta_compile_series/types.py diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs index a3bd567c9c..cdabaf37dc 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; use crate::project_reuse::_helpers::entry::{read_delta_base, read_delta_output, write_entry}; use crate::project_reuse::_helpers::files::{project_file_changes, unverified_paths}; -use crate::project_reuse::models::{ProjectFileChanges, StoredProjectFile}; +use crate::project_reuse::models::StoredProjectFile; use crate::project_reuse::tests::helpers::{ image, image_file, stamp, stored_inputs, stored_output, }; @@ -250,15 +250,19 @@ fn given_project_file_when_computing_delta_changes_then_only_content_changes_cou let changes = project_file_changes(&stored, ¤t, (&digests, &unverified)); - let expected = ProjectFileChanges { - changed: test_case - .expected_changed - .iter() - .map(|path| (*path).to_owned()) - .collect(), - added: vec!["models/customers.sql".to_owned()], - removed: Vec::new(), - }; - assert_eq!(changes, expected, "{}", test_case.description); + assert_eq!( + changes.changed, test_case.expected_changed, + "{}", + test_case.description + ); + assert_eq!( + (changes.added, changes.removed), + ( + vec!["models/customers.sql".to_owned()], + Vec::::new() + ), + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 83242dc1d4..daffde72a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -274,8 +274,10 @@ pub(crate) fn start_model_analysis_session( adapter_rules: Option<(Py, Py)>, ) -> PyResult { let py: Python<'_> = catalog.py(); - let mut request: SessionRequest = session_request(request); - crate::bindings::_helpers::compile_delta::context::inject_relations(&mut request)?; + let mut request: SessionRequest = + crate::bindings::_helpers::compile_delta::context::inject_relations(session_request( + request, + ))?; let rule_failure: RuleFailure = RuleFailure::default(); request.nullability_callback = adapter_rules .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 486acbfff3..b950805395 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -458,12 +458,12 @@ fn location_json<'py>(location: &Bound<'py, PyAny>) -> PyResult(diagnostic: &Bound<'py, PyAny>) -> PyResult> { diagnostic_json(diagnostic) } +/// `_diagnostic_to_json`: one diagnostic's JSON fields, omitting the unset ones. fn diagnostic_json<'py>(diagnostic: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = diagnostic.py(); let payload: Bound<'py, PyDict> = dict( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 9fc2c87329..fc2242caca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -53,6 +53,8 @@ const HIT: &str = "hit"; const CHECKED_HIT: &str = "hit"; const CHECKED_BYPASS: &str = "bypass"; const MISS: &str = "miss"; +/// The logging format that prints its one argument as given. +const LOG_TEXT: &str = "%s"; const BYPASS: &str = "bypass"; fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { @@ -63,6 +65,12 @@ fn reuse_logger(py: Python<'_>) -> PyResult> { imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) } +/// Log one message at debug level on the compile reuse logger. +pub(crate) fn reuse_debug(py: Python<'_>, message: &str) -> PyResult<()> { + let _ = reuse_logger(py)?.call_method1("debug", (LOG_TEXT, message))?; + Ok(()) +} + /// The reuse constants the native check, the stored compile and the launch record share. pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult> { let rules: Bound<'_, PyDict> = PyDict::new(py); @@ -407,12 +415,10 @@ fn attempt_compile_reuse<'py>(request: &Bound<'py, PyAny>) -> PyResult(request: &Bound<'py, PyAny>) -> PyResult( request: &Bound<'py, PyAny>, attempt: &Bound<'py, PyAny>, @@ -515,16 +520,17 @@ fn compiled_with_delta<'py>( context::reset(); let project_dir: Bound<'py, PyAny> = native_field(attempt, "project_dir")?; let cache: Bound<'py, PyAny> = cache_directory(&project_dir)?; - let compiled_root: PathBuf = project_dir - .call_method1("__truediv__", (TARGET_DIRECTORY_NAME,))? - .call_method1("__truediv__", (COMPILED_DIRECTORY_NAME,))? - .call_method0("absolute")? - .extract()?; + let compiled_root: PathBuf = std::path::absolute( + project_dir + .extract::()? + .join(TARGET_DIRECTORY_NAME) + .join(COMPILED_DIRECTORY_NAME), + )?; match delta { DeltaOutcome::Finished(finished) if !verifying => { let (files, models) = (finished.changed_files, finished.rerun_models); let committing: std::time::Instant = std::time::Instant::now(); - let code: i64 = driver::commit(*finished, attempt)?; + let code: i64 = driver::commit(*finished, (request, attempt))?; driver::trace(py, "delta record and print", committing)?; let seconds: f64 = elapsed_ms(py, started)? as f64 / MILLISECONDS_PER_SECOND; print_stderr( @@ -538,7 +544,7 @@ fn compiled_with_delta<'py>( } DeltaOutcome::Raised(error) if !verifying => return Err(error), DeltaOutcome::Fallback(reason) => { - reuse_logger(py)?.call_method1("debug", ("Delta compile fell back: %s", &reason))?; + reuse_debug(py, &format!("Delta compile fell back: {reason}"))?; if verifying { print_stderr( py, @@ -548,15 +554,15 @@ fn compiled_with_delta<'py>( context::start_capture(); return run_compile_command(py, request.clone(), Some(attempt.clone())); } - DeltaOutcome::Finished(finished) => verify::expect(verify::Expected { + DeltaOutcome::Finished(finished) => verify::hold_expected(verify::Expected { stdout: Some(finished.stdout.clone()), - exit_code: Some(finished.phases.outcome.exit_code), - artifacts_written: finished.phases.outcome.artifacts_written, + exit_code: Some(finished.phases.0.exit_code), + artifacts_written: finished.phases.0.artifacts_written, raised: None, compiled: verify::compiled_digests(&compiled_root), compiled_root: compiled_root.to_string_lossy().into_owned(), }), - DeltaOutcome::Raised(error) => verify::expect(verify::Expected { + DeltaOutcome::Raised(error) => verify::hold_expected(verify::Expected { raised: Some(error.to_string()), compiled_root: compiled_root.to_string_lossy().into_owned(), ..verify::Expected::default() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index e05e862fab..67885bd8b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -1,12 +1,13 @@ //! What one full compile printed and left in each artifact, and storing it for compile reuse. +use sqlbuild_cache::project_reuse::types::ImageSections; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Mutex, PoisonError}; use pyo3::exceptions::{PyException, PyOSError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_core::posix_path::main::posix_abspath::posix_abspath; @@ -253,11 +254,30 @@ impl RecordedCompileOutput { } /// Output already printed or recorded elsewhere: its stderr lines and its report. - pub(crate) fn recorded(stderr_lines: Vec>, stdout: Py) -> Self { + pub(crate) fn recorded(py: Python<'_>, stderr_lines: &[String], stdout: &str) -> Self { + let lines: Vec> = stderr_lines + .iter() + .map(|line| PyString::new(py, line).unbind()) + .collect(); Self { - output: Mutex::new((stderr_lines, Some(stdout))), + output: Mutex::new((lines, Some(PyString::new(py, stdout).unbind()))), } } + + /// The recorded stderr lines and report, as text. + pub(crate) fn texts(&self, py: Python<'_>) -> PyResult<(Vec, Option)> { + let output = self.locked(); + let lines: Vec = output + .0 + .iter() + .map(|line| Ok(line.bind(py).to_str()?.to_owned())) + .collect::>()?; + let stdout: Option = match &output.1 { + Some(text) => Some(text.bind(py).to_str()?.to_owned()), + None => None, + }; + Ok((lines, stdout)) + } } /// `print(*values, file=sys.)`. @@ -504,21 +524,25 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { ], )?; if let Some(mut capture) = crate::bindings::_helpers::compile_delta::context::take_capture() { - capture.environment_names = native_field(&facts.input_reads, "environment_names")? - .try_iter()? - .map(|name| name?.extract::()) - .collect::>()?; + capture.environment_names = + native_field(&facts.input_reads, "environment_names")?.extract()?; let json_output: bool = facts.json_output.is_truthy()?; match crate::bindings::_helpers::compile_delta::image::ProjectImage::from_capture( capture, json_output, ) { - Ok(image) => crate::bindings::_helpers::compile_delta::context::set_pending_image( - image.encode().ok(), - ), + Ok(image) => { + let mut pending: Option = None; + if let Ok(sections) = image.encode() { + pending = Some(sections); + } + crate::bindings::_helpers::compile_delta::context::set_pending_image(pending); + } Err(reason) => { - let _ = reuse_logger(py)? - .call_method1("debug", ("Compile reuse kept no project image: %s", reason))?; + crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug( + py, + &format!("Compile reuse kept no project image: {reason}"), + )?; } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 69eb3824a2..d0db9f18af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -1,7 +1,4 @@ -//! What a full compile captures for its project image, and what a delta compile injects. -//! -//! Compiles run one at a time per process, but rules run on their own threads, so both the -//! capture and the injection live in process-wide slots rather than thread-local ones. +//! Process-wide slots for what a full compile captures and what a delta compile injects. use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; @@ -131,6 +128,11 @@ pub(crate) struct DeltaInjection { static CAPTURE: Mutex> = Mutex::new(None); static INJECTION: Mutex>> = Mutex::new(None); +/// Why the running delta compile cannot continue, once something ruled it out. +static UNSUPPORTED: Mutex> = Mutex::new(None); +static PENDING_IMAGE: Mutex> = Mutex::new(None); +static RECORD_MERGE: Mutex> = Mutex::new(None); + fn locked(slot: &Mutex) -> MutexGuard<'_, T> { slot.lock().unwrap_or_else(PoisonError::into_inner) } @@ -194,9 +196,7 @@ pub(crate) fn capture_unsupported(reason: &str) { }); } -/// Run `body` with `injection` visible to the stages a delta compile reruns. -/// -/// Returns the injection back with the body's result, or `None` if something still holds it. +/// Run `body` with `injection` visible to the rerun stages; returns it unless still held. pub(crate) fn with_injection( injection: DeltaInjection, body: impl FnOnce() -> T, @@ -206,7 +206,12 @@ pub(crate) fn with_injection( let injection: Option> = locked(&INJECTION).take(); ( result, - injection.and_then(|injection| Arc::try_unwrap(injection).ok()), + injection.and_then(|injection| { + let Ok(injection) = Arc::try_unwrap(injection) else { + return None; + }; + Some(injection) + }), ) } @@ -215,9 +220,6 @@ pub(crate) fn injection() -> Option> { locked(&INJECTION).clone() } -/// Why the running delta compile cannot continue, once something ruled it out. -static UNSUPPORTED: Mutex> = Mutex::new(None); - /// Rule out the running delta compile: returns the error that unwinds to its fallback. pub(crate) fn delta_unsupported(reason: &str) -> pyo3::PyErr { let mut slot = locked(&UNSUPPORTED); @@ -255,14 +257,13 @@ pub(crate) fn injected_fact_digest(key: &[String]) -> Option { injection.fact_digests.get(key).cloned() } -/// Give a delta compile's analysis session the whole project's facts for relations it does not -/// analyse, and the whole project's analysis order. -pub(crate) fn inject_relations(request: &mut SessionRequest) -> pyo3::PyResult<()> { +/// Give a delta's analysis session the whole project's relation facts and analysis order. +pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult { if !request.dynamic_families_by_table.is_empty() { capture_unsupported("a model declares dynamic column families"); } let Some(injection) = injection() else { - return Ok(()); + return Ok(request); }; if !request.dynamic_families_by_table.is_empty() { return Err(delta_unsupported( @@ -290,7 +291,7 @@ pub(crate) fn inject_relations(request: &mut SessionRequest) -> pyo3::PyResult<( .cloned(), ); } - Ok(()) + Ok(request) } /// What a delta compile's reuse record keeps of the stored compile it patched. @@ -302,9 +303,6 @@ pub(crate) struct RecordMerge { pub(crate) environment_names: Vec, } -static PENDING_IMAGE: Mutex> = Mutex::new(None); -static RECORD_MERGE: Mutex> = Mutex::new(None); - /// Hand the next reuse record the image to keep beside it. pub(crate) fn set_pending_image(image: Option) { *locked(&PENDING_IMAGE) = image; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 406fc7a80e..d35267019f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -1,17 +1,17 @@ -//! Compile only the changed models and SQL tests, patching the stored compile's image and -//! report; anything this cannot reproduce exactly falls back to a full compile. +//! Compile only changed models and SQL tests against the stored compile, or fall back. use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::PathBuf; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyString, PyStringMethods}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyStringMethods; use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; use sqlbuild_cache::project_reuse::main::read_delta_base::read_delta_base; use sqlbuild_cache::project_reuse::main::read_delta_output::read_delta_output; use sqlbuild_cache::project_reuse::models::{DeltaBase, DeltaCandidate, ProjectFileChanges}; use sqlbuild_cache::project_reuse::types::ImageSections; +use crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug; use crate::bindings::_helpers::compile_command::reuse_recording::RecordedCompileOutput; use crate::bindings::_helpers::compile_delta::context::{ CapturedDiagnostic, CapturedModel, CapturedTest, DeltaInjection, ImageCapture, RecordMerge, @@ -24,7 +24,9 @@ use crate::bindings::_helpers::compile_delta::image::{ BaseRules, ImageGate, ImageRelations, ImageRules, PendingRules, ProjectImage, }; use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; -use crate::bindings::_helpers::compile_driver::command::{PhasesOutcome, unrecorded_compile}; +use crate::bindings::_helpers::compile_driver::command::{ + PhasesOutcome, resolved_request, unrecorded_compile, +}; use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; use crate::bindings::_helpers::records::fields::field as native_field; @@ -49,6 +51,14 @@ const UNKNOWN_SOURCE: &str = "unknown source"; const SOURCE_KIND: &str = "source"; const TESTS_ROOT: &str = "tests/"; const SQL_SUFFIX: &str = ".sql"; +/// How many changed, added and removed paths the candidates debug line names. +const CANDIDATES_LOGGED: usize = 5; +const UNREADABLE_IMAGE: &str = "the project image cannot be read"; +const TOO_MANY_MODELS: &str = "too many models would compile again"; +/// The attribute holding a Python enum member's value. +const ENUM_VALUE: &str = "value"; +/// The lineage mode a delta compile reproduces; richer modes compile in full. +const FAST_LINEAGE: &str = "fast"; /// Under verification, the delta's own report is kept here for inspection. const VERIFY_REPORT_DIRECTORY: &str = "target/cache/compiler-native-v1"; const VERIFY_REPORT_FILE: &str = "delta-report.out"; @@ -65,9 +75,8 @@ pub(crate) fn delta_enabled() -> bool { /// A delta compile that finished, ready to be stored and printed or compared. pub(crate) struct Finished<'py> { - pub(crate) request: Bound<'py, PyAny>, pub(crate) stdout: String, - pub(crate) stderr_lines: Vec>, + pub(crate) stderr_lines: Vec, pub(crate) phases: PhasesOutcome<'py>, pub(crate) image: Option, pub(crate) merge: RecordMerge, @@ -107,7 +116,7 @@ fn ineligible(request: &Bound<'_, PyAny>) -> PyResult> { } } let lineage: Bound<'_, PyAny> = native_field(request, "lineage_mode")?; - if !lineage.is_none() && lineage.getattr("value")?.str()?.to_str()? != "fast" { + if !lineage.is_none() && native_field(&lineage, ENUM_VALUE)?.str()?.to_str()? != FAST_LINEAGE { return Ok(Some("rich lineage was requested")); } let flags: Bound<'_, PyAny> = native_field(request, "profile_flags")?; @@ -131,12 +140,12 @@ fn resource_sql(path: &str, root: &str) -> bool { && !path.split('/').any(|part| part.starts_with('_')) } -/// The stored compile's inputs and image, read from the attempt's slot; its report is read -/// only once the delta has run. +/// The stored compile's inputs and image; its report is read only after the delta runs. struct Base { store_path: PathBuf, stored: DeltaBase, image: ProjectImage, + project_dir: PathBuf, } /// The project files the reuse check found changed, and the slot it checked. @@ -197,6 +206,21 @@ impl Rerun { self } + fn with_models(mut self, image: &ProjectImage, added: Vec) -> Self { + let paths: HashMap<&str, &str> = image + .models + .iter() + .map(|model| (model.name.as_str(), model.relative_path.as_str())) + .collect(); + for name in added { + if let Some(path) = paths.get(name.as_str()) { + self.model_paths.insert((*path).to_owned()); + } + self.models.insert(name); + } + self + } + fn names(&self) -> BTreeSet { let mut names: BTreeSet = self.models.clone(); names.extend(self.model_paths.iter().cloned()); @@ -219,11 +243,11 @@ impl Rerun { /// The injection of the image's whole-project facts, moved out of `image` while the delta runs. fn lent_injection( - image: &mut ProjectImage, + mut image: ProjectImage, project_dir: PathBuf, store_path: &std::path::Path, -) -> DeltaInjection { - DeltaInjection { +) -> (ProjectImage, DeltaInjection) { + let injection: DeltaInjection = DeltaInjection { yaml_store: store_path .parent() .and_then(std::path::Path::parent) @@ -241,10 +265,7 @@ fn lent_injection( .map(|model| model.name.clone()) .collect(), relations: std::mem::take(&mut image.relations).into(), - rules: std::sync::Mutex::new(match image.rules.take() { - Some(ImageRules::Decoding(rules)) => Some(rules), - Some(ImageRules::Retained(_)) | None => None, - }), + rules: std::sync::Mutex::new(image.rules.take().and_then(ImageRules::pending)), fact_digests: std::mem::take(&mut image.fact_digests) .into_iter() .collect(), @@ -254,17 +275,19 @@ fn lent_injection( lint_model_keys: image.lint_model_keys.take(), lint_file_keys: image.lint_file_keys.take(), ..DeltaInjection::default() - } + }; + (image, injection) } /// Return the facts `lent_injection` moved out of `image`. -fn returned(image: &mut ProjectImage, injection: DeltaInjection) { +fn returned(mut image: ProjectImage, injection: DeltaInjection) -> ProjectImage { image.relations = injection.relations.into(); image.fact_digests = injection.fact_digests.into_iter().collect(); image.model_subjects = injection.model_subjects; image.model_shapes = injection.model_shapes; image.lint_model_keys = injection.lint_model_keys; image.lint_file_keys = injection.lint_file_keys; + image } /// `injection` set up to rerun `rerun`. @@ -298,9 +321,9 @@ fn shape_of(relations: &RelationTables, name: &str) -> [Option)>, + mut base: Vec<(String, Vec<(String, String)>)>, fresh: Vec<(String, Vec<(String, String)>)>, -) { +) -> Vec<(String, Vec<(String, String)>)> { let mut positions: HashMap = base .iter() .enumerate() @@ -315,12 +338,13 @@ fn merged_rows( } } } + base } /// One restricted compile of `rerun`, with what it captured. struct Round<'py> { - request: Bound<'py, PyAny>, - output: Bound<'py, PyAny>, + stdout: Option, + stderr_lines: Vec, phases: PhasesOutcome<'py>, capture: ImageCapture, } @@ -353,7 +377,7 @@ fn ran_round<'py>( if let Some(reason) = unsupported { return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(reason))); } - let (request, output, phases) = match ran { + let (_, output, phases) = match ran { Ok(ran) => ran, Err(error) => { return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(format!( @@ -361,6 +385,10 @@ fn ran_round<'py>( )))); } }; + let (stderr_lines, stdout): (Vec, Option) = output + .cast::()? + .get() + .texts(output.py())?; let phases: PhasesOutcome<'py> = match phases { Ok(phases) => phases, Err(error) => { @@ -391,14 +419,23 @@ fn ran_round<'py>( return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(reason.clone()))); } Ok(RoundOutcome::Ran(Box::new(Round { - request, - output, + stdout, + stderr_lines, phases, capture, }))) } -/// Compile the changed files against the stored compile, or say why that is not possible. +/// A delta compile ready to run its rounds: the stored compile, the changes and the first rerun. +struct Prepared { + base: Base, + changed_files: usize, + rerun: Rerun, + changed_tests: BTreeSet, + limit: usize, +} + +/// Compile only the changed files against the stored compile, or say why it cannot. pub(crate) fn run<'py>( request: &Bound<'py, PyAny>, attempt: &Bound<'py, PyAny>, @@ -409,85 +446,134 @@ pub(crate) fn run<'py>( if let Some(reason) = ineligible(request)? { return fallback(reason); } + let py: Python<'py> = request.py(); let started: std::time::Instant = std::time::Instant::now(); - let (changes, store_path): (ProjectFileChanges, PathBuf) = match changes_of(attempt)? { - Ok(found) => found, + let prepared: Prepared = match prepared(request, attempt)? { + Ok(prepared) => prepared, Err(reason) => return fallback(reason), }; - request - .py() - .import("logging")? - .getattr("getLogger")? - .call1(("sqlbuild.compile.delta",))? - .call_method1( - "debug", - ( - "Delta compile candidates: changed %s, added %s, removed %s", - changes - .changed - .iter() - .take(5) - .cloned() - .collect::>(), - changes - .added - .iter() - .take(5) - .cloned() - .collect::>(), - changes - .removed - .iter() - .take(5) - .cloned() - .collect::>(), - ), - )?; + trace(py, "delta base read", started)?; + let changed_files: usize = prepared.changed_files; + let (ran, rerun, base): (Round, Rerun, Base) = match rounds(request, attempt, prepared)? { + Ok(found) => found, + Err(outcome) => return Ok(outcome), + }; + trace(py, "delta rounds", started)?; + let outcome: PyResult> = finished(ran, base, &rerun, (py, changed_files)); + trace(py, "delta report", started)?; + outcome +} + +/// The first `CANDIDATES_LOGGED` of `paths`. +fn first_paths(paths: &[String]) -> &[String] { + &paths[..paths.len().min(CANDIDATES_LOGGED)] +} + +/// Why `changes` rule a delta out before anything is read, if they do. +fn changes_ruled_out(changes: &ProjectFileChanges) -> Option { if !changes.added.is_empty() || !changes.removed.is_empty() { - return fallback("files were added or removed"); + return Some("files were added or removed".to_owned()); } if changes.changed.is_empty() { - return fallback("no project file changed"); + return Some("no project file changed".to_owned()); } - if let Some(path) = changes + changes .changed .iter() .find(|path| !resource_sql(path, MODELS_ROOT) && !resource_sql(path, TESTS_ROOT)) - { - return fallback(format!("{path} is not a model or SQL test file")); + .map(|path| format!("{path} is not a model or SQL test file")) +} + +/// The stored compile, its image and the first rerun set, or why the gate rules a delta out. +fn prepared( + request: &Bound<'_, PyAny>, + attempt: &Bound<'_, PyAny>, +) -> PyResult> { + let (changes, store_path): (ProjectFileChanges, PathBuf) = match changes_of(attempt)? { + Ok(found) => found, + Err(reason) => return Ok(Err(reason)), + }; + reuse_debug( + request.py(), + &format!( + "Delta compile candidates: changed {:?}, added {:?}, removed {:?}", + first_paths(&changes.changed), + first_paths(&changes.added), + first_paths(&changes.removed), + ), + )?; + if let Some(reason) = changes_ruled_out(&changes) { + return Ok(Err(reason)); } let Some(mut stored) = read_delta_base(&store_path) else { - return fallback("the stored compile has no project image"); + return Ok(Err("the stored compile has no project image".to_owned())); }; let Some(gate) = ImageGate::decode(&stored.image) else { - return fallback("the project image cannot be read"); + return Ok(Err(UNREADABLE_IMAGE.to_owned())); }; if !ERRORS_ON_DELTA_PATH && gate.has_errors { - return fallback("the stored compile reported errors"); + return Ok(Err("the stored compile reported errors".to_owned())); } - let json_output: bool = native_field(request, "json_output")?.is_truthy()?; - if gate.json_output != json_output { - return fallback("the stored compile has the other output format"); + if gate.json_output != native_field(request, "json_output")?.is_truthy()? { + return Ok(Err( + "the stored compile has the other output format".to_owned() + )); } let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; - let (mut rerun, changed_tests) = match first_rerun(&gate, &changes, &project_dir) { + let (rerun, changed_tests) = match first_rerun(&gate, &changes, &project_dir) { Ok(found) => found, - Err(reason) => return fallback(reason), + Err(reason) => return Ok(Err(reason)), }; let limit: usize = MAX_RERUN_MODELS.max(gate.models.len() * MAX_RERUN_PERCENT / 100); if rerun.models.len() > limit { - return fallback("too many models would compile again"); + return Ok(Err(TOO_MANY_MODELS.to_owned())); } - let Some(mut image) = ProjectImage::decode(&mut stored.image, gate) else { - return fallback("the project image cannot be read"); + let Some(image) = ProjectImage::decode(std::mem::take(&mut stored.image), gate) else { + return Ok(Err(UNREADABLE_IMAGE.to_owned())); }; - stored.image = ImageSections::new(); - trace(request.py(), "delta base read", started)?; - let mut injection: DeltaInjection = lent_injection(&mut image, project_dir, &store_path); - let mut last: Option> = None; + Ok(Ok(Prepared { + base: Base { + store_path, + stored, + image, + project_dir, + }, + changed_files: changes.changed.len(), + rerun, + changed_tests, + limit, + })) +} + +/// A fallback that stops the rounds. +fn stopped<'py, T>(reason: impl Into) -> PyResult>> { + Ok(Err(DeltaOutcome::Fallback(reason.into()))) +} + +/// Run rounds until no rerun model changes the shape a model outside the rerun set reads. +fn rounds<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, + prepared: Prepared, +) -> PyResult, Rerun, Base), DeltaOutcome<'py>>> { + let Prepared { + base, + mut rerun, + changed_tests, + limit, + .. + } = prepared; + let Base { + store_path, + stored, + image, + project_dir, + } = base; + let (image, mut injection): (ProjectImage, DeltaInjection) = + lent_injection(image, project_dir.clone(), &store_path); for _ in 0..MAX_ROUNDS { if rerun.models.len() > limit { - return fallback("too many models would compile again"); + return stopped(TOO_MANY_MODELS); } rerun = rerun.with_tests(&image, &changed_tests); if image @@ -495,81 +581,32 @@ pub(crate) fn run<'py>( .iter() .any(|diagnostic| kept_only_by_full_compiles(&rerun, diagnostic)) { - return fallback("the stored compile reports problems in files the edit leaves alone"); + return stopped("the stored compile reports problems in files the edit leaves alone"); } - let sources_read: Option> = - (!image.diagnostics.iter().any(outside_resources)).then(|| { - image - .models - .iter() - .filter(|model| rerun.models.contains(&model.name)) - .flat_map(|model| &model.upstream) - .filter(|(kind, _)| kind == SOURCE_KIND) - .map(|(_, name)| name.clone()) - .chain(image.pinned_sources.iter().cloned()) - .chain( - rerun - .test_paths - .iter() - .filter_map(|path| image.test_sources.get(path)) - .flatten() - .cloned(), - ) - .collect() - }); - rerun.sources = sources_read.clone(); - let injection_for_round: DeltaInjection = for_rerun(injection, &rerun, sources_read); + rerun.sources = sources_read(&image, &rerun); + let injection_for_round: DeltaInjection = + for_rerun(injection, &rerun, rerun.sources.clone()); let (outcome, back) = round(request, attempt, (injection_for_round, &rerun))?; let ran: Round<'py> = match outcome { RoundOutcome::Ran(ran) => *ran, - RoundOutcome::Stop(outcome) => return Ok(outcome), + RoundOutcome::Stop(outcome) => return Ok(Err(outcome)), }; let Some(back) = back else { - return fallback("the delta compile's injection is still in use"); + return stopped("the delta compile's injection is still in use"); }; injection = back; - if rerun.models.is_empty() { - last = Some(ran); - break; - } - let Some(relations) = ran.capture.relations.as_ref() else { - return fallback("the delta compile analysed no models"); + let added: Vec = match reshaped_dependents(&ran, &image, &injection, &rerun) { + Ok(added) => added, + Err(reason) => return stopped(reason), }; - let stored_relations: &RelationTables = &injection.relations; - if relations.cyclic != stored_relations.cyclic - || relations.dependency_ordered != stored_relations.dependency_ordered - { - return fallback("the project's analysis order changed"); - } - let stored_models: HashMap<&str, &CapturedModel> = image - .models - .iter() - .map(|model| (model.name.as_str(), model)) - .collect(); - for model in &ran.capture.models { - if stored_models.get(model.name.as_str()) != Some(&model) { - return fallback(format!("{} changed its references", model.name)); - } - } - for path in &rerun.model_paths { - if ran.capture.usages.get(path) != image.usages.get(path) { - return fallback(format!("{path} changed the declarations it uses")); - } - } - let reshaped: BTreeSet = rerun - .models - .iter() - .filter(|name| shape_of(relations, name) != shape_of(stored_relations, name)) - .cloned() - .collect(); - let added: Vec = image - .dependents(&reshaped) - .into_iter() - .filter(|name| !rerun.models.contains(name)) - .collect(); if added.is_empty() { - last = Some(ran); - break; + let base: Base = Base { + store_path, + stored, + image: returned(image, injection), + project_dir, + }; + return Ok(Ok((ran, rerun, base))); } let rules: Option = ran .capture @@ -577,34 +614,93 @@ pub(crate) fn run<'py>( .as_ref() .and_then(|rules| BaseRules::decoded(&rules.retained)); let Some(rules) = rules else { - return fallback("the delta compile retained no rules request"); + return stopped("the delta compile retained no rules request"); }; injection.rules = std::sync::Mutex::new(Some(PendingRules::Ready(Box::new(rules)))); - for name in added { - if let Some(model) = stored_models.get(name.as_str()) { - rerun.model_paths.insert(model.relative_path.clone()); - } - rerun.models.insert(name); - } - last = None; + rerun = rerun.with_models(&image, added); } - returned(&mut image, injection); - let base: Base = Base { - store_path, - stored, - image, - }; - let Some(ran) = last else { - return fallback("changed shapes reach too far"); + stopped("changed shapes reach too far") +} + +/// The sources the rerun models, functions, audits and rerun SQL tests read; `None` for all. +fn sources_read(image: &ProjectImage, rerun: &Rerun) -> Option> { + if image.diagnostics.iter().any(outside_resources) { + return None; + } + Some( + image + .models + .iter() + .filter(|model| rerun.models.contains(&model.name)) + .flat_map(|model| &model.upstream) + .filter(|(kind, _)| kind == SOURCE_KIND) + .map(|(_, name)| name.clone()) + .chain(image.pinned_sources.iter().cloned()) + .chain( + rerun + .test_paths + .iter() + .filter_map(|path| image.test_sources.get(path)) + .flatten() + .cloned(), + ) + .collect(), + ) +} + +/// The models outside `rerun` whose inputs' shapes the round changed, or why it cannot go on. +fn reshaped_dependents( + ran: &Round<'_>, + image: &ProjectImage, + injection: &DeltaInjection, + rerun: &Rerun, +) -> Result, String> { + if rerun.models.is_empty() { + return Ok(Vec::new()); + } + let Some(relations) = ran.capture.relations.as_ref() else { + return Err("the delta compile analysed no models".to_owned()); }; - trace(request.py(), "delta rounds", started)?; - let outcome = finished(ran, base, &rerun, changes.changed.len()); - trace(request.py(), "delta report", started)?; - outcome + let stored_relations: &RelationTables = &injection.relations; + if relations.cyclic != stored_relations.cyclic + || relations.dependency_ordered != stored_relations.dependency_ordered + { + return Err("the project's analysis order changed".to_owned()); + } + let stored_models: HashMap<&str, &CapturedModel> = image + .models + .iter() + .map(|model| (model.name.as_str(), model)) + .collect(); + if let Some(model) = ran + .capture + .models + .iter() + .find(|model| stored_models.get(model.name.as_str()) != Some(model)) + { + return Err(format!("{} changed its references", model.name)); + } + if let Some(path) = rerun + .model_paths + .iter() + .find(|path| ran.capture.usages.get(*path) != image.usages.get(*path)) + { + return Err(format!("{path} changed the declarations it uses")); + } + let reshaped: BTreeSet = rerun + .models + .iter() + .filter(|name| shape_of(relations, name) != shape_of(stored_relations, name)) + .cloned() + .collect(); + Ok(image + .dependents(&reshaped) + .into_iter() + .filter(|name| !rerun.models.contains(name)) + .collect()) } -/// The models and SQL tests the changed files rerun, and the changed tests; or why the gate -/// rules a delta out. +/// The models and SQL tests the changed files rerun, or why the gate rules a delta out. fn first_rerun( gate: &ImageGate, changes: &ProjectFileChanges, @@ -687,8 +783,7 @@ fn first_rerun( Ok((rerun, changed_tests)) } -/// Whether a stored diagnostic concerns a model or SQL test the delta does not compile again -/// and comes from a phase other than rules, which only a full compile reports again. +/// Whether only a full compile would repeat a stored non-rule diagnostic outside the rerun. fn kept_only_by_full_compiles(rerun: &Rerun, diagnostic: &CapturedDiagnostic) -> bool { !rerun.owns(diagnostic) && !diagnostic.json.contains(RULE_PHASE) @@ -698,8 +793,7 @@ fn kept_only_by_full_compiles(rerun: &Rerun, diagnostic: &CapturedDiagnostic) -> .any(|path| path.starts_with(MODELS_ROOT) || path.starts_with(TESTS_ROOT)) } -/// Whether a stored diagnostic concerns a file other than a model or SQL test, which a delta -/// compile then reads in full. +/// Whether a stored diagnostic concerns a file other than a model or SQL test. fn outside_resources(diagnostic: &CapturedDiagnostic) -> bool { diagnostic .paths @@ -711,11 +805,10 @@ fn outside_resources(diagnostic: &CapturedDiagnostic) -> bool { /// Log how long the delta has run, for profiling. pub(crate) fn trace(py: Python<'_>, phase: &str, started: std::time::Instant) -> PyResult<()> { let milliseconds: u128 = started.elapsed().as_millis(); - py.import("logging")? - .getattr("getLogger")? - .call1(("sqlbuild.compile.delta",))? - .call_method1("debug", ("%s after %d ms", phase, milliseconds))?; - Ok(()) + reuse_debug( + py, + &format!("Delta compile {phase} after {milliseconds} ms"), + ) } /// Every diagnostic not owned by the rerun set, which the delta must leave as it was. @@ -731,10 +824,9 @@ fn finished<'py>( ran: Round<'py>, base: Base, rerun: &Rerun, - changed_files: usize, + (py, changed_files): (Python<'py>, usize), ) -> PyResult> { - let py: Python<'py> = ran.request.py(); - if !ERRORS_ON_DELTA_PATH && ran.phases.outcome.exit_code != 0 { + if !ERRORS_ON_DELTA_PATH && ran.phases.0.exit_code != 0 { let first: &str = ran .capture .diagnostics @@ -768,28 +860,35 @@ fn finished<'py>( only(&fresh, &stored) )); } - let recorded: Bound<'py, PyAny> = ran.output.getattr("stdout")?; - if recorded.is_none() { + let Some(delta_stdout) = ran.stdout else { return fallback("the delta compile recorded no report"); - } - let delta_stdout: String = recorded.extract()?; + }; if env_enabled(DELTA_VERIFY_ENV_VAR) { - let project_dir: PathBuf = native_field(&ran.request, "project_dir")?.extract()?; - let directory: PathBuf = project_dir.join(VERIFY_REPORT_DIRECTORY); + let directory: PathBuf = base.project_dir.join(VERIFY_REPORT_DIRECTORY); let _ = std::fs::create_dir_all(&directory); let _ = std::fs::write(directory.join(VERIFY_REPORT_FILE), &delta_stdout); } - let stderr_lines: Vec> = ran.output.getattr("stderr_lines")?.extract()?; + let stderr_lines: Vec = ran.stderr_lines; let merge: RecordMerge = RecordMerge { kept: base.stored.inputs.target_files.clone(), environment_names: base.image.environment_names.clone(), }; let json_output: bool = base.image.json_output; - let (store_path, stored, image) = (base.store_path, base.stored, base.image); + let Base { + store_path, + stored, + image, + .. + } = base; let capture: ImageCapture = ran.capture; let (image, stdout): (Option, Result) = py.detach(|| { rayon::join( - || patched_image(image, rerun, capture).encode().ok(), + || { + let Ok(sections) = patched_image(image, rerun, capture).encode() else { + return None; + }; + Some(sections) + }, || { if !json_output { return Ok(delta_stdout); @@ -812,7 +911,6 @@ fn finished<'py>( }; let _ = py; Ok(DeltaOutcome::Finished(Box::new(Finished { - request: ran.request, stdout, stderr_lines, phases: ran.phases, @@ -851,10 +949,18 @@ fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> Pr } if let Some(relations) = capture.relations { let fresh: ImageRelations = relations.into(); - merged_rows(&mut image.relations.types, fresh.types); - merged_rows(&mut image.relations.nullability, fresh.nullability); - merged_rows(&mut image.relations.complete, fresh.complete); - merged_rows(&mut image.relations.schemas, fresh.schemas); + image.relations.types = + merged_rows(std::mem::take(&mut image.relations.types), fresh.types); + image.relations.nullability = merged_rows( + std::mem::take(&mut image.relations.nullability), + fresh.nullability, + ); + image.relations.complete = merged_rows( + std::mem::take(&mut image.relations.complete), + fresh.complete, + ); + image.relations.schemas = + merged_rows(std::mem::take(&mut image.relations.schemas), fresh.schemas); let mut legacy: BTreeSet = image.relations.legacy_models.iter().cloned().collect(); for name in &rerun.models { legacy.remove(name); @@ -890,10 +996,13 @@ fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> Pr } /// Store and print a finished delta compile; returns its exit code. -pub(crate) fn commit(finished: Finished<'_>, attempt: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn commit( + finished: Finished<'_>, + (request, attempt): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { let py: Python<'_> = attempt.py(); + let request: Bound<'_, PyAny> = resolved_request(request)?; let Finished { - request, stdout, stderr_lines, phases, @@ -904,10 +1013,7 @@ pub(crate) fn commit(finished: Finished<'_>, attempt: &Bound<'_, PyAny>) -> PyRe let report: &str = stdout.strip_suffix('\n').unwrap_or(&stdout); let output: Bound<'_, PyAny> = Bound::new( py, - RecordedCompileOutput::recorded( - stderr_lines, - PyString::new(py, &format!("{report}\n")).unbind(), - ), + RecordedCompileOutput::recorded(py, &stderr_lines, &format!("{report}\n")), )? .into_any(); crate::bindings::_helpers::compile_delta::context::set_pending_image(image); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs index d8e658ae4b..9cc07cefd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs @@ -42,46 +42,59 @@ const SEED_KIND: &str = "seed"; const KEPT_DIGEST_BYTES: usize = 16; const HEX: &[u8; 16] = b"0123456789abcdef"; -/// Append an unambiguous encoding of `value`: a tag byte, then length-prefixed parts. -fn encode_value(value: &AuthoredValue, out: &mut Vec) { - let text = |tag: u8, text: &str, out: &mut Vec| { - out.push(tag); - out.extend_from_slice(&(text.len() as u64).to_le_bytes()); - out.extend_from_slice(text.as_bytes()); - }; - let list = |tag: u8, items: &[AuthoredValue], out: &mut Vec| { - out.push(tag); - out.extend_from_slice(&(items.len() as u64).to_le_bytes()); +/// An unambiguous encoding of header values: a tag byte, then length-prefixed parts. +#[derive(Default)] +struct ValueEncoding { + bytes: Vec, +} + +impl ValueEncoding { + fn text(&mut self, tag: u8, text: &str) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(text.len() as u64).to_le_bytes()); + self.bytes.extend_from_slice(text.as_bytes()); + } + + fn list(&mut self, tag: u8, items: &[AuthoredValue]) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(items.len() as u64).to_le_bytes()); for item in items { - encode_value(item, out); + self.value(item); } - }; - let pairs = |tag: u8, items: &[(String, AuthoredValue)], out: &mut Vec| { - out.push(tag); - out.extend_from_slice(&(items.len() as u64).to_le_bytes()); + } + + fn pairs(&mut self, tag: u8, items: &[(String, AuthoredValue)]) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(items.len() as u64).to_le_bytes()); for (key, item) in items { - text(b'k', key, out); - encode_value(item, out); + self.text(b'k', key); + self.value(item); } - }; - match value { - AuthoredValue::Null => out.push(b'n'), - AuthoredValue::Boolean(flag) => out.extend_from_slice(&[b'b', u8::from(*flag)]), - AuthoredValue::BareWord(word) => text(b'w', word, out), - AuthoredValue::String(string) => text(b's', string, out), - AuthoredValue::List(items) => list(b'l', items, out), - AuthoredValue::Map(items) => pairs(b'm', items, out), - AuthoredValue::Set(items) => list(b'e', items, out), - AuthoredValue::Tuple(items) => list(b't', items, out), - AuthoredValue::TypedConstant(items) => pairs(b'c', items, out), - AuthoredValue::InlineSqlHook(sql) => text(b'i', sql, out), - AuthoredValue::NamedSqlHook(name, items) => { - text(b'h', name, out); - pairs(b'a', items, out); - } - AuthoredValue::PythonHook(name, items) => { - text(b'p', name, out); - pairs(b'a', items, out); + } + + fn value(&mut self, value: &AuthoredValue) { + match value { + AuthoredValue::Null => self.bytes.push(b'n'), + AuthoredValue::Boolean(flag) => self.bytes.extend_from_slice(&[b'b', u8::from(*flag)]), + AuthoredValue::BareWord(word) => self.text(b'w', word), + AuthoredValue::String(string) => self.text(b's', string), + AuthoredValue::List(items) => self.list(b'l', items), + AuthoredValue::Map(items) => self.pairs(b'm', items), + AuthoredValue::Set(items) => self.list(b'e', items), + AuthoredValue::Tuple(items) => self.list(b't', items), + AuthoredValue::TypedConstant(items) => self.pairs(b'c', items), + AuthoredValue::InlineSqlHook(sql) => self.text(b'i', sql), + AuthoredValue::NamedSqlHook(name, items) => { + self.text(b'h', name); + self.pairs(b'a', items); + } + AuthoredValue::PythonHook(name, items) => { + self.text(b'p', name); + self.pairs(b'a', items); + } } } } @@ -103,14 +116,19 @@ fn references_read(query_sql: &str, upstream: &[(String, String)]) -> bool { _ => None, }; match (kind, &reference.package) { - (Some(kind), None) => upstream - .iter() - .any(|(stored_kind, name)| stored_kind == kind && *name == reference.name), + (Some(kind), None) => stored_upstream(upstream, kind, &reference.name), _ => true, } }) } +/// Whether `upstream` holds the `kind` resource `name`. +fn stored_upstream(upstream: &[(String, String)], kind: &str, name: &str) -> bool { + upstream + .iter() + .any(|(stored_kind, stored_name)| stored_kind == kind && stored_name == name) +} + fn short_hex(digest: &[u8]) -> String { let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); for byte in &digest[..KEPT_DIGEST_BYTES] { @@ -121,13 +139,12 @@ fn short_hex(digest: &[u8]) -> String { } fn header_keys(values: &[(String, AuthoredValue)]) -> HeaderKeys { - let mut buffer: Vec = Vec::new(); let mut keys: HeaderKeys = values .iter() .map(|(key, value)| { - buffer.clear(); - encode_value(value, &mut buffer); - let hex: String = short_hex(&bytes_digest(&buffer)); + let mut encoding: ValueEncoding = ValueEncoding::default(); + encoding.value(value); + let hex: String = short_hex(&bytes_digest(&encoding.bytes)); (key.clone(), hex) }) .collect(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 769edf4ade..68a4e86404 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -15,10 +15,12 @@ use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; /// Changes whenever the image's meaning changes, so an older image is never read. pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v6"; -const GATE_SECTION: &str = "gate"; -const FACTS_SECTION: &str = "facts"; +pub(crate) const GATE_SECTION: &str = "gate"; +/// The upstream kind of a model another model reads. +const MODEL_KIND: &str = "model"; +pub(crate) const FACTS_SECTION: &str = "facts"; const RULES_SECTION: &str = "rules"; -const RULE_DIGESTS_SECTION: &str = "rule-digests"; +pub(crate) const RULE_DIGESTS_SECTION: &str = "rule-digests"; /// Everything a delta compile reads of the compile it patches. #[derive(Debug, Default)] @@ -54,7 +56,7 @@ pub(crate) struct BaseRules { impl BaseRules { /// The rows a compile retained, decoded. pub(crate) fn decoded(retained: &RetainedRows) -> Option { - let request: EvaluateRequest = serde_json::from_slice(&retained.request_json).ok()?; + let request: EvaluateRequest = from_json(&retained.request_json)?; (request.models.len() == retained.digests.len()).then(|| Self { request, digests: retained.digests.clone(), @@ -69,26 +71,56 @@ pub(crate) enum ImageRules { Retained(RetainedRows), } -/// A stored rules request decoding on its own thread while the delta compile starts, or -/// decoded already. +/// A stored rules request, decoding on its own thread or decoded already. #[derive(Debug)] pub(crate) enum PendingRules { Decoding(std::thread::JoinHandle>), Ready(Box), } +impl ImageRules { + /// The rules request still decoding from a stored image, if this is one. + pub(crate) fn pending(self) -> Option { + match self { + Self::Decoding(rules) => Some(rules), + Self::Retained(_) => None, + } + } + + /// The rows a compile just retained, if this is them. + pub(crate) fn retained(self) -> Option { + match self { + Self::Retained(rules) => Some(rules), + Self::Decoding(_) => None, + } + } +} + impl PendingRules { /// The decoded request, or `None` when it could not be decoded. pub(crate) fn resolved(self) -> Option { match self { - Self::Decoding(handle) => handle.join().ok().flatten(), + Self::Decoding(handle) => { + let Ok(rules) = handle.join() else { + return None; + }; + rules + } Self::Ready(rules) => Some(*rules), } } } +/// The value `bytes` hold as JSON, or `None` when they are malformed. +fn from_json(bytes: &[u8]) -> Option { + let Ok(value) = serde_json::from_slice(bytes) else { + return None; + }; + Some(value) +} + fn decoded_rules(request: &[u8], digests: &[u8]) -> Option { - let request: EvaluateRequest = serde_json::from_slice(request).ok()?; + let request: EvaluateRequest = from_json(request)?; let digests: Vec = decoded_strings(digests)?; (digests.len() == request.models.len()).then_some(BaseRules { request, digests }) } @@ -418,7 +450,7 @@ impl StoredFacts { .into_iter() .collect(); let (diagnostics, text_models, environment_names): StoredMisc = - serde_json::from_slice(decoder.bytes()?).ok()?; + from_json(decoder.bytes()?)?; decoder.finish()?; Some(Self { usages, @@ -537,10 +569,7 @@ impl ProjectImage { lint_model_keys: self.lint_model_keys, lint_file_keys: self.lint_file_keys, }; - let rules: Option = match self.rules { - Some(ImageRules::Retained(rules)) => Some(rules), - Some(ImageRules::Decoding(_)) | None => None, - }; + let rules: Option = self.rules.and_then(ImageRules::retained); let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); let rules = rules.map(|rules| (rules.request_json, encoded_strings(&rules.digests))); let mut sections: ImageSections = vec![ @@ -554,11 +583,8 @@ impl ProjectImage { Ok(sections) } - /// The image stored in `sections` behind `gate`, or `None` when it is unreadable. - /// - /// The rules request, the largest section, is taken out of `sections` and decodes on its - /// own thread. - pub(crate) fn decode(sections: &mut ImageSections, gate: ImageGate) -> Option { + /// The image behind `gate`, its rules request decoding on its own thread; `None` if unreadable. + pub(crate) fn decode(mut sections: ImageSections, gate: ImageGate) -> Option { let mut taken = |name: &str| { sections .iter_mut() @@ -572,7 +598,7 @@ impl ProjectImage { (None, None) => None, _ => return None, }; - let facts: StoredFacts = StoredFacts::decode(section(sections, FACTS_SECTION)?)?; + let facts: StoredFacts = StoredFacts::decode(section(§ions, FACTS_SECTION)?)?; let types: Shapes = facts.types; let relations: ImageRelations = ImageRelations { complete: restored(facts.complete, &types)?, @@ -611,13 +637,16 @@ impl ProjectImage { self.models .iter() .filter(|model| !names.contains(&model.name)) - .filter(|model| { - model - .upstream - .iter() - .any(|(kind, name)| kind == "model" && names.contains(name)) - }) + .filter(|model| reads_any_model(model, names)) .map(|model| model.name.clone()) .collect() } } + +/// Whether `model` reads any of the models `names`. +fn reads_any_model(model: &CapturedModel, names: &BTreeSet) -> bool { + model + .upstream + .iter() + .any(|(kind, name)| kind == MODEL_KIND && names.contains(name)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs index f2c98be98d..3b9378d1c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs @@ -1,12 +1,28 @@ -//! The project image's binary sections: the native store's length-prefixed records, with each -//! distinct string written once and referred to by its index. +//! Image sections as length-prefixed records over a table of each distinct string. use rustc_hash::FxHashMap; +use sqlbuild_cache::store::errors::StoreDecodeError; use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; /// The most items a list reserves room for before reading them, against corrupt counts. const MAX_RESERVED_ITEMS: usize = 1 << 16; +/// A stored value, or `None` when the section is damaged. +fn read(value: Result) -> Option { + let Ok(value) = value else { + return None; + }; + Some(value) +} + +/// A stored count or index, or `None` when it does not fit this platform. +fn length(value: u64) -> Option { + let Ok(length) = usize::try_from(value) else { + return None; + }; + Some(length) +} + /// Writes one section: its values refer to a table of the distinct strings they hold. #[derive(Default)] pub(crate) struct Encoder<'a> { @@ -80,13 +96,13 @@ pub(crate) struct Decoder<'a> { impl<'a> Decoder<'a> { pub(crate) fn new(section: &'a [u8]) -> Option { let mut reader: RecordReader<'a> = RecordReader::new(section); - let count: usize = usize::try_from(reader.u64().ok()?).ok()?; + let count: usize = length(read(reader.u64())?)?; let mut strings: Vec = Vec::with_capacity(count.min(section.len())); for _ in 0..count { - strings.push(reader.string().ok()?); + strings.push(read(reader.string())?); } - let body: &'a [u8] = reader.bytes().ok()?; - reader.finish().ok()?; + let body: &'a [u8] = read(reader.bytes())?; + read(reader.finish())?; Some(Self { strings, body: RecordReader::new(body), @@ -94,7 +110,7 @@ impl<'a> Decoder<'a> { } pub(crate) fn string(&mut self) -> Option { - let id: usize = usize::try_from(self.body.u64().ok()?).ok()?; + let id: usize = length(read(self.body.u64())?)?; self.strings.get(id).cloned() } @@ -107,11 +123,11 @@ impl<'a> Decoder<'a> { } pub(crate) fn count(&mut self) -> Option { - usize::try_from(self.body.u64().ok()?).ok() + length(read(self.body.u64())?) } pub(crate) fn flag(&mut self) -> Option { - match self.body.u64().ok()? { + match read(self.body.u64())? { 0 => Some(false), 1 => Some(true), _ => None, @@ -119,7 +135,7 @@ impl<'a> Decoder<'a> { } pub(crate) fn bytes(&mut self) -> Option<&'a [u8]> { - self.body.bytes().ok() + read(self.body.bytes()) } pub(crate) fn list( @@ -144,6 +160,6 @@ impl<'a> Decoder<'a> { /// Whether every value was read. pub(crate) fn finish(self) -> Option<()> { - self.body.finish().ok() + read(self.body.finish()) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs index 08c0c46d78..95d66e6339 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs @@ -12,8 +12,7 @@ use sqlbuild_core::format_command::constants::LINT_DIRECTORY_NAMES; use crate::bindings::_helpers::rules::lint_pipeline::LintFile; -/// Each model file's lint cache key: recorded for the image, and in a delta compile the whole -/// project's keys with the recompiled models' keys fresh. +/// Each model file's lint key, recorded for the image or, in a delta, patched for reruns. pub(crate) fn model_identities(fresh: Vec<(String, String)>) -> PyResult> { let Some(injection) = injection() else { if capturing() { @@ -44,8 +43,7 @@ pub(crate) fn model_identities(fresh: Vec<(String, String)>) -> PyResult PyResult> { let Some(injection) = injection() else { return Ok(Vec::new()); @@ -66,8 +64,7 @@ pub(crate) fn changed_files(project_dir: &Path) -> PyResult> { Ok(files) } -/// Each non-model file's lint cache key, in walk order: recorded for the image, and in a delta -/// compile the whole project's keys with the changed files' keys fresh. +/// Each non-model file's lint key in walk order, recorded or, in a delta, patched. pub(crate) fn file_keys( project_dir: &Path, fresh: Vec<(PathBuf, String, Option)>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs index 5380d7f686..1e2e6a90b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs @@ -10,6 +10,3 @@ pub(crate) mod rules_rows; pub(crate) mod splice; pub(crate) mod verify; pub(crate) mod yaml_cache; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs index eb0f2cd61f..a0821ea13c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -12,10 +12,15 @@ use crate::bindings::_helpers::compile_delta::context::{ capturing, injection, }; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::records::fields::field as native_field; -/// The text report's model lines: captured for the image, and in a delta compile the whole -/// project's lines with the rerun models' lines replaced. +const SOURCES_KEY: &str = "sources"; +const NAME_KEY: &str = "name"; +/// The graph key kind of a model. +const MODEL_KIND: &str = "model"; + +/// The text report's model lines, captured or, in a delta, patched for the rerun models. pub(crate) fn text_model_lines(models: Vec) -> Vec { let models: Vec = match injection() { None => models, @@ -83,17 +88,13 @@ fn record_compile_facts_inner( return Ok(()); } let native: Bound<'_, PyAny> = native_field(graph, "native")?; + let graph_native: &NativeProjectGraph = native.cast::()?.get(); let mut models: Vec = Vec::new(); for model in native_field(&native_field(graph, "project")?, "models")?.try_iter()? { let model: Bound<'_, PyAny> = model?; let name: String = native_field(&model, "name")?.extract()?; - let mut upstream: Vec<(String, String)> = Vec::new(); - for item in native - .call_method1("upstream", (("model", name.as_str()),))? - .try_iter()? - { - upstream.push(item?.extract()?); - } + let upstream: Vec<(String, String)> = + graph_native.upstream((MODEL_KIND.to_owned(), name.clone())); models.push(CapturedModel { name, relative_path: native_field(&model, "relative_path")? @@ -124,7 +125,10 @@ fn record_compile_facts_inner( pub(crate) fn captured_diagnostic(diagnostic: &Bound<'_, PyAny>) -> PyResult { let item = diagnostic_item(diagnostic)?; let json: String = plain_json(item.as_any())?; - let value: serde_json::Value = serde_json::from_str(&json).unwrap_or_default(); + let value: serde_json::Value = match serde_json::from_str(&json) { + Ok(value) => value, + Err(_malformed) => serde_json::Value::Null, + }; let text = |value: Option<&serde_json::Value>| -> Option { value.and_then(serde_json::Value::as_str).map(str::to_owned) }; @@ -243,8 +247,7 @@ pub(crate) fn record_model_shapes( guarded(|| record_model_shapes_inner(project, shapes)) } -/// In a delta compile, the authored text of each model or SQL test file a diagnostic points -/// into but the delta did not compile, read from the project as the stored compile saw it. +/// In a delta, add the text of files that stored diagnostics point into but were not rerun. pub(crate) fn complete_source_texts( project: &Bound<'_, PyAny>, diagnostics: &[Bound<'_, PyAny>], @@ -278,8 +281,7 @@ pub(crate) fn complete_source_texts( Ok(()) } -/// In a delta compile that reads only some sources, drop every other source entry from loaded -/// source YAML documents before they are built. +/// In a delta reading only some sources, drop every other loaded source YAML entry. pub(crate) fn retain_read_sources( outcomes: &mut [sqlbuild_discovery::yaml_files::models::YamlFileOutcome], ) { @@ -324,6 +326,3 @@ pub(crate) fn retain_read_sources( } } } - -const SOURCES_KEY: &str = "sources"; -const NAME_KEY: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs index 579f4c1e07..fa160532fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs @@ -8,8 +8,7 @@ use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact}; use crate::bindings::_helpers::compile_delta::context::{delta_unsupported, injection}; use crate::bindings::_helpers::compile_delta::image::{BaseRules, PendingRules}; -/// `request` unchanged outside a delta compile; inside one, the whole project's request with -/// the rerun models' rows and the rerun SQL tests' facts taken from `request`. +/// In a delta, the whole project's rules request with the rerun rows taken from `request`. pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult { let Some(injection) = injection() else { return Ok(request); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs index 894796f19f..da0c310a07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs @@ -12,14 +12,15 @@ const WORD: usize = 8; const LOW_BITS: u64 = 0x0101_0101_0101_0101; const HIGH_BITS: u64 = 0x8080_8080_8080_8080; +const DELTA_SUMMARY_KEYS: [&str; 2] = ["errors", "warnings"]; + /// Whether any byte of `word` equals `byte`. fn has_byte(word: u64, byte: u8) -> bool { let matched: u64 = word ^ (LOW_BITS * u64::from(byte)); matched.wrapping_sub(LOW_BITS) & !matched & HIGH_BITS != 0 } -/// The first quote or backslash at or after `index`, or the text's end; words without either -/// are skipped whole. +/// The first quote or backslash at or after `index`, or the end, skipping whole clean words. fn string_stop(text: &[u8], mut index: usize) -> usize { while index + WORD <= text.len() { let mut word: [u8; WORD] = [0; WORD]; @@ -87,6 +88,14 @@ fn skip_space(text: &[u8], mut index: usize) -> usize { index } +/// The value `text` holds as JSON, or `None` when it is malformed. +fn from_json(text: &str) -> Option { + let Ok(value) = serde_json::from_str(text) else { + return None; + }; + Some(value) +} + /// The `(key, value span)` members of the object spanning `span`. fn members(text: &str, span: Span) -> Option> { let bytes: &[u8] = text.as_bytes(); @@ -97,7 +106,7 @@ fn members(text: &str, span: Span) -> Option> { let mut index: usize = skip_space(bytes, span.0 + 1); while index < span.1 && bytes[index] != b'}' { let key_end: usize = value_end(bytes, index)?; - let key: String = serde_json::from_str(&text[index..key_end]).ok()?; + let key: String = from_json(&text[index..key_end])?; index = skip_space(bytes, key_end); if bytes.get(index) != Some(&b':') { return None; @@ -156,10 +165,7 @@ pub(crate) struct SpliceScope<'a> { pub(crate) sources: Option<&'a BTreeSet>, } -/// The string at `path` in the item spanning `span`: the first member of each key in turn. -/// -/// Reports keep each item's own keys before any nested object, so the first `"key": ` after -/// the previous key is that key's member; anything else is read as JSON. +/// The string at `path` in the item at `span`, items keeping own keys before nested objects. fn item_field(text: &str, span: Span, path: &[&str]) -> Option { let bytes: &[u8] = text.as_bytes(); let mut start: usize = span.0; @@ -175,19 +181,18 @@ fn item_field(text: &str, span: Span, path: &[&str]) -> Option { return slow_item_field(text, span, path); } let end: usize = value_end(bytes, start)?; - serde_json::from_str(&text[start..end]).ok() + from_json(&text[start..end]) } fn slow_item_field(text: &str, span: Span, path: &[&str]) -> Option { - let mut value: Value = serde_json::from_str(&text[span.0..span.1]).ok()?; + let mut value: Value = from_json(&text[span.0..span.1])?; for key in path { value = value.get(*key)?.clone(); } value.as_str().map(str::to_owned) } -/// The stored report with the delta's items for the rerun resources and the delta's -/// diagnostics, error counts and timings; `None` when the two reports do not line up. +/// The stored report patched with the delta's rerun items, diagnostics, counts and timings. pub(crate) fn spliced_report( base: &str, delta: &str, @@ -209,7 +214,7 @@ pub(crate) fn spliced_report( "has_errors" | "diagnostics" | "compile_timings" => text(delta, delta_span).to_owned(), "summary" => spliced_summary(base, *base_span, delta, *delta_span) .ok_or("the summaries do not line up")?, - "resources" => spliced_resources(base, *base_span, delta, *delta_span, scope) + "resources" => spliced_resources((base, *base_span), (delta, *delta_span), scope) .map_err(|kind| format!("the {kind} do not line up"))?, _ => { if text(base, base_span) != text(delta, delta_span) { @@ -226,8 +231,6 @@ pub(crate) fn spliced_report( Ok(output) } -const DELTA_SUMMARY_KEYS: [&str; 2] = ["errors", "warnings"]; - fn spliced_summary(base: &str, base_span: Span, delta: &str, delta_span: Span) -> Option { let base_members: Vec<(String, Span)> = members(base, base_span)?; let delta_members: Vec<(String, Span)> = members(delta, delta_span)?; @@ -247,10 +250,8 @@ fn spliced_summary(base: &str, base_span: Span, delta: &str, delta_span: Span) - } fn spliced_resources( - base: &str, - base_span: Span, - delta: &str, - delta_span: Span, + (base, base_span): (&str, Span), + (delta, delta_span): (&str, Span), scope: &SpliceScope<'_>, ) -> Result { let resources: String = "resources".to_owned(); @@ -262,25 +263,29 @@ fn spliced_resources( let delta_list: &Span = member(&delta_members, key).ok_or(key.clone())?; let replacement: Option> = match (key.as_str(), scope.sources) { ("sources", Some(sources)) => { - spliced_items(base, *span, delta, *delta_list, |document, item| { + spliced_items((base, *span), (delta, *delta_list), |document, item| { item_field(document, item, &["name"]).filter(|name| sources.contains(name)) }) } - ("models", _) => spliced_items(base, *span, delta, *delta_list, |document, item| { - item_field(document, item, &["name"]).filter(|name| scope.models.contains(name)) - }), - ("tests", _) => spliced_items(base, *span, delta, *delta_list, |document, item| { + ("models", _) => { + spliced_items((base, *span), (delta, *delta_list), |document, item| { + item_field(document, item, &["name"]).filter(|name| scope.models.contains(name)) + }) + } + ("tests", _) => spliced_items((base, *span), (delta, *delta_list), |document, item| { item_field(document, item, &["name"]).filter(|name| scope.tests.contains(name)) }), - ("audits", _) => spliced_items(base, *span, delta, *delta_list, |document, item| { - let owner: Option = - item_field(document, item, &["attached_to", "resource_type"]); - let name: Option = item_field(document, item, &["attached_to", "name"]); - (owner.as_deref() == Some("model")) - .then_some(name) - .flatten() - .filter(|name| scope.models.contains(name)) - }), + ("audits", _) => { + spliced_items((base, *span), (delta, *delta_list), |document, item| { + let owner: Option = + item_field(document, item, &["attached_to", "resource_type"]); + let name: Option = item_field(document, item, &["attached_to", "name"]); + (owner.as_deref() == Some("model")) + .then_some(name) + .flatten() + .filter(|name| scope.models.contains(name)) + }) + } _ => { if base[span.0..span.1] != delta[delta_list.0..delta_list.1] { return Err(key.clone()); @@ -299,13 +304,10 @@ fn spliced_resources( Ok(output) } -/// The base list with each item `owned` marks replaced, in order, by the delta's items `owned` -/// marks with the same owner; every other delta item must equal a base item. +/// The base list with owned items replaced by the delta's items of the same owner. fn spliced_items( - base: &str, - base_span: Span, - delta: &str, - delta_span: Span, + (base, base_span): (&str, Span), + (delta, delta_span): (&str, Span), owned: impl Fn(&str, Span) -> Option, ) -> Option> { let base_items: Vec = elements(base, base_span)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs index ad35ee8133..5ab8ccb6e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs @@ -1,17 +1,17 @@ -//! The hidden differential check: run each delta compile, then the full compile it replaces, -//! and fail when their reports, exit codes or compiled artifacts differ. +//! The hidden check that each delta compile matches the full compile it replaces. use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use std::sync::{Mutex, MutexGuard, PoisonError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::PyStringMethods; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_cache::digest::main::digest_files::digest_files; use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_cache::project_reuse::constants::NATIVE_DIGEST_PREFIX; -use crate::bindings::_helpers::compile_command::reuse_recording::CompileArtifactWrites; +use crate::bindings::_helpers::compile_command::reuse_recording::{ + CompileArtifactWrites, RecordedCompileOutput, +}; /// The exit code of a compile whose delta differed from its full compile. pub(crate) const MISMATCH_EXIT_CODE: i64 = 70; @@ -45,30 +45,34 @@ fn locked(slot: &Mutex) -> MutexGuard<'_, T> { slot.lock().unwrap_or_else(PoisonError::into_inner) } -fn files_under(directory: &Path, found: &mut Vec) { +/// Every file under `directory`, depth first. +fn files_under(directory: &Path) -> Vec { let Ok(entries) = std::fs::read_dir(directory) else { - return; + return Vec::new(); }; + let mut found: Vec = Vec::new(); for entry in entries.flatten() { let path: PathBuf = entry.path(); match entry.file_type() { - Ok(kind) if kind.is_dir() => files_under(&path, found), + Ok(kind) if kind.is_dir() => found.extend(files_under(&path)), Ok(kind) if kind.is_file() => found.push(path), _ => {} } } + found } /// Every file under `root` with its native digest, keyed by absolute path text. pub(crate) fn compiled_digests(root: &Path) -> BTreeMap { - let mut paths: Vec = Vec::new(); - files_under(root, &mut paths); + let paths: Vec = files_under(root); let digests = digest_files(&paths); paths .into_iter() .zip(digests) .filter_map(|(path, digest)| { - let digest = digest.ok()?; + let Ok(digest) = digest else { + return None; + }; Some(( path.to_string_lossy().into_owned(), format!("{NATIVE_DIGEST_PREFIX}{}", hex_digest(&digest)), @@ -78,7 +82,7 @@ pub(crate) fn compiled_digests(root: &Path) -> BTreeMap { } /// Hold the delta's result for comparison once the full compile finishes. -pub(crate) fn expect(expected: Expected) { +pub(crate) fn hold_expected(expected: Expected) { *locked(&EXPECTED) = Some(expected); *locked(&ACTUAL) = None; } @@ -92,17 +96,16 @@ pub(crate) fn full_compile_finished( if locked(&EXPECTED).is_none() { return Ok(()); } - let stdout: Bound<'_, PyAny> = output.getattr("stdout")?; + let (_, stdout): (Vec, Option) = output + .cast::()? + .get() + .texts(output.py())?; let written: HashMap> = artifact_writes .cast::() .map(|writes| writes.get().written_digests()) .unwrap_or_default(); *locked(&ACTUAL) = Some(Actual { - stdout: if stdout.is_none() { - None - } else { - Some(stdout.str()?.to_str()?.to_owned()) - }, + stdout, exit_code: Some(exit_code), artifacts_written, written, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs index 70da741d8a..01fa3cb588 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs @@ -1,5 +1,4 @@ -//! Loaded YAML documents a delta compile keeps by content digest, so the unchanged YAML it -//! reads again is decoded rather than parsed. +//! Loaded YAML documents a delta compile keeps by content digest instead of parsing again. use std::path::PathBuf; @@ -19,16 +18,6 @@ pub(crate) const YAML_STORE_FILE_NAME: &str = "yaml-documents.bin"; /// Bumped whenever the stored encoding changes; older stores open empty. pub(crate) const YAML_STORE_ENVIRONMENT: &str = "yaml-documents-v1"; -/// Where a delta compile keeps loaded YAML documents. -pub(crate) fn yaml_store_path(cache_dir: &std::path::Path) -> PathBuf { - cache_dir.join(YAML_STORE_FILE_NAME) -} - -/// The store key of a document's text. -pub(crate) fn document_key(contents: &str) -> ContentDigest { - bytes_digest(contents.as_bytes()) -} - const NULL: usize = 0; const BOOL: usize = 1; const INTEGER: usize = 2; @@ -43,75 +32,87 @@ const MAP: usize = 10; const NO_OFFSET: usize = 0; const OFFSET: usize = 1; -fn put_date(encoder: &mut Encoder<'_>, date: &ConfigDate) { - encoder.count(usize::from(date.year)); - encoder.count(usize::from(date.month)); - encoder.count(usize::from(date.day)); +/// Where a delta compile keeps loaded YAML documents. +pub(crate) fn yaml_store_path(cache_dir: &std::path::Path) -> PathBuf { + cache_dir.join(YAML_STORE_FILE_NAME) } -fn put_time(encoder: &mut Encoder<'_>, time: &ConfigTime) { - encoder.count(usize::from(time.hour)); - encoder.count(usize::from(time.minute)); - encoder.count(usize::from(time.second)); - encoder.bytes(&time.microsecond.to_le_bytes()); +/// The store key of a document's text. +pub(crate) fn document_key(contents: &str) -> ContentDigest { + bytes_digest(contents.as_bytes()) } -fn put_value<'a>(encoder: &mut Encoder<'a>, value: &'a ConfigValue) { - match value { - ConfigValue::Null => encoder.count(NULL), - ConfigValue::Bool(flag) => { - encoder.count(BOOL); - encoder.flag(*flag); - } - ConfigValue::Integer(number) => { - encoder.count(INTEGER); - encoder.bytes(&number.to_le_bytes()); - } - ConfigValue::BigInteger(digits) => { - encoder.count(BIG_INTEGER); - encoder.string(digits); - } - ConfigValue::Float(number) => { - encoder.count(FLOAT); - encoder.bytes(&number.to_bits().to_le_bytes()); - } - ConfigValue::String(text) => { - encoder.count(STRING); - encoder.string(text); - } - ConfigValue::Date(date) => { - encoder.count(DATE); - put_date(encoder, date); - } - ConfigValue::DateTime(moment) => { - encoder.count(DATE_TIME); - put_date(encoder, &moment.date); - put_time(encoder, &moment.time); - match moment.utc_offset_seconds { - Some(offset) => { - encoder.count(OFFSET); - encoder.bytes(&offset.to_le_bytes()); +impl<'a> Encoder<'a> { + fn date(&mut self, date: &ConfigDate) { + self.count(usize::from(date.year)); + self.count(usize::from(date.month)); + self.count(usize::from(date.day)); + } + + fn time(&mut self, time: &ConfigTime) { + self.count(usize::from(time.hour)); + self.count(usize::from(time.minute)); + self.count(usize::from(time.second)); + self.bytes(&time.microsecond.to_le_bytes()); + } + + fn document(&mut self, value: &'a ConfigValue) { + match value { + ConfigValue::Null => self.count(NULL), + ConfigValue::Bool(flag) => { + self.count(BOOL); + self.flag(*flag); + } + ConfigValue::Integer(number) => { + self.count(INTEGER); + self.bytes(&number.to_le_bytes()); + } + ConfigValue::BigInteger(digits) => { + self.count(BIG_INTEGER); + self.string(digits); + } + ConfigValue::Float(number) => { + self.count(FLOAT); + self.bytes(&number.to_bits().to_le_bytes()); + } + ConfigValue::String(text) => { + self.count(STRING); + self.string(text); + } + ConfigValue::Date(date) => { + self.count(DATE); + self.date(date); + } + ConfigValue::DateTime(moment) => { + self.count(DATE_TIME); + self.date(&moment.date); + self.time(&moment.time); + match moment.utc_offset_seconds { + Some(offset) => { + self.count(OFFSET); + self.bytes(&offset.to_le_bytes()); + } + None => self.count(NO_OFFSET), } - None => encoder.count(NO_OFFSET), } - } - ConfigValue::Time(time) => { - encoder.count(TIME); - put_time(encoder, time); - } - ConfigValue::List(items) => { - encoder.count(LIST); - encoder.count(items.len()); - for item in items { - put_value(encoder, item); + ConfigValue::Time(time) => { + self.count(TIME); + self.time(time); } - } - ConfigValue::Map(entries) => { - encoder.count(MAP); - encoder.count(entries.len()); - for (key, item) in entries { - put_value(encoder, key); - put_value(encoder, item); + ConfigValue::List(items) => { + self.count(LIST); + self.count(items.len()); + for item in items { + self.document(item); + } + } + ConfigValue::Map(entries) => { + self.count(MAP); + self.count(entries.len()); + for (key, item) in entries { + self.document(key); + self.document(item); + } } } } @@ -120,64 +121,74 @@ fn put_value<'a>(encoder: &mut Encoder<'a>, value: &'a ConfigValue) { /// A loaded document as stored. pub(crate) fn encoded_document(value: &ConfigValue) -> Vec { let mut encoder: Encoder<'_> = Encoder::default(); - put_value(&mut encoder, value); + encoder.document(value); encoder.finish() } -fn fixed(decoder: &mut Decoder<'_>) -> Option<[u8; N]> { - decoder.bytes()?.try_into().ok() -} +impl Decoder<'_> { + fn fixed(&mut self) -> Option<[u8; N]> { + let Ok(bytes) = self.bytes()?.try_into() else { + return None; + }; + Some(bytes) + } -fn small>(decoder: &mut Decoder<'_>) -> Option { - T::try_from(decoder.count()?).ok() -} + fn small>(&mut self) -> Option { + let Ok(value) = T::try_from(self.count()?) else { + return None; + }; + Some(value) + } -fn date(decoder: &mut Decoder<'_>) -> Option { - Some(ConfigDate { - year: small(decoder)?, - month: small(decoder)?, - day: small(decoder)?, - }) -} + fn date(&mut self) -> Option { + Some(ConfigDate { + year: self.small()?, + month: self.small()?, + day: self.small()?, + }) + } -fn time(decoder: &mut Decoder<'_>) -> Option { - Some(ConfigTime { - hour: small(decoder)?, - minute: small(decoder)?, - second: small(decoder)?, - microsecond: u32::from_le_bytes(fixed(decoder)?), - }) -} + fn time(&mut self) -> Option { + Some(ConfigTime { + hour: self.small()?, + minute: self.small()?, + second: self.small()?, + microsecond: u32::from_le_bytes(self.fixed()?), + }) + } -fn value(decoder: &mut Decoder<'_>) -> Option { - Some(match decoder.count()? { - NULL => ConfigValue::Null, - BOOL => ConfigValue::Bool(decoder.flag()?), - INTEGER => ConfigValue::Integer(i64::from_le_bytes(fixed(decoder)?)), - BIG_INTEGER => ConfigValue::BigInteger(decoder.string()?), - FLOAT => ConfigValue::Float(f64::from_bits(u64::from_le_bytes(fixed(decoder)?))), - STRING => ConfigValue::String(decoder.string()?), - DATE => ConfigValue::Date(date(decoder)?), - DATE_TIME => ConfigValue::DateTime(ConfigDateTime { - date: date(decoder)?, - time: time(decoder)?, - utc_offset_seconds: match decoder.count()? { - NO_OFFSET => None, - OFFSET => Some(i32::from_le_bytes(fixed(decoder)?)), - _ => return None, - }, - }), - TIME => ConfigValue::Time(time(decoder)?), - LIST => ConfigValue::List(decoder.list(value)?), - MAP => ConfigValue::Map(decoder.list(|decoder| Some((value(decoder)?, value(decoder)?)))?), - _ => return None, - }) + fn document(&mut self) -> Option { + Some(match self.count()? { + NULL => ConfigValue::Null, + BOOL => ConfigValue::Bool(self.flag()?), + INTEGER => ConfigValue::Integer(i64::from_le_bytes(self.fixed()?)), + BIG_INTEGER => ConfigValue::BigInteger(self.string()?), + FLOAT => ConfigValue::Float(f64::from_bits(u64::from_le_bytes(self.fixed()?))), + STRING => ConfigValue::String(self.string()?), + DATE => ConfigValue::Date(self.date()?), + DATE_TIME => ConfigValue::DateTime(ConfigDateTime { + date: self.date()?, + time: self.time()?, + utc_offset_seconds: match self.count()? { + NO_OFFSET => None, + OFFSET => Some(i32::from_le_bytes(self.fixed()?)), + _ => return None, + }, + }), + TIME => ConfigValue::Time(self.time()?), + LIST => ConfigValue::List(self.list(Self::document)?), + MAP => ConfigValue::Map( + self.list(|decoder| Some((decoder.document()?, decoder.document()?)))?, + ), + _ => return None, + }) + } } /// A stored document, or `None` when it cannot be decoded. pub(crate) fn decoded_document(stored: &[u8]) -> Option { let mut decoder: Decoder<'_> = Decoder::new(stored)?; - let document: ConfigValue = value(&mut decoder)?; + let document: ConfigValue = decoder.document()?; decoder.finish()?; Some(document) } @@ -188,20 +199,78 @@ pub(crate) struct DocumentStore { pub(crate) path: PathBuf, } -/// In a delta compile, the document store beside its reuse slots; `None` otherwise or when it -/// cannot be opened, which only means parsing again. +impl DocumentStore { + /// Each file's document, decoded from the store or loaded by `load` and stored. + pub(crate) fn load_with( + &mut self, + paths: &[String], + absolute: impl Fn(&str) -> PathBuf, + load: impl FnOnce(&[String]) -> Result, E>, + ) -> Result, E> { + let mut outcomes: Vec> = Vec::with_capacity(paths.len()); + let mut missed: Vec = Vec::new(); + for path in paths { + let stored: Option = stored_outcome(&mut self.store, &absolute(path)); + if stored.is_none() { + missed.push(path.clone()); + } + outcomes.push(stored); + } + let mut loaded = if missed.is_empty() { + Vec::new() + } else { + load(&missed)? + } + .into_iter(); + let mut complete: Vec = Vec::with_capacity(paths.len()); + for outcome in outcomes { + let outcome: YamlFileOutcome = match outcome { + Some(outcome) => outcome, + None => { + let Some(outcome) = loaded.next() else { + break; + }; + if let YamlFileOutcome::Loaded { contents, value } = &outcome { + self.store + .put(document_key(contents), encoded_document(value)); + } + outcome + } + }; + complete.push(outcome); + } + Ok(complete) + } +} + +/// In a delta compile, the document store beside its reuse slots, when it opens. pub(crate) fn open_documents(py: pyo3::Python<'_>) -> Option { let path: PathBuf = crate::bindings::_helpers::compile_delta::context::injection()? .yaml_store .clone()?; - let store: NativeStore = crate::bindings::_helpers::cache::native_store::open_store( + match crate::bindings::_helpers::cache::native_store::open_store( py, &path, YAML_STORE_KIND, YAML_STORE_ENVIRONMENT, - ) - .ok()?; - Some(DocumentStore { store, path }) + ) { + Ok(store) => Some(DocumentStore { store, path }), + Err(_unopened) => None, + } +} + +/// The stored document of the file at `path` when its current text is stored. +fn stored_outcome(store: &mut NativeStore, path: &std::path::Path) -> Option { + let bytes: Vec = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(_unreadable) => return None, + }; + let contents: String = match decode_text(&bytes) { + Ok(contents) => contents, + Err(_undecodable) => return None, + }; + let value: ConfigValue = decoded_document(store.get(&document_key(&contents))?)?; + Some(YamlFileOutcome::Loaded { contents, value }) } /// Save the documents this compile loaded; failing to only means parsing them again. @@ -213,54 +282,3 @@ pub(crate) fn save_documents(py: pyo3::Python<'_>, documents: &DocumentStore) { &[], ); } - -/// Each file's loaded document: decoded from `documents` when its text is stored, otherwise -/// loaded by `load` and stored. -pub(crate) fn load_with( - documents: &mut DocumentStore, - paths: &[String], - absolute: impl Fn(&str) -> PathBuf, - load: impl FnOnce(&[String]) -> Result, E>, -) -> Result, E> { - let mut outcomes: Vec> = Vec::with_capacity(paths.len()); - let mut missed: Vec = Vec::new(); - for path in paths { - let stored: Option = std::fs::read(absolute(path)) - .ok() - .and_then(|bytes| decode_text(&bytes).ok()) - .and_then(|contents| { - let value: ConfigValue = - decoded_document(documents.store.get(&document_key(&contents))?)?; - Some(YamlFileOutcome::Loaded { contents, value }) - }); - if stored.is_none() { - missed.push(path.clone()); - } - outcomes.push(stored); - } - let mut loaded = if missed.is_empty() { - Vec::new() - } else { - load(&missed)? - } - .into_iter(); - let mut complete: Vec = Vec::with_capacity(paths.len()); - for outcome in outcomes { - let outcome: YamlFileOutcome = match outcome { - Some(outcome) => outcome, - None => { - let Some(outcome) = loaded.next() else { - break; - }; - if let YamlFileOutcome::Loaded { contents, value } = &outcome { - documents - .store - .put(document_key(contents), encoded_document(value)); - } - outcome - } - }; - complete.push(outcome); - } - Ok(complete) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 813f1473b2..cc4d880e2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -64,12 +64,8 @@ pub(crate) struct CommandOutcome<'py> { pub(crate) dag_artifact_path: Bound<'py, PyAny>, } -/// One compile's outcome and recorders, before it is stored and printed. -pub(crate) struct PhasesOutcome<'py> { - pub(crate) outcome: CommandOutcome<'py>, - pub(crate) input_reads: Bound<'py, PyAny>, - pub(crate) artifact_writes: Bound<'py, PyAny>, -} +/// One compile's outcome, input reads and artifact writes, before it is stored and printed. +pub(crate) type PhasesOutcome<'py> = (CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>); /// The per-invocation recorders a compile runs inside. struct Recorders<'py> { @@ -91,38 +87,23 @@ pub(crate) fn run_compile_command<'py>( Some(attempt) => native_field(attempt, "started")?.extract()?, None => monotonic(py)?, }; - let path_class: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?; - let effective_request: Bound<'py, PyAny> = if native_field(&request, "project_dir")?.is_none() { - replaced( - &request.clone(), - &[("project_dir", path_class.call_method0("cwd")?)], - )? - } else { - request.clone() - }; + let effective_request: Bound<'py, PyAny> = resolved_request(&request)?; let attempt: Bound<'py, PyAny> = match reuse_attempt { Some(attempt) => attempt, None => bypass_attempt(py, &effective_request, total_start)?, }; let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); let status: Status<'py> = Status(compile_status(&request)?); - let gc: Bound<'py, PyModule> = py.import("gc")?; - let was_enabled: bool = gc.call_method0("isenabled")?.is_truthy()?; - gc.call_method0("disable")?; - let exit_code: PyResult = recorded_compile( - (&effective_request, &attempt), - (&output, &status), - total_start, - ); - let deferred: bool = RETAINED_UNTIL_EXIT.get(py).is_some(); - if deferred || was_enabled { - gc.call_method0(if deferred { "freeze" } else { "enable" })?; - } - exit_code + collecting_later(py, || { + recorded_compile( + (&effective_request, &attempt), + (&output, &status), + total_start, + ) + })? } -/// The project-directory-resolved request, a fresh output recorder and the compile's phases, -/// run as `run_compile_command` runs them but neither stored nor printed. +/// The resolved request, output recorder and phases of a compile neither stored nor printed. pub(crate) fn unrecorded_compile<'py>( request: &Bound<'py, PyAny>, attempt: &Bound<'py, PyAny>, @@ -133,31 +114,39 @@ pub(crate) fn unrecorded_compile<'py>( )> { let py: Python<'py> = request.py(); let total_start: f64 = native_field(attempt, "started")?.extract()?; - let effective_request: Bound<'py, PyAny> = if native_field(request, "project_dir")?.is_none() { - replaced( - &request.clone(), - &[( - "project_dir", - imported(py, "pathlib", "Path")?.call_method0("cwd")?, - )], - )? - } else { - request.clone() - }; + let effective_request: Bound<'py, PyAny> = resolved_request(request)?; let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); let status: Status<'py> = Status(py.None().into_bound(py)); - let gc: Bound<'py, PyModule> = py.import("gc")?; + let phases: PyResult> = collecting_later(py, || { + compile_phases( + (&effective_request, attempt), + (&output, &status), + total_start, + ) + })?; + Ok((effective_request, output, phases)) +} + +/// `request` with its project directory resolved to the working directory when unset. +pub(crate) fn resolved_request<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + if !native_field(request, "project_dir")?.is_none() { + return Ok(request.clone()); + } + let cwd: Bound<'py, PyAny> = imported(request.py(), "pathlib", "Path")?.call_method0("cwd")?; + replaced(&request.clone(), &[("project_dir", cwd)]) +} + +/// Run `body` with Python's garbage collector paused, then resume or freeze it as compiles do. +fn collecting_later(py: Python<'_>, body: impl FnOnce() -> T) -> PyResult { + let gc: Bound<'_, PyModule> = py.import("gc")?; let was_enabled: bool = gc.call_method0("isenabled")?.is_truthy()?; gc.call_method0("disable")?; - let phases: PyResult> = compile_phases( - (&effective_request, attempt), - (&output, &status), - total_start, - ); - if was_enabled { - gc.call_method0("enable")?; + let result: T = body(); + let deferred: bool = RETAINED_UNTIL_EXIT.get(py).is_some(); + if deferred || was_enabled { + gc.call_method0(if deferred { "freeze" } else { "enable" })?; } - Ok((effective_request, output, phases)) + Ok(result) } /// `CompileReuseAttempt` of a compile that did not consult the reuse record. @@ -218,8 +207,8 @@ fn recorded_compile<'py>( let phases: PhasesOutcome<'py> = phases?; crate::bindings::_helpers::compile_delta::verify::full_compile_finished( output, - (phases.outcome.exit_code, phases.outcome.artifacts_written), - &phases.artifact_writes, + (phases.0.exit_code, phases.0.artifacts_written), + &phases.2, )?; store_and_print((request, attempt), output, phases) } @@ -231,11 +220,7 @@ pub(crate) fn store_and_print<'py>( phases: PhasesOutcome<'py>, ) -> PyResult { let py: Python<'py> = request.py(); - let PhasesOutcome { - outcome, - input_reads, - artifact_writes, - } = phases; + let (outcome, input_reads, artifact_writes): PhasesOutcome<'py> = phases; let stored: PyResult<()> = write_reusable_compile( py, Some(keywords( @@ -324,11 +309,7 @@ pub(crate) fn compile_phases<'py>( }, ) })?; - Ok(PhasesOutcome { - outcome, - input_reads, - artifact_writes, - }) + Ok((outcome, input_reads, artifact_writes)) } /// `_run_compile_with_status`: analyze, run the output phases and record the command output. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 6cd405d255..f116a462bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -244,9 +244,11 @@ fn assembled_inputs<'py>( } else { &project_config }; - if !native_field(delta_settings, "settings")? - .getattr(DELTA_COMPILE_SETTING_KEY)? - .is_truthy()? + if !native_field( + &native_field(delta_settings, "settings")?, + DELTA_COMPILE_SETTING_KEY, + )? + .is_truthy()? { crate::bindings::_helpers::compile_delta::context::stop_capture(); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index 1e48ff6bb5..4fcf280d58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -172,8 +172,7 @@ fn load_files<'py>( let failure_kind: FailureKind = kind.failure_kind(); match documents.as_mut() { Some(documents) => py.detach(|| { - crate::bindings::_helpers::compile_delta::yaml_cache::load_with( - documents, + documents.load_with( &named, |path| inner.absolute(path), |missed| load_yaml(&root, inner, missed, failure_kind), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index 6a951e4a9b..62e7a80269 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -214,7 +214,7 @@ impl NativeProjectGraph { } /// The key's direct upstream lineage keys, in authored dependency order. - fn upstream(&self, key: ObjectKey) -> Vec { + pub(crate) fn upstream(&self, key: ObjectKey) -> Vec { edge_keys(&self.graph, &key, EdgeDirection::Upstream).to_vec() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index a334bcd8cd..ed839bc92a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -346,6 +346,29 @@ fn prepared_result<'a>( Some(prepared.result(py)) } +/// The files a whole-project lint reads: prepared, walked, or in a delta only the changed ones. +fn linted_files( + run: &SqlRun<'_, '_>, + prepared_files: Option>, +) -> PyResult> { + let py: Python<'_> = run.project.py(); + let project_dir: &Path = &run.project_dir; + let prepared = prepared_result(run, py); + if crate::bindings::_helpers::compile_delta::context::injection().is_some() { + return crate::bindings::_helpers::compile_delta::lint_hooks::changed_files(project_dir); + } + Ok(match prepared { + Some(Ok(result)) => result.outcome.files.clone(), + Some(Err(error)) if !error.is_instance_of::(py) => { + return Err(error.clone_ref(py)); + } + Some(Err(_)) | None => match prepared_files { + Some(files) => files, + None => lint_sources(project_dir, None, None)?, + }, + }) +} + /// Each analysed model's findings, cached or linted, then cache accounting. fn evaluated_sql_rules( run: &SqlRun<'_, '_>, @@ -386,20 +409,7 @@ fn evaluated_sql_rules( for model in project_attribute(&run.project, "models")?.try_iter()? { model_paths.insert(project_dir.join(model_posix_path(&model?)?)); } - let delta = crate::bindings::_helpers::compile_delta::context::injection(); - let files: Vec = match prepared_result(run, py) { - _ if delta.is_some() => { - crate::bindings::_helpers::compile_delta::lint_hooks::changed_files(project_dir)? - } - Some(Ok(result)) => result.outcome.files.clone(), - Some(Err(error)) if !error.is_instance_of::(py) => { - return Err(error.clone_ref(py)); - } - Some(Err(_)) | None => match prepared_files { - Some(files) => files, - None => lint_sources(project_dir, None, None)?, - }, - }; + let files: Vec = linted_files(run, prepared_files)?; let mut file_keys: Vec<(PathBuf, String, Option)> = Vec::new(); for file in &files { if model_paths.contains(&file.path) { diff --git a/crates/sqlbuild-python/src/bindings/mod.rs b/crates/sqlbuild-python/src/bindings/mod.rs index d887a75d22..6a1afca4b9 100644 --- a/crates/sqlbuild-python/src/bindings/mod.rs +++ b/crates/sqlbuild-python/src/bindings/mod.rs @@ -2,3 +2,6 @@ pub(crate) mod _helpers; pub mod main; pub(crate) mod models; pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests.rs rename to crates/sqlbuild-python/src/bindings/tests.rs index f7ab89f124..6a055292ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_image.rs"] mod image; #[path = "tests/test_splice.rs"] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs similarity index 54% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs rename to crates/sqlbuild-python/src/bindings/tests/helpers.rs index 03efd9fb17..6f3e431bce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_image.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -1,31 +1,35 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; +use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_config::models::ConfigValue; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::compile_delta::context::{ CapturedDiagnostic, CapturedModel, CapturedTest, }; use crate::bindings::_helpers::compile_delta::image::{ - BaseRules, IMAGE_FORMAT, ImageGate, ImageRelations, ImageRules, PendingRules, ProjectImage, + BaseRules, FACTS_SECTION, GATE_SECTION, IMAGE_FORMAT, ImageRelations, ImageRules, PendingRules, + ProjectImage, RULE_DIGESTS_SECTION, }; -use crate::bindings::_helpers::compile_delta::tests::test_types::DamagedImageTestCase; -fn pairs(values: &[(&str, &str)]) -> Vec<(String, String)> { +const OTHER_FORMAT: &str = "sqlbuild-project-image-v0"; + +pub(crate) fn pairs(values: &[(&str, &str)]) -> Vec<(String, String)> { values .iter() .map(|(left, right)| ((*left).to_owned(), (*right).to_owned())) .collect() } -fn strings(values: &[&str]) -> Vec { +pub(crate) fn strings(values: &[&str]) -> Vec { values.iter().map(|value| (*value).to_owned()).collect() } /// An image holding at least one value of every kind it stores. -fn image() -> ProjectImage { +pub(crate) fn image() -> ProjectImage { let order_types: Vec<(String, String)> = pairs(&[("order_id", "BIGINT"), ("amount", "DOUBLE")]); let customer_types: Vec<(String, String)> = pairs(&[("customer_id", "BIGINT")]); - let request: Vec = serde_json::to_vec(&EvaluateRequest::default()).expect("request"); + let request: Vec = serde_json::to_vec(&EvaluateRequest::default()).unwrap_or_default(); ProjectImage { format: String::new(), json_output: true, @@ -106,7 +110,7 @@ fn image() -> ProjectImage { } /// Every stored fact of `image` but its rules, in a comparable form. -fn facts(image: &ProjectImage) -> String { +pub(crate) fn facts(image: &ProjectImage) -> String { format!( "{:?}", ( @@ -134,103 +138,76 @@ fn facts(image: &ProjectImage) -> String { ) } -fn current_format() -> String { +pub(crate) fn current_format() -> String { IMAGE_FORMAT.to_owned() } -fn encoded() -> Vec<(String, Vec)> { - let mut stored: ProjectImage = image(); - stored.format = current_format(); - stored.encode().expect("encoded") +pub(crate) fn encoded(image: ProjectImage) -> ImageSections { + let stored: ProjectImage = ProjectImage { + format: current_format(), + ..image + }; + stored.encode().unwrap_or_default() +} + +/// The image `image()` without any of its optional facts. +pub(crate) fn sparse_image() -> ProjectImage { + ProjectImage { + model_subjects: None, + text_models: None, + lint_model_keys: None, + lint_file_keys: None, + rules: None, + ..image() + } } -#[test] -fn given_project_image_when_encoded_and_decoded_then_every_fact_and_the_rules_survive() { - let mut sections = encoded(); - let gate: ImageGate = ImageGate::decode(§ions).expect("gate"); - assert!(gate.json_output); - assert!(!gate.has_errors); - assert!(gate.has_model_subjects); - let decoded: ProjectImage = ProjectImage::decode(&mut sections, gate).expect("image"); - let mut expected: ProjectImage = image(); - expected.format = current_format(); - assert_eq!(decoded.format, expected.format); - assert_eq!(facts(&decoded), facts(&expected)); - let Some(ImageRules::Decoding(rules)) = decoded.rules else { - panic!("the rules request was not decoding"); +/// The rules request a decoded image holds, resolved. +pub(crate) fn decoded_rules(image: ProjectImage) -> Option { + image + .rules + .and_then(ImageRules::pending) + .and_then(PendingRules::resolved) +} + +/// `sections` with the gate of an image of another format. +pub(crate) fn with_other_format(sections: &mut ImageSections) { + let other: ProjectImage = ProjectImage { + format: OTHER_FORMAT.to_owned(), + ..ProjectImage::default() }; - let rules: BaseRules = PendingRules::resolved(rules).expect("rules"); - assert!(rules.request.models.is_empty()); - assert!(rules.digests.is_empty()); + let gate: Vec = other + .encode() + .unwrap_or_default() + .into_iter() + .find(|(name, _)| name == GATE_SECTION) + .map(|(_, bytes)| bytes) + .unwrap_or_default(); + sections + .iter_mut() + .filter(|(name, _)| name == GATE_SECTION) + .for_each(|section| section.1 = gate.clone()); } -#[test] -fn given_damaged_image_when_decoded_then_it_is_never_read() { - let test_cases = [ - DamagedImageTestCase { - description: "intact", - damage: |_| {}, - expected_gate: true, - expected_image: true, - }, - DamagedImageTestCase { - description: "truncated facts", - damage: |sections| { - let facts = sections - .iter_mut() - .find(|(name, _)| name == "facts") - .expect("facts"); - facts.1.truncate(facts.1.len() / 2); - }, - expected_gate: true, - expected_image: false, - }, - DamagedImageTestCase { - description: "rules request without its digests", - damage: |sections| sections.retain(|(name, _)| name != "rule-digests"), - expected_gate: true, - expected_image: false, - }, - DamagedImageTestCase { - description: "gate of another format", - damage: |sections| { - let other: ProjectImage = ProjectImage { - format: "sqlbuild-project-image-v0".to_owned(), - ..ProjectImage::default() - }; - let gate = other - .encode() - .expect("encoded") - .into_iter() - .find(|(name, _)| name == "gate") - .expect("gate"); - for section in sections.iter_mut() { - if section.0 == "gate" { - section.1 = gate.1.clone(); - } - } - }, - expected_gate: false, - expected_image: false, - }, - ]; - for test_case in test_cases { - let mut sections = encoded(); - (test_case.damage)(&mut sections); - let gate: Option = ImageGate::decode(§ions); - assert_eq!( - gate.is_some(), - test_case.expected_gate, - "{}", - test_case.description - ); - let decoded: Option = - gate.and_then(|gate| ProjectImage::decode(&mut sections, gate)); - assert_eq!( - decoded.is_some(), - test_case.expected_image, - "{}", - test_case.description - ); - } +/// `sections` with their facts cut in half. +pub(crate) fn with_truncated_facts(sections: &mut ImageSections) { + sections + .iter_mut() + .filter(|(name, _)| name == FACTS_SECTION) + .for_each(|(_, bytes)| bytes.truncate(bytes.len() / 2)); +} + +/// `sections` without the rules request's digests. +pub(crate) fn without_rule_digests(sections: &mut ImageSections) { + sections.retain(|(name, _)| name != RULE_DIGESTS_SECTION); +} + +/// The names in `values`, as a set. +pub(crate) fn names(values: &[&str]) -> BTreeSet { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// A YAML string value. +pub(crate) fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) } diff --git a/crates/sqlbuild-python/src/bindings/tests/test_image.rs b/crates/sqlbuild-python/src/bindings/tests/test_image.rs new file mode 100644 index 0000000000..16c9cbbe23 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_image.rs @@ -0,0 +1,111 @@ +use sqlbuild_cache::project_reuse::types::ImageSections; + +use crate::bindings::_helpers::compile_delta::image::{ImageGate, ProjectImage}; +use crate::bindings::tests::helpers::{ + current_format, decoded_rules, encoded, facts, image, sparse_image, with_other_format, + with_truncated_facts, without_rule_digests, +}; +use crate::bindings::tests::test_types::{DamagedImageTestCase, ImageRoundTripTestCase}; + +#[test] +fn given_project_image_when_encoded_and_decoded_then_every_fact_and_the_rules_survive() { + let test_cases = [ + ImageRoundTripTestCase { + description: "every fact and a retained rules request", + image, + expected_model_subjects: true, + expected_rule_models: Some(0), + }, + ImageRoundTripTestCase { + description: "no optional facts and no rules request", + image: sparse_image, + expected_model_subjects: false, + expected_rule_models: None, + }, + ]; + for test_case in test_cases { + let sections: ImageSections = encoded((test_case.image)()); + + let gate: Option = ImageGate::decode(§ions); + let gate_facts: Option<(bool, bool, bool)> = gate + .as_ref() + .map(|gate| (gate.json_output, gate.has_errors, gate.has_model_subjects)); + let decoded: Option = + gate.and_then(|gate| ProjectImage::decode(sections, gate)); + let decoded_facts: Option<(String, String)> = decoded + .as_ref() + .map(|image| (image.format.clone(), facts(image))); + let rule_models: Option = decoded + .and_then(decoded_rules) + .map(|rules| rules.request.models.len() + rules.digests.len()); + + assert_eq!( + gate_facts, + Some((true, false, test_case.expected_model_subjects)), + "{}", + test_case.description + ); + assert_eq!( + decoded_facts, + Some((current_format(), facts(&(test_case.image)()))), + "{}", + test_case.description + ); + assert_eq!( + rule_models, test_case.expected_rule_models, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_damaged_image_when_decoded_then_it_is_never_read() { + let test_cases = [ + DamagedImageTestCase { + description: "intact", + damage: |_| {}, + expected_gate: true, + expected_image: true, + }, + DamagedImageTestCase { + description: "truncated facts", + damage: with_truncated_facts, + expected_gate: true, + expected_image: false, + }, + DamagedImageTestCase { + description: "rules request without its digests", + damage: without_rule_digests, + expected_gate: true, + expected_image: false, + }, + DamagedImageTestCase { + description: "gate of another format", + damage: with_other_format, + expected_gate: false, + expected_image: false, + }, + ]; + for test_case in test_cases { + let mut sections: ImageSections = encoded(image()); + (test_case.damage)(&mut sections); + + let gate: Option = ImageGate::decode(§ions); + let gate_read: bool = gate.is_some(); + let decoded: Option = + gate.and_then(|gate| ProjectImage::decode(sections, gate)); + + assert_eq!( + gate_read, test_case.expected_gate, + "{}", + test_case.description + ); + assert_eq!( + decoded.is_some(), + test_case.expected_image, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_splice.rs b/crates/sqlbuild-python/src/bindings/tests/test_splice.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_splice.rs rename to crates/sqlbuild-python/src/bindings/tests/test_splice.rs index 667f913b50..116346ca4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_splice.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_splice.rs @@ -1,14 +1,11 @@ use std::collections::BTreeSet; use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; -use crate::bindings::_helpers::compile_delta::tests::test_types::SpliceTestCase; +use crate::bindings::tests::helpers::names; +use crate::bindings::tests::test_types::SpliceTestCase; const BASE: &str = r#"{"has_errors": false, "summary": {"models": 2, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 1}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "not_null", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "compiled"}]}, "diagnostics": [], "compile_timings": {"total_ms": 9}}"#; -fn names(values: &[&str]) -> BTreeSet { - values.iter().map(|value| (*value).to_owned()).collect() -} - #[test] fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_sections_change() { let test_cases = [ @@ -19,7 +16,7 @@ fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_ models: &["orders"], tests: &[], sources: None, - expected: Some( + expected_report: Some( r#"{"has_errors": true, "summary": {"models": 2, "errors": 1, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 3}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "unique", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "compiled"}]}, "diagnostics": [{"code": "E1"}], "compile_timings": {"total_ms": 1}}"#, ), }, @@ -30,7 +27,7 @@ fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_ models: &[], tests: &["orders_test"], sources: Some(&["raw_customers"]), - expected: Some( + expected_report: Some( r#"{"has_errors": false, "summary": {"models": 2, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 1}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 5}], "audits": [{"name": "not_null", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "changed"}]}, "diagnostics": [], "compile_timings": {"total_ms": 1}}"#, ), }, @@ -41,7 +38,7 @@ fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_ models: &["orders"], tests: &[], sources: None, - expected: None, + expected_report: None, }, SpliceTestCase { description: "a whole-project section that differs cannot be spliced", @@ -50,7 +47,7 @@ fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_ models: &[], tests: &[], sources: None, - expected: None, + expected_report: None, }, SpliceTestCase { description: "reports with different sections cannot be spliced", @@ -59,7 +56,7 @@ fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_ models: &[], tests: &[], sources: None, - expected: None, + expected_report: None, }, ]; for test_case in test_cases { @@ -75,7 +72,7 @@ fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_ spliced_report(test_case.base, test_case.delta, &scope); assert_eq!( spliced.ok().as_deref(), - test_case.expected, + test_case.expected_report, "{}", test_case.description ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_types.rs rename to crates/sqlbuild-python/src/bindings/tests/test_types.rs index 73216caed5..57e91b9a5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -1,5 +1,8 @@ +use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_config::models::ConfigValue; +use crate::bindings::_helpers::compile_delta::image::ProjectImage; + /// A stored report, a delta report and what splicing the delta into the stored one gives. pub(crate) struct SpliceTestCase { pub(crate) description: &'static str, @@ -9,19 +12,30 @@ pub(crate) struct SpliceTestCase { pub(crate) tests: &'static [&'static str], pub(crate) sources: Option<&'static [&'static str]>, /// The spliced report, or `None` when the reports must not line up. - pub(crate) expected: Option<&'static str>, + pub(crate) expected_report: Option<&'static str>, } /// A loaded YAML document kept by the delta compile's document store. pub(crate) struct DocumentTestCase { pub(crate) description: &'static str, pub(crate) document: ConfigValue, + /// How many truncations of the stored document still read back as it. + pub(crate) expected_truncations_read: usize, } /// Stored image bytes damaged before they are read back. pub(crate) struct DamagedImageTestCase { pub(crate) description: &'static str, - pub(crate) damage: fn(&mut Vec<(String, Vec)>), + pub(crate) damage: fn(&mut ImageSections), pub(crate) expected_gate: bool, pub(crate) expected_image: bool, } + +/// An image encoded and decoded again. +pub(crate) struct ImageRoundTripTestCase { + pub(crate) description: &'static str, + pub(crate) image: fn() -> ProjectImage, + pub(crate) expected_model_subjects: bool, + /// The retained rules request's model and digest count, or `None` without one. + pub(crate) expected_rule_models: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_yaml_cache.rs b/crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_yaml_cache.rs rename to crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs index 56ae88c523..60ea19a5f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/tests/test_yaml_cache.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs @@ -1,11 +1,8 @@ use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; -use crate::bindings::_helpers::compile_delta::tests::test_types::DocumentTestCase; use crate::bindings::_helpers::compile_delta::yaml_cache::{decoded_document, encoded_document}; - -fn text(value: &str) -> ConfigValue { - ConfigValue::String(value.to_owned()) -} +use crate::bindings::tests::helpers::text; +use crate::bindings::tests::test_types::DocumentTestCase; #[test] fn given_loaded_document_when_stored_and_read_back_then_it_is_unchanged() { @@ -23,10 +20,12 @@ fn given_loaded_document_when_stored_and_read_back_then_it_is_unchanged() { let test_cases = [ DocumentTestCase { description: "empty document", + expected_truncations_read: 0, document: ConfigValue::Null, }, DocumentTestCase { description: "sources document with repeated strings", + expected_truncations_read: 0, document: ConfigValue::Map(vec![( text("sources"), ConfigValue::List(vec![ @@ -43,6 +42,7 @@ fn given_loaded_document_when_stored_and_read_back_then_it_is_unchanged() { }, DocumentTestCase { description: "every scalar kind", + expected_truncations_read: 0, document: ConfigValue::List(vec![ ConfigValue::Bool(true), ConfigValue::Bool(false), @@ -67,6 +67,7 @@ fn given_loaded_document_when_stored_and_read_back_then_it_is_unchanged() { }, DocumentTestCase { description: "mapping with non-string keys", + expected_truncations_read: 0, document: ConfigValue::Map(vec![ (ConfigValue::Integer(1), text("one")), (ConfigValue::Null, ConfigValue::List(Vec::new())), @@ -75,19 +76,24 @@ fn given_loaded_document_when_stored_and_read_back_then_it_is_unchanged() { ]; for test_case in test_cases { let stored: Vec = encoded_document(&test_case.document); + + let decoded: Option = decoded_document(&stored); + let truncations_read: usize = (0..stored.len()) + .filter(|length| { + decoded_document(&stored[..*length]).as_ref() == Some(&test_case.document) + }) + .count(); + + assert_eq!( + decoded.as_ref(), + Some(&test_case.document), + "{}", + test_case.description + ); assert_eq!( - decoded_document(&stored), - Some(test_case.document.clone()), + truncations_read, test_case.expected_truncations_read, "{}", test_case.description ); - for length in 0..stored.len() { - assert_ne!( - decoded_document(&stored[..length]), - Some(test_case.document.clone()), - "{}: truncated to {length} bytes", - test_case.description - ); - } } } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs index e720d8fa80..05e627214e 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs @@ -62,22 +62,22 @@ fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_sta "{}", test_case.description ); - let findings: Option> = match &evaluated { - Ok(evaluation) => Some(evaluation_summary(evaluation).0), - Err(_refused) => None, - }; + let findings: Option> = evaluated + .as_ref() + .ok() + .map(|evaluation| evaluation_summary(evaluation).0); assert_eq!( findings == Some(evaluation_summary(&first).0), test_case.expected_same_findings, "{}", test_case.description ); - let missed: Option<&str> = match &evaluated { - Err(RowsError::Rules(message)) => Some(message.as_str()), - _ => None, - }; + let missed: Option = evaluated.as_ref().err().map(|error| format!("{error:?}")); assert_eq!( - missed, test_case.expected_missed, + missed, + test_case + .expected_missed + .map(|message| format!("{:?}", RowsError::Rules(message.to_owned()))), "{}", test_case.description ); diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs index 71c4365e9a..d9673c87f1 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs @@ -790,10 +790,10 @@ fn stored_summaries(path: &Path) -> Option { Ok(bytes) => bytes, Err(_unreadable) => return None, }; - match serde_json::from_slice::(&bytes) { - Ok(stored) => Some(stored), - Err(_damaged) => None, - } + let Ok(stored) = serde_json::from_slice::(&bytes) else { + return None; + }; + Some(stored) } /// The stored fixture summaries, kept only for one native build. diff --git a/scripts/delta_compile_series/__init__.py b/scripts/delta_compile_series/__init__.py new file mode 100644 index 0000000000..2dd71392bd --- /dev/null +++ b/scripts/delta_compile_series/__init__.py @@ -0,0 +1 @@ +"""Seeded edit series that check delta compiles against full compiles.""" diff --git a/scripts/delta_compile_series/_helpers/__init__.py b/scripts/delta_compile_series/_helpers/__init__.py new file mode 100644 index 0000000000..0e330898e1 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/__init__.py @@ -0,0 +1 @@ +"""Edit and outcome helpers for delta compile series.""" diff --git a/scripts/delta_compile_series/_helpers/edits.py b/scripts/delta_compile_series/_helpers/edits.py new file mode 100644 index 0000000000..2f941de525 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/edits.py @@ -0,0 +1,55 @@ +"""Apply and revert the edits of a delta compile series.""" + +from __future__ import annotations + +import random +import re +from pathlib import Path + +from scripts.delta_compile_series.constants import ( + EDIT_ATTEMPTS, + PRIVATE_PATH_MARKER, + PROJECT_CONFIG, + RANDOM_VALUE_LIMIT, + SQL_GLOB, +) +from scripts.delta_compile_series.exceptions import DeltaCompileSeriesError +from scripts.delta_compile_series.models import DeltaCompileEdit +from scripts.delta_compile_series.types import EditTarget + + +def project_files(*, project_dir: Path, edit: DeltaCompileEdit) -> list[Path]: + """The files `edit` may change.""" + + if edit.target is EditTarget.PROJECT_CONFIG: + return [project_dir / PROJECT_CONFIG] + root: Path = project_dir / edit.target.value + return sorted( + path for path in root.rglob(SQL_GLOB) if PRIVATE_PATH_MARKER not in path.as_posix() + ) + + +def applied(*, path: Path, edit: DeltaCompileEdit, rng: random.Random) -> bool: + """Apply `edit` to `path`; return whether it changed the file.""" + + text: str = path.read_text(encoding="utf-8") + replacement: str = edit.replacement.replace("{value}", str(rng.randint(1, RANDOM_VALUE_LIMIT))) + edited, count = re.subn(edit.pattern, replacement, text, count=1) + if count == 0 or edited == text: + return False + path.write_text(edited, encoding="utf-8") + return True + + +def edited_file( + *, project_dir: Path, edit: DeltaCompileEdit, rng: random.Random +) -> tuple[Path, str]: + """Apply `edit` to a randomly chosen file it changes; return the file and its old text.""" + + candidates: list[Path] = project_files(project_dir=project_dir, edit=edit) + for _ in range(EDIT_ATTEMPTS): + path: Path = rng.choice(candidates) + original: str = path.read_text(encoding="utf-8") + if applied(path=path, edit=edit, rng=rng): + return path, original + raise DeltaCompileSeriesError(f"no file takes the {edit.kind} edit") diff --git a/scripts/delta_compile_series/_helpers/outcomes.py b/scripts/delta_compile_series/_helpers/outcomes.py new file mode 100644 index 0000000000..7ff4ce718e --- /dev/null +++ b/scripts/delta_compile_series/_helpers/outcomes.py @@ -0,0 +1,43 @@ +"""Judge what one compile of a delta compile series did.""" + +from __future__ import annotations + +import re + +from scripts.delta_compile_series.constants import ( + COMPILE_REUSE_HIT_LINE, + DELTA_FALLBACK_PATTERN, + DELTA_MISMATCH_PATTERN, + DELTA_VERIFIED_LINE, + MISMATCH_EXIT_CODE, + STALE_PROBLEMS, +) +from scripts.delta_compile_series.models import DeltaCompileEdit, DeltaCompileOutcome + + +def fallback_reason(*, outcome: DeltaCompileOutcome) -> str | None: + """Why the compile fell back from a delta, when it says.""" + + reasons: list[str] = re.findall(DELTA_FALLBACK_PATTERN, outcome.stderr, flags=re.MULTILINE) + return reasons[0] if reasons else None + + +def judged( + *, outcome: DeltaCompileOutcome, edit: DeltaCompileEdit, reverted: bool, others_edited: bool +) -> tuple[bool, str | None]: + """Whether the compile was a verified delta, and what was unexpected about it, if anything.""" + + mismatches: list[str] = re.findall(DELTA_MISMATCH_PATTERN, outcome.stderr, flags=re.MULTILINE) + if outcome.returncode == MISMATCH_EXIT_CODE or mismatches: + return False, f"mismatch: {mismatches}" + if COMPILE_REUSE_HIT_LINE in outcome.stderr: + return False, None if reverted else "replayed a compile after an edit" + reason: str | None = fallback_reason(outcome=outcome) + if others_edited and reason is not None and STALE_PROBLEMS in reason: + return False, None + if edit.fallback_reason is None: + verified: bool = DELTA_VERIFIED_LINE in outcome.stderr + return verified, None if verified else f"fell back: {reason}" + if reason is None or edit.fallback_reason not in reason: + return False, f"expected a fallback for {edit.fallback_reason!r}, got {reason!r}" + return False, None diff --git a/scripts/delta_compile_series/_helpers/sequence.py b/scripts/delta_compile_series/_helpers/sequence.py new file mode 100644 index 0000000000..c89d857ef9 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/sequence.py @@ -0,0 +1,52 @@ +"""Apply the named edits of a fixed delta compile sequence.""" + +from __future__ import annotations + +import random +from pathlib import Path + +from scripts.delta_compile_series._helpers.edits import applied, project_files +from scripts.delta_compile_series.constants import ( + BODY_EDIT, + BODY_MODEL_INDEX, + BODY_SEED, + DOUBLE_CAST, + DOUBLE_COLUMN, + FLOAT_CAST, + FLOAT_COLUMN, +) +from scripts.delta_compile_series.exceptions import DeltaCompileSeriesError +from scripts.delta_compile_series.types import SequenceEdit + + +def reshaped(*, project_dir: Path) -> Path: + """Change the first model whose first quantity is a DOUBLE to output a FLOAT instead.""" + + models: list[Path] = project_files(project_dir=project_dir, edit=BODY_EDIT) + path: Path = next(path for path in models if DOUBLE_COLUMN in path.read_text("utf-8")) + path.write_text( + path.read_text(encoding="utf-8") + .replace(DOUBLE_COLUMN, FLOAT_COLUMN, 1) + .replace(DOUBLE_CAST, FLOAT_CAST), + encoding="utf-8", + ) + return path + + +def edited_body(*, project_dir: Path) -> Path: + """Change a constant in one model's query.""" + + model: Path = project_files(project_dir=project_dir, edit=BODY_EDIT)[BODY_MODEL_INDEX] + if not applied(path=model, edit=BODY_EDIT, rng=random.Random(BODY_SEED)): + raise DeltaCompileSeriesError(f"the body edit does not apply to {model.name}") + return model + + +def sequence_edited(*, project_dir: Path, edit: SequenceEdit) -> Path | None: + """Apply `edit`; return the file it changed, if any.""" + + if edit is SequenceEdit.BODY: + return edited_body(project_dir=project_dir) + if edit is SequenceEdit.SHAPE: + return reshaped(project_dir=project_dir) + return None diff --git a/scripts/delta_compile_series/constants.py b/scripts/delta_compile_series/constants.py new file mode 100644 index 0000000000..291b2c4dc6 --- /dev/null +++ b/scripts/delta_compile_series/constants.py @@ -0,0 +1,71 @@ +"""Edits and markers of delta compile series.""" + +from __future__ import annotations + +from scripts.delta_compile_series.models import DeltaCompileEdit +from scripts.delta_compile_series.types import EditTarget + +DELTA_COMPILE_LINE: str = "against the previous compile" +DELTA_VERIFIED_LINE: str = "delta compile verified" +DELTA_FALLBACK_PATTERN: str = r"^delta compile fell back: (.*)$" +DELTA_MISMATCH_PATTERN: str = r"^delta compile mismatch: .*$" +COMPILE_REUSE_HIT_LINE: str = "Inputs unchanged; reused the previous compile" +MISMATCH_EXIT_CODE: int = 70 +REVERT_PROBABILITY: float = 0.5 +EDIT_ATTEMPTS: int = 50 +DELTA_WEIGHT: int = 3 +FALLBACK_WEIGHT: int = 1 +STALE_PROBLEMS: str = "reports problems in files the edit leaves alone" +PROJECT_CONFIG: str = "sqlbuild_project.toml" +RANDOM_VALUE_LIMIT: int = 10**6 + +EDITS: tuple[DeltaCompileEdit, ...] = ( + DeltaCompileEdit("body_constant", r"COALESCE\(b\.amount, \d+\)", "COALESCE(b.amount, {value})"), + DeltaCompileEdit("body_comment", r"\Z", "\n-- reviewed {value}\n"), + DeltaCompileEdit( + "description", r'MODEL \(description "[^"]*"', 'MODEL (description "Edited {value}."' + ), + DeltaCompileEdit( + "column_description", + r'description "Order quantity measure"', + 'description "Edited measure {value}"', + ), + DeltaCompileEdit("type_error", r"quantity_000 \(type DOUBLE", "quantity_000 (type VARCHAR"), + DeltaCompileEdit("parse_error", r"projected AS \(SELECT", "projected AS (SELEC"), + DeltaCompileEdit( + "test_expectation", + r"(__expected__\w+ AS \(SELECT CAST\()\d+", + r"\g<1>{value}", + target=EditTarget.TESTS, + ), + DeltaCompileEdit( + "materialization", + r"materialized table", + "materialized view", + fallback_reason="custom rules select models by their materialization", + ), + DeltaCompileEdit("contract", r"contract enforced, ", "", fallback_reason="a custom rule reads"), + DeltaCompileEdit( + "unknown_reference", + r'__ref\("([^"]+)"\)', + r'__ref("\1_missing")', + fallback_reason="changed what it references", + ), + DeltaCompileEdit( + "project_config", + r"\Z", + "\n# reviewed {value}\n", + target=EditTarget.PROJECT_CONFIG, + fallback_reason="sqlbuild_project.toml is not a model or SQL test file", + ), +) +BODY_EDIT: DeltaCompileEdit = EDITS[0] +PRIVATE_PATH_MARKER: str = "/_" +SQL_GLOB: str = "*.sql" +FAILURE_TAIL_CHARACTERS: int = 2000 +BODY_MODEL_INDEX: int = 11 +BODY_SEED: int = 3 +DOUBLE_COLUMN: str = "quantity_000 (type DOUBLE" +FLOAT_COLUMN: str = "quantity_000 (type FLOAT" +DOUBLE_CAST: str = "AS DOUBLE) AS quantity_000" +FLOAT_CAST: str = "AS FLOAT) AS quantity_000" diff --git a/scripts/delta_compile_series/exceptions.py b/scripts/delta_compile_series/exceptions.py new file mode 100644 index 0000000000..219f801bd0 --- /dev/null +++ b/scripts/delta_compile_series/exceptions.py @@ -0,0 +1,5 @@ +"""Delta compile series errors.""" + + +class DeltaCompileSeriesError(RuntimeError): + """Raised when a series cannot apply an edit or its first compile fails.""" diff --git a/scripts/delta_compile_series/main/__init__.py b/scripts/delta_compile_series/main/__init__.py new file mode 100644 index 0000000000..0be72e24e5 --- /dev/null +++ b/scripts/delta_compile_series/main/__init__.py @@ -0,0 +1 @@ +"""Entry points for delta compile series.""" diff --git a/scripts/delta_compile_series/main/run_delta_compile_sequence.py b/scripts/delta_compile_series/main/run_delta_compile_sequence.py new file mode 100644 index 0000000000..4f32ae1559 --- /dev/null +++ b/scripts/delta_compile_series/main/run_delta_compile_sequence.py @@ -0,0 +1,41 @@ +"""Compile after each of a fixed sequence of named edits, then once in full.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +from scripts.delta_compile_series._helpers.sequence import sequence_edited +from scripts.delta_compile_series.constants import ( + COMPILE_REUSE_HIT_LINE, + DELTA_COMPILE_LINE, + FAILURE_TAIL_CHARACTERS, +) +from scripts.delta_compile_series.exceptions import DeltaCompileSeriesError +from scripts.delta_compile_series.models import DeltaCompileOutcome, DeltaCompileSequence +from scripts.delta_compile_series.types import SequenceEdit + + +def run_delta_compile_sequence( + *, + project_dir: Path, + edits: tuple[SequenceEdit, ...], + compile_project: Callable[[], DeltaCompileOutcome], + compile_in_full: Callable[[], DeltaCompileOutcome], +) -> DeltaCompileSequence: + """Compile once, then after each edit, then in full.""" + + first: DeltaCompileOutcome = compile_project() + if first.returncode != 0: + raise DeltaCompileSeriesError(first.stderr[-FAILURE_TAIL_CHARACTERS:]) + outcomes: list[DeltaCompileOutcome] = [] + for edit in edits: + _ = sequence_edited(project_dir=project_dir, edit=edit) + outcomes.append(compile_project()) + full: DeltaCompileOutcome = compile_in_full() + return DeltaCompileSequence( + delta_lines=tuple(DELTA_COMPILE_LINE in outcome.stderr for outcome in outcomes), + replayed_last=COMPILE_REUSE_HIT_LINE in outcomes[-1].stderr, + last_report=outcomes[-1].stdout, + full_report=full.stdout, + ) diff --git a/scripts/delta_compile_series/main/run_delta_compile_series.py b/scripts/delta_compile_series/main/run_delta_compile_series.py new file mode 100644 index 0000000000..e5b5b83c83 --- /dev/null +++ b/scripts/delta_compile_series/main/run_delta_compile_series.py @@ -0,0 +1,58 @@ +"""Run a seeded series of edits, compiling and judging after each one.""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from pathlib import Path + +from scripts.delta_compile_series._helpers.edits import edited_file +from scripts.delta_compile_series._helpers.outcomes import judged +from scripts.delta_compile_series.constants import ( + DELTA_WEIGHT, + EDITS, + FALLBACK_WEIGHT, + REVERT_PROBABILITY, +) +from scripts.delta_compile_series.models import ( + DeltaCompileEdit, + DeltaCompileOutcome, + DeltaCompileSeries, +) + + +def run_delta_compile_series( + *, project_dir: Path, seed: int, steps: int, compile_project: Callable[[], DeltaCompileOutcome] +) -> DeltaCompileSeries: + """Edit or revert `steps` times, compiling after each, and judge every compile.""" + + rng: random.Random = random.Random(seed) + weights: list[int] = [ + FALLBACK_WEIGHT if edit.fallback_reason else DELTA_WEIGHT for edit in EDITS + ] + pending: list[tuple[Path, str, DeltaCompileEdit]] = [] + described: list[str] = [] + unexpected: list[str] = [] + deltas: int = 0 + failing: bool = compile_project().returncode != 0 + for index in range(steps): + reverted: bool = bool(pending) and (failing or rng.random() < REVERT_PROBABILITY) + if reverted: + path, original, edit = pending.pop() + path.write_text(original, encoding="utf-8") + else: + edit: DeltaCompileEdit = rng.choices(EDITS, weights=weights)[0] + path, original = edited_file(project_dir=project_dir, edit=edit, rng=rng) + pending.append((path, original, edit)) + step: str = f"step {index}: {'revert ' if reverted else ''}{edit.kind} in {path.name}" + described.append(step) + outcome: DeltaCompileOutcome = compile_project() + others_edited: bool = any(other != path for other, _, _ in pending) + delta, problem = judged( + outcome=outcome, edit=edit, reverted=reverted, others_edited=others_edited + ) + deltas += int(delta) + if problem is not None: + unexpected.append(f"{step}: {problem}") + failing = outcome.returncode != 0 + return DeltaCompileSeries(steps=tuple(described), deltas=deltas, unexpected=tuple(unexpected)) diff --git a/scripts/delta_compile_series/main/write_delta_compile_project.py b/scripts/delta_compile_series/main/write_delta_compile_project.py new file mode 100644 index 0000000000..b0975074d7 --- /dev/null +++ b/scripts/delta_compile_series/main/write_delta_compile_project.py @@ -0,0 +1,19 @@ +"""Write the dense synthetic project on DuckDB for delta compile series.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from scripts.delta_compile_series.constants import PROJECT_CONFIG + + +def write_delta_compile_project(*, project_dir: Path, model_count: int) -> None: + """Write sources, seeds, functions, macros, models, SQL tests and custom rules on DuckDB.""" + + write_dense_compile_project(project_dir=project_dir, model_count=model_count) + config: Path = project_dir / PROJECT_CONFIG + config.write_text( + config.read_text(encoding="utf-8").replace('adapter = "snowflake"', 'adapter = "duckdb"'), + encoding="utf-8", + ) diff --git a/scripts/delta_compile_series/models.py b/scripts/delta_compile_series/models.py new file mode 100644 index 0000000000..26628e2dfc --- /dev/null +++ b/scripts/delta_compile_series/models.py @@ -0,0 +1,46 @@ +"""Edits, compile outcomes and series results of delta compile series.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.delta_compile_series.types import EditTarget + + +@dataclass(frozen=True) +class DeltaCompileEdit: + """One edit a series can make, and the fallback reason it must give when not a delta.""" + + kind: str + pattern: str + replacement: str + target: EditTarget = EditTarget.MODELS + fallback_reason: str | None = None + + +@dataclass(frozen=True) +class DeltaCompileOutcome: + """One compile's exit code and output.""" + + returncode: int + stdout: str + stderr: str + + +@dataclass(frozen=True) +class DeltaCompileSeries: + """What a seeded series found: each step, the deltas it took and anything unexpected.""" + + steps: tuple[str, ...] + deltas: int + unexpected: tuple[str, ...] + + +@dataclass(frozen=True) +class DeltaCompileSequence: + """What a fixed sequence of edits printed: which compiles were deltas and the last report.""" + + delta_lines: tuple[bool, ...] + replayed_last: bool + last_report: str + full_report: str diff --git a/scripts/delta_compile_series/types.py b/scripts/delta_compile_series/types.py new file mode 100644 index 0000000000..7b07a6c4b1 --- /dev/null +++ b/scripts/delta_compile_series/types.py @@ -0,0 +1,21 @@ +"""Enumerations of delta compile series.""" + +from __future__ import annotations + +from enum import StrEnum + + +class EditTarget(StrEnum): + """Which project files an edit changes.""" + + MODELS = "models" + TESTS = "tests" + PROJECT_CONFIG = "project_config" + + +class SequenceEdit(StrEnum): + """One step of a fixed edit sequence.""" + + BODY = "body" + SHAPE = "shape" + NONE = "none" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 2a24618817..7cc088907e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -9,6 +9,7 @@ import pytest +from scripts.delta_compile_series.types import SequenceEdit from tests.integration.src.sqlbuild.compiler.pipeline._test_types import PreparedCompile if TYPE_CHECKING: @@ -1105,24 +1106,23 @@ class EarlyBuiltinRulesEditTestCase: expected_rule_cache_outcomes: tuple[tuple[int, int], ...] -@dataclass(frozen=True) -class DeltaCompileEdit: - """One edit a seeded delta compile series can make, and whether it may compile as a delta.""" - - kind: str - pattern: str - replacement: str - on_test_file: bool = False - # A fallback reason fragment when the edit must compile in full; `None` when it is a delta. - fallback_reason: str | None = None - - @dataclass(frozen=True) class DeltaCompileSeriesTestCase: description: str seed: int json_output: bool steps: int + expected_unexpected_outcomes: tuple[str, ...] = () + expected_minimum_deltas: int = 3 + + +@dataclass(frozen=True) +class DeltaCompileSequenceTestCase: + description: str + edits: tuple[SequenceEdit, ...] + expected_delta_lines: tuple[bool, ...] + expected_replayed_last: bool + expected_same_as_full: bool @dataclass(frozen=True) @@ -1130,3 +1130,5 @@ class DeltaCompileOptOutTestCase: description: str settings: str env: dict[str, str] + expected_delta_line: bool = False + expected_images: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 1f08108c46..971d60f100 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -38,6 +38,7 @@ semantic_compile_fingerprint, ) from scripts.compiler_differential.constants import FAILURE_BASE_FILES +from scripts.delta_compile_series.models import DeltaCompileOutcome from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from sqlbuild._native import ContractValidationResult, compiler_cache_directory from sqlbuild._native import run_cli as main @@ -5095,45 +5096,20 @@ def json_diagnostic_codes(stdout: str) -> tuple[str, ...]: document: dict[str, Any] = json.loads(stdout or "{}") return tuple(str(diagnostic["code"]) for diagnostic in document.get("diagnostics", ())) - - -DELTA_COMPILE_LINE: str = "against the previous compile" -DELTA_VERIFIED_LINE: str = "delta compile verified" -DELTA_FALLBACK_PREFIX: str = "delta compile fell back: " -DELTA_MISMATCH_PREFIX: str = "delta compile mismatch: " - - -class DeltaCompileRun(NamedTuple): - """One fresh-process compile with compile reuse and, optionally, the delta verifier.""" - - returncode: int - stdout: str - stderr: str - - @property - def verified(self) -> bool: - return DELTA_VERIFIED_LINE in self.stderr - - @property - def mismatches(self) -> list[str]: - return [line for line in self.stderr.splitlines() if line.startswith(DELTA_MISMATCH_PREFIX)] - - @property - def fallback_reason(self) -> str | None: - for line in self.stderr.splitlines(): - if line.startswith(DELTA_FALLBACK_PREFIX): - return line.removeprefix(DELTA_FALLBACK_PREFIX) - return None - - def run_delta_compile( - *, project_dir: Path, json_output: bool, env: dict[str, str] | None = None -) -> DeltaCompileRun: + *, project_dir: Path, args: tuple[str, ...], env: dict[str, str] +) -> DeltaCompileOutcome: """Compile in a fresh process with compile reuse enabled and the given extra environment.""" result: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir, - args=("compile", "--json") if json_output else ("compile",), - env={REUSE_DISABLE_ENV_VAR: "0", **(env or {})}, + project_dir=project_dir, args=args, env={REUSE_DISABLE_ENV_VAR: "0", **env} + ) + return DeltaCompileOutcome( + returncode=result.returncode, stdout=result.stdout, stderr=result.stderr ) - return DeltaCompileRun(returncode=result.returncode, stdout=result.stdout, stderr=result.stderr) + + +def without_compile_timings(*, report: str) -> str: + """A JSON report without its compile timings, which differ between compiles.""" + + return _COMPILE_TIMINGS_PATTERN.sub("", report) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 50cc23d903..9eb5ad798c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -2,149 +2,46 @@ from __future__ import annotations -import random -import re +from functools import partial from pathlib import Path import pytest -from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from scripts.delta_compile_series.constants import DELTA_COMPILE_LINE, PROJECT_CONFIG +from scripts.delta_compile_series.main.run_delta_compile_sequence import ( + run_delta_compile_sequence, +) +from scripts.delta_compile_series.main.run_delta_compile_series import run_delta_compile_series +from scripts.delta_compile_series.main.write_delta_compile_project import ( + write_delta_compile_project, +) +from scripts.delta_compile_series.models import ( + DeltaCompileOutcome, + DeltaCompileSequence, + DeltaCompileSeries, +) +from scripts.delta_compile_series.types import SequenceEdit from sqlbuild._native import DELTA_COMPILE_DISABLE_ENV_VAR, DELTA_COMPILE_VERIFY_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( - DeltaCompileEdit, DeltaCompileOptOutTestCase, + DeltaCompileSequenceTestCase, DeltaCompileSeriesTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( - COMPILE_REUSE_HIT_LINE, - DELTA_COMPILE_LINE, - DeltaCompileRun, replace_project_text, run_delta_compile, + without_compile_timings, ) _MODEL_COUNT: int = 300 -_VERIFY_ENV: dict[str, str] = {DELTA_COMPILE_VERIFY_ENV_VAR: "1"} -_MISMATCH_EXIT_CODE: int = 70 -_REVERT_PROBABILITY: float = 0.5 -_MINIMUM_DELTAS: int = 3 -# Edits that compile as deltas are drawn more often than those that must compile in full. -_DELTA_WEIGHT: int = 3 -_FALLBACK_WEIGHT: int = 1 -_EDIT_ATTEMPTS: int = 50 -_PROJECT_CONFIG: str = "sqlbuild_project.toml" -# Problems a full compile reported in files a delta does not compile again keep it full. -_STALE_PROBLEMS: str = "reports problems in files the edit leaves alone" +_JSON_COMPILE: tuple[str, ...] = ("compile", "--json") +_TEXT_COMPILE: tuple[str, ...] = ("compile",) _REUSE_ROOT: str = "target/cache/compiler-native-v1/project-reuse-v3" - -_EDITS: tuple[DeltaCompileEdit, ...] = ( - DeltaCompileEdit("body_constant", r"COALESCE\(b\.amount, \d+\)", "COALESCE(b.amount, {value})"), - DeltaCompileEdit("body_comment", r"\Z", "\n-- reviewed {value}\n"), - DeltaCompileEdit( - "description", r'MODEL \(description "[^"]*"', 'MODEL (description "Edited {value}."' - ), - DeltaCompileEdit( - "column_description", - r'description "Order quantity measure"', - 'description "Edited measure {value}"', - ), - DeltaCompileEdit("type_error", r"quantity_000 \(type DOUBLE", "quantity_000 (type VARCHAR"), - DeltaCompileEdit("parse_error", r"projected AS \(SELECT", "projected AS (SELEC"), - DeltaCompileEdit( - "test_expectation", - r"(__expected__\w+ AS \(SELECT CAST\()\d+", - r"\g<1>{value}", - on_test_file=True, - ), - DeltaCompileEdit( - "materialization", - r"materialized table", - "materialized view", - fallback_reason="custom rules select models by their materialization", - ), - DeltaCompileEdit( - "contract", - r"contract enforced, ", - "", - fallback_reason="a custom rule reads", - ), - DeltaCompileEdit( - "unknown_reference", - r'__ref\("([^"]+)"\)', - r'__ref("\1_missing")', - fallback_reason="changed what it references", - ), -) -_CONFIG_EDIT: DeltaCompileEdit = DeltaCompileEdit( - "project_config", - r"\Z", - "\n# reviewed {value}\n", - fallback_reason="sqlbuild_project.toml is not a model or SQL test file", +_EDITED_MODEL: str = ( + "models/sales000/intermediate/clean/batch00000/sales000__int_clean__orders_00003.sql" ) -def write_delta_compile_project(*, project_dir: Path, model_count: int) -> None: - """Write the dense synthetic project on DuckDB: sources, seeds, functions, macros, SQL tests - and custom rules.""" - - write_dense_compile_project(project_dir=project_dir, model_count=model_count) - replace_project_text( - project_dir, _PROJECT_CONFIG, 'adapter = "snowflake"', 'adapter = "duckdb"' - ) - - -def _project_files(project_dir: Path, on_test_file: bool) -> list[Path]: - root: Path = project_dir / ("tests" if on_test_file else "models") - return sorted(path for path in root.rglob("*.sql") if "/_" not in path.as_posix()) - - -def _applied(path: Path, edit: DeltaCompileEdit, rng: random.Random) -> bool: - text: str = path.read_text(encoding="utf-8") - replacement: str = edit.replacement.replace("{value}", str(rng.randint(1, 10**6))) - edited, count = re.subn(edit.pattern, replacement, text, count=1) - if count == 0 or edited == text: - return False - path.write_text(edited, encoding="utf-8") - return True - - -def _edited_file(project_dir: Path, edit: DeltaCompileEdit, rng: random.Random) -> tuple[Path, str]: - """Apply `edit` to a randomly chosen file it changes; return the file and its old text.""" - - if edit is _CONFIG_EDIT: - candidates: list[Path] = [project_dir / _PROJECT_CONFIG] - else: - candidates = _project_files(project_dir, edit.on_test_file) - for _ in range(_EDIT_ATTEMPTS): - path: Path = rng.choice(candidates) - original: str = path.read_text(encoding="utf-8") - if _applied(path, edit, rng): - return path, original - raise AssertionError(f"no file takes the {edit.kind} edit") - - -def _assert_outcome( - run: DeltaCompileRun, edit: DeltaCompileEdit, step: str, other_files_edited: bool -) -> bool: - """Assert `run` compiled `edit` as it must; return whether it compiled as a delta.""" - - assert run.returncode != _MISMATCH_EXIT_CODE, f"{step}: {run.mismatches}" - assert run.mismatches == [], step - if COMPILE_REUSE_HIT_LINE in run.stderr: - # A revert to the state of a compile that stored nothing, such as one that raised, - # replays the compile stored before it. - assert "revert" in step, step - return False - reason: str | None = run.fallback_reason - if other_files_edited and reason is not None and _STALE_PROBLEMS in reason: - return False - if edit.fallback_reason is None: - assert run.verified, f"{step}: fell back: {reason}\n{run.stderr[-2000:]}" - return True - assert reason is not None and edit.fallback_reason in reason, f"{step}: {reason}" - return False - - @pytest.mark.parametrize( "test_case", [ @@ -161,89 +58,70 @@ def test_given_seeded_edit_series_when_compiling_each_edit_then_deltas_match_ful ) -> None: project_dir: Path = tmp_path / "dense_orders" write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) - rng: random.Random = random.Random(test_case.seed) - baseline: DeltaCompileRun = run_delta_compile( - project_dir=project_dir, json_output=test_case.json_output, env=_VERIFY_ENV + args: tuple[str, ...] = (_TEXT_COMPILE, _JSON_COMPILE)[int(test_case.json_output)] + + series: DeltaCompileSeries = run_delta_compile_series( + project_dir=project_dir, + seed=test_case.seed, + steps=test_case.steps, + compile_project=partial( + run_delta_compile, + project_dir=project_dir, + args=args, + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + ), ) - assert baseline.returncode == 0, baseline.stderr[-2000:] - pending: list[tuple[Path, str, DeltaCompileEdit]] = [] - edits: tuple[DeltaCompileEdit, ...] = (*_EDITS, _CONFIG_EDIT) - weights: list[int] = [ - _FALLBACK_WEIGHT if edit.fallback_reason else _DELTA_WEIGHT for edit in edits - ] - failing: bool = False - deltas: int = 0 - for index in range(test_case.steps): - if pending and (failing or rng.random() < _REVERT_PROBABILITY): - path, original, edit = pending.pop() - path.write_text(original, encoding="utf-8") - step: str = f"step {index}: revert {edit.kind} in {path.name}" - else: - edit = rng.choices(edits, weights=weights)[0] - path, original = _edited_file(project_dir, edit, rng) - pending.append((path, original, edit)) - step = f"step {index}: {edit.kind} in {path.name}" - run: DeltaCompileRun = run_delta_compile( - project_dir=project_dir, json_output=test_case.json_output, env=_VERIFY_ENV - ) - other_files_edited: bool = any(other != path for other, _, _ in pending) - deltas += _assert_outcome(run, edit, step, other_files_edited) - failing = run.returncode != 0 - assert deltas >= _MINIMUM_DELTAS + assert series.unexpected == test_case.expected_unexpected_outcomes, series.steps + assert series.deltas >= test_case.expected_minimum_deltas, series.steps -def test_given_unchanged_project_when_compiling_after_a_delta_then_the_delta_is_replayed( - tmp_path: Path, -) -> None: - project_dir: Path = tmp_path / "dense_orders" - write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) - full: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) - edited: Path = _project_files(project_dir, on_test_file=False)[7] - rng: random.Random = random.Random(5) - assert _applied(edited, _EDITS[0], rng) - delta: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) - replayed: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) - - assert full.returncode == delta.returncode == replayed.returncode == 0 - assert DELTA_COMPILE_LINE in delta.stderr - assert DELTA_COMPILE_LINE not in replayed.stderr - assert COMPILE_REUSE_HIT_LINE in replayed.stderr - assert replayed.stdout.count('"compile_timings"') == 1 - assert _without_timings(replayed.stdout) == _without_timings(delta.stdout) - -def test_given_edit_changing_downstream_shapes_when_compiling_the_next_edit_then_both_are_deltas( - tmp_path: Path, +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileSequenceTestCase( + description="an unchanged project replays the delta", + edits=(SequenceEdit.BODY, SequenceEdit.NONE), + expected_delta_lines=(True, False), + expected_replayed_last=True, + expected_same_as_full=True, + ), + DeltaCompileSequenceTestCase( + description="an edit after a multi-round delta is a delta too", + edits=(SequenceEdit.SHAPE, SequenceEdit.BODY), + expected_delta_lines=(True, True), + expected_replayed_last=False, + expected_same_as_full=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_edit_sequence_when_compiling_each_edit_then_deltas_store_and_replay_like_full_compiles( + test_case: DeltaCompileSequenceTestCase, tmp_path: Path ) -> None: project_dir: Path = tmp_path / "dense_orders" write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) - models: list[Path] = _project_files(project_dir, on_test_file=False) - reshaped: Path = next( - path for path in models if "quantity_000 (type DOUBLE" in path.read_text(encoding="utf-8") - ) - baseline: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) - reshaped.write_text( - reshaped.read_text(encoding="utf-8") - .replace("quantity_000 (type DOUBLE", "quantity_000 (type FLOAT", 1) - .replace("AS DOUBLE) AS quantity_000", "AS FLOAT) AS quantity_000"), - encoding="utf-8", - ) - shape_edit: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) - assert _applied(models[11], _EDITS[0], random.Random(3)) - next_edit: DeltaCompileRun = run_delta_compile(project_dir=project_dir, json_output=True) - full: DeltaCompileRun = run_delta_compile( - project_dir=project_dir, json_output=True, env={DELTA_COMPILE_DISABLE_ENV_VAR: "1"} - ) - - assert baseline.returncode == 0, baseline.stderr[-2000:] - assert re.search(r"\(([2-9]|\d\d+) model\(s\)", shape_edit.stderr), shape_edit.stderr[-2000:] - assert DELTA_COMPILE_LINE in next_edit.stderr, next_edit.stderr[-2000:] - assert DELTA_COMPILE_LINE not in full.stderr - assert _without_timings(next_edit.stdout) == _without_timings(full.stdout) + sequence: DeltaCompileSequence = run_delta_compile_sequence( + project_dir=project_dir, + edits=test_case.edits, + compile_project=partial( + run_delta_compile, project_dir=project_dir, args=_JSON_COMPILE, env={} + ), + compile_in_full=partial( + run_delta_compile, + project_dir=project_dir, + args=_JSON_COMPILE, + env={DELTA_COMPILE_DISABLE_ENV_VAR: "1"}, + ), + ) -def _without_timings(report: str) -> str: - return re.sub(r'"compile_timings": \{[^}]*\}', "", report) + assert sequence.delta_lines == test_case.expected_delta_lines + assert sequence.replayed_last == test_case.expected_replayed_last + assert ( + without_compile_timings(report=sequence.last_report) + == without_compile_timings(report=sequence.full_report) + ) == test_case.expected_same_as_full @pytest.mark.parametrize( @@ -263,24 +141,25 @@ def test_given_delta_compiles_turned_off_when_compiling_an_edit_then_it_compiles ) -> None: project_dir: Path = tmp_path / "dense_orders" write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) - config: Path = project_dir / _PROJECT_CONFIG + config: Path = project_dir / PROJECT_CONFIG config.write_text(config.read_text(encoding="utf-8") + test_case.settings, encoding="utf-8") - first: DeltaCompileRun = run_delta_compile( - project_dir=project_dir, json_output=False, env=test_case.env + first: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=_TEXT_COMPILE, env=test_case.env ) replace_project_text( - project_dir, - _project_files(project_dir, on_test_file=False)[3].relative_to(project_dir).as_posix(), - "COALESCE(b.amount, ", - "COALESCE(b.amount, 1 + ", + project_dir, _EDITED_MODEL, "COALESCE(b.amount, ", "COALESCE(b.amount, 1 + " ) - edited: DeltaCompileRun = run_delta_compile( - project_dir=project_dir, json_output=False, env=test_case.env + + edited: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=_TEXT_COMPILE, env=test_case.env ) - assert first.returncode == edited.returncode == 0, edited.stderr[-2000:] - assert DELTA_COMPILE_LINE not in edited.stderr - assert sorted((project_dir / _REUSE_ROOT).glob("*.image")) == [] + assert (first.returncode, edited.returncode) == (0, 0), edited.stderr[-2000:] + assert (DELTA_COMPILE_LINE in edited.stderr) == test_case.expected_delta_line + assert ( + tuple(sorted(path.name for path in (project_dir / _REUSE_ROOT).glob("*.image"))) + == test_case.expected_images + ) if __name__ == "__main__": From 1615168225bb1ee40e05db4a0ee42c567794fb28 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:05:35 +0100 Subject: [PATCH 24/32] perf(compile): keep the retained rules rows in the native record format Full compiles serialized every model's rules row a second time as JSON, without its SQL, for the project image. The rows are now written once in the native length-prefixed record format next to the model-free request JSON and read back in sqlbuild-rules, cutting the image overhead of a full compile after an edit to under 2%. --- Cargo.lock | 1 + .../bindings/_helpers/compile_delta/image.rs | 48 ++- .../src/bindings/tests/helpers.rs | 3 + crates/sqlbuild-rules/Cargo.toml | 1 + .../src/engine/_helpers/memo.rs | 66 +-- .../sqlbuild-rules/src/engine/_helpers/mod.rs | 1 + .../src/engine/_helpers/retained.rs | 408 ++++++++++++++++++ crates/sqlbuild-rules/src/engine/main/mod.rs | 1 + .../src/engine/main/retained_request.rs | 9 + .../src/engine/tests/helpers.rs | 93 +++- .../src/engine/tests/test_retained_rows.rs | 59 ++- .../src/engine/tests/test_types.rs | 8 + crates/sqlbuild-rules/src/models.rs | 4 +- 13 files changed, 629 insertions(+), 73 deletions(-) create mode 100644 crates/sqlbuild-rules/src/engine/_helpers/retained.rs create mode 100644 crates/sqlbuild-rules/src/engine/main/retained_request.rs diff --git a/Cargo.lock b/Cargo.lock index de1498a7da..2dc2a64115 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1306,6 +1306,7 @@ dependencies = [ "sha2", "sqlbuild-analysis", "sqlbuild-attachments", + "sqlbuild-cache", "sqlbuild-core", "sqlbuild-scopes", "sqlbuild-sqltext", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index 68a4e86404..d3e4e5f49c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeMap, BTreeSet}; use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_rules::engine::main::retained_request::retained_request; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::compile_delta::context::{ @@ -14,13 +15,14 @@ use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; /// Changes whenever the image's meaning changes, so an older image is never read. -pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v6"; +pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v7"; pub(crate) const GATE_SECTION: &str = "gate"; /// The upstream kind of a model another model reads. const MODEL_KIND: &str = "model"; pub(crate) const FACTS_SECTION: &str = "facts"; const RULES_SECTION: &str = "rules"; pub(crate) const RULE_DIGESTS_SECTION: &str = "rule-digests"; +const RULE_ROWS_SECTION: &str = "rule-rows"; /// Everything a delta compile reads of the compile it patches. #[derive(Debug, Default)] @@ -56,9 +58,8 @@ pub(crate) struct BaseRules { impl BaseRules { /// The rows a compile retained, decoded. pub(crate) fn decoded(retained: &RetainedRows) -> Option { - let request: EvaluateRequest = from_json(&retained.request_json)?; - (request.models.len() == retained.digests.len()).then(|| Self { - request, + Some(Self { + request: retained_request(retained)?, digests: retained.digests.clone(), }) } @@ -119,10 +120,13 @@ fn from_json(bytes: &[u8]) -> Option { Some(value) } -fn decoded_rules(request: &[u8], digests: &[u8]) -> Option { - let request: EvaluateRequest = from_json(request)?; - let digests: Vec = decoded_strings(digests)?; - (digests.len() == request.models.len()).then_some(BaseRules { request, digests }) +fn decoded_rules(request_json: Vec, rows: Vec, digests: &[u8]) -> Option { + let retained: RetainedRows = RetainedRows { + request_json, + rows, + digests: decoded_strings(digests)?, + }; + BaseRules::decoded(&retained) } /// The analysis session's final relation facts, as stored. @@ -571,13 +575,20 @@ impl ProjectImage { }; let rules: Option = self.rules.and_then(ImageRules::retained); let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); - let rules = rules.map(|rules| (rules.request_json, encoded_strings(&rules.digests))); + let rules = rules.map(|rules| { + ( + rules.request_json, + rules.rows, + encoded_strings(&rules.digests), + ) + }); let mut sections: ImageSections = vec![ (GATE_SECTION.to_owned(), gate), (FACTS_SECTION.to_owned(), facts?), ]; - if let Some((request, digests)) = rules { + if let Some((request, rows, digests)) = rules { sections.push((RULES_SECTION.to_owned(), request)); + sections.push((RULE_ROWS_SECTION.to_owned(), rows)); sections.push((RULE_DIGESTS_SECTION.to_owned(), digests)); } Ok(sections) @@ -591,11 +602,18 @@ impl ProjectImage { .find(|(section, _)| section == name) .map(|(_, bytes)| std::mem::take(bytes)) }; - let rules: Option = match (taken(RULES_SECTION), taken(RULE_DIGESTS_SECTION)) { - (Some(request), Some(digests)) => Some(ImageRules::Decoding(PendingRules::Decoding( - std::thread::spawn(move || decoded_rules(&request, &digests)), - ))), - (None, None) => None, + let stored_rules = ( + taken(RULES_SECTION), + taken(RULE_ROWS_SECTION), + taken(RULE_DIGESTS_SECTION), + ); + let rules: Option = match stored_rules { + (Some(request), Some(rows), Some(digests)) => { + Some(ImageRules::Decoding(PendingRules::Decoding( + std::thread::spawn(move || decoded_rules(request, rows, &digests)), + ))) + } + (None, None, None) => None, _ => return None, }; let facts: StoredFacts = StoredFacts::decode(section(§ions, FACTS_SECTION)?)?; diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index 6f3e431bce..c19e16eed1 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -12,6 +12,8 @@ use crate::bindings::_helpers::compile_delta::image::{ ProjectImage, RULE_DIGESTS_SECTION, }; +/// The rows of a request without models: a zero model count. +const RULE_ROWS: [u8; 8] = [0; 8]; const OTHER_FORMAT: &str = "sqlbuild-project-image-v0"; pub(crate) fn pairs(values: &[(&str, &str)]) -> Vec<(String, String)> { @@ -104,6 +106,7 @@ pub(crate) fn image() -> ProjectImage { )]), rules: Some(ImageRules::Retained(RetainedRows { request_json: request, + rows: RULE_ROWS.to_vec(), digests: Vec::new(), })), } diff --git a/crates/sqlbuild-rules/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml index 6fb65866b2..559fa48c1a 100644 --- a/crates/sqlbuild-rules/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -20,6 +20,7 @@ serde_json.workspace = true sha2.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true +sqlbuild-cache.workspace = true sqlbuild-core.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index ed6eaece1a..d6e217a141 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -9,13 +9,10 @@ use serde_json::Value; use crate::constants::{API_VERSION, NATIVE_ROWS_MEMO_VERSION}; use crate::engine::_helpers::cache::PendingBucketWrite; use crate::engine::_helpers::evaluation::{EvaluationControl, evaluate_request_with}; +use crate::engine::_helpers::retained::encoded_rows; use crate::errors::RowsError; use crate::models::{EvaluateRequest, Model, RetainedRows, RowsEvaluation, RowsRequest}; -/// Where the model-free request's JSON holds its empty model list; strings escape their quotes. -const EMPTY_MODELS: &[u8] = b",\"models\":[],\"sql_tests\":"; -const MODELS_KEY: &[u8] = b",\"models\":["; - /// The shared key prefix of fields a request serializes but evaluation derives again. const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; const DERIVED_FIELDS: [&str; 2] = ["_test_count\":", "\":"]; @@ -56,16 +53,10 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result>)> = std::mem::take(&mut request.models) + let models: Vec = std::mem::take(&mut request.models) .into_iter() - .map(|model| digested(model, retain)) + .map(digested) .collect::>()?; - let mut row_json: Vec> = Vec::new(); - let mut models: Vec = Vec::with_capacity(digested_rows.len()); - for (model, json) in digested_rows { - models.push(model); - row_json.extend(json); - } let mut request_json: Option> = None; let memo: Option<(String, PathBuf)> = match memo { Some((code_identity, path)) => { @@ -80,7 +71,7 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result = if retain { - retained_rows(&request, &models, (request_json, row_json)) + retained_rows(&request, &models, request_json) } else { None }; @@ -93,39 +84,24 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result>, Vec>), + request_json: Option>, ) -> Option { - if row_json.len() != models.len() { - return None; - } let request_json: Vec = match request_json { Some(json) => json, - None => match serde_json::to_vec(request) { - Ok(json) => json, - Err(_unencodable) => return None, - }, - }; - let at: usize = request_json - .windows(EMPTY_MODELS.len()) - .position(|window| window == EMPTY_MODELS)?; - let rows: usize = row_json.iter().map(Vec::len).sum::() + row_json.len(); - let mut json: Vec = Vec::with_capacity(request_json.len() + rows); - json.extend_from_slice(&request_json[..at]); - json.extend_from_slice(MODELS_KEY); - for (index, row) in row_json.iter().enumerate() { - if index > 0 { - json.push(b','); + None => { + let Ok(json) = serde_json::to_vec(request) else { + return None; + }; + json } - json.extend_from_slice(row); - } - json.push(b']'); - json.extend_from_slice(&request_json[at + MODELS_KEY.len() + 1..]); + }; Some(RetainedRows { - request_json: without_derived_fields(json), + request_json: without_derived_fields(request_json), + rows: encoded_rows(models), digests: models .iter() .map(|model| model.payload_digest.clone().unwrap_or_default()) @@ -268,22 +244,14 @@ fn speculation_key( Ok(hasher.finalize().to_hex().to_string()) } -/// `model` with the BLAKE3 digest of its exact facts and, with `retain`, its JSON without SQL. -fn digested(mut model: Model, retain: bool) -> Result<(Model, Option>), RowsError> { +/// `model` with the BLAKE3 digest of its exact facts, which keys its cached findings. +fn digested(mut model: Model) -> Result { if model.payload_digest.is_none() { let mut hasher: blake3::Hasher = blake3::Hasher::new(); serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; model.payload_digest = Some(hasher.finalize().to_hex().to_string()); } - if !retain { - return Ok((model, None)); - } - let query_sql: String = std::mem::take(&mut model.query_sql); - let authored_sql: String = std::mem::take(&mut model.authored_sql); - let stripped: Result, serde_json::Error> = serde_json::to_vec(&model); - model.query_sql = query_sql; - model.authored_sql = authored_sql; - Ok((model, Some(stripped.map_err(rules_error)?))) + Ok(model) } /// The memo version, build, request and model digests, keeping the request's JSON in `kept`. diff --git a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs index ec8f8aa27d..234f764470 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs @@ -3,5 +3,6 @@ pub mod custom_host; pub mod evaluation; pub(crate) mod memo; pub(crate) mod project_rows; +pub(crate) mod retained; pub(crate) mod rows; pub(crate) mod speculation; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs new file mode 100644 index 0000000000..15c69a0ea5 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -0,0 +1,408 @@ +//! Retained model rows in the native record format, without their SQL text. + +use std::collections::BTreeMap; +use std::str::FromStr; + +use serde_json::{Map, Number, Value}; +use sqlbuild_cache::store::errors::StoreDecodeError; +use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; + +use crate::models::{ + Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Model, Reference, + RetainedRows, +}; + +const NULL: u64 = 0; +const FALSE: u64 = 1; +const TRUE: u64 = 2; +const NUMBER: u64 = 3; +const STRING: u64 = 4; +const ARRAY: u64 = 5; +const OBJECT: u64 = 6; +const ABSENT: u64 = 0; +const PRESENT: u64 = 1; +/// Items reserved up front per decoded list at most, whatever count a damaged row claims. +const MAX_RESERVED_ITEMS: usize = 1 << 12; + +/// Writes model rows as length-prefixed records. +#[derive(Default)] +struct RowWriter { + record: RecordWriter, +} + +impl RowWriter { + fn flag(&mut self, value: bool) { + self.record.put_u64(u64::from(value)); + } + + fn count(&mut self, value: usize) { + self.record.put_u64(value as u64); + } + + fn text(&mut self, value: &str) { + self.record.put_str(value); + } + + fn optional_text(&mut self, value: Option<&str>) { + match value { + Some(text) => { + self.record.put_u64(PRESENT); + self.text(text); + } + None => self.record.put_u64(ABSENT), + } + } + + fn texts(&mut self, values: &[String]) { + self.count(values.len()); + for value in values { + self.text(value); + } + } + + fn value(&mut self, value: &Value) { + match value { + Value::Null => self.record.put_u64(NULL), + Value::Bool(flag) => self.record.put_u64(if *flag { TRUE } else { FALSE }), + Value::Number(number) => { + self.record.put_u64(NUMBER); + self.text(&number.to_string()); + } + Value::String(text) => { + self.record.put_u64(STRING); + self.text(text); + } + Value::Array(items) => { + self.record.put_u64(ARRAY); + self.count(items.len()); + for item in items { + self.value(item); + } + } + Value::Object(entries) => { + self.record.put_u64(OBJECT); + self.count(entries.len()); + for (key, item) in entries { + self.text(key); + self.value(item); + } + } + } + } + + fn optional_value(&mut self, value: Option<&Value>) { + match value { + Some(value) => { + self.record.put_u64(PRESENT); + self.value(value); + } + None => self.record.put_u64(ABSENT), + } + } + + fn declarations(&mut self, declarations: &[Declaration]) { + self.count(declarations.len()); + for declaration in declarations { + let Declaration { + name, + relative_path, + members, + value, + value_type, + render_as, + } = declaration; + self.text(name); + self.text(relative_path); + self.count(members.len()); + for DeclarationMember { name, value } in members { + self.text(name); + self.value(value); + } + self.optional_value(value.as_ref()); + self.optional_text(value_type.as_deref()); + self.optional_text(render_as.as_deref()); + } + } + + /// One model's row; its SQL text, digest and derived counts are left out. + fn model(&mut self, model: &Model) { + let Model { + sql_analysis_disabled, + name, + relative_path, + query_sql: _, + authored_sql: _, + config, + authored_config_keys, + logical_schema, + references, + columns, + dynamic_columns, + dynamic_columns_proven, + bare_dynamic_pivot, + enum_columns, + enum_declarations, + constant_declarations, + declared_audit_count, + targeting_test_count, + empty_input_only_test_count: _, + payload_digest: _, + stripped: _, + } = model; + self.flag(*sql_analysis_disabled); + self.text(name); + self.text(relative_path); + self.count(config.len()); + for (key, value) in config { + self.text(key); + self.value(value); + } + self.texts(authored_config_keys); + self.optional_text(logical_schema.as_deref()); + self.count(references.len()); + for Reference { + ref_kind, + ref_name, + ref_package, + } in references + { + self.text(ref_kind); + self.text(ref_name); + self.optional_text(ref_package.as_deref()); + } + self.count(columns.len()); + for Column { + name, + data_type, + nullable, + audit_count, + type_proven, + } in columns + { + self.text(name); + self.text(data_type); + self.record.put_u64(match nullable { + None => NULL, + Some(false) => FALSE, + Some(true) => TRUE, + }); + self.record.put_u64(u64::from(*audit_count)); + self.flag(*type_proven); + } + self.count(dynamic_columns.len()); + for DynamicColumnFamily { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + type_proven, + } in dynamic_columns + { + for text in [name, pivot_column, value_column, aggregate, data_type] { + self.text(text); + } + self.optional_text(name_pattern.as_deref()); + self.flag(*type_proven); + } + self.flag(*dynamic_columns_proven); + self.flag(*bare_dynamic_pivot); + self.texts(enum_columns); + self.declarations(enum_declarations); + self.declarations(constant_declarations); + self.record.put_u64(u64::from(*declared_audit_count)); + self.record.put_u64(u64::from(*targeting_test_count)); + } +} + +/// Reads the rows a [`RowWriter`] wrote. +struct RowReader<'a> { + record: RecordReader<'a>, +} + +type Read = Result; + +impl RowReader<'_> { + fn count(&mut self) -> Read { + usize::try_from(self.record.u64()?).map_err(|_| StoreDecodeError) + } + + fn small(&mut self) -> Read { + u32::try_from(self.record.u64()?).map_err(|_| StoreDecodeError) + } + + fn flag(&mut self) -> Read { + match self.record.u64()? { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(StoreDecodeError), + } + } + + fn text(&mut self) -> Read { + self.record.string() + } + + fn present(&mut self) -> Read { + match self.record.u64()? { + ABSENT => Ok(false), + PRESENT => Ok(true), + _ => Err(StoreDecodeError), + } + } + + fn optional_text(&mut self) -> Read> { + Ok(if self.present()? { + Some(self.text()?) + } else { + None + }) + } + + fn list(&mut self, mut item: impl FnMut(&mut Self) -> Read) -> Read> { + let count: usize = self.count()?; + let mut items: Vec = Vec::with_capacity(count.min(MAX_RESERVED_ITEMS)); + for _ in 0..count { + items.push(item(self)?); + } + Ok(items) + } + + fn value(&mut self) -> Read { + Ok(match self.record.u64()? { + NULL => Value::Null, + FALSE => Value::Bool(false), + TRUE => Value::Bool(true), + NUMBER => Value::Number(Number::from_str(&self.text()?).map_err(|_| StoreDecodeError)?), + STRING => Value::String(self.text()?), + ARRAY => Value::Array(self.list(Self::value)?), + OBJECT => { + let count: usize = self.count()?; + let mut entries: Map = Map::new(); + for _ in 0..count { + let key: String = self.text()?; + entries.insert(key, self.value()?); + } + Value::Object(entries) + } + _ => return Err(StoreDecodeError), + }) + } + + fn optional_value(&mut self) -> Read> { + Ok(if self.present()? { + Some(self.value()?) + } else { + None + }) + } + + fn declaration(&mut self) -> Read { + Ok(Declaration { + name: self.text()?, + relative_path: self.text()?, + members: self.list(|reader| { + Ok(DeclarationMember { + name: reader.text()?, + value: reader.value()?, + }) + })?, + value: self.optional_value()?, + value_type: self.optional_text()?, + render_as: self.optional_text()?, + }) + } + + fn column(&mut self) -> Read { + Ok(Column { + name: self.text()?, + data_type: self.text()?, + nullable: match self.record.u64()? { + NULL => None, + FALSE => Some(false), + TRUE => Some(true), + _ => return Err(StoreDecodeError), + }, + audit_count: self.small()?, + type_proven: self.flag()?, + }) + } + + fn dynamic_column(&mut self) -> Read { + Ok(DynamicColumnFamily { + name: self.text()?, + pivot_column: self.text()?, + value_column: self.text()?, + aggregate: self.text()?, + data_type: self.text()?, + name_pattern: self.optional_text()?, + type_proven: self.flag()?, + }) + } + + fn model(&mut self) -> Read { + Ok(Model { + sql_analysis_disabled: self.flag()?, + name: self.text()?, + relative_path: self.text()?, + query_sql: String::new(), + authored_sql: String::new(), + config: { + let count: usize = self.count()?; + let mut config: BTreeMap = BTreeMap::new(); + for _ in 0..count { + let key: String = self.text()?; + config.insert(key, self.value()?); + } + config + }, + authored_config_keys: self.list(Self::text)?, + logical_schema: self.optional_text()?, + references: self.list(|reader| { + Ok(Reference { + ref_kind: reader.text()?, + ref_name: reader.text()?, + ref_package: reader.optional_text()?, + }) + })?, + columns: self.list(Self::column)?, + dynamic_columns: self.list(Self::dynamic_column)?, + dynamic_columns_proven: self.flag()?, + bare_dynamic_pivot: self.flag()?, + enum_columns: self.list(Self::text)?, + enum_declarations: self.list(Self::declaration)?, + constant_declarations: self.list(Self::declaration)?, + declared_audit_count: self.small()?, + targeting_test_count: self.small()?, + empty_input_only_test_count: 0, + payload_digest: None, + stripped: false, + }) + } +} + +/// Every model's row without its SQL text, in model order. +pub(crate) fn encoded_rows(models: &[Model]) -> Vec { + let mut writer: RowWriter = RowWriter::default(); + writer.count(models.len()); + for model in models { + writer.model(model); + } + writer.record.into_bytes() +} + +/// The request `retained` holds, each model without its SQL text, digest or derived counts. +pub(crate) fn retained_request(retained: &RetainedRows) -> Read { + let mut request: EvaluateRequest = + serde_json::from_slice(&retained.request_json).map_err(|_| StoreDecodeError)?; + let mut reader: RowReader<'_> = RowReader { + record: RecordReader::new(&retained.rows), + }; + request.models = reader.list(RowReader::model)?; + reader.record.finish()?; + if request.models.len() != retained.digests.len() { + return Err(StoreDecodeError); + } + Ok(request) +} diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index a20ecdbdd4..1237dba54e 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -5,4 +5,5 @@ pub mod discard_speculative_rows; pub mod evaluate_rows; pub mod finalize_rows; pub mod project_model_rows; +pub mod retained_request; pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/retained_request.rs b/crates/sqlbuild-rules/src/engine/main/retained_request.rs new file mode 100644 index 0000000000..1023e8ebc4 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/retained_request.rs @@ -0,0 +1,9 @@ +use crate::models::{EvaluateRequest, RetainedRows}; + +/// The request a compile retained, its models without SQL text; `None` when it is damaged. +pub fn retained_request(retained: &RetainedRows) -> Option { + let Ok(request) = crate::engine::_helpers::retained::retained_request(retained) else { + return None; + }; + Some(request) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index f9482f9c95..d68e0cc521 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use serde_json::{Value, json}; @@ -8,9 +9,10 @@ use std::sync::atomic::AtomicBool; use crate::engine::main::answer_speculative_rows::answer_speculative_rows; use crate::engine::main::discard_speculative_rows::discard_speculative_rows; +use crate::engine::main::retained_request::retained_request; use crate::models::{ - EvaluateRequest, Fault, RetainedRows, RowsEvaluation, RowsRequest, SpeculativeAnswer, - SpeculativeRows, + Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Fault, Model, + Reference, RetainedRows, RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows, }; use crate::engine::_helpers::evaluation::evaluate_json; @@ -642,11 +644,94 @@ pub(crate) fn query_facts_faults( /// The retained request with each row's digest restored, as a delta compile rebuilds it. pub(crate) fn restored_rows(retained: &RetainedRows) -> Result { - let mut request: EvaluateRequest = serde_json::from_slice(&retained.request_json) - .map_err(|error| format!("the retained request does not decode: {error}"))?; + let mut request: EvaluateRequest = + retained_request(retained).ok_or("the retained request does not decode")?; for (model, digest) in request.models.iter_mut().zip(&retained.digests) { model.payload_digest = Some(digest.clone()); model.stripped = true; } Ok(request) } + +/// A model with a value in every field the retained rows keep, and SQL text they drop. +pub(crate) fn every_field_model() -> Model { + let declaration: Declaration = Declaration { + name: "status".to_owned(), + relative_path: "enums/status.sql".to_owned(), + members: vec![DeclarationMember { + name: "open".to_owned(), + value: json!({"code": 1, "ratio": -0.25, "tags": ["a", null, true]}), + }], + value: Some(json!(12345678901234567890_u64)), + value_type: Some("BIGINT".to_owned()), + render_as: None, + }; + Model { + sql_analysis_disabled: true, + name: "orders".to_owned(), + relative_path: "models/orders.sql".to_owned(), + query_sql: "SELECT 1".to_owned(), + authored_sql: "SELECT 1 -- authored".to_owned(), + config: BTreeMap::from([ + ("materialized".to_owned(), json!("table")), + ("tags".to_owned(), json!(["finance", "daily"])), + ]), + authored_config_keys: vec!["materialized".to_owned()], + logical_schema: Some("analytics".to_owned()), + references: vec![Reference { + ref_kind: "source".to_owned(), + ref_name: "raw_orders".to_owned(), + ref_package: Some("shop".to_owned()), + }], + columns: vec![Column { + name: "amount ✓".to_owned(), + data_type: "DOUBLE".to_owned(), + nullable: Some(false), + audit_count: 2, + type_proven: true, + }], + dynamic_columns: vec![DynamicColumnFamily { + name: "amounts".to_owned(), + pivot_column: "region".to_owned(), + value_column: "amount".to_owned(), + aggregate: "sum".to_owned(), + data_type: "DOUBLE".to_owned(), + name_pattern: Some("amount_{region}".to_owned()), + type_proven: false, + }], + dynamic_columns_proven: true, + bare_dynamic_pivot: true, + enum_columns: vec!["status".to_owned()], + enum_declarations: vec![declaration.clone()], + constant_declarations: vec![declaration], + declared_audit_count: 3, + targeting_test_count: 4, + empty_input_only_test_count: 0, + payload_digest: Some("digest".to_owned()), + stripped: false, + } +} + +/// Two models: one with every field set, one with none. +pub(crate) fn coded_models() -> Vec { + vec![every_field_model(), Model::default()] +} + +/// No models. +pub(crate) fn no_models() -> Vec { + Vec::new() +} + +/// `models` as retained rows read back would hold them: SQL text and digests dropped. +pub(crate) fn without_sql(models: Vec) -> Value { + let stripped: Vec = models + .into_iter() + .map(|model| Model { + query_sql: String::new(), + authored_sql: String::new(), + payload_digest: None, + ..model + }) + .collect(); + serde_json::to_value(stripped).unwrap_or_default() +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs index 05e627214e..eb8f0503f8 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs @@ -1,13 +1,16 @@ +use serde_json::Value; use tempfile::TempDir; use crate::engine::_helpers::evaluation::STRIPPED_ROW_MISSED; +use crate::engine::_helpers::retained::{encoded_rows, retained_request}; use crate::engine::main::evaluate_rows::evaluate_rows; +use crate::engine::tests::helpers::{coded_models, no_models, without_sql}; use crate::engine::tests::helpers::{ evaluation_summary, orders_models, restored_rows, speculative_rows_request, }; -use crate::engine::tests::test_types::RetainedRowsTestCase; +use crate::engine::tests::test_types::{RetainedRowsTestCase, RowCodecTestCase}; use crate::errors::RowsError; -use crate::models::{EvaluateRequest, Fault, RetainedRows, RowsEvaluation, RowsRequest}; +use crate::models::{EvaluateRequest, Fault, Model, RetainedRows, RowsEvaluation, RowsRequest}; #[test] fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_stand_in() @@ -34,8 +37,7 @@ fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_sta request.retain = true; let first = evaluate_rows(request).map_err(|_| "first evaluation failed".to_owned())?; let retained: RetainedRows = first.retained.clone().ok_or("nothing was retained")?; - let kept: EvaluateRequest = serde_json::from_slice(&retained.request_json) - .map_err(|error| format!("the retained request does not decode: {error}"))?; + let kept: EvaluateRequest = restored_rows(&retained)?; let other_dir = TempDir::new().map_err(|error| error.to_string())?; let mut stripped: EvaluateRequest = restored_rows(&retained)?; stripped.project_dir = [project_dir.path(), other_dir.path()] @@ -84,3 +86,52 @@ fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_sta } Ok(()) } + +#[test] +fn given_model_rows_when_encoded_and_decoded_then_every_fact_but_the_sql_survives() { + let test_cases = [ + RowCodecTestCase { + description: "a model with every field set and a default model", + models: coded_models, + expected_truncations_read: 0, + }, + RowCodecTestCase { + description: "no models", + models: no_models, + expected_truncations_read: 0, + }, + ]; + for test_case in test_cases { + let models: Vec = (test_case.models)(); + let retained: RetainedRows = RetainedRows { + request_json: serde_json::to_vec(&EvaluateRequest::default()).unwrap_or_default(), + rows: encoded_rows(&models), + digests: vec![String::new(); models.len()], + }; + + let decoded: Option = retained_request(&retained) + .ok() + .and_then(|request| serde_json::to_value(request.models).ok()); + let truncations_read: usize = (0..retained.rows.len()) + .filter(|length| { + retained_request(&RetainedRows { + rows: retained.rows[..*length].to_vec(), + ..retained.clone() + }) + .is_ok() + }) + .count(); + + assert_eq!( + decoded, + Some(without_sql(models)), + "{}", + test_case.description + ); + assert_eq!( + truncations_read, test_case.expected_truncations_read, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 7753d9c716..c22ae94525 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -290,3 +290,11 @@ pub(crate) struct RetainedRowsTestCase { pub(crate) expected_same_findings: bool, pub(crate) expected_missed: Option<&'static str>, } + +/// Model rows written in the native record format and read back. +pub(crate) struct RowCodecTestCase { + pub(crate) description: &'static str, + pub(crate) models: fn() -> Vec, + /// How many truncations of the rows still read back. + pub(crate) expected_truncations_read: usize, +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 9e0ab232b9..744677e574 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -933,10 +933,12 @@ pub struct RowsRequest { pub retain: bool, } -/// A request kept for the project image as `EvaluateRequest` JSON without its models' SQL text. +/// A request kept for the project image: its model-free JSON and its rows without SQL text. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct RetainedRows { pub request_json: Vec, + /// Each model's row without its SQL text, in the native record format, in model order. + pub rows: Vec, /// Each model's payload digest, which keys its cached findings, in model order. pub digests: Vec, } From 3ce44f48d40ea8aa260486de1e97d8a1b0c4a178 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:39:11 +0100 Subject: [PATCH 25/32] fix(compile): make delta compiles print exactly what full compiles print Delta compiles diverged from full compiles in three ways: - Partial semantic-check reasons came only from the rerun models, so the text report dropped notices for unchanged models (models with SQL analysis off, dependents of an invalid model), and JSON reports with partial models always fell back. Each model's coverage facts are now kept in the project image (format v8). A delta merges the stored facts with the rerun models' fresh facts and recomputes the invalid-dependency reasons over the whole project. Text and JSON both render from that merged set, and the JSON splice takes the delta's section. - A SQL test edited to expect or mock another model was compiled without that model, which gave a false missing-model error. An edited test now reruns its stored chain, the chain's direct upstream models and every model its text names. The delta falls back when the test's chain resolves differently, or when a round reports a project model it did not compile as missing. - A fallback after a delta round printed the lifecycle lines twice. The progress projector now holds a delta's lines back. It writes only the longest round's lines, and only once the delta finishes; delta stderr notes print after them. The edit series gains SQL-analysis, invalid-dependency and SQL-test retarget edits, a dialect without type checks, and a mode that compares each compile's stdout and stderr with a full compile of a copy of the project. --- .../src/bindings/_helpers/cli/progress.rs | 43 ++++- .../_helpers/compile_command/report.rs | 3 + .../_helpers/compile_command/reuse_attempt.rs | 24 ++- .../compile_command/reuse_recording.rs | 6 +- .../_helpers/compile_delta/context.rs | 7 + .../_helpers/compile_delta/coverage.rs | 124 ++++++++++++++ .../bindings/_helpers/compile_delta/driver.rs | 155 +++++++++++++++++- .../bindings/_helpers/compile_delta/image.rs | 13 +- .../bindings/_helpers/compile_delta/mod.rs | 1 + .../_helpers/compile_delta/report_hooks.rs | 38 ++++- .../bindings/_helpers/compile_delta/splice.rs | 4 +- .../_helpers/semantic_shapes/coverage.rs | 148 ++++++++--------- crates/sqlbuild-python/src/bindings/tests.rs | 2 + .../src/bindings/tests/helpers.rs | 30 ++++ .../src/bindings/tests/test_coverage.rs | 115 +++++++++++++ .../src/bindings/tests/test_splice.rs | 11 ++ .../src/bindings/tests/test_types.rs | 18 ++ .../delta_compile_series/_helpers/outcomes.py | 54 +++++- scripts/delta_compile_series/constants.py | 18 ++ .../main/run_delta_compile_series.py | 54 +++++- .../main/write_delta_compile_project.py | 12 +- .../cli/commands/main/compile/_test_types.py | 12 ++ .../cli/commands/main/compile/helpers.py | 14 +- .../main/compile/test_compile_delta.py | 155 +++++++++++++++++- 24 files changed, 954 insertions(+), 107 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_delta/coverage.rs create mode 100644 crates/sqlbuild-python/src/bindings/tests/test_coverage.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index 5878950bff..9e4501a2cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -1,6 +1,6 @@ //! The invocation-scoped native CLI projection of canonical lifecycle facts. -use std::sync::{Mutex, MutexGuard}; +use std::sync::{Mutex, MutexGuard, PoisonError}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -43,6 +43,8 @@ pub(crate) struct NativeProgressProjector { state: Mutex, stream: Py, coordinator: Py, + /// The lines held back instead of written, while something holds them. + held: Mutex>>, } fn payload_value<'py>(payload: &Bound<'py, PyAny>, key: &str) -> PyResult> { @@ -169,6 +171,37 @@ pub(crate) fn terminal_stream(stream: &Bound<'_, PyAny>) -> PyResult { } impl NativeProgressProjector { + fn held(&self) -> MutexGuard<'_, Option>> { + self.held.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Hold back every line from now on instead of writing it. + pub(crate) fn hold_lines(&self) { + *self.held() = Some(Vec::new()); + } + + /// Replace the lines held back so far with `lines`, returning them; nothing while not holding. + pub(crate) fn exchange_held_lines(&self, lines: Vec) -> Vec { + match self.held().as_mut() { + Some(held) => std::mem::replace(held, lines), + None => Vec::new(), + } + } + + /// Write lines as they come again, returning those held back since `hold_lines`. + pub(crate) fn release_lines(&self) -> Vec { + self.held().take().unwrap_or_default() + } + + /// Write lines held back earlier, as they would have been written. + pub(crate) fn write_held(&self, py: Python<'_>, lines: &[String]) -> PyResult<()> { + let coordinator: &Bound<'_, PyAny> = self.coordinator.bind(py); + for line in lines { + persist(coordinator, self.stream.bind(py), line)?; + } + Ok(()) + } + fn locked(&self) -> PyResult> { self.state .lock() @@ -192,7 +225,12 @@ impl NativeProgressProjector { (lines, value) }; for line in lines { - persist(coordinator, self.stream.bind(py), &format!("{line}\n"))?; + let text: String = format!("{line}\n"); + if let Some(held) = self.held().as_mut() { + held.push(text); + continue; + } + persist(coordinator, self.stream.bind(py), &text)?; } Ok(value) })(); @@ -236,6 +274,7 @@ impl NativeProgressProjector { })), stream: stream.unbind(), coordinator: coordinator.unbind(), + held: Mutex::new(None), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index b950805395..d06d17d092 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -163,6 +163,9 @@ impl<'py> ReportInputs<'py> { return Ok(PyDict::new(py)); } let reasons: Bound<'py, PyDict> = semantic_coverage(&self.project)?; + if crate::bindings::_helpers::compile_delta::context::injection().is_some() { + return Ok(reasons); + } let Some(names) = self.selected_names(MODEL_TYPE)? else { return Ok(reasons); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index fc2242caca..a4b5351a56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -14,6 +14,7 @@ use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FORMAT}; use crate::bindings::_helpers::cli::launch_state::launch_record_path; +use crate::bindings::_helpers::cli::progress::{NativeProgressProjector, current_projector}; use crate::bindings::_helpers::compile_command::constants::{ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, COMPILE_COMMAND_NAME, COMPILED_DIRECTORY_NAME, COMPUTE_LOG_DIRECTORY_NAME, EXCLUDED_DIRECTORIES, @@ -513,10 +514,25 @@ fn compiled_with_delta<'py>( let py: Python<'py> = request.py(); let verifying: bool = env_enabled(DELTA_VERIFY_ENV_VAR); let started: f64 = monotonic(py)?; - let delta: DeltaOutcome<'py> = match driver::run(request, attempt) { + let projector: Option> = current_projector(py)?; + if let Some(projector) = &projector { + projector.get().hold_lines(); + } + let ran: PyResult> = driver::run(request, attempt); + let held: Vec = projector + .as_ref() + .map(|projector| projector.get().release_lines()) + .unwrap_or_default(); + let delta: DeltaOutcome<'py> = match ran { Ok(delta) => delta, Err(error) => DeltaOutcome::Fallback(format!("the delta compile failed: {error}")), }; + let write_held = || -> PyResult<()> { + match &projector { + Some(projector) => projector.get().write_held(py, &held), + None => Ok(()), + } + }; context::reset(); let project_dir: Bound<'py, PyAny> = native_field(attempt, "project_dir")?; let cache: Bound<'py, PyAny> = cache_directory(&project_dir)?; @@ -528,6 +544,7 @@ fn compiled_with_delta<'py>( )?; match delta { DeltaOutcome::Finished(finished) if !verifying => { + write_held()?; let (files, models) = (finished.changed_files, finished.rerun_models); let committing: std::time::Instant = std::time::Instant::now(); let code: i64 = driver::commit(*finished, (request, attempt))?; @@ -542,7 +559,10 @@ fn compiled_with_delta<'py>( )?; return Ok(code); } - DeltaOutcome::Raised(error) if !verifying => return Err(error), + DeltaOutcome::Raised(error) if !verifying => { + write_held()?; + return Err(error); + } DeltaOutcome::Fallback(reason) => { reuse_debug(py, &format!("Delta compile fell back: {reason}"))?; if verifying { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 67885bd8b8..7b1471283a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -297,9 +297,11 @@ pub(crate) fn print_to<'py>( #[pymethods] impl RecordedCompileOutput { - /// Print one stderr result line, such as a note, and record it. + /// Print one stderr result line, such as a note, and record it; a delta only records it. pub(crate) fn print_stderr_line(&self, line: Bound<'_, PyString>) -> PyResult<()> { - print_to(line.py(), "stderr", &line)?; + if crate::bindings::_helpers::compile_delta::context::injection().is_none() { + print_to(line.py(), "stderr", &line)?; + } self.locked().0.push(line.unbind()); Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index d0db9f18af..b860cbe7a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -7,6 +7,7 @@ use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, Sess use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_rules::models::RetainedRows; +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; use crate::bindings::_helpers::compile_delta::image::PendingRules; /// The facts a full compile leaves for its project image beyond its analysis record. @@ -47,6 +48,8 @@ pub(crate) struct ImageCapture { pub(crate) lint_file_keys: Option)>>, /// The closed shape each model published for semantic checks, in publication order. pub(crate) model_shapes: Option>, + /// The notable semantic coverage facts of every model, when semantic checks reported them. + pub(crate) coverage: Option>, /// Why this compile cannot be the base of a delta compile, when something ruled it out. pub(crate) unsupported: Option, } @@ -123,6 +126,10 @@ pub(crate) struct DeltaInjection { pub(crate) model_shapes: Vec, /// The whole project's text report model lines, with the rerun models' lines to replace. pub(crate) text_models: Vec<(String, bool, i64)>, + /// The whole project's notable semantic coverage facts, as the image recorded them. + pub(crate) coverage: Option>, + /// Every project model in project order, with the models it reads. + pub(crate) coverage_order: Vec<(String, Vec)>, } static CAPTURE: Mutex> = Mutex::new(None); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/coverage.rs new file mode 100644 index 0000000000..0b40ab3fd1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/coverage.rs @@ -0,0 +1,124 @@ +//! Partial semantic-check reasons from per-model facts, shared by full and delta compiles. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +/// The reason a model that opts out of SQL analysis is partial. +pub(crate) const ANALYSIS_DISABLED: &str = "SQL analysis disabled"; +const INVALID_DEPENDENCY: &str = "depends on invalid model"; + +/// What one model contributes to the partial-check reasons, apart from its dependencies. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct ModelCoverage { + pub(crate) name: String, + /// The model opts out of SQL analysis. + pub(crate) disabled: bool, + /// The model has a binding error, which makes every model reading it partial. + pub(crate) blocked: bool, + /// Its own reasons, sorted; those from invalid dependencies are derived. + pub(crate) partial: Vec, +} + +impl ModelCoverage { + /// Whether the model adds anything to the reasons. + pub(crate) fn notable(&self) -> bool { + self.disabled || self.blocked || !self.partial.is_empty() + } +} + +/// Each model's partial-check reasons, in `order`, which pairs each model with its upstream. +pub(crate) fn coverage_reasons( + order: &[(String, Vec)], + facts: &[ModelCoverage], +) -> Vec<(String, Vec)> { + let facts: HashMap<&str, &ModelCoverage> = facts + .iter() + .map(|model| (model.name.as_str(), model)) + .collect(); + let mut blocked_by: HashMap<&str, HashSet<&str>> = facts + .values() + .filter(|model| model.blocked) + .map(|model| (model.name.as_str(), HashSet::from([model.name.as_str()]))) + .collect(); + if !blocked_by.is_empty() { + for _ in order { + let mut changed: bool = false; + for (name, upstream) in order { + let name: &str = name.as_str(); + let mut causes: HashSet<&str> = blocked_by.get(name).cloned().unwrap_or_default(); + for upstream in upstream { + causes.extend( + blocked_by + .get(upstream.as_str()) + .cloned() + .unwrap_or_default(), + ); + } + if !causes.is_empty() && blocked_by.get(name) != Some(&causes) { + blocked_by.insert(name, causes); + changed = true; + } + } + if !changed { + break; + } + } + } + let mut reasons: Vec<(String, Vec)> = Vec::new(); + for (name, _) in order { + let name: &str = name.as_str(); + let model: Option<&&ModelCoverage> = facts.get(name); + if model.is_some_and(|model| model.disabled) { + reasons.push((name.to_owned(), vec![ANALYSIS_DISABLED.to_owned()])); + continue; + } + let mut partial: BTreeSet = model + .map(|model| model.partial.iter().cloned().collect()) + .unwrap_or_default(); + if let Some(causes) = blocked_by.get(name) { + for cause in causes.iter().filter(|cause| **cause != name) { + partial.insert(format!("{INVALID_DEPENDENCY} {cause}")); + } + } + if !partial.is_empty() { + reasons.push((name.to_owned(), partial.into_iter().collect())); + } + } + reasons +} + +/// The whole project's notable facts: `stored` with the fresh models' replaced by `fresh`. +pub(crate) fn merged_coverage( + stored: &[ModelCoverage], + fresh: &[ModelCoverage], +) -> Vec { + let rerun: HashSet<&str> = fresh.iter().map(|model| model.name.as_str()).collect(); + stored + .iter() + .filter(|model| !rerun.contains(model.name.as_str())) + .chain(fresh.iter().filter(|model| model.notable())) + .cloned() + .collect() +} + +/// Every project model with its upstream names, the fresh models' replacing the stored. +pub(crate) fn merged_order( + stored: &[(String, Vec)], + fresh: &[(String, Vec)], +) -> Vec<(String, Vec)> { + let fresh: HashMap<&str, &Vec> = fresh + .iter() + .map(|(name, upstream)| (name.as_str(), upstream)) + .collect(); + stored + .iter() + .map(|(name, upstream)| { + ( + name.clone(), + fresh + .get(name.as_str()) + .map_or(upstream, |fresh| *fresh) + .clone(), + ) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index d35267019f..014c93c3f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyStringMethods; use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; use sqlbuild_cache::project_reuse::main::read_delta_base::read_delta_base; @@ -11,8 +11,11 @@ use sqlbuild_cache::project_reuse::main::read_delta_output::read_delta_output; use sqlbuild_cache::project_reuse::models::{DeltaBase, DeltaCandidate, ProjectFileChanges}; use sqlbuild_cache::project_reuse::types::ImageSections; +use crate::bindings::_helpers::cli::progress::{NativeProgressProjector, current_projector}; use crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug; -use crate::bindings::_helpers::compile_command::reuse_recording::RecordedCompileOutput; +use crate::bindings::_helpers::compile_command::reuse_recording::{ + RecordedCompileOutput, print_to, +}; use crate::bindings::_helpers::compile_delta::context::{ CapturedDiagnostic, CapturedModel, CapturedTest, DeltaInjection, ImageCapture, RecordMerge, start_capture, take_capture, take_delta_unsupported, with_injection, @@ -49,8 +52,17 @@ const RULE_PHASE: &str = "\"phase\": \"rule\""; const UNKNOWN_SOURCE: &str = "unknown source"; /// The upstream kind of a source a model reads. const SOURCE_KIND: &str = "source"; +/// The upstream kind of a model a model reads. +const MODEL_KIND: &str = "model"; const TESTS_ROOT: &str = "tests/"; +const STDERR: &str = "stderr"; const SQL_SUFFIX: &str = ".sql"; +/// How generated SQL test CTE names mark their prefix, as in `__ref__orders`. +const CTE_PREFIX_MARK: &str = "__"; +const QUOTE: char = '\''; +/// What errors for a reference to a model the project lacks say. +const MISSING_MODEL_PHRASES: [&str; 2] = ["unknown model", "which does not exist"]; +const UNCOMPILED_MODEL: &str = "the delta compile reported a model it did not compile as missing"; /// How many changed, added and removed paths the candidates debug line names. const CANDIDATES_LOGGED: usize = 5; const UNREADABLE_IMAGE: &str = "the project image cannot be read"; @@ -272,6 +284,12 @@ fn lent_injection( model_subjects: image.model_subjects.take(), text_models: image.text_models.clone().unwrap_or_default(), model_shapes: std::mem::take(&mut image.model_shapes), + coverage: image.coverage.take(), + coverage_order: image + .models + .iter() + .map(|model| (model.name.clone(), upstream_models(&model.upstream))) + .collect(), lint_model_keys: image.lint_model_keys.take(), lint_file_keys: image.lint_file_keys.take(), ..DeltaInjection::default() @@ -285,6 +303,7 @@ fn returned(mut image: ProjectImage, injection: DeltaInjection) -> ProjectImage image.fact_digests = injection.fact_digests.into_iter().collect(); image.model_subjects = injection.model_subjects; image.model_shapes = injection.model_shapes; + image.coverage = injection.coverage; image.lint_model_keys = injection.lint_model_keys; image.lint_file_keys = injection.lint_file_keys; image @@ -354,13 +373,29 @@ enum RoundOutcome<'py> { Stop(DeltaOutcome<'py>), } +/// Compile the rerun set once, keeping the longest round's lines: every phase a full compile runs. fn round<'py>( request: &Bound<'py, PyAny>, attempt: &Bound<'py, PyAny>, (injection, rerun): (DeltaInjection, &Rerun), ) -> PyResult<(RoundOutcome<'py>, Option)> { start_capture(); + let projector: Option> = current_projector(request.py())?; + let earlier: Vec = projector + .as_ref() + .map(|projector| projector.get().exchange_held_lines(Vec::new())) + .unwrap_or_default(); let (ran, injection) = with_injection(injection, || unrecorded_compile(request, attempt)); + if let Some(projector) = &projector { + let latest: Vec = projector.get().exchange_held_lines(Vec::new()); + let _ = projector + .get() + .exchange_held_lines(if latest.len() < earlier.len() { + earlier + } else { + latest + }); + } Ok((ran_round(ran, rerun)?, injection)) } @@ -587,10 +622,29 @@ fn rounds<'py>( let injection_for_round: DeltaInjection = for_rerun(injection, &rerun, rerun.sources.clone()); let (outcome, back) = round(request, attempt, (injection_for_round, &rerun))?; + let model_names: &[String] = back + .as_ref() + .map_or(&[], |injection| injection.model_names.as_slice()); let ran: Round<'py> = match outcome { RoundOutcome::Ran(ran) => *ran, + RoundOutcome::Stop(DeltaOutcome::Raised(error)) + if names_uncompiled_model(&error.to_string(), model_names, &rerun.models) => + { + return stopped(UNCOMPILED_MODEL); + } RoundOutcome::Stop(outcome) => return Ok(Err(outcome)), }; + if ran + .capture + .diagnostics + .iter() + .any(|diagnostic| names_uncompiled_model(&diagnostic.json, model_names, &rerun.models)) + { + return stopped(UNCOMPILED_MODEL); + } + if let Some(path) = rechained_test(&ran.capture.test_chains, &image.tests, &changed_tests) { + return stopped(format!("{path} now covers other models")); + } let Some(back) = back else { return stopped("the delta compile's injection is still in use"); }; @@ -768,21 +822,108 @@ fn first_rerun( .iter() .map(|(path, name)| (*name, *path)) .collect(); + let upstream_by_name: HashMap<&str, &[(String, String)]> = gate + .models + .iter() + .map(|model| (model.name.as_str(), model.upstream.as_slice())) + .collect(); for test in &gate.tests { if !changed_tests.contains(&test.relative_path) { continue; } - for name in &test.chain { + let ring = test.chain.iter().flat_map(|name| { + upstream_models( + upstream_by_name + .get(name.as_str()) + .copied() + .unwrap_or_default(), + ) + }); + let named: BTreeSet = + test_model_references(project_dir, &test.relative_path, &path_by_model)?; + for name in test.chain.iter().cloned().chain(ring).chain(named) { let Some(path) = path_by_model.get(name.as_str()) else { return Err(format!("{} tests a model the image lacks", test.key)); }; - rerun.models.insert(name.clone()); rerun.model_paths.insert((*path).to_owned()); + rerun.models.insert(name); } } Ok((rerun, changed_tests)) } +/// The first edited SQL test whose model chain the delta resolved differently than stored. +fn rechained_test<'a>( + fresh: &[CapturedTest], + stored: &'a [CapturedTest], + changed_tests: &BTreeSet, +) -> Option<&'a str> { + let sorted = |chain: &[String]| -> BTreeSet { chain.iter().cloned().collect() }; + let fresh: HashMap<&str, BTreeSet> = fresh + .iter() + .map(|test| (test.key.as_str(), sorted(&test.chain))) + .collect(); + stored + .iter() + .filter(|test| changed_tests.contains(&test.relative_path)) + .find(|test| fresh.get(test.key.as_str()) != Some(&sorted(&test.chain))) + .map(|test| test.relative_path.as_str()) +} + +/// Every project model the SQL test file at `path` names, a harmless superset of its references. +fn test_model_references( + project_dir: &std::path::Path, + path: &str, + path_by_model: &HashMap<&str, &str>, +) -> Result, String> { + let Ok(text) = std::fs::read_to_string(project_dir.join(path)) else { + return Err(format!("{path} cannot be read")); + }; + let mut names: BTreeSet = BTreeSet::new(); + for word in text.split(|character: char| !(character.is_alphanumeric() || character == '_')) { + let unprefixed: &str = word + .strip_prefix(CTE_PREFIX_MARK) + .and_then(|rest| rest.split_once(CTE_PREFIX_MARK)) + .map_or(word, |(_, name)| name); + for candidate in [word, unprefixed] { + if path_by_model.contains_key(candidate) { + names.insert(candidate.to_owned()); + } + } + } + Ok(names) +} + +/// Whether `message` reports as missing a project model the delta did not compile. +pub(crate) fn names_uncompiled_model( + message: &str, + model_names: &[String], + rerun_models: &BTreeSet, +) -> bool { + MISSING_MODEL_PHRASES + .iter() + .any(|phrase| message.contains(phrase)) + && message + .split(QUOTE) + .skip(1) + .step_by(2) + .filter(|name| !rerun_models.contains(*name)) + .any(|name| project_model(model_names, name)) +} + +fn project_model(model_names: &[String], name: &str) -> bool { + model_names.iter().any(|model| model == name) +} + +/// The names of the models among `upstream`. +fn upstream_models(upstream: &[(String, String)]) -> Vec { + upstream + .iter() + .filter(|(kind, _)| kind == MODEL_KIND) + .map(|(_, name)| name.clone()) + .collect() +} + /// Whether only a full compile would repeat a stored non-rule diagnostic outside the rerun. fn kept_only_by_full_compiles(rerun: &Rerun, diagnostic: &CapturedDiagnostic) -> bool { !rerun.owns(diagnostic) @@ -981,6 +1122,9 @@ fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> Pr if let Some(shapes) = capture.model_shapes { image.model_shapes = shapes; } + if capture.coverage.is_some() { + image.coverage = capture.coverage; + } image.header_keys.extend(capture.header_keys); let mut pinned: BTreeSet = std::mem::take(&mut image.pinned_sources) .into_iter() @@ -1016,6 +1160,9 @@ pub(crate) fn commit( RecordedCompileOutput::recorded(py, &stderr_lines, &format!("{report}\n")), )? .into_any(); + for line in &stderr_lines { + print_to(py, STDERR, line.as_str())?; + } crate::bindings::_helpers::compile_delta::context::set_pending_image(image); crate::bindings::_helpers::compile_delta::context::set_record_merge(Some(merge)); crate::bindings::_helpers::compile_driver::command::store_and_print( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index d3e4e5f49c..dd32ec6f5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -11,11 +11,12 @@ use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::compile_delta::context::{ CapturedDiagnostic, CapturedModel, CapturedTest, ImageCapture, ModelShape, }; +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; /// Changes whenever the image's meaning changes, so an older image is never read. -pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v7"; +pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v8"; pub(crate) const GATE_SECTION: &str = "gate"; /// The upstream kind of a model another model reads. const MODEL_KIND: &str = "model"; @@ -41,6 +42,7 @@ pub(crate) struct ProjectImage { pub(crate) pinned_sources: Vec, pub(crate) test_sources: BTreeMap>, pub(crate) model_shapes: Vec, + pub(crate) coverage: Option>, pub(crate) lint_model_keys: Option>, pub(crate) lint_file_keys: Option)>>, pub(crate) header_keys: BTreeMap, @@ -238,6 +240,7 @@ struct StoredFacts { pinned_sources: Vec, test_sources: BTreeMap>, model_shapes: StoredShapes, + coverage: Option>, lint_model_keys: Option>, lint_file_keys: Option)>>, } @@ -254,6 +257,7 @@ type StoredMisc = ( Vec, Option>, Vec, + Option>, ); fn put_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a Shapes) { @@ -366,6 +370,7 @@ impl StoredFacts { &self.diagnostics, &self.text_models, &self.environment_names, + &self.coverage, )) .map_err(|error| error.to_string())?; let mut encoder: Encoder<'_> = Encoder::default(); @@ -453,7 +458,7 @@ impl StoredFacts { .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? .into_iter() .collect(); - let (diagnostics, text_models, environment_names): StoredMisc = + let (diagnostics, text_models, environment_names, coverage): StoredMisc = from_json(decoder.bytes()?)?; decoder.finish()?; Some(Self { @@ -472,6 +477,7 @@ impl StoredFacts { pinned_sources, test_sources, model_shapes, + coverage, lint_model_keys, lint_file_keys, }) @@ -531,6 +537,7 @@ impl ProjectImage { .map(|(path, names)| (path, names.into_iter().collect())) .collect(), model_shapes, + coverage: capture.coverage, lint_model_keys: capture.lint_model_keys, lint_file_keys: capture.lint_file_keys, header_keys: capture.header_keys, @@ -568,6 +575,7 @@ impl ProjectImage { diagnostics: self.diagnostics, text_models: self.text_models, environment_names: self.environment_names, + coverage: self.coverage, pinned_sources: self.pinned_sources, test_sources: self.test_sources, lint_model_keys: self.lint_model_keys, @@ -643,6 +651,7 @@ impl ProjectImage { pinned_sources: facts.pinned_sources, test_sources: facts.test_sources, model_shapes, + coverage: facts.coverage, lint_model_keys: facts.lint_model_keys, lint_file_keys: facts.lint_file_keys, header_keys: gate.header_keys, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs index 1e2e6a90b7..50b7b9c359 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod context; +pub(crate) mod coverage; pub(crate) mod driver; pub(crate) mod header_keys; pub(crate) mod image; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs index a0821ea13c..8ea8654232 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -9,7 +9,10 @@ use sqlbuild_core::cli_text::models::ModelLine; use crate::bindings::_helpers::compile_command::report::diagnostic_item; use crate::bindings::_helpers::compile_delta::context::{ CapturedDiagnostic, CapturedModel, CapturedTest, ModelShape, capture, capture_unsupported, - capturing, injection, + capturing, delta_unsupported, injection, +}; +use crate::bindings::_helpers::compile_delta::coverage::{ + ModelCoverage, coverage_reasons, merged_coverage, merged_order, }; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; @@ -150,6 +153,39 @@ pub(crate) fn captured_diagnostic(diagnostic: &Bound<'_, PyAny>) -> PyResult)>, + fresh: Vec, +) -> PyResult)>> { + let (order, facts): (Vec<(String, Vec)>, Vec) = match injection() { + None => ( + order, + fresh.into_iter().filter(ModelCoverage::notable).collect(), + ), + Some(injection) => { + let Some(stored) = &injection.coverage else { + return Err(delta_unsupported( + "the stored compile recorded no semantic coverage", + )); + }; + ( + merged_order(&injection.coverage_order, &order), + merged_coverage(stored, &fresh), + ) + } + }; + let reasons: Vec<(String, Vec)> = coverage_reasons(&order, &facts); + if capturing() { + capture(|capture| { + if capture.coverage.is_none() { + capture.coverage = Some(facts); + } + }); + } + Ok(reasons) +} + /// In a delta compile, publish the shapes of every model it does not compile again first. pub(crate) fn inject_model_shapes( models: &[Bound<'_, PyAny>], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs index da0c310a07..bd71d20971 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs @@ -211,7 +211,9 @@ pub(crate) fn spliced_report( let mut cursor: usize = 0; for ((key, base_span), (_, delta_span)) in base_members.iter().zip(&delta_members) { let replacement: String = match key.as_str() { - "has_errors" | "diagnostics" | "compile_timings" => text(delta, delta_span).to_owned(), + "has_errors" | "diagnostics" | "semantic_checks_partial" | "compile_timings" => { + text(delta, delta_span).to_owned() + } "summary" => spliced_summary(base, *base_span, delta, *delta_span) .ok_or("the summaries do not line up")?, "resources" => spliced_resources((base, *base_span), (delta, *delta_span), scope) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs index d756c0ffa4..d3dc4ed58f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs @@ -1,12 +1,14 @@ //! Per-model reasons a semantic check was only partial, for machine reports. -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::BTreeSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::analysis::model_reads::{inferred_rows, star_flags}; +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; +use crate::bindings::_helpers::compile_delta::report_hooks::semantic_reasons; use crate::bindings::_helpers::compiled_project::compiled_model_view::InferredRow; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; @@ -24,97 +26,89 @@ pub(crate) fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult = project_shapes(project, None)?; - let models: Vec> = native_field(project, "models")? - .try_iter()? - .collect::>()?; - let mut names: Vec = Vec::with_capacity(models.len()); - let mut references: Vec> = Vec::with_capacity(models.len()); - let mut blocked_by: HashMap> = HashMap::new(); - for model in &models { - let name: String = native_field(model, "name")?.extract()?; + let dialect: Bound<'py, PyAny> = native_field(project, "sql_analysis_dialect")?; + let dialect: String = if dialect.is_truthy()? { + dialect.extract()? + } else { + "generic".to_owned() + }; + let mut order: Vec<(String, Vec)> = Vec::new(); + let mut facts: Vec = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let name: String = native_field(&model, "name")?.extract()?; let mut rows: Vec<(String, String)> = Vec::new(); - for row in native_field(model, "reference_rows")?.try_iter()? { + for row in native_field(&model, "reference_rows")?.try_iter()? { let row: Bound<'py, PyAny> = row?; rows.push(( row.get_item(0)?.str()?.extract()?, row.get_item(1)?.extract()?, )); } - for item in native_field(model, "binding_diagnostics")?.try_iter()? { + let mut blocked: bool = false; + for item in native_field(&model, "binding_diagnostics")?.try_iter()? { if native_field(&item?, "is_error")?.is_truthy()? { - blocked_by.insert(name.clone(), HashSet::from([name.clone()])); + blocked = true; break; } } - names.push(name); - references.push(rows); + let opted: Bound<'py, PyAny> = native_field(&native_field(&model, "config")?, "values")? + .call_method1("get", ("sql_analysis",))?; + let disabled: bool = opted.is(PyBool::new(py, false)); + let partial: BTreeSet = if disabled { + BTreeSet::new() + } else { + own_reasons((&model, &name), &rows, (&shapes, &dialect))? + }; + facts.push(ModelCoverage { + name: name.clone(), + disabled, + blocked, + partial: partial.into_iter().collect(), + }); + order.push(( + name, + rows.into_iter().map(|(_, upstream)| upstream).collect(), + )); } - if !blocked_by.is_empty() { - for _ in &names { - let mut changed: bool = false; - for (name, rows) in names.iter().zip(&references) { - let mut causes: HashSet = blocked_by.get(name).cloned().unwrap_or_default(); - for (_, upstream) in rows { - causes.extend(blocked_by.get(upstream).cloned().unwrap_or_default()); - } - if !causes.is_empty() && blocked_by.get(name) != Some(&causes) { - blocked_by.insert(name.clone(), causes); - changed = true; - } - } - if !changed { - break; - } - } + for (name, partial) in semantic_reasons(order, facts)? { + reasons.set_item(name, PyTuple::new(py, partial)?)?; } - let dialect: Bound<'py, PyAny> = native_field(project, "sql_analysis_dialect")?; - let dialect: String = if dialect.is_truthy()? { - dialect.extract()? - } else { - "generic".to_owned() - }; - for ((model, name), rows) in models.iter().zip(&names).zip(&references) { - let opted: Bound<'py, PyAny> = native_field(&native_field(model, "config")?, "values")? - .call_method1("get", ("sql_analysis",))?; - if opted.is(PyBool::new(py, false)) { - reasons.set_item(name, PyTuple::new(py, ["SQL analysis disabled"])?)?; - continue; - } - let mut partial: BTreeSet = BTreeSet::new(); - for (kind, upstream) in rows { - if OPEN_KINDS.contains(&kind.as_str()) && !shapes.contains(upstream)? { - partial.insert(if kind == SOURCE_KIND { - format!("open source: {upstream}") - } else { - format!("open input: {upstream}") - }); - } - } - if let Some(causes) = blocked_by.get(name) { - for cause in causes.iter().filter(|cause| *cause != name) { - partial.insert(format!("depends on invalid model {cause}")); - } - } - let inferred: Vec = inferred_rows(model)?.unwrap_or_default(); - if inferred.is_empty() { - partial.insert("output shape could not be inferred".to_owned()); - } else if star_flags(model)?.0 && !shapes.contains(name)? { - partial.insert("unresolved star over open inputs".to_owned()); - } - if inferred - .iter() - .any(|(_, column_type, _)| column_type.is_none()) - { - partial.insert("some output types are unknown".to_owned()); - } - if !TYPE_CHECKED_DIALECTS.contains(&dialect.as_str()) { - partial.insert("type checking awaits dialect coercion support".to_owned()); - } - if !partial.is_empty() { - reasons.set_item(name, PyTuple::new(py, partial)?)?; + Ok(reasons) +} + +/// A model's partial-check reasons apart from those its invalid dependencies give it. +fn own_reasons<'py>( + (model, name): (&Bound<'py, PyAny>, &str), + rows: &[(String, String)], + (shapes, dialect): (&Bound<'py, PyDict>, &str), +) -> PyResult> { + let mut partial: BTreeSet = BTreeSet::new(); + for (kind, upstream) in rows { + if OPEN_KINDS.contains(&kind.as_str()) && !shapes.contains(upstream)? { + partial.insert(if kind == SOURCE_KIND { + format!("open source: {upstream}") + } else { + format!("open input: {upstream}") + }); } } - Ok(reasons) + let inferred: Vec = inferred_rows(model)?.unwrap_or_default(); + if inferred.is_empty() { + partial.insert("output shape could not be inferred".to_owned()); + } else if star_flags(model)?.0 && !shapes.contains(name)? { + partial.insert("unresolved star over open inputs".to_owned()); + } + if inferred + .iter() + .any(|(_, column_type, _)| column_type.is_none()) + { + partial.insert("some output types are unknown".to_owned()); + } + if !TYPE_CHECKED_DIALECTS.contains(&dialect) { + partial.insert("type checking awaits dialect coercion support".to_owned()); + } + Ok(partial) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs index 6a055292ef..44f522ea77 100644 --- a/crates/sqlbuild-python/src/bindings/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_coverage.rs"] +mod coverage; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_image.rs"] diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index c19e16eed1..f4fb041c20 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -7,6 +7,7 @@ use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; use crate::bindings::_helpers::compile_delta::context::{ CapturedDiagnostic, CapturedModel, CapturedTest, }; +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; use crate::bindings::_helpers::compile_delta::image::{ BaseRules, FACTS_SECTION, GATE_SECTION, IMAGE_FORMAT, ImageRelations, ImageRules, PendingRules, ProjectImage, RULE_DIGESTS_SECTION, @@ -95,6 +96,12 @@ pub(crate) fn image() -> ProjectImage { ("orders".to_owned(), order_types), ("customers".to_owned(), pairs(&[("customer_id", "VARCHAR")])), ], + coverage: Some(vec![ModelCoverage { + name: "orders".to_owned(), + disabled: false, + blocked: true, + partial: strings(&["open source: raw_orders"]), + }]), lint_model_keys: Some(pairs(&[("models/orders.sql", "lint-key")])), lint_file_keys: Some(vec![ ("models/orders.sql".to_owned(), Some("file-key".to_owned())), @@ -133,6 +140,7 @@ pub(crate) fn facts(image: &ProjectImage) -> String { &image.pinned_sources, &image.test_sources, &image.model_shapes, + &image.coverage, &image.lint_model_keys, &image.lint_file_keys, &image.header_keys, @@ -214,3 +222,25 @@ pub(crate) fn names(values: &[&str]) -> BTreeSet { pub(crate) fn text(value: &str) -> ConfigValue { ConfigValue::String(value.to_owned()) } + +/// One model's semantic coverage facts. +pub(crate) fn coverage( + name: &str, + (disabled, blocked): (bool, bool), + partial: &[&str], +) -> ModelCoverage { + ModelCoverage { + name: name.to_owned(), + disabled, + blocked, + partial: strings(partial), + } +} + +/// Models paired with their upstream names, or with their reasons. +pub(crate) fn order(models: &[(&str, &[&str])]) -> Vec<(String, Vec)> { + models + .iter() + .map(|(name, upstream)| ((*name).to_owned(), strings(upstream))) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs b/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs new file mode 100644 index 0000000000..6edbf806ef --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs @@ -0,0 +1,115 @@ +use std::collections::BTreeSet; + +use crate::bindings::_helpers::compile_delta::coverage::{ + ModelCoverage, coverage_reasons, merged_coverage, merged_order, +}; +use crate::bindings::_helpers::compile_delta::driver::names_uncompiled_model; +use crate::bindings::tests::helpers::{coverage, names, order, strings}; +use crate::bindings::tests::test_types::{CoverageTestCase, UncompiledModelTestCase}; + +const DISABLED: &str = "SQL analysis disabled"; +const OPEN_INPUT: &str = "open input: raw_orders"; + +#[test] +fn given_stored_and_fresh_coverage_when_merging_then_reasons_match_a_whole_project_compile() { + let chain: &[(&str, &[&str])] = &[ + ("orders", &[]), + ("payments", &["orders"]), + ("customers", &["payments"]), + ("returns", &[]), + ]; + let test_cases = [ + CoverageTestCase { + description: "a stored model without SQL analysis keeps its reason", + stored: vec![coverage("returns", (true, false), &[])], + fresh: vec![coverage("orders", (false, false), &[])], + stored_order: order(chain), + fresh_order: order(&[("orders", &[])]), + expected_reasons: order(&[("returns", &[DISABLED])]), + }, + CoverageTestCase { + description: "a rerun model turning invalid makes every stored dependent partial", + stored: vec![coverage("customers", (false, false), &[OPEN_INPUT])], + fresh: vec![coverage("orders", (false, true), &[])], + stored_order: order(chain), + fresh_order: order(&[("orders", &[])]), + expected_reasons: order(&[ + ("payments", &["depends on invalid model orders"]), + ( + "customers", + &["depends on invalid model orders", OPEN_INPUT], + ), + ]), + }, + CoverageTestCase { + description: "a rerun model fixed drops the reasons its dependents had from it", + stored: vec![coverage("orders", (false, true), &[])], + fresh: vec![coverage("orders", (false, false), &[])], + stored_order: order(chain), + fresh_order: order(&[("orders", &[])]), + expected_reasons: vec![], + }, + CoverageTestCase { + description: "a rerun model reading a stored invalid model becomes partial", + stored: vec![coverage("returns", (false, true), &[])], + fresh: vec![coverage("orders", (false, false), &[])], + stored_order: order(chain), + fresh_order: order(&[("orders", &["returns"])]), + expected_reasons: order(&[ + ("orders", &["depends on invalid model returns"]), + ("payments", &["depends on invalid model returns"]), + ("customers", &["depends on invalid model returns"]), + ]), + }, + ]; + + for test_case in test_cases { + let facts: Vec = merged_coverage(&test_case.stored, &test_case.fresh); + let project_order: Vec<(String, Vec)> = + merged_order(&test_case.stored_order, &test_case.fresh_order); + + assert_eq!( + coverage_reasons(&project_order, &facts), + test_case.expected_reasons, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_missing_model_errors_when_judging_a_delta_then_only_uncompiled_models_rule_it_out() { + let test_cases = [ + UncompiledModelTestCase { + description: "an expectation of a project model the delta did not compile", + message: "test 'orders_check' expects model 'payments' which does not exist", + expected_uncompiled: true, + }, + UncompiledModelTestCase { + description: "a reference to a model the project lacks is a real error", + message: "references unknown model 'refunds'", + expected_uncompiled: false, + }, + UncompiledModelTestCase { + description: "a reference to a model the delta compiled is a real error", + message: "references unknown model 'orders'", + expected_uncompiled: false, + }, + UncompiledModelTestCase { + description: "an unrelated error naming a project model", + message: "model 'payments' has a type error", + expected_uncompiled: false, + }, + ]; + let model_names: Vec = strings(&["orders", "payments"]); + let rerun: BTreeSet = names(&["orders"]); + + for test_case in test_cases { + assert_eq!( + names_uncompiled_model(test_case.message, &model_names, &rerun), + test_case.expected_uncompiled, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_splice.rs b/crates/sqlbuild-python/src/bindings/tests/test_splice.rs index 116346ca4a..705202a2fe 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_splice.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_splice.rs @@ -40,6 +40,17 @@ fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_ sources: None, expected_report: None, }, + SpliceTestCase { + description: "the delta's partial-check reasons, already the whole project's, replace the stored", + base: r#"{"has_errors": false, "semantic_checks_partial": {"customers": ["open input: orders"]}, "resources": {}, "diagnostics": []}"#, + delta: r#"{"has_errors": true, "semantic_checks_partial": {"customers": ["depends on invalid model orders"]}, "resources": {}, "diagnostics": [{"code": "B002"}]}"#, + models: &["orders"], + tests: &[], + sources: None, + expected_report: Some( + r#"{"has_errors": true, "semantic_checks_partial": {"customers": ["depends on invalid model orders"]}, "resources": {}, "diagnostics": [{"code": "B002"}]}"#, + ), + }, SpliceTestCase { description: "a whole-project section that differs cannot be spliced", base: r#"{"has_errors": false, "target": "dev", "summary": {"errors": 0, "warnings": 0}, "resources": {}, "diagnostics": [], "compile_timings": {}}"#, diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs index 57e91b9a5b..eb32f89cee 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -1,6 +1,7 @@ use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_config::models::ConfigValue; +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; use crate::bindings::_helpers::compile_delta::image::ProjectImage; /// A stored report, a delta report and what splicing the delta into the stored one gives. @@ -39,3 +40,20 @@ pub(crate) struct ImageRoundTripTestCase { /// The retained rules request's model and digest count, or `None` without one. pub(crate) expected_rule_models: Option, } + +/// Stored and fresh semantic coverage facts, and the whole project's partial-check reasons. +pub(crate) struct CoverageTestCase { + pub(crate) description: &'static str, + pub(crate) stored: Vec, + pub(crate) fresh: Vec, + pub(crate) stored_order: Vec<(String, Vec)>, + pub(crate) fresh_order: Vec<(String, Vec)>, + pub(crate) expected_reasons: Vec<(String, Vec)>, +} + +/// An error message a delta round reported. +pub(crate) struct UncompiledModelTestCase { + pub(crate) description: &'static str, + pub(crate) message: &'static str, + pub(crate) expected_uncompiled: bool, +} diff --git a/scripts/delta_compile_series/_helpers/outcomes.py b/scripts/delta_compile_series/_helpers/outcomes.py index 7ff4ce718e..7f69434af1 100644 --- a/scripts/delta_compile_series/_helpers/outcomes.py +++ b/scripts/delta_compile_series/_helpers/outcomes.py @@ -3,17 +3,24 @@ from __future__ import annotations import re +from itertools import zip_longest +from pathlib import Path from scripts.delta_compile_series.constants import ( COMPILE_REUSE_HIT_LINE, + COMPILE_TIMINGS_PATTERN, + DELTA_COMPILE_LINE, DELTA_FALLBACK_PATTERN, DELTA_MISMATCH_PATTERN, DELTA_VERIFIED_LINE, + DURATION_PATTERN, MISMATCH_EXIT_CODE, STALE_PROBLEMS, ) from scripts.delta_compile_series.models import DeltaCompileEdit, DeltaCompileOutcome +_STREAMS: tuple[str, ...] = ("exit code", "stdout", "stderr") + def fallback_reason(*, outcome: DeltaCompileOutcome) -> str | None: """Why the compile fell back from a delta, when it says.""" @@ -23,7 +30,12 @@ def fallback_reason(*, outcome: DeltaCompileOutcome) -> str | None: def judged( - *, outcome: DeltaCompileOutcome, edit: DeltaCompileEdit, reverted: bool, others_edited: bool + *, + outcome: DeltaCompileOutcome, + edit: DeltaCompileEdit, + reverted: bool, + others_edited: bool, + after_failure: bool = False, ) -> tuple[bool, str | None]: """Whether the compile was a verified delta, and what was unexpected about it, if anything.""" @@ -38,6 +50,46 @@ def judged( if edit.fallback_reason is None: verified: bool = DELTA_VERIFIED_LINE in outcome.stderr return verified, None if verified else f"fell back: {reason}" + if reverted and after_failure and reason is not None: + return False, None if reason is None or edit.fallback_reason not in reason: return False, f"expected a fallback for {edit.fallback_reason!r}, got {reason!r}" return False, None + + +def _seen(*, outcome: DeltaCompileOutcome, project_dir: Path, other_dir: Path) -> tuple[str, ...]: + """What a user sees of a compile, without durations, timings and the delta's own line.""" + + def shown(text: str) -> str: + text = COMPILE_TIMINGS_PATTERN.sub("", text.replace(str(other_dir), str(project_dir))) + kept: list[str] = [line for line in text.splitlines() if DELTA_COMPILE_LINE not in line] + return DURATION_PATTERN.sub("", "\n".join(kept)) + + return (str(outcome.returncode), shown(outcome.stdout), shown(outcome.stderr)) + + +def full_compile_difference( + *, + outcome: DeltaCompileOutcome, + full: DeltaCompileOutcome, + project_dir: Path, + full_project_dir: Path, +) -> str | None: + """The first line a user sees differently than after a full compile of a copy, if any.""" + + if COMPILE_REUSE_HIT_LINE in outcome.stderr: + return None + + seen: tuple[str, ...] = _seen(outcome=outcome, project_dir=project_dir, other_dir=project_dir) + expected: tuple[str, ...] = _seen( + outcome=full, project_dir=project_dir, other_dir=full_project_dir + ) + for stream, left, right in zip(_STREAMS, seen, expected, strict=True): + if left != right: + first: tuple[str, str] = next( + pair + for pair in zip_longest(left.splitlines(), right.splitlines(), fillvalue="") + if pair[0] != pair[1] + ) + return f"{stream} differs from a full compile: {first[0]!r} vs {first[1]!r}" + return None diff --git a/scripts/delta_compile_series/constants.py b/scripts/delta_compile_series/constants.py index 291b2c4dc6..d82dbe34cc 100644 --- a/scripts/delta_compile_series/constants.py +++ b/scripts/delta_compile_series/constants.py @@ -2,6 +2,8 @@ from __future__ import annotations +import re + from scripts.delta_compile_series.models import DeltaCompileEdit from scripts.delta_compile_series.types import EditTarget @@ -18,6 +20,9 @@ STALE_PROBLEMS: str = "reports problems in files the edit leaves alone" PROJECT_CONFIG: str = "sqlbuild_project.toml" RANDOM_VALUE_LIMIT: int = 10**6 +FULL_COPY_SUFFIX: str = "_full_copy" +DURATION_PATTERN: re.Pattern[str] = re.compile(r"\s*\(\d+(?:\.\d+)?s\)") +COMPILE_TIMINGS_PATTERN: re.Pattern[str] = re.compile(r'\n "compile_timings": \{[^{}]*\n \}') EDITS: tuple[DeltaCompileEdit, ...] = ( DeltaCompileEdit("body_constant", r"COALESCE\(b\.amount, \d+\)", "COALESCE(b.amount, {value})"), @@ -38,6 +43,19 @@ r"\g<1>{value}", target=EditTarget.TESTS, ), + DeltaCompileEdit( + "analysis_off", + r"MODEL \(description", + "MODEL (sql_analysis false, description", + fallback_reason="changed its sql_analysis header key", + ), + DeltaCompileEdit( + "test_retarget", + r"(__expected__\w+_\d{4})[1-9]( AS)", + r"\g<1>0\g<2>", + target=EditTarget.TESTS, + fallback_reason="now covers other models", + ), DeltaCompileEdit( "materialization", r"materialized table", diff --git a/scripts/delta_compile_series/main/run_delta_compile_series.py b/scripts/delta_compile_series/main/run_delta_compile_series.py index e5b5b83c83..ac38d9beba 100644 --- a/scripts/delta_compile_series/main/run_delta_compile_series.py +++ b/scripts/delta_compile_series/main/run_delta_compile_series.py @@ -3,15 +3,18 @@ from __future__ import annotations import random +import shutil from collections.abc import Callable from pathlib import Path from scripts.delta_compile_series._helpers.edits import edited_file -from scripts.delta_compile_series._helpers.outcomes import judged +from scripts.delta_compile_series._helpers.outcomes import full_compile_difference, judged from scripts.delta_compile_series.constants import ( + DELTA_COMPILE_LINE, DELTA_WEIGHT, EDITS, FALLBACK_WEIGHT, + FULL_COPY_SUFFIX, REVERT_PROBABILITY, ) from scripts.delta_compile_series.models import ( @@ -21,10 +24,33 @@ ) +def _compared( + *, + project_dir: Path, + compile_project: Callable[[], DeltaCompileOutcome], + compile_in_full: Callable[[Path], DeltaCompileOutcome], +) -> tuple[DeltaCompileOutcome, str | None]: + """Compile, and compile a copy of the project as it was in full; how what they print differs.""" + + copy: Path = project_dir.with_name(project_dir.name + FULL_COPY_SUFFIX) + shutil.rmtree(copy, ignore_errors=True) + _ = shutil.copytree(project_dir, copy, symlinks=True) + outcome: DeltaCompileOutcome = compile_project() + problem: str | None = full_compile_difference( + outcome=outcome, full=compile_in_full(copy), project_dir=project_dir, full_project_dir=copy + ) + return outcome, problem + + def run_delta_compile_series( - *, project_dir: Path, seed: int, steps: int, compile_project: Callable[[], DeltaCompileOutcome] + *, + project_dir: Path, + seed: int, + steps: int, + compile_project: Callable[[], DeltaCompileOutcome], + compile_in_full: Callable[[Path], DeltaCompileOutcome] | None = None, ) -> DeltaCompileSeries: - """Edit or revert `steps` times, compiling after each, and judge every compile.""" + """Edit or revert `steps` times, compiling after each, and judge or compare every compile.""" rng: random.Random = random.Random(seed) weights: list[int] = [ @@ -46,11 +72,23 @@ def run_delta_compile_series( pending.append((path, original, edit)) step: str = f"step {index}: {'revert ' if reverted else ''}{edit.kind} in {path.name}" described.append(step) - outcome: DeltaCompileOutcome = compile_project() - others_edited: bool = any(other != path for other, _, _ in pending) - delta, problem = judged( - outcome=outcome, edit=edit, reverted=reverted, others_edited=others_edited - ) + if compile_in_full is None: + outcome: DeltaCompileOutcome = compile_project() + others_edited: bool = any(other != path for other, _, _ in pending) + delta, problem = judged( + outcome=outcome, + edit=edit, + reverted=reverted, + others_edited=others_edited, + after_failure=failing, + ) + else: + outcome, problem = _compared( + project_dir=project_dir, + compile_project=compile_project, + compile_in_full=compile_in_full, + ) + delta: bool = DELTA_COMPILE_LINE in outcome.stderr deltas += int(delta) if problem is not None: unexpected.append(f"{step}: {problem}") diff --git a/scripts/delta_compile_series/main/write_delta_compile_project.py b/scripts/delta_compile_series/main/write_delta_compile_project.py index b0975074d7..338b4c56bb 100644 --- a/scripts/delta_compile_series/main/write_delta_compile_project.py +++ b/scripts/delta_compile_series/main/write_delta_compile_project.py @@ -1,4 +1,4 @@ -"""Write the dense synthetic project on DuckDB for delta compile series.""" +"""Write the dense synthetic project for delta compile series, on DuckDB by default.""" from __future__ import annotations @@ -8,12 +8,16 @@ from scripts.delta_compile_series.constants import PROJECT_CONFIG -def write_delta_compile_project(*, project_dir: Path, model_count: int) -> None: - """Write sources, seeds, functions, macros, models, SQL tests and custom rules on DuckDB.""" +def write_delta_compile_project( + *, project_dir: Path, model_count: int, adapter: str = "duckdb" +) -> None: + """Write sources, seeds, functions, macros, models, SQL tests and custom rules on `adapter`.""" write_dense_compile_project(project_dir=project_dir, model_count=model_count) config: Path = project_dir / PROJECT_CONFIG config.write_text( - config.read_text(encoding="utf-8").replace('adapter = "snowflake"', 'adapter = "duckdb"'), + config.read_text(encoding="utf-8").replace( + 'adapter = "snowflake"', f'adapter = "{adapter}"' + ), encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 7cc088907e..a29f38892e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1112,10 +1112,22 @@ class DeltaCompileSeriesTestCase: seed: int json_output: bool steps: int + adapter: str = "duckdb" expected_unexpected_outcomes: tuple[str, ...] = () expected_minimum_deltas: int = 3 +@dataclass(frozen=True) +class DeltaCompileFullCopyTestCase: + description: str + edits: tuple[tuple[str, str, str], ...] + expected_delta_line: bool + stored_edits: tuple[tuple[str, str, str], ...] = () + json_output: bool = False + adapter: str = "duckdb" + expected_difference: None = None + + @dataclass(frozen=True) class DeltaCompileSequenceTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 971d60f100..9909a14262 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -40,7 +40,11 @@ from scripts.compiler_differential.constants import FAILURE_BASE_FILES from scripts.delta_compile_series.models import DeltaCompileOutcome from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR -from sqlbuild._native import ContractValidationResult, compiler_cache_directory +from sqlbuild._native import ( + DELTA_COMPILE_DISABLE_ENV_VAR, + ContractValidationResult, + compiler_cache_directory, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -5109,6 +5113,14 @@ def run_delta_compile( ) +def run_full_compile(project_dir: Path, *, args: tuple[str, ...]) -> DeltaCompileOutcome: + """Compile the project at `project_dir` in full, with delta compiles off.""" + + return run_delta_compile( + project_dir=project_dir, args=args, env={DELTA_COMPILE_DISABLE_ENV_VAR: "1"} + ) + + def without_compile_timings(*, report: str) -> str: """A JSON report without its compile timings, which differ between compiles.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 9eb5ad798c..90ea08b5ec 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -2,12 +2,21 @@ from __future__ import annotations +import shutil from functools import partial from pathlib import Path import pytest -from scripts.delta_compile_series.constants import DELTA_COMPILE_LINE, PROJECT_CONFIG +from scripts.delta_compile_series._helpers.outcomes import full_compile_difference +from scripts.delta_compile_series.constants import ( + DELTA_COMPILE_LINE, + DOUBLE_CAST, + DOUBLE_COLUMN, + FLOAT_CAST, + FLOAT_COLUMN, + PROJECT_CONFIG, +) from scripts.delta_compile_series.main.run_delta_compile_sequence import ( run_delta_compile_sequence, ) @@ -23,6 +32,7 @@ from scripts.delta_compile_series.types import SequenceEdit from sqlbuild._native import DELTA_COMPILE_DISABLE_ENV_VAR, DELTA_COMPILE_VERIFY_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + DeltaCompileFullCopyTestCase, DeltaCompileOptOutTestCase, DeltaCompileSequenceTestCase, DeltaCompileSeriesTestCase, @@ -30,12 +40,35 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( replace_project_text, run_delta_compile, + run_full_compile, without_compile_timings, ) _MODEL_COUNT: int = 300 _JSON_COMPILE: tuple[str, ...] = ("compile", "--json") _TEXT_COMPILE: tuple[str, ...] = ("compile",) +_UNTYPED_ADAPTER: str = "sqlserver" +_SMALL_MODEL_COUNT: int = 30 +_SMALL_MODEL_ROOT: str = "models/sales000/intermediate/clean" +_ANALYSED_MODEL: str = f"{_SMALL_MODEL_ROOT}/batch00000/sales000__int_clean__orders_00001.sql" +_SMALL_EDITED_MODEL: str = f"{_SMALL_MODEL_ROOT}/batch00001/sales000__int_clean__orders_00005.sql" +_RESHAPED_MODEL: str = f"{_SMALL_MODEL_ROOT}/batch00002/sales000__int_clean__orders_00010.sql" +_SMALL_EDITED_TEST: str = ( + "tests/unit/sales000/intermediate/clean/batch00001/" + "test_sales000_int_clean_orders_00005__preserves_order_id.sql" +) +_EXPECTED_MODEL_CTE: str = "__expected__sales000__int_clean__orders_00005 AS" +_RETARGETED_MODEL_CTE: str = "__expected__sales000__int_clean__orders_00020 AS" +_BINDING_ERROR_EDIT: tuple[str, str, str] = ( + _SMALL_EDITED_MODEL, + "COALESCE(b.amount, ", + "COALESCE(b.missing_col, ", +) +_BODY_EDIT: tuple[str, str, str] = ( + _SMALL_EDITED_MODEL, + "COALESCE(b.amount, ", + "COALESCE(b.amount, 1 + ", +) _REUSE_ROOT: str = "target/cache/compiler-native-v1/project-reuse-v3" _EDITED_MODEL: str = ( "models/sales000/intermediate/clean/batch00000/sales000__int_clean__orders_00003.sql" @@ -50,6 +83,13 @@ DeltaCompileSeriesTestCase( description="long_json_series", seed=13, json_output=True, steps=20 ), + DeltaCompileSeriesTestCase( + description="partial_checks_text_report", + seed=14, + json_output=False, + steps=12, + adapter=_UNTYPED_ADAPTER, + ), ], ids=lambda case: case.description, ) @@ -57,7 +97,9 @@ def test_given_seeded_edit_series_when_compiling_each_edit_then_deltas_match_ful test_case: DeltaCompileSeriesTestCase, tmp_path: Path ) -> None: project_dir: Path = tmp_path / "dense_orders" - write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) + write_delta_compile_project( + project_dir=project_dir, model_count=_MODEL_COUNT, adapter=test_case.adapter + ) args: tuple[str, ...] = (_TEXT_COMPILE, _JSON_COMPILE)[int(test_case.json_output)] series: DeltaCompileSeries = run_delta_compile_series( @@ -76,6 +118,115 @@ def test_given_seeded_edit_series_when_compiling_each_edit_then_deltas_match_ful assert series.deltas >= test_case.expected_minimum_deltas, series.steps +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileSeriesTestCase(description="text_report", seed=15, json_output=False, steps=12), + DeltaCompileSeriesTestCase( + description="partial_checks_json_report", + seed=16, + json_output=True, + steps=12, + adapter=_UNTYPED_ADAPTER, + ), + ], + ids=lambda case: case.description, +) +def test_given_seeded_edit_series_when_compiling_each_edit_then_output_matches_full_compiles_of_copies( + test_case: DeltaCompileSeriesTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project( + project_dir=project_dir, model_count=_MODEL_COUNT, adapter=test_case.adapter + ) + args: tuple[str, ...] = (_TEXT_COMPILE, _JSON_COMPILE)[int(test_case.json_output)] + + series: DeltaCompileSeries = run_delta_compile_series( + project_dir=project_dir, + seed=test_case.seed, + steps=test_case.steps, + compile_project=partial(run_delta_compile, project_dir=project_dir, args=args, env={}), + compile_in_full=partial(run_full_compile, args=args), + ) + + assert series.unexpected == test_case.expected_unexpected_outcomes, series.steps + assert series.deltas >= test_case.expected_minimum_deltas, series.steps + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileFullCopyTestCase( + description="a model without SQL analysis keeps its partial-check notice", + stored_edits=((_ANALYSED_MODEL, "MODEL (", "MODEL (sql_analysis false, "),), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileFullCopyTestCase( + description="an invalid model makes its unchanged dependents partial", + edits=(_BINDING_ERROR_EDIT,), + expected_delta_line=True, + ), + DeltaCompileFullCopyTestCase( + description="an invalid model makes its unchanged dependents partial in JSON", + edits=(_BINDING_ERROR_EDIT,), + expected_delta_line=True, + json_output=True, + ), + DeltaCompileFullCopyTestCase( + description="a dialect without type checks reports every model partial", + edits=(_BODY_EDIT,), + expected_delta_line=True, + json_output=True, + adapter=_UNTYPED_ADAPTER, + ), + DeltaCompileFullCopyTestCase( + description="a SQL test expecting another model falls back after its round", + edits=((_SMALL_EDITED_TEST, _EXPECTED_MODEL_CTE, _RETARGETED_MODEL_CTE),), + expected_delta_line=False, + ), + DeltaCompileFullCopyTestCase( + description="a shape change printed once over several rounds", + edits=( + (_RESHAPED_MODEL, DOUBLE_COLUMN, FLOAT_COLUMN), + (_RESHAPED_MODEL, DOUBLE_CAST, FLOAT_CAST), + (_RESHAPED_MODEL, DOUBLE_CAST, FLOAT_CAST), + ), + expected_delta_line=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_compile_when_compiling_an_edit_then_it_prints_what_a_full_compile_prints( + test_case: DeltaCompileFullCopyTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project( + project_dir=project_dir, model_count=_SMALL_MODEL_COUNT, adapter=test_case.adapter + ) + args: tuple[str, ...] = (_TEXT_COMPILE, _JSON_COMPILE)[int(test_case.json_output)] + for relative_path, old, new in test_case.stored_edits: + replace_project_text(project_dir, relative_path, old, new) + _ = run_delta_compile(project_dir=project_dir, args=args, env={}) + for relative_path, old, new in test_case.edits: + replace_project_text(project_dir, relative_path, old, new) + copy: Path = tmp_path / "full_copy" + _ = shutil.copytree(project_dir, copy, symlinks=True) + + edited: DeltaCompileOutcome = run_delta_compile(project_dir=project_dir, args=args, env={}) + + assert (DELTA_COMPILE_LINE in edited.stderr) == test_case.expected_delta_line, edited.stderr + assert ( + full_compile_difference( + outcome=edited, + full=run_full_compile(copy, args=args), + project_dir=project_dir, + full_project_dir=copy, + ) + == test_case.expected_difference + ) + + @pytest.mark.parametrize( "test_case", [ From d63907bba40c7e1aeb69f427e09390d1914e6d54 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:17:25 +0100 Subject: [PATCH 26/32] test(compile): guard the delta edit loop on the dense project The dense all-rules jobs at 3k and 10k models now compile the project once, then edit one model body, one model header and one SQL test, compiling after each edit. Each of those compiles must: - take the delta path, so it analyses only a handful of models according to its compile_timings counters; - stay under a wall cap; - keep its CPU within a ratio of a delta-disabled full compile on the same runner. Then project-config edits alternate between delta-enabled compiles, which fall back, and delta-disabled ones. The median CPU of the fallbacks must stay within 5% of the delta-disabled compiles. The dense job's timeout grows to 30 minutes to fit the 10k loop. --- .github/workflows/verify.yml | 5 +- .../_helpers/dense_project.py | 21 +++ .../cli/commands/main/compile/_test_types.py | 13 ++ .../cli/commands/main/compile/helpers.py | 135 ++++++++++++++++++ .../compile/test_dense_compile_performance.py | 99 +++++++++++-- 5 files changed, 261 insertions(+), 12 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2d8fc4e0d6..4430b01a54 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -571,7 +571,8 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 20 + # The 3k and 10k cells also run the delta edit-loop guard, about five minutes at 10k. + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -592,7 +593,7 @@ jobs: run: uv python install 3.12 - name: Install dependencies run: uv sync --locked --all-groups --all-extras - - name: Run dense all-rules guard under 4 GiB + - name: Run dense all-rules and delta edit-loop guards under 4 GiB env: BENCHMARK_MODELS: ${{ matrix.models }} MEMORY_LIMIT_BYTES: "4294967296" diff --git a/scripts/cold_compile_performance/_helpers/dense_project.py b/scripts/cold_compile_performance/_helpers/dense_project.py index 0d7ef833c5..28e8fe8d76 100644 --- a/scripts/cold_compile_performance/_helpers/dense_project.py +++ b/scripts/cold_compile_performance/_helpers/dense_project.py @@ -26,6 +26,27 @@ def dense_model_name(index: int) -> str: return f"sales{index // 1000:03d}__int_clean__orders_{index:05d}" +def dense_model_path(*, project_dir: Path, index: int) -> Path: + """The file of the generated dense model at `index`.""" + + return ( + project_dir + / "models" + / f"sales{index // 1000:03d}" + / "intermediate" + / "clean" + / f"batch{index // 5:05d}" + / f"{dense_model_name(index)}.sql" + ) + + +def dense_test_path(*, project_dir: Path, index: int) -> Path: + """The SQL test file of the generated dense model at `index`.""" + + name: str = dense_model_name(index).replace("__", "_") + return next((project_dir / "tests").rglob(f"test_{name}__*.sql")) + + def dense_column_count(index: int) -> int: rank: int = (index * 17) % 1000 for upper, width in ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index a29f38892e..12bfcd9dd3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -276,6 +276,19 @@ class DenseWarmEditCompileGuardTestCase: expected_edit_rule_cache_misses: int +@dataclass(frozen=True) +class DenseDeltaEditLoopGuardTestCase: + description: str + model_count: int + edited_model_index: int + full_pairs: int + expected_max_full_seconds: float + expected_max_delta_seconds: float + expected_max_delta_cpu_ratio: float + expected_max_fallback_cpu_ratio: float + expected_max_rss_bytes: int + + @dataclass(frozen=True) class RuleGatedTestDiagnosticsTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 9909a14262..bdbfe12b15 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -20,6 +20,7 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager, redirect_stdout from dataclasses import replace +from functools import partial from io import StringIO from pathlib import Path from types import FrameType, ModuleType @@ -666,12 +667,14 @@ def _run_fresh_process_compile_benchmark( expected_max_wall_seconds: float, compile_args: tuple[str, ...], retained_payload_keys: tuple[str, ...] = _RETAINED_BENCHMARK_PAYLOAD_KEYS, + env: dict[str, str] | None = None, ) -> FreshProcessCompileBenchmarkResult: run: _TimedSqbRun = _run_timed_sqb( project_dir=project_dir, label=label, sqb_args=("compile", "--json", *compile_args), expected_max_wall_seconds=expected_max_wall_seconds, + env=env, ) assert isinstance(run.payload, dict) payload: dict[str, object] = cast(dict[str, object], run.payload) @@ -707,6 +710,7 @@ def _run_timed_sqb( label: str, sqb_args: tuple[str, ...], expected_max_wall_seconds: float, + env: dict[str, str] | None = None, ) -> _TimedSqbRun: measurement_path: Path = project_dir.parent / f"{label}-measurement.txt" output_path: Path = project_dir.parent / f"{label}.out" @@ -737,6 +741,7 @@ def _run_timed_sqb( stdout=output_file, stderr=stderr_file, start_new_session=True, + env={**os.environ, **(env or {})}, ) as process: try: returncode: int = process.wait(timeout=expected_max_wall_seconds + 10.0) @@ -942,6 +947,136 @@ def run_dense_warm_edit_benchmark( return measurements +class DeltaEditLoopMeasurements(NamedTuple): + """Fresh-process compiles of an edit loop: deltas, delta fallbacks and delta-disabled ones.""" + + deltas: dict[str, FreshProcessCompileBenchmarkResult] + fallbacks: tuple[FreshProcessCompileBenchmarkResult, ...] + full: tuple[FreshProcessCompileBenchmarkResult, ...] + + +DELTA_EDIT_LOOP_ENV: dict[str, str] = {REUSE_DISABLE_ENV_VAR: "0"} +_DELTA_DISABLED_ENV: dict[str, str] = {**DELTA_EDIT_LOOP_ENV, DELTA_COMPILE_DISABLE_ENV_VAR: "1"} +_WARM_UP_PAIRS: int = 1 +# Each pair swaps which compile runs first, so neither always runs on a warmer system. +_ALTERNATING_CONFIG_RUNS: tuple[tuple[tuple[str, dict[str, str]], ...], ...] = ( + (("fallback", DELTA_EDIT_LOOP_ENV), ("full", _DELTA_DISABLED_ENV)), + (("full", _DELTA_DISABLED_ENV), ("fallback", DELTA_EDIT_LOOP_ENV)), +) +_DENSE_DESCRIPTION: str = 'description "Generated dense model."' +_DENSE_EXPECTED_VALUE: re.Pattern[str] = re.compile(r"(__expected__\w+ AS \(SELECT CAST\()\d+") +_ANALYSIS_COUNTS: tuple[str, ...] = ( + "analysis_batch_cache_hits", + "analysis_entry_cache_hits", + "analysis_cache_misses", + "analysis_cache_bypasses", +) + + +def analysed_model_count(result: FreshProcessCompileBenchmarkResult) -> int: + """How many models the compile analysed, from cache or not; a delta analyses only its own.""" + + timings: dict[str, int] = cast(dict[str, int], result.payload["compile_timings"]) + return sum(timings[name] for name in _ANALYSIS_COUNTS) + + +def _edit_loop_compile( + *, project_dir: Path, label: str, max_seconds: float, env: dict[str, str] +) -> FreshProcessCompileBenchmarkResult: + return _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label=label, + expected_max_wall_seconds=max_seconds, + compile_args=(), + env=env, + ) + + +def _edited_test_expectation(path: Path) -> None: + contents: str = path.read_text(encoding="utf-8") + edited, count = _DENSE_EXPECTED_VALUE.subn(r"\g<1>97", contents, count=1) + assert count == 1 and edited != contents, path + path.write_text(edited, encoding="utf-8") + + +def run_dense_delta_edit_loop_benchmark( + *, + project_dir: Path, + edited_files: tuple[Path, Path, Path], + full_pairs: int, + expected_max_full_seconds: float, + expected_max_delta_seconds: float, +) -> DeltaEditLoopMeasurements: + """Compile, edit a body, a header and a SQL test, then edit config with and without deltas.""" + + body_model, header_model, test_file = edited_files + _ = _edit_loop_compile( + project_dir=project_dir, + label="delta-loop-first", + max_seconds=2 * expected_max_full_seconds, + env=DELTA_EDIT_LOOP_ENV, + ) + deltas: dict[str, FreshProcessCompileBenchmarkResult] = {} + for label, edit in ( + ( + "body", + partial( + _replace_benchmark_text, + path=body_model, + old="COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)", + new="COALESCE(b.amount, 1000) + CAST(b.id AS DOUBLE)", + ), + ), + ( + "header", + partial( + _replace_benchmark_text, + path=header_model, + old=_DENSE_DESCRIPTION, + new='description "Edited dense model."', + ), + ), + ( + "sql_test", + partial(_edited_test_expectation, test_file), + ), + ): + edit() + deltas[label] = _edit_loop_compile( + project_dir=project_dir, + label=f"delta-loop-{label}", + max_seconds=expected_max_delta_seconds, + env=DELTA_EDIT_LOOP_ENV, + ) + runs: dict[str, list[FreshProcessCompileBenchmarkResult]] = {"fallback": [], "full": []} + for pair in range(_WARM_UP_PAIRS + full_pairs): + for kind, env in _ALTERNATING_CONFIG_RUNS[pair % len(_ALTERNATING_CONFIG_RUNS)]: + config: Path = project_dir / "sqlbuild_project.toml" + config.write_text( + config.read_text(encoding="utf-8") + f"# {kind} edit {pair}\n", + encoding="utf-8", + ) + runs[kind].append( + _edit_loop_compile( + project_dir=project_dir, + label=f"delta-loop-{kind}-{pair}", + max_seconds=expected_max_full_seconds, + env=env, + ) + ) + return DeltaEditLoopMeasurements( + deltas=deltas, + fallbacks=tuple(runs["fallback"][_WARM_UP_PAIRS:]), + full=tuple(runs["full"][_WARM_UP_PAIRS:]), + ) + + +def median_cpu_seconds(results: tuple[FreshProcessCompileBenchmarkResult, ...]) -> float: + """The median CPU seconds of same-runner compiles.""" + + return statistics.median(result.cpu_seconds for result in results) + + def _append_benchmark_edit(path: Path, label: str) -> None: path.write_text( path.read_text(encoding="utf-8") + f"\n-- one {label} edit\n", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index e5e44d18b7..d927f3e040 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -8,7 +8,8 @@ import pytest from scripts.cold_compile_performance._helpers.dense_project import ( - dense_model_name, + dense_model_path, + dense_test_path, write_dense_compile_project, ) from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( @@ -23,18 +24,25 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, + DenseDeltaEditLoopGuardTestCase, DenseWarmEditCompileGuardTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + DeltaEditLoopMeasurements, FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, + analysed_model_count, fresh_process_compile_cache_metrics, measure_model_sql_bytes, + median_cpu_seconds, + run_dense_delta_edit_loop_benchmark, run_dense_warm_edit_benchmark, ) _MIB: int = 1024 * 1024 _GIB: int = 1024 * _MIB +# A delta reruns the edited model or the edited SQL test's few models; a full compile all of them. +_MAX_DELTA_ANALYSED_MODELS: int = 10 @pytest.mark.performance @@ -174,15 +182,8 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches assert_required_cgroup_memory_limit() project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) - index: int = test_case.edited_model_index - edited_model_path: Path = ( - project_dir - / "models" - / f"sales{index // 1000:03d}" - / "intermediate" - / "clean" - / f"batch{index // 5:05d}" - / f"{dense_model_name(index)}.sql" + edited_model_path: Path = dense_model_path( + project_dir=project_dir, index=test_case.edited_model_index ) measurements: dict[str, FreshProcessCompileBenchmarkResult] = run_dense_warm_edit_benchmark( project_dir=project_dir, @@ -227,3 +228,81 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) + + +@pytest.mark.performance +@pytest.mark.cold_compile_performance +@pytest.mark.parametrize( + "test_case", + ( + DenseDeltaEditLoopGuardTestCase( + description="dense_models_3000_delta_edit_loop", + model_count=3000, + edited_model_index=1521, + full_pairs=4, + expected_max_full_seconds=15.0, + expected_max_delta_seconds=4.5, + expected_max_delta_cpu_ratio=0.375, + expected_max_fallback_cpu_ratio=1.05, + expected_max_rss_bytes=3 * _GIB // 2, + ), + DenseDeltaEditLoopGuardTestCase( + description="dense_models_10000_delta_edit_loop", + model_count=10000, + edited_model_index=5021, + full_pairs=3, + expected_max_full_seconds=60.0, + expected_max_delta_seconds=9.0, + expected_max_delta_cpu_ratio=0.30, + expected_max_fallback_cpu_ratio=1.05, + expected_max_rss_bytes=15 * _GIB // 4, + ), + ), + ids=lambda case: case.description, +) +def test_given_dense_project_when_editing_one_file_then_delta_compiles_stay_cheap( + test_case: DenseDeltaEditLoopGuardTestCase, + tmp_path: Path, +) -> None: + assert_required_cgroup_memory_limit() + project_dir: Path = tmp_path / "dense_orders" + write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) + index: int = test_case.edited_model_index + + loop: DeltaEditLoopMeasurements = run_dense_delta_edit_loop_benchmark( + project_dir=project_dir, + edited_files=( + dense_model_path(project_dir=project_dir, index=index), + dense_model_path(project_dir=project_dir, index=index + 1), + dense_test_path(project_dir=project_dir, index=index + 2), + ), + full_pairs=test_case.full_pairs, + expected_max_full_seconds=test_case.expected_max_full_seconds, + expected_max_delta_seconds=test_case.expected_max_delta_seconds, + ) + + full_cpu: float = median_cpu_seconds(loop.full) + fallback_cpu: float = median_cpu_seconds(loop.fallbacks) + for label, result in loop.deltas.items(): + print( + f"delta {label} models={test_case.model_count} wall={result.elapsed_seconds:.2f}s " + f"cpu={result.cpu_seconds:.2f}s cpu_ratio={result.cpu_seconds / full_cpu:.3f} " + f"analysed={analysed_model_count(result)}", + flush=True, + ) + print( + f"delta loop models={test_case.model_count} full_cpu={full_cpu:.2f}s " + f"fallback_cpu={fallback_cpu:.2f}s fallback_ratio={fallback_cpu / full_cpu:.3f}", + flush=True, + ) + for result in (*loop.deltas.values(), *loop.fallbacks, *loop.full): + assert result.payload["has_errors"] is False + assert result.payload["diagnostics"] == [] + assert result.peak_rss_bytes < test_case.expected_max_rss_bytes + for result in loop.deltas.values(): + assert 0 < analysed_model_count(result) <= _MAX_DELTA_ANALYSED_MODELS, result + assert result.elapsed_seconds < test_case.expected_max_delta_seconds, result + assert result.cpu_seconds <= test_case.expected_max_delta_cpu_ratio * full_cpu, result + for result in (*loop.fallbacks, *loop.full): + assert analysed_model_count(result) == test_case.model_count, result + assert fallback_cpu <= test_case.expected_max_fallback_cpu_ratio * full_cpu From ab68fd9bf24fc6b230a8739917b25387df63a875 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:17:26 +0100 Subject: [PATCH 27/32] test(compile): tighten the fresh-process, cache and dense compile wall budgets The wall budgets sat two to three times above what CI measures. Each budget is now the larger of 1.3 times the median and 1.15 times the maximum of the last 20 successful CI runs of the job, rounded up to half a second. --- .../compile/test_dense_compile_performance.py | 16 ++++++++-------- ..._fresh_process_compile_cache_performance.py | 18 +++++++++--------- .../test_fresh_process_compile_performance.py | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index d927f3e040..c2c786313e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -53,28 +53,28 @@ DenseCompileGuardTestCase( "dense_models_1000_all_rules", 1000, - 13.0, + 7.5, 800 * _MIB, "de23b14d87e954a65c2da91918fce062d158419edd63cc8cf3dc938b3e47cd44", ), DenseCompileGuardTestCase( "dense_models_3000_all_rules", 3000, - 25.0, + 16.5, 3 * _GIB // 2, "196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed", ), DenseCompileGuardTestCase( "dense_models_5000_all_rules", 5000, - 42.0, + 26.0, 9 * _GIB // 4, "3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22", ), DenseCompileGuardTestCase( "dense_models_10000_all_rules", 10000, - 77.0, + 51.0, 15 * _GIB // 4, "f2a574af6e8068a0634fa3c078b168dff9a475ada5d71502482ad2926c90e1c3", ), @@ -154,8 +154,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_3000_warm_and_one_edit", 3000, 1521, - 11.0, - 12.0, + 8.0, + 8.5, 3 * _GIB // 2, "196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed", "1afe754dd8c66bc36e38a40c66e9183ecbaf365fd8d87a784069cd615c3ce64a", @@ -165,8 +165,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_5000_warm_and_one_edit", 5000, 2521, - 19.0, - 21.0, + 13.0, + 13.5, 9 * _GIB // 4, "3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22", "dfbada4c59a3ab8ec7733784890d74c7b64e9b0adcb03364baaabef5bdba5574", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 84b298e45c..601275d93e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -50,9 +50,9 @@ macro_count=500, test_count=2_945, audit_count=5_056, - expected_cold_max_wall_seconds=22.0, - expected_warm_max_wall_seconds=11.0, - expected_edit_max_wall_seconds=11.5, + expected_cold_max_wall_seconds=10.5, + expected_warm_max_wall_seconds=6.0, + expected_edit_max_wall_seconds=6.0, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, @@ -84,9 +84,9 @@ macro_count=834, test_count=4_908, audit_count=8_427, - expected_cold_max_wall_seconds=35.0, - expected_warm_max_wall_seconds=18.0, - expected_edit_max_wall_seconds=18.0, + expected_cold_max_wall_seconds=18.0, + expected_warm_max_wall_seconds=10.5, + expected_edit_max_wall_seconds=10.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, @@ -118,9 +118,9 @@ macro_count=1_667, test_count=9_816, audit_count=16_855, - expected_cold_max_wall_seconds=68.0, - expected_warm_max_wall_seconds=37.5, - expected_edit_max_wall_seconds=37.5, + expected_cold_max_wall_seconds=34.5, + expected_warm_max_wall_seconds=19.5, + expected_edit_max_wall_seconds=19.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_LARGE_CACHE_WRITE_CPU_OVERHEAD_RATIO, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index 568b961a17..0431b89012 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -36,7 +36,7 @@ macro_count=37, test_count=2_945, audit_count=5_056, - expected_max_wall_seconds=17.0, + expected_max_wall_seconds=9.5, expected_max_rss_bytes=int(1.5 * _GIB), expected_semantic_fingerprint=( "d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543" @@ -51,7 +51,7 @@ macro_count=61, test_count=4_908, audit_count=8_427, - expected_max_wall_seconds=25.0, + expected_max_wall_seconds=16.0, expected_max_rss_bytes=int(1.75 * _GIB), expected_semantic_fingerprint=( "874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46" @@ -66,7 +66,7 @@ macro_count=123, test_count=9_816, audit_count=16_855, - expected_max_wall_seconds=50.0, + expected_max_wall_seconds=31.0, expected_max_rss_bytes=int(2.5 * _GIB), expected_semantic_fingerprint=( "5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6" From 023bc9d99c51cc39ab9cd19d7160395a7e74f8aa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:34:52 +0100 Subject: [PATCH 28/32] test: list delta_compile among the allowed settings in the compiler golden --- tests/goldens/compiler/failure/local-config-unknown-key.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goldens/compiler/failure/local-config-unknown-key.json b/tests/goldens/compiler/failure/local-config-unknown-key.json index 5aea9e667d..d8b2adc939 100644 --- a/tests/goldens/compiler/failure/local-config-unknown-key.json +++ b/tests/goldens/compiler/failure/local-config-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: auto_load_sources, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", + "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: auto_load_sources, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", "help": "did you mean 'sql_analysis'?" } ] From 4fb68eacd19e5150fba3143853336d3395a308ef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:00:42 +0100 Subject: [PATCH 29/32] fix(compile): log the reuse check's digested file count with its argument again The delta driver refactor formatted the count into the message, so tools and tests reading the compile reuse debug log's "digested %d project files" records no longer saw the check's reads. The message is logged with the count as its argument again, through the shared compile reuse debug helper. --- .../_helpers/compile_command/reuse_attempt.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index a4b5351a56..c1ecb51e7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -68,7 +68,16 @@ fn reuse_logger(py: Python<'_>) -> PyResult> { /// Log one message at debug level on the compile reuse logger. pub(crate) fn reuse_debug(py: Python<'_>, message: &str) -> PyResult<()> { - let _ = reuse_logger(py)?.call_method1("debug", (LOG_TEXT, message))?; + reuse_debug_with(py, LOG_TEXT, message) +} + +/// Log `argument` through the logging `format` at debug level on the compile reuse logger. +fn reuse_debug_with<'py>( + py: Python<'py>, + format: &str, + argument: impl pyo3::IntoPyObject<'py>, +) -> PyResult<()> { + let _ = reuse_logger(py)?.call_method1("debug", (format, argument))?; Ok(()) } @@ -416,10 +425,10 @@ fn attempt_compile_reuse<'py>(request: &Bound<'py, PyAny>) -> PyResult Date: Mon, 19 Oct 2026 12:00:42 +0100 Subject: [PATCH 30/32] fix(compile): keep delta compiles from replaying stale stored macro calls A delta compile's reuse record listed only the module files its own process imported. A stored macro call that the delta did not run kept its lazily imported module out of the record, so editing that module replayed stale output. The delta's record now also keeps every module file of the compile it patched. Three related problems made the delta fall back, or cost the following full compile its stored calls: - A SQL test rerun because its chain touched a rerun model was compiled without the chain's other models, which gave false missing-model errors. A rerun test now brings its whole stored chain into the rerun. - Sources filled by a loader were dropped with the other unread sources, so their loaders were reported as undescribed resources. Such sources are now always discovered. - A delta round claimed the process's first compile, the only one allowed to use the macro call store, so the full compile after a fallback re-ran every stored macro. The claim now passes on to that full compile and is kept only when the delta finishes. --- .../_helpers/compile_command/reuse_attempt.rs | 3 ++ .../_helpers/compile_delta/context.rs | 2 + .../bindings/_helpers/compile_delta/driver.rs | 32 +++++++++++- .../_helpers/compile_outputs/reuse.rs | 1 + .../_helpers/discovery/yaml_entries.rs | 19 +++++++ .../_helpers/macro_bridge/first_compile.rs | 52 +++++++++++++++++++ .../src/bindings/_helpers/macro_bridge/mod.rs | 1 + .../_helpers/macro_bridge/registration.rs | 3 +- src/sqlbuild/_native.pyi | 1 + .../compiler/macro_bridge/constants.py | 2 - .../macro_bridge/main/_claim_first_compile.py | 8 +-- .../cli/commands/main/compile/_test_types.py | 11 ++++ .../cli/commands/main/compile/helpers.py | 2 + .../compile/test_macro_store_module_edits.py | 37 +++++++++++++ 14 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index c1ecb51e7a..1544d28982 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -41,6 +41,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::compile_outputs::json_report::plain_json; use crate::bindings::_helpers::compile_outputs::reuse::{ReuseCheckPy, check_compile_reuse}; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; +use crate::bindings::_helpers::macro_bridge::first_compile::keep_delta_claim; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; @@ -553,6 +554,7 @@ fn compiled_with_delta<'py>( )?; match delta { DeltaOutcome::Finished(finished) if !verifying => { + keep_delta_claim(); write_held()?; let (files, models) = (finished.changed_files, finished.rerun_models); let committing: std::time::Instant = std::time::Instant::now(); @@ -569,6 +571,7 @@ fn compiled_with_delta<'py>( return Ok(code); } DeltaOutcome::Raised(error) if !verifying => { + keep_delta_claim(); write_held()?; return Err(error); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index b860cbe7a1..1a1fa34a25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -308,6 +308,8 @@ pub(crate) struct RecordMerge { pub(crate) kept: Vec, /// The template environment names the stored compile read. pub(crate) environment_names: Vec, + /// The module files the stored compile depended on, such as those behind stored macro calls. + pub(crate) module_paths: Vec, } /// Hand the next reuse record the image to keep beside it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 014c93c3f9..a8a22ee3a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -190,6 +190,7 @@ struct Rerun { } impl Rerun { + /// The rerun set grown by every SQL test it touches and every model those tests run. fn with_tests(mut self, image: &ProjectImage, changed_tests: &BTreeSet) -> Self { self.test_paths = changed_tests.clone(); let test_files: BTreeSet<&str> = image @@ -204,9 +205,29 @@ impl Rerun { } } } - for test in &image.tests { - if test.chain.iter().any(|name| self.models.contains(name)) { + let model_paths: HashMap<&str, &str> = image + .models + .iter() + .map(|model| (model.name.as_str(), model.relative_path.as_str())) + .collect(); + let mut growing: bool = true; + while growing { + growing = false; + for test in &image.tests { + if !self.test_paths.contains(&test.relative_path) + && !test.chain.iter().any(|name| self.models.contains(name)) + { + continue; + } self.test_paths.insert(test.relative_path.clone()); + for name in &test.chain { + if let Some(path) = model_paths.get(name.as_str()) + && self.models.insert(name.clone()) + { + self.model_paths.insert((*path).to_owned()); + growing = true; + } + } } } self.tests = image @@ -1013,6 +1034,13 @@ fn finished<'py>( let merge: RecordMerge = RecordMerge { kept: base.stored.inputs.target_files.clone(), environment_names: base.image.environment_names.clone(), + module_paths: base + .stored + .inputs + .modules + .iter() + .map(|(path, _, _)| path.clone()) + .collect(), }; let json_output: bool = base.image.json_output; let Base { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index 7983364ffd..2c61cac5ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -236,6 +236,7 @@ impl NativeReuseAttempt { .extend(merge.environment_names); compile.template_environment_names.sort(); compile.template_environment_names.dedup(); + compile.module_paths.extend(merge.module_paths); } let attempt = self.attempt.lock().map_err(poisoned)?; let result: RecordResult = py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index 3a121e2b00..9734f1a56b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -351,6 +351,9 @@ pub(crate) fn parse_sources_document<'py>( let referenced: Option> = crate::bindings::_helpers::compile_delta::context::injection() .and_then(|injection| injection.source_names.clone()); + crate::bindings::_helpers::compile_delta::context::record_source_reads( + loaded_sources(&sources).iter().map(String::as_str), + ); let mut built: Vec> = Vec::with_capacity(sources.len()); for source in &sources { if let Some(referenced) = &referenced @@ -364,6 +367,22 @@ pub(crate) fn parse_sources_document<'py>( PyTuple::new(py, built) } +/// The sources a loader fills, which own that loader wherever the project is compiled. +fn loaded_sources(sources: &[ParsedSourceEntry>]) -> Vec { + sources + .iter() + .filter(|source| { + source.managed || source.loader.is_some() || source.integration_loader.is_some() + }) + .filter_map(|source| { + let Ok(name) = source.name.0.extract::() else { + return None; + }; + Some(name) + }) + .collect() +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_schema_document, module)?)?; module.add_function(wrap_pyfunction!(parse_sources_document, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs new file mode 100644 index 0000000000..69314bf43e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs @@ -0,0 +1,52 @@ +//! Which compile of this process may reuse stored macro calls: only its first. + +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_delta::context::injection; + +/// Whether a compile claimed the first-compile slot, and whether a delta compile only borrowed it. +#[derive(Debug)] +struct FirstCompile { + claimed: bool, + lent_to_delta: bool, +} + +static FIRST_COMPILE: Mutex = Mutex::new(FirstCompile { + claimed: false, + lent_to_delta: false, +}); + +fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Record that a compile started; true only for this process's first compile invocation. +#[pyfunction] +pub(crate) fn claim_first_compile() -> bool { + let mut first: MutexGuard<'_, FirstCompile> = locked(&FIRST_COMPILE); + if injection().is_some() { + let granted: bool = !first.claimed || first.lent_to_delta; + first.claimed = true; + first.lent_to_delta = granted; + return granted; + } + if first.lent_to_delta { + first.lent_to_delta = false; + return true; + } + let granted: bool = !first.claimed; + first.claimed = true; + granted +} + +/// Keep the delta compile's claim: it finished, so no full compile of this invocation follows. +pub(crate) fn keep_delta_claim() { + locked(&FIRST_COMPILE).lent_to_delta = false; +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(claim_first_compile, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs index 008b35b5c2..8623ff9f42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod first_compile; pub(crate) mod module_digests; pub(crate) mod registration; pub(crate) mod store_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs index b483dec47b..eddce4b97a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::macro_bridge::{module_digests, store_keys}; +use crate::bindings::_helpers::macro_bridge::{first_compile, module_digests, store_keys}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + first_compile::register(module)?; module_digests::register(module)?; store_keys::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bc9c83bb49..1e057364aa 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4068,6 +4068,7 @@ def compile_frontier[T]( native_stage: Callable[[], T] | None = None, ) -> T: ... def render_stage_capture(value: object) -> str: ... +def claim_first_compile() -> bool: ... def digest_module_files(paths: Iterable[str], /) -> dict[str, str] | None: ... def module_digests_metadata(digests: dict[str, str], /) -> bytes: ... def unchanged_module_digests( diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 0226aab70f..d3edb4ac37 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -2,13 +2,11 @@ from __future__ import annotations -import itertools from contextvars import ContextVar ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( "sqlbuild_active_macro_bridge", default=None ) -COMPILES_STARTED: itertools.count[int] = itertools.count() STORE_MODULE_PATHS: set[str] = set() MACRO_CALL_STORE_FILE_NAME: str = "macro-calls.bin" MACRO_CALL_STORE_ENVIRONMENT_VERSION: str = "macro-call-store-v2" diff --git a/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py b/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py index ef5d3668c6..1c28dabe64 100644 --- a/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py +++ b/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py @@ -1,11 +1,11 @@ -"""Count the compiles this process has started, so only its first one uses the macro call store.""" +"""Ask whether this compile is the process's first, so only that one uses the macro call store.""" from __future__ import annotations -from sqlbuild.compiler.macro_bridge.constants import COMPILES_STARTED +from sqlbuild._native import claim_first_compile as claim_native_first_compile def claim_first_compile() -> bool: - """Record that a compile started; True only for the first compile of this process.""" + """Record that a compile started; True only for the first compile invocation of this process.""" - return next(COMPILES_STARTED) == 0 + return claim_native_first_compile() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 12bfcd9dd3..a07b1346fa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -817,6 +817,17 @@ class BrokenEditInvalidationTestCase: expected_matches_uncached: bool +@dataclass(frozen=True) +class StoredMacroDeltaFallbackTestCase: + """A project edit whose delta compile falls back after compiling its round.""" + + description: str + edited_path: str + old: str + new: str + expected_outcome: tuple[int, int, bool, int] + + @dataclass(frozen=True) class StoredMacroModuleEditTestCase: """An outside module behind a stored macro call, edited after a compile that reused the call.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index bdbfe12b15..da20018bd0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5235,6 +5235,8 @@ def json_diagnostic_codes(stdout: str) -> tuple[str, ...]: document: dict[str, Any] = json.loads(stdout or "{}") return tuple(str(diagnostic["code"]) for diagnostic in document.get("diagnostics", ())) + + def run_delta_compile( *, project_dir: Path, args: tuple[str, ...], env: dict[str, str] ) -> DeltaCompileOutcome: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_store_module_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_store_module_edits.py index a0d232512b..b0e0ac297f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_store_module_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_store_module_edits.py @@ -7,6 +7,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + StoredMacroDeltaFallbackTestCase, StoredMacroModuleEditTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( @@ -15,6 +16,7 @@ add_stored_flavor_macro, compiled_text, logged_calls, + replace_project_text, run_reuse_compile, staging_comment, write_external_modules, @@ -90,5 +92,40 @@ def test_given_stored_call_reused_when_its_lazily_imported_module_changes_then_o ) +@pytest.mark.parametrize( + "test_case", + [ + StoredMacroDeltaFallbackTestCase( + description="a model error found by the delta round", + edited_path="models/staging/stg_customers.sql", + old=" email,\n", + new=" email,\n unknown_column,\n", + expected_outcome=(0, 1, False, 1), + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_call_when_a_delta_falls_back_after_its_round_then_the_full_compile_reuses_it( + compile_reuse_project: Path, tmp_path: Path, test_case: StoredMacroDeltaFallbackTestCase +) -> None: + extlib: Path = tmp_path / "extlib" + log_path: Path = tmp_path / "flavor-calls.log" + write_external_modules(extlib=extlib, files=_MODULE) + env: dict[str, str] = add_stored_flavor_macro( + project_dir=compile_reuse_project, extlib=extlib, import_name="extflavor", log_path=log_path + ) + cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + replace_project_text(compile_reuse_project, test_case.edited_path, test_case.old, test_case.new) + + edited: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + + assert ( + cold.returncode, + edited.returncode, + edited.reused, + logged_calls(log_path), + ) == test_case.expected_outcome, edited.stderr + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 160031eb32872e4b0c7a11d7b8e319168200767a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:22:55 +0100 Subject: [PATCH 31/32] fix(compile): match path defaults against every project model in a delta compile A delta compile discovers only the models it reruns, so cross-file validation saw a path_defaults key whose folder held no rerun model as unmatched and raised a discovery conflict. Every edit on such a project fell back to a full compile, slightly slower than compiling in full outright. Path defaults are now matched against the whole project's model paths from the stored compile, which a delta shares because added or removed files fall back. --- .../_helpers/path_defaults.rs | 17 ++++++++++++----- .../src/project_validation/models.rs | 2 ++ .../tests/test_validate_project_inputs.rs | 13 +++++++++++++ .../bindings/_helpers/compile_delta/context.rs | 2 ++ .../bindings/_helpers/compile_delta/driver.rs | 5 +++++ .../_helpers/discovery/project_facts.rs | 2 ++ .../commands/main/compile/test_compile_delta.py | 12 ++++++++++++ 7 files changed, 48 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs index 0568696b6a..a18f0b2287 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs @@ -13,11 +13,18 @@ pub(crate) fn validate_path_defaults(facts: &ProjectFacts) -> Result<(), Validat if facts.path_default_keys.is_empty() { return Ok(()); } - let model_paths: Vec = facts - .models - .iter() - .map(|model| { - let path: String = model.file.path.replace('\\', "/"); + let discovered: Vec<&str> = match &facts.project_model_paths { + Some(paths) => paths.iter().map(String::as_str).collect(), + None => facts + .models + .iter() + .map(|model| model.file.path.as_str()) + .collect(), + }; + let model_paths: Vec = discovered + .into_iter() + .map(|path| { + let path: String = path.replace('\\', "/"); path.strip_prefix(MODELS_PREFIX) .map_or(path.clone(), str::to_owned) }) diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index 1a0f4fdb09..58c65bb869 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -174,6 +174,8 @@ pub struct ProjectFacts { pub event_exporters: Vec, pub providers: Vec, pub path_default_keys: Vec, + /// Every model path of the whole project when discovery read only some of its models. + pub project_model_paths: Option>, /// Python's `csv.field_size_limit()`, or `None` for its default. pub csv_field_limit: Option, } diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs index c6a9d1bd52..f07daa347f 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs @@ -118,6 +118,19 @@ fn given_discovered_facts_when_validating_then_reports_python_first_conflict() { include: marts", ), }, + ValidationTestCase { + description: "a path default matching only undiscovered project models passes", + facts: ProjectFacts { + models: vec![model_at("orders", "models/marts/orders.sql")], + path_default_keys: vec!["staging".to_owned()], + project_model_paths: Some(vec![ + "models/marts/orders.sql".to_owned(), + "models/staging/customers.sql".to_owned(), + ]), + ..ProjectFacts::default() + }, + expected_failure: Ok(()), + }, ValidationTestCase { description: "accepts Windows-style model paths that match path defaults", facts: path_default_facts(&["models\\staging\\orders.sql"], &["staging"]), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs index 1a1fa34a25..1f2d422c3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -103,6 +103,8 @@ pub(crate) struct DeltaInjection { pub(crate) project_dir: std::path::PathBuf, /// Every model and SQL test file of the whole project, as project-relative POSIX paths. pub(crate) source_paths: BTreeSet, + /// Every model file of the whole project, which path defaults are matched against. + pub(crate) project_model_paths: Vec, /// Every model name of the whole project, which references resolve against. pub(crate) model_names: Vec, /// The whole project's relation facts, injected for relations the delta does not analyse. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index a8a22ee3a2..6051989506 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -292,6 +292,11 @@ fn lent_injection( .map(|model| model.relative_path.clone()) .chain(image.tests.iter().map(|test| test.relative_path.clone())) .collect(), + project_model_paths: image + .models + .iter() + .map(|model| model.relative_path.clone()) + .collect(), model_names: image .models .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index fe0176e6ef..4a8806e1bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -1,5 +1,6 @@ //! The facts cross-file validation reads, taken from Python's `DiscoveredProjectInputs`. +use crate::bindings::_helpers::compile_delta::context::injection; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use pyo3::FromPyObject; @@ -69,6 +70,7 @@ pub(crate) fn project_facts( .try_iter()? .map(|key| key?.extract()) .collect::>()?, + project_model_paths: injection().map(|injection| injection.project_model_paths.clone()), csv_field_limit: Some(csv_field_limit(py)?), }) } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 90ea08b5ec..55c6ad429a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -69,6 +69,12 @@ "COALESCE(b.amount, ", "COALESCE(b.amount, 1 + ", ) +_PATH_DEFAULT_EDIT: tuple[str, str, str] = ( + "sqlbuild_project.toml", + "[defaults]", + '[path_defaults."sales000/intermediate/clean/batch00000"]\nmaterialized = "table"\n\n' + "[defaults]", +) _REUSE_ROOT: str = "target/cache/compiler-native-v1/project-reuse-v3" _EDITED_MODEL: str = ( "models/sales000/intermediate/clean/batch00000/sales000__int_clean__orders_00003.sql" @@ -180,6 +186,12 @@ def test_given_seeded_edit_series_when_compiling_each_edit_then_output_matches_f json_output=True, adapter=_UNTYPED_ADAPTER, ), + DeltaCompileFullCopyTestCase( + description="a path default matching only unchanged models", + stored_edits=(_PATH_DEFAULT_EDIT,), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), DeltaCompileFullCopyTestCase( description="a SQL test expecting another model falls back after its round", edits=((_SMALL_EDITED_TEST, _EXPECTED_MODEL_CTE, _RETARGETED_MODEL_CTE),), From cb2f077db33934a20fc684dd1a0be6e8de4c105d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:22:55 +0100 Subject: [PATCH 32/32] test(compile): accept a delta compile's fresh analysis in the same-runner edit ratio A delta compile analyses only the models it reruns, without the analysis cache, so the ratio script's cache precondition rejected head's one-model edit as an uncached compile. Each run now records how many models its analysis counters account for. An edit run that analysed fewer models than the project has is a delta, and passes when it analysed at least one model afresh. Full compiles keep the existing checks, so a bypassing full compile is still rejected. --- .../_helpers/measure.py | 20 ++++++-- .../compile_performance_ratio/constants.py | 6 +++ scripts/compile_performance_ratio/models.py | 2 + .../_helpers/_test_types.py | 2 + .../_helpers/test_measure.py | 50 ++++++++++++++++++- 5 files changed, 75 insertions(+), 5 deletions(-) diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 1c64a16985..32b4209acc 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -17,6 +17,7 @@ from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit from scripts.compile_performance_ratio.constants import ( + ANALYSED_MODEL_COUNTERS, ANALYSIS_CACHE_BYPASSES, ANALYSIS_CACHE_MISSES, BASE_GENERATOR_ENTRY, @@ -166,7 +167,7 @@ def compare_builds( mode=mode, compile_args=compile_args, ) - check_cache_use(run=run, mode=mode) + check_cache_use(run=run, mode=mode, models=models) results[label].append(run) if mode != COLD_MODE: _, head_python, head_dir = builds[1] @@ -213,13 +214,21 @@ def _comparison( ) -def check_cache_use(*, run: CompileRun, mode: str) -> None: - """Fail a bypassing run, a warm run with misses, or an edit run that analysed nothing.""" +def check_cache_use(*, run: CompileRun, mode: str, models: int) -> None: + """Fail a bypassing full compile, a warm run with misses, or an edit that analysed nothing.""" misses: int | None = run.analysis_cache_misses if misses is None or mode == COLD_MODE: return bypasses: int = run.analysis_cache_bypasses or 0 + delta: bool = run.analysed_models is not None and run.analysed_models < models + if mode == EDIT_MODE and delta: + if misses + bypasses == 0: + raise CompileComparisonError( + f"{run.label} one-model edit delta compile analysed no model afresh, so the " + "edit did not invalidate the edited model" + ) + return if bypasses != 0: raise CompileComparisonError( f"{run.label} {mode} compile bypassed the analysis cache for {bypasses} models, so " @@ -328,6 +337,11 @@ def _compile_once( timings_ms={name: value for name, value in numeric.items() if name in REPORTED_PHASES}, analysis_cache_misses=numeric.get(ANALYSIS_CACHE_MISSES), analysis_cache_bypasses=numeric.get(ANALYSIS_CACHE_BYPASSES), + analysed_models=( + sum(numeric[name] for name in ANALYSED_MODEL_COUNTERS) + if all(name in numeric for name in ANALYSED_MODEL_COUNTERS) + else None + ), report=json.dumps(payload, sort_keys=True), ) diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index 95054308b2..eb7a87d241 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -20,6 +20,12 @@ } ANALYSIS_CACHE_MISSES: str = "analysis_cache_misses" ANALYSIS_CACHE_BYPASSES: str = "analysis_cache_bypasses" +ANALYSED_MODEL_COUNTERS: tuple[str, ...] = ( + "analysis_batch_cache_hits", + "analysis_entry_cache_hits", + ANALYSIS_CACHE_MISSES, + ANALYSIS_CACHE_BYPASSES, +) MODELS_DIRECTORY: str = "models" MODEL_FILE_PATTERN: str = "*.sql" EDIT_COMMENT: str = "-- Benchmark edit {revision}.\n" diff --git a/scripts/compile_performance_ratio/models.py b/scripts/compile_performance_ratio/models.py index 88f83587ba..bfd5b230c4 100644 --- a/scripts/compile_performance_ratio/models.py +++ b/scripts/compile_performance_ratio/models.py @@ -13,6 +13,8 @@ class CompileRun: timings_ms: dict[str, int] analysis_cache_misses: int | None = None analysis_cache_bypasses: int | None = None + analysed_models: int | None = None + """How many models the compile analysed; a delta compile analyses only the models it reran.""" report: str = "" """The JSON report without its timings, which an uncached compile must reproduce.""" diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py index 26693a6d92..79cf282263 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py @@ -51,6 +51,7 @@ class CacheUseTestCase: description: str run: CompileRun mode: str + models: int = 3000 expected_result: None = None @@ -60,6 +61,7 @@ class CacheUseErrorTestCase: run: CompileRun mode: str expected_message: str + models: int = 3000 @dataclass(frozen=True) diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py b/tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py index 84886bf562..1a3c470cb6 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py @@ -78,6 +78,20 @@ def test_given_compile_runs_when_taking_phase_medians_then_reports_only_complete timings_ms={}, analysis_cache_misses=1, analysis_cache_bypasses=0, + analysed_models=3000, + ), + mode=EDIT_MODE, + ), + CacheUseTestCase( + description="a delta edit analyses only the edited model, afresh", + run=CompileRun( + label="head", + wall_seconds=1.0, + cpu_seconds=1.0, + timings_ms={}, + analysis_cache_misses=0, + analysis_cache_bypasses=1, + analysed_models=1, ), mode=EDIT_MODE, ), @@ -110,7 +124,10 @@ def test_given_compile_runs_when_taking_phase_medians_then_reports_only_complete def test_given_observed_cache_use_when_checking_then_accepts_the_run( test_case: CacheUseTestCase, ) -> None: - assert check_cache_use(run=test_case.run, mode=test_case.mode) is test_case.expected_result + assert ( + check_cache_use(run=test_case.run, mode=test_case.mode, models=test_case.models) + is test_case.expected_result + ) @pytest.mark.parametrize( @@ -138,10 +155,39 @@ def test_given_observed_cache_use_when_checking_then_accepts_the_run( timings_ms={}, analysis_cache_misses=0, analysis_cache_bypasses=3000, + analysed_models=3000, ), mode=EDIT_MODE, expected_message="bypassed the analysis cache for 3000 models", ), + CacheUseErrorTestCase( + description="a delta edit that analysed nothing afresh did not observe the edit", + run=CompileRun( + label="head", + wall_seconds=1.0, + cpu_seconds=1.0, + timings_ms={}, + analysis_cache_misses=0, + analysis_cache_bypasses=0, + analysed_models=1, + ), + mode=EDIT_MODE, + expected_message="delta compile analysed no model afresh", + ), + CacheUseErrorTestCase( + description="a warm compile that bypassed the cache, even for few models, is rejected", + run=CompileRun( + label="head", + wall_seconds=1.0, + cpu_seconds=1.0, + timings_ms={}, + analysis_cache_misses=0, + analysis_cache_bypasses=1, + analysed_models=1, + ), + mode=WARM_MODE, + expected_message="bypassed the analysis cache for 1 models", + ), CacheUseErrorTestCase( description="a warm compile that missed the cache is not an unchanged compile", run=CompileRun( @@ -162,7 +208,7 @@ def test_given_unobserved_cache_use_when_checking_then_rejects_the_run( test_case: CacheUseErrorTestCase, ) -> None: with pytest.raises(CompileComparisonError, match=test_case.expected_message): - check_cache_use(run=test_case.run, mode=test_case.mode) + check_cache_use(run=test_case.run, mode=test_case.mode, models=test_case.models) @pytest.mark.parametrize(