diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2d8fc4e0d6..4430b01a54 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -571,7 +571,8 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 20 + # The 3k and 10k cells also run the delta edit-loop guard, about five minutes at 10k. + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -592,7 +593,7 @@ jobs: run: uv python install 3.12 - name: Install dependencies run: uv sync --locked --all-groups --all-extras - - name: Run dense all-rules guard under 4 GiB + - name: Run dense all-rules and delta edit-loop guards under 4 GiB env: BENCHMARK_MODELS: ${{ matrix.models }} MEMORY_LIMIT_BYTES: "4294967296" diff --git a/Cargo.lock b/Cargo.lock index 37ac81b26c..2dc2a64115 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1244,6 +1244,7 @@ dependencies = [ "mimalloc", "pyo3", "rayon", + "rustc-hash", "serde", "serde_json", "sqlbuild-analysis", @@ -1305,6 +1306,7 @@ dependencies = [ "sha2", "sqlbuild-analysis", "sqlbuild-attachments", + "sqlbuild-cache", "sqlbuild-core", "sqlbuild-scopes", "sqlbuild-sqltext", diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs index 47310708ff..e604264fd8 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/catalog_state.rs @@ -115,6 +115,17 @@ impl SessionCatalog { self.schemas.get(name) } + /// Every closed shape the catalog holds, sorted by name. + pub(crate) fn schema_rows(&self) -> Shapes { + let mut rows: Shapes = self + .schemas + .iter() + .map(|(name, shape)| (name.clone(), shape.clone())) + .collect(); + rows.sort_by(|left, right| left.0.cmp(&right.0)); + rows + } + /// `(shared members, shared members re-analysed alone)` so far. pub(crate) fn sharing(&self) -> (usize, usize) { (self.shared_members, self.reanalysed_members) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 74e8b174c9..66f3c1196b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -33,7 +33,7 @@ use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_ use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, DeferredCleaning, FinishedSession, LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables, - SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, + RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::ProjectCatalog; @@ -49,13 +49,18 @@ impl AnalysisSession { .ok_or_else(|| native_failure(SESSION_CONTEXT, "two analysed models share a name"))?; let scheduled: Option>> = waves(&producers); let complete_shapes: ShapeTable = ShapeTable::from_shapes(&request.complete_schemas); - let dependency_ordered: bool = scheduled.is_some() - && request - .models - .iter() - .flat_map(|model| model.required_names.iter()) - .any(|name| complete_shapes.get(name).is_none_or(Vec::is_empty)); - let cyclic: bool = scheduled.is_none(); + let (dependency_ordered, cyclic): (bool, bool) = match request.forced_order { + Some(order) => order, + None => ( + scheduled.is_some() + && request + .models + .iter() + .flat_map(|model| model.required_names.iter()) + .any(|name| complete_shapes.get(name).is_none_or(Vec::is_empty)), + scheduled.is_none(), + ), + }; let waves: Vec> = scheduled.unwrap_or_default(); Ok(Self { catalog: SessionCatalog::new(catalog, &request.catalog_schemas), @@ -76,6 +81,7 @@ impl AnalysisSession { publications: Vec::new(), failures: Vec::new(), cache: None, + legacy_models: Vec::new(), request, }) } @@ -118,6 +124,16 @@ impl AnalysisSession { .into_iter() .collect::>() .ok_or("the session left a model unanalysed")?; + let relations: Option = + self.request.retain_relations.then(|| RelationTables { + types: owned_rows(&self.available_types), + nullability: owned_rows(&self.available_nullability), + complete: owned_rows(&self.complete_shapes), + schemas: self.catalog.schema_rows(), + dependency_ordered: self.dependency_ordered, + cyclic: self.cyclic, + legacy_models: self.legacy_models.clone(), + }); let (schema_additions, analysis_names) = self.catalog.into_changes(); let mut request: SessionRequest = self.request; for ((outcome, model), pending) in models @@ -155,6 +171,7 @@ impl AnalysisSession { pool, models: model_facts, dynamic_proofs, + relations, }; let outcome = SessionOutcome { models, @@ -254,7 +271,9 @@ impl AnalysisSession { let legacy: LegacyAnalysis = self.native_legacy(request, &result.cleaned_sql) .map_err(|reason| native_failure(&model_context(request), &reason))?; + let name: String = request.name.clone(); self.record_legacy_tables(*model, &mut tables); + self.legacy_models.push(name); legacy_outcome(legacy, lineage, diagnostics) } }; @@ -669,6 +688,15 @@ fn batch_member<'a>(model: &'a ModelRequest, schema: &'a Shapes) -> BatchMember< } } +/// A table's shapes in insertion order, owned. +fn owned_rows(table: &ShapeTable) -> Shapes { + table + .ordered() + .into_iter() + .map(|(name, shape)| (name.to_owned(), shape.clone())) + .collect() +} + /// Python's `{column.name: column.type or "UNKNOWN"}`. pub(crate) fn shape_columns(columns: &[ColumnFact]) -> Pairs { dict_from_pairs(columns.iter().map(shape_column)) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index c5a18ee5d3..3a625a8170 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -107,6 +107,8 @@ pub struct FinishedSession { pub(crate) models: HashMap, /// Each model's dynamic pivot proof from the session's own relation facts, by model name. pub(crate) dynamic_proofs: HashMap, + /// The relation facts the session ended with, when the request asked to keep them. + pub(crate) relations: Option, } /// The session's typed result for one compiled model, borrowed from the finished session. @@ -121,6 +123,12 @@ pub struct SessionModelAnalysis<'a> { } impl FinishedSession { + /// The relation facts the session ended with, when its request asked to keep them. + #[must_use] + pub fn relation_tables(&self) -> Option<&RelationTables> { + self.relations.as_ref() + } + /// The session's typed columns and dynamic proof for model `name`, without re-analysis. #[must_use] pub fn model_analysis(&self, name: &str) -> SessionModelAnalysis<'_> { @@ -173,6 +181,24 @@ pub struct SessionRequest { /// Python's `dynamic_families_by_table`, in dict order. pub dynamic_families_by_table: Vec<(String, Vec)>, pub models: Vec, + /// The whole project's `(dependency ordered, cyclic)`, for a session of only some models. + pub forced_order: Option<(bool, bool)>, + /// Keep the relation facts the session ends with, for the project image. + pub retain_relations: bool, +} + +/// The relation facts a finished session ends with, and how it ordered its analysis. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RelationTables { + pub types: Shapes, + pub nullability: Shapes, + pub complete: Shapes, + /// The binding catalog's closed shapes, sorted by name. + pub schemas: Shapes, + pub dependency_ordered: bool, + pub cyclic: bool, + /// Models whose analysis took the legacy path, which reads every relation's facts. + pub legacy_models: Vec, } /// One inferred output column; nullability is Python's `InferredNullability` value. @@ -323,6 +349,8 @@ pub struct AnalysisSession { pub(crate) publications: Shapes, pub(crate) failures: Vec, pub(crate) cache: Option, + /// Models whose analysis took the legacy path. + pub(crate) legacy_models: Vec, } /// Which of Python's column analyses one query outside the session takes. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs index b4f9db0702..95de9dfe5c 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs @@ -4,6 +4,8 @@ mod analysis_cache; mod analysis_session; #[path = "tests/test_contract_proof_json.rs"] mod contract_proof_json; +#[path = "tests/test_delta_session.rs"] +mod delta_session; #[path = "tests/test_expression_shapes.rs"] mod expression_shapes; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 16b5f60f76..28908a0405 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -1,3 +1,4 @@ +use crate::assembly::analysis_session::models::RelationTables; use std::collections::HashMap; use crate::assembly::analysis_session::_helpers::cte_facts::{ @@ -123,6 +124,8 @@ pub(crate) fn orders_request(models: Vec) -> SessionRequest { nullability_rules: Some(Vec::new()), nullability_callback: None, rich_type_inference: true, + forced_order: None, + retain_relations: false, column_types: raw_orders.clone(), column_nullability: shapes(&[( "raw_orders", @@ -384,6 +387,7 @@ pub(crate) fn finished_session(tables: PivotTables) -> FinishedSession { pool: catalog("duckdb", &Vec::new()).analysis_pool(), models: HashMap::new(), dynamic_proofs: HashMap::new(), + relations: None, } } @@ -654,3 +658,30 @@ pub(crate) fn column_view(finished: &ModelColumns) -> ColumnView<'_> { finished.proven_star, ) } + +/// `request` with `relations`' facts added for every relation it does not analyse. +pub(crate) fn injected_relations( + mut request: SessionRequest, + relations: &RelationTables, +) -> SessionRequest { + let analysed: Vec = request + .models + .iter() + .map(|model| model.name.clone()) + .collect(); + for (table, facts) in [ + (&mut request.column_types, &relations.types), + (&mut request.column_nullability, &relations.nullability), + (&mut request.complete_schemas, &relations.complete), + ] { + let present: Vec = table.iter().map(|(name, _)| name.clone()).collect(); + table.extend( + facts + .iter() + .filter(|(name, _)| !analysed.contains(name) && !present.contains(name)) + .cloned(), + ); + } + request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); + request +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs new file mode 100644 index 0000000000..2588501a63 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_delta_session.rs @@ -0,0 +1,60 @@ +use crate::assembly::analysis_session::models::{ModelRequest, RelationTables, SessionRequest}; +use crate::assembly::analysis_session::tests::helpers::{ + completed, injected_relations, model_requests, orders_request, started, +}; +use crate::assembly::analysis_session::tests::test_types::{DeltaSessionTestCase, ModelSpec}; + +const ORDER_MODELS: [ModelSpec; 3] = [ + ( + "stg_orders", + "SELECT order_id, amount * 2 AS doubled FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], + ), + ( + "orders_mart", + "SELECT doubled, order_id + 1 AS next_id FROM __ref(\"stg_orders\")", + &[], + &["stg_orders"], + ), + ( + "orders_star", + "SELECT * FROM __ref(\"stg_orders\")", + &[], + &["stg_orders"], + ), +]; + +#[test] +fn given_whole_project_relations_when_analysing_one_model_then_it_matches_the_whole_session() { + let test_cases = [ + DeltaSessionTestCase { + description: "a model reading another model's columns", + model_index: 1, + expected_whole_session_outcome: true, + }, + DeltaSessionTestCase { + description: "a model selecting every column of another model", + model_index: 2, + expected_whole_session_outcome: true, + }, + ]; + let mut whole: SessionRequest = orders_request(model_requests(&ORDER_MODELS)); + whole.retain_relations = true; + let (_, whole_outcomes, finished) = completed(started(whole)); + let relations: RelationTables = finished.relation_tables().cloned().expect("relations kept"); + for test_case in test_cases { + let spec: &ModelSpec = &ORDER_MODELS[test_case.model_index]; + let one: Vec = model_requests(std::slice::from_ref(spec)); + let request: SessionRequest = injected_relations(orders_request(one), &relations); + + let (_, outcomes, _) = completed(started(request)); + + assert_eq!( + outcomes == vec![whole_outcomes[test_case.model_index].clone()], + test_case.expected_whole_session_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index 21e234622c..c74ecd0b9e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -201,3 +201,10 @@ pub(crate) struct ContractProofJsonTestCase { pub(crate) proof: ContractProof, pub(crate) expected_json: &'static str, } + +/// One model of a session analysed alone against the whole session's relation facts. +pub(crate) struct DeltaSessionTestCase { + pub(crate) description: &'static str, + pub(crate) model_index: usize, + pub(crate) expected_whole_session_outcome: bool, +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs index 689056c288..0ceec82d72 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs @@ -8,24 +8,31 @@ use crate::project_reuse::_helpers::entry::{ read_output, read_slot, remove_entry, rewrite_inputs, write_entry, }; use crate::project_reuse::_helpers::files::{ - carried_forward_digests, compare_project_files, needs_refresh, restamped_paths, - stored_project_files, with_missing_digests, + carried_forward_digests, compare_project_files, needs_refresh, project_file_changes, + restamped_paths, stored_project_files, unverified_paths, with_missing_digests, }; use crate::project_reuse::_helpers::identity::{ environment_digest, module_stamps, module_stamps_unchanged, search_path_stamps, settings_inputs_digest, tracked_environment_names, }; +use crate::project_reuse::_helpers::image::{image_path, mark_used}; use crate::project_reuse::_helpers::target::{target_files_unchanged, verified_target_files}; use crate::project_reuse::_helpers::timings::compile_timings_span; use crate::project_reuse::errors::ReuseRecordError; use crate::project_reuse::models::{ - CompileRecord, RecordOutcome, RecordResult, ReuseAttempt, ReuseCheck, ReuseOutcome, ReuseRules, - SettingsInputs, StoredInputs, StoredOutput, + CompileRecord, DeltaCandidate, RecordOutcome, RecordResult, ReuseAttempt, ReuseCheck, + ReuseOutcome, ReuseRules, SettingsInputs, StoredInputs, StoredOutput, }; use crate::project_snapshot::main::snapshot_project_files::snapshot_project_files; use crate::project_snapshot::main::text_path::text_path; use crate::project_snapshot::models::PathStamp; +const NO_STORED_COMPILE: &str = "no stored compile"; +const REUSE_BYPASSED: &str = "compile reuse is off"; +const OUTPUT_UNREADABLE: &str = "the stored output cannot be replayed"; +const IDENTITY_CHANGED: &str = "the invocation, runtime, environment or settings changed"; +const TARGET_CHANGED: &str = "the compiled target changed"; + fn now_ns() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -65,8 +72,10 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { replay: None, replay_failed: false, digested_files: 0, + delta: DeltaCandidate::Unavailable(NO_STORED_COMPILE), }; if bypassed(&check) { + attempt.delta = DeltaCandidate::Unavailable(REUSE_BYPASSED); return attempt; } attempt.outcome = ReuseOutcome::Miss; @@ -87,6 +96,7 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { return attempt; } attempt.outcome = ReuseOutcome::Hit; + mark_used(&image_path(&check.store_path)); if needs_refresh(&inputs.project_files, &attempt.snapshot) { let refreshed: StoredInputs = StoredInputs { project_files: stored_project_files( @@ -103,6 +113,29 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { return attempt; } attempt.restamped = restamped_paths(&inputs.project_files, &attempt.snapshot); + if comparison.unchanged { + attempt.delta = DeltaCandidate::Unavailable(OUTPUT_UNREADABLE); + } else if !comparison.identity_unchanged { + attempt.delta = DeltaCandidate::Unavailable(IDENTITY_CHANGED); + } else if !target_files_unchanged( + &inputs.target_files, + &attempt.project_dir, + inputs.target_tree, + rules, + ) { + attempt.delta = DeltaCandidate::Unavailable(TARGET_CHANGED); + } else { + let unverified: HashSet = + unverified_paths(&inputs.project_files, &attempt.snapshot); + let known: usize = attempt.digests.len(); + attempt.digests = with_missing_digests(&attempt, &unverified, rules.racy_window_ns); + attempt.digested_files += attempt.digests.len() - known; + attempt.delta = DeltaCandidate::Files(project_file_changes( + &inputs.project_files, + &attempt.snapshot, + (&attempt.digests, &unverified), + )); + } } let known: usize = attempt.digests.len(); attempt.digests = with_missing_digests(&attempt, &HashSet::new(), rules.racy_window_ns); @@ -113,6 +146,7 @@ pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { /// Whether a stored compile still matches, with the digests still valid either way. struct Comparison { unchanged: bool, + identity_unchanged: bool, digests: HashMap, digested_files: usize, } @@ -136,6 +170,7 @@ fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> (false, HashMap::new(), 0) }; Comparison { + identity_unchanged, unchanged: files_unchanged && target_files_unchanged( &inputs.target_files, @@ -228,8 +263,7 @@ fn store( } write_entry( &attempt.store_path, - &inputs, - &output, + (&inputs, &output, record.image.as_ref()), rules.max_stored_entries, )?; Ok(RecordResult { diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs index 4f6bf849e1..c042aea9ae 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs @@ -4,11 +4,17 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use crate::digest::types::ContentDigest; +use crate::project_reuse::_helpers::image::{image_path, prune_images, read_image, write_image}; use crate::project_reuse::_helpers::sections::{ SectionFile, decode_sections, encode_sections, write_atomically, }; -use crate::project_reuse::constants::{REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX}; -use crate::project_reuse::models::{SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile}; +use crate::project_reuse::constants::{ + REUSE_MAX_STORED_IMAGES, REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX, +}; +use crate::project_reuse::models::{ + DeltaBase, SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile, +}; +use crate::project_reuse::types::ImageSections; use crate::project_snapshot::models::PathStamp; use crate::store::errors::StoreDecodeError; use crate::store::models::{RecordReader, RecordWriter}; @@ -20,6 +26,7 @@ const INPUTS_SECTION: &str = "inputs"; const OUTPUT_IDENTITY_SECTION: &str = "output"; const OUTPUT_META_SECTION: &str = "meta"; const STDOUT_SECTION: &str = "stdout"; +const IMAGE_IDENTITY_SECTION: &str = "image"; /// How long an output with no slot is kept, so a compile still writing its slot keeps it. const ORPHANED_OUTPUT_GRACE: Duration = Duration::from_secs(600); @@ -28,6 +35,7 @@ const ORPHANED_OUTPUT_GRACE: Duration = Duration::from_secs(600); pub(crate) struct StoredSlot { pub(crate) inputs: StoredInputs, output_identity: ContentDigest, + image_identity: Option, sections: Vec<(String, Vec)>, } @@ -54,9 +62,14 @@ fn decoded_slot(bytes: &[u8]) -> Result { let output_identity: ContentDigest = required(&file, OUTPUT_IDENTITY_SECTION)? .try_into() .map_err(|_| StoreDecodeError)?; + let image_identity: Option = match file.section(IMAGE_IDENTITY_SECTION) { + Some(bytes) => Some(bytes.try_into().map_err(|_| StoreDecodeError)?), + None => None, + }; Ok(StoredSlot { inputs, output_identity, + image_identity, sections: file .sections .into_iter() @@ -103,31 +116,76 @@ pub(crate) fn read_entry(path: &Path) -> Option<(StoredInputs, StoredOutput)> { Some((slot.inputs, output)) } -/// Replace the slot with one stored compile, then keep at most `max_entries` slots. +/// The stored inputs and image of the slot, or `None` when either cannot be read. +pub(crate) fn read_delta_base(path: &Path) -> Option { + let slot: StoredSlot = read_slot(path)?; + let image: ImageSections = read_slot_image(path, &slot)?; + Some(DeltaBase { + inputs: slot.inputs, + image, + output_identity: slot.output_identity, + }) +} + +/// The output of the stored compile `base` was read from, or `None` when it no longer is. +pub(crate) fn read_delta_output(path: &Path, base: &DeltaBase) -> Option { + let bytes: Vec = match std::fs::read(output_path(path)) { + Ok(bytes) => bytes, + Err(_unreadable) => return None, + }; + let Ok(output) = decoded_output(&bytes, &base.output_identity) else { + return None; + }; + Some(output) +} + +/// The image the slot names, or `None` when it has none or it cannot be read. +pub(crate) fn read_slot_image(path: &Path, slot: &StoredSlot) -> Option { + read_image(path, slot.image_identity.as_ref()?) +} + +/// Replace the slot with one stored compile and its image, then keep at most `max_entries` slots. pub(crate) fn write_entry( path: &Path, - inputs: &StoredInputs, - output: &StoredOutput, + (inputs, output, image): (&StoredInputs, &StoredOutput, Option<&ImageSections>), max_entries: usize, ) -> std::io::Result<()> { - let meta: Vec = encode_output(output); - let (encoded, identity) = encode_sections( - OUTPUT_ROLE, - &[ - (OUTPUT_META_SECTION, &meta), - (STDOUT_SECTION, output.stdout.as_bytes()), - ], - ); - write_atomically(&output_path(path), &encoded)?; - drop(encoded); - write_slot( - path, - &[ - (INPUTS_SECTION, &encode_inputs(inputs)), - (OUTPUT_IDENTITY_SECTION, &identity), - ], - )?; + let write_output = || -> std::io::Result { + let meta: Vec = encode_output(output); + let (encoded, identity) = encode_sections( + OUTPUT_ROLE, + &[ + (OUTPUT_META_SECTION, &meta), + (STDOUT_SECTION, output.stdout.as_bytes()), + ], + ); + write_atomically(&output_path(path), &encoded)?; + Ok(identity) + }; + let write_named_image = || -> std::io::Result> { + match image { + Some(sections) => Ok(Some(write_image(path, sections)?)), + None => { + let _ = std::fs::remove_file(image_path(path)); + Ok(None) + } + } + }; + let (identity, image_identity) = rayon::join(write_output, write_named_image); + let (identity, image_identity) = (identity?, image_identity?); + let encoded_inputs: Vec = encode_inputs(inputs); + let mut sections: Vec<(&str, &[u8])> = vec![ + (INPUTS_SECTION, &encoded_inputs), + (OUTPUT_IDENTITY_SECTION, &identity), + ]; + if let Some(image_identity) = &image_identity { + sections.push((IMAGE_IDENTITY_SECTION, image_identity)); + } + write_slot(path, §ions)?; prune_entries(path, max_entries); + if let Some(directory) = path.parent() { + prune_images(directory, path, REUSE_MAX_STORED_IMAGES); + } Ok(()) } @@ -153,10 +211,11 @@ fn write_slot(path: &Path, sections: &[(&str, &[u8])]) -> std::io::Result<()> { write_atomically(path, &encoded) } -/// Remove the slot and its output. +/// Remove the slot, its output and its image. pub(crate) fn remove_entry(path: &Path) { let _ = std::fs::remove_file(path); let _ = std::fs::remove_file(output_path(path)); + let _ = std::fs::remove_file(image_path(path)); } /// Keep at most `max_entries` slots, newest first, and remove outputs no slot names any more. diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs index 69c98dbfb4..ce48822f3e 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use crate::digest::main::digest_files::digest_files; use crate::digest::main::hex_digest::hex_digest; use crate::project_reuse::constants::NATIVE_DIGEST_PREFIX; -use crate::project_reuse::models::{ReuseAttempt, StoredProjectFile}; +use crate::project_reuse::models::{ProjectFileChanges, ReuseAttempt, StoredProjectFile}; use crate::project_snapshot::main::text_path::text_path; use crate::project_snapshot::models::PathStamp; @@ -202,3 +202,60 @@ pub(crate) fn pending_digest_bytes( .map(|stamp| stamp.size) .sum() } + +/// Paths whose content, kind or presence differs; files in `unverified` compare by digest. +pub(crate) fn project_file_changes( + stored: &[StoredProjectFile], + current: &[PathStamp], + (digests, unverified): (&HashMap, &HashSet), +) -> ProjectFileChanges { + let previous_by_path: HashMap<&str, &StoredProjectFile> = by_path(stored); + let mut changes: ProjectFileChanges = ProjectFileChanges::default(); + let mut seen: HashSet<&str> = HashSet::with_capacity(current.len()); + for stamp in current { + let path: &str = stamp.relative_path.as_str(); + seen.insert(path); + let Some(previous) = previous_by_path.get(path) else { + changes.added.push(stamp.relative_path.clone()); + continue; + }; + if previous.stamp.kind != stamp.kind || previous.stamp.link != stamp.link { + changes.changed.push(stamp.relative_path.clone()); + continue; + } + if !hashable(stamp) || !unverified.contains(path) { + continue; + } + let current_digest: Option<&String> = digests.get(path); + if current_digest.is_none() || current_digest != previous.digest.as_ref() { + changes.changed.push(stamp.relative_path.clone()); + } + } + for file in stored { + if !seen.contains(file.stamp.relative_path.as_str()) { + changes.removed.push(file.stamp.relative_path.clone()); + } + } + changes.changed.sort(); + changes.added.sort(); + changes.removed.sort(); + changes +} + +/// Hashable current files whose stored digest may no longer apply: restamped or stored racy. +pub(crate) fn unverified_paths( + stored: &[StoredProjectFile], + current: &[PathStamp], +) -> HashSet { + let previous_by_path: HashMap<&str, &StoredProjectFile> = by_path(stored); + current + .iter() + .filter(|stamp| hashable(stamp)) + .filter(|stamp| { + previous_by_path + .get(stamp.relative_path.as_str()) + .is_some_and(|previous| previous.racy || previous.stamp != **stamp) + }) + .map(|stamp| stamp.relative_path.clone()) + .collect() +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs new file mode 100644 index 0000000000..b4b5cb273a --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/image.rs @@ -0,0 +1,94 @@ +//! The project image beside a slot: the facts a delta compile patches, kept for recent slots only. + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use crate::digest::types::ContentDigest; +use crate::project_reuse::_helpers::sections::{ + decode_sections, encode_sections, write_atomically, +}; +use crate::project_reuse::constants::{ + REUSE_IMAGE_SUFFIX, REUSE_OUTPUT_SUFFIX, REUSE_STORE_SUFFIX, +}; +use crate::project_reuse::types::ImageSections; + +const IMAGE_ROLE: &str = "project-reuse-image"; + +/// The image file paired with the slot file at `path`. +pub(crate) fn image_path(path: &Path) -> PathBuf { + path.with_extension(REUSE_IMAGE_SUFFIX) +} + +/// Write `sections` as the slot's image; returns the identity the slot names it by. +pub(crate) fn write_image(path: &Path, sections: &ImageSections) -> std::io::Result { + let borrowed: Vec<(&str, &[u8])> = sections + .iter() + .map(|(name, bytes)| (name.as_str(), bytes.as_slice())) + .collect(); + let (encoded, identity) = encode_sections(IMAGE_ROLE, &borrowed); + write_atomically(&image_path(path), &encoded)?; + Ok(identity) +} + +/// The slot's image, marked as used, or `None` when it is missing, damaged or another compile's. +pub(crate) fn read_image(path: &Path, identity: &ContentDigest) -> Option { + let file_path: PathBuf = image_path(path); + let bytes: Vec = match std::fs::read(&file_path) { + Ok(bytes) => bytes, + Err(_unreadable) => return None, + }; + let Ok(file) = decode_sections(&bytes, IMAGE_ROLE) else { + return None; + }; + if file.identity != *identity { + return None; + } + let sections: ImageSections = file + .sections + .into_iter() + .map(|(name, bytes)| (name, bytes.to_vec())) + .collect(); + mark_used(&file_path); + Some(sections) +} + +/// Mark an image as just used; failing to is harmless and only affects pruning order. +pub(crate) fn mark_used(path: &Path) { + if let Ok(file) = std::fs::OpenOptions::new().append(true).open(path) { + let _ = file.set_modified(SystemTime::now()); + } +} + +/// Keep the images of the `keep_images` most recently used slots, and drop images no slot names. +pub(crate) fn prune_images(directory: &Path, keep: &Path, keep_images: usize) { + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + let keep_image: PathBuf = image_path(keep); + let mut images: Vec<(SystemTime, PathBuf)> = Vec::new(); + for path in entries.flatten().map(|entry| entry.path()) { + if !path + .extension() + .is_some_and(|suffix| suffix == REUSE_IMAGE_SUFFIX) + { + continue; + } + let slot: PathBuf = path.with_extension(REUSE_STORE_SUFFIX); + let output: PathBuf = path.with_extension(REUSE_OUTPUT_SUFFIX); + if !slot.exists() && !output.exists() { + let _ = std::fs::remove_file(&path); + continue; + } + if path == keep_image { + continue; + } + let Ok(modified) = std::fs::metadata(&path).and_then(|metadata| metadata.modified()) else { + continue; + }; + images.push((modified, path)); + } + images.sort_by(|left, right| right.cmp(left)); + for (_, stale) in images.into_iter().skip(keep_images.saturating_sub(1)) { + let _ = std::fs::remove_file(stale); + } +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs index 30f44c1302..0556229271 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod attempt; pub(crate) mod entry; pub(crate) mod files; pub(crate) mod identity; +pub(crate) mod image; pub(crate) mod sections; pub(crate) mod target; pub(crate) mod timings; diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs index edfecaebac..649a4737ae 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs @@ -1,6 +1,6 @@ //! Stat identity of the artifacts a compile leaves in target/. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use crate::project_reuse::_helpers::files::file_digest; @@ -96,6 +96,26 @@ fn matches_recorded( } } +/// Each expected artifact's stamp: vouched when kept as vouched for, otherwise read. +fn vouched_stamps(expected: &BTreeMap<&str, &RecordedArtifact>, vouched: &[PathStamp]) -> Stamps { + let vouched: HashMap<&str, &PathStamp> = vouched + .iter() + .map(|stamp| (stamp.relative_path.as_str(), stamp)) + .collect(); + expected + .iter() + .filter_map(|(path, artifact)| match artifact { + RecordedArtifact::Kept { size, mtime_ns } => match vouched.get(path) { + Some(stamp) if stamp.size == *size && stamp.mtime_ns == *mtime_ns => { + Some(((*path).to_owned(), (*stamp).clone())) + } + _ => keyed_stamp(path), + }, + RecordedArtifact::Written { .. } => keyed_stamp(path), + }) + .collect() +} + /// Stamp this compile's artifacts, or `None` when any changed since it wrote them. pub(crate) fn verified_target_files( project_dir: &Path, @@ -113,7 +133,10 @@ pub(crate) fn verified_target_files( .map(|(path, artifact)| (path.as_str(), artifact)) .collect(); let paths: Vec<&str> = expected.keys().copied().collect(); - let stamps: Stamps = target_stamps(project_dir, &paths, written, rules); + let stamps: Stamps = match &record.vouched_target_stamps { + Some(vouched) => vouched_stamps(&expected, vouched), + None => target_stamps(project_dir, &paths, written, rules), + }; let keys_match: bool = stamps.len() == expected.len() && stamps .keys() diff --git a/crates/sqlbuild-cache/src/project_reuse/constants.rs b/crates/sqlbuild-cache/src/project_reuse/constants.rs index c5407b5c90..17ffaad603 100644 --- a/crates/sqlbuild-cache/src/project_reuse/constants.rs +++ b/crates/sqlbuild-cache/src/project_reuse/constants.rs @@ -2,5 +2,9 @@ pub const REUSE_STORE_SUFFIX: &str = "store"; /// Suffix of the file beside each slot file holding the stored compile's output. pub const REUSE_OUTPUT_SUFFIX: &str = "output"; +/// Suffix of the file beside a recently used slot holding the facts a delta compile patches. +pub const REUSE_IMAGE_SUFFIX: &str = "image"; +/// How many of the most recently used slots keep their image. +pub const REUSE_MAX_STORED_IMAGES: usize = 2; /// Prefix of digests computed natively, so they never equal a Python digest. pub const NATIVE_DIGEST_PREFIX: &str = "b3:"; diff --git a/crates/sqlbuild-cache/src/project_reuse/main/mod.rs b/crates/sqlbuild-cache/src/project_reuse/main/mod.rs index f6d51418fe..ba6988e321 100644 --- a/crates/sqlbuild-cache/src/project_reuse/main/mod.rs +++ b/crates/sqlbuild-cache/src/project_reuse/main/mod.rs @@ -1,4 +1,6 @@ pub mod check_reuse; pub mod pending_digest_bytes; +pub mod read_delta_base; +pub mod read_delta_output; pub mod record_reuse; pub mod replay_stdout; diff --git a/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs new file mode 100644 index 0000000000..f4347e59f8 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_base.rs @@ -0,0 +1,8 @@ +use std::path::Path; + +use crate::project_reuse::models::DeltaBase; + +/// The stored compile at `store_path` with its image, the base a delta compile patches. +pub fn read_delta_base(store_path: &Path) -> Option { + crate::project_reuse::_helpers::entry::read_delta_base(store_path) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/main/read_delta_output.rs b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_output.rs new file mode 100644 index 0000000000..ebbb734700 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/read_delta_output.rs @@ -0,0 +1,8 @@ +use std::path::Path; + +use crate::project_reuse::models::{DeltaBase, StoredOutput}; + +/// The output of the stored compile `base` was read from, or `None` when it no longer is. +pub fn read_delta_output(store_path: &Path, base: &DeltaBase) -> Option { + crate::project_reuse::_helpers::entry::read_delta_output(store_path, base) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/mod.rs b/crates/sqlbuild-cache/src/project_reuse/mod.rs index 2a66cc14e3..76910b98c4 100644 --- a/crates/sqlbuild-cache/src/project_reuse/mod.rs +++ b/crates/sqlbuild-cache/src/project_reuse/mod.rs @@ -5,6 +5,7 @@ pub mod constants; pub mod errors; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-cache/src/project_reuse/models.rs b/crates/sqlbuild-cache/src/project_reuse/models.rs index 7a2ffde470..893ec924f6 100644 --- a/crates/sqlbuild-cache/src/project_reuse/models.rs +++ b/crates/sqlbuild-cache/src/project_reuse/models.rs @@ -1,8 +1,10 @@ //! The stored compile, the facts it is checked against, and one invocation's reuse state. +use crate::project_reuse::types::ImageSections; use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::PathBuf; +use crate::digest::types::ContentDigest; use crate::project_snapshot::models::{PathStamp, SnapshotRules}; /// Constants the Python host owns, passed once so both sides agree. @@ -69,6 +71,31 @@ pub struct StoredInputs { pub settings_digest: String, } +/// The stored compile a delta compile patches: its inputs and image, and which output it names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeltaBase { + pub inputs: StoredInputs, + pub image: ImageSections, + pub output_identity: ContentDigest, +} + +/// The project paths that differ from the stored compile's, each list sorted. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ProjectFileChanges { + pub changed: Vec, + pub added: Vec, + pub removed: Vec, +} + +/// Whether a missed check may be compiled as a delta of the stored compile, and from what. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeltaCandidate { + /// Only project file contents differ from the stored compile. + Files(ProjectFileChanges), + /// Why the stored compile cannot be the base of a delta compile. + Unavailable(&'static str), +} + /// The command output of one stored compile. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct StoredOutput { @@ -105,6 +132,8 @@ pub struct ReuseAttempt { pub replay_failed: bool, /// Project files the check read to compare or complete content digests. pub digested_files: usize, + /// Whether a miss may be compiled as a delta of the stored compile. + pub delta: DeltaCandidate, } /// What the host asks of one reuse check. @@ -137,6 +166,10 @@ pub struct CompileRecord { pub artifacts_written: bool, pub dag_path: Option, pub json_output: bool, + /// The project image to keep beside the stored compile, when one was built. + pub image: Option, + /// Verified stamps of kept artifacts, which spare recording a walk of the compiled tree. + pub vouched_target_stamps: Option>, } /// What one compile left at one artifact path. diff --git a/crates/sqlbuild-cache/src/project_reuse/tests.rs b/crates/sqlbuild-cache/src/project_reuse/tests.rs index 98085a469c..66a11ad735 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests.rs @@ -2,6 +2,8 @@ mod file_changes; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_project_image.rs"] +mod project_image; #[path = "tests/test_project_reuse.rs"] mod project_reuse; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs b/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs index 14b3da8483..4359a23b90 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs @@ -1,9 +1,10 @@ use std::collections::BTreeMap; -use std::path::Path; +use std::path::{Path, PathBuf}; use crate::project_reuse::_helpers::entry::write_entry; use crate::project_reuse::_helpers::sections::{decode_sections, encode_sections}; use crate::project_reuse::models::{SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile}; +use crate::project_reuse::types::ImageSections; use crate::project_snapshot::models::PathStamp; pub(crate) fn stamp(relative_path: &str, kind: &'static str, link: Option<&str>) -> PathStamp { @@ -64,7 +65,12 @@ pub(crate) fn stored_output() -> StoredOutput { pub(crate) fn store_aged_slots(directory: &Path, slots: u64, max_entries: usize) { for slot in 0..slots { let path = directory.join(format!("slot{slot}.store")); - write_entry(&path, &stored_inputs(), &stored_output(), max_entries).expect("stored"); + write_entry( + &path, + (&stored_inputs(), &stored_output(), None), + max_entries, + ) + .expect("stored"); let stamp = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(slot + 1); std::fs::File::options() .write(true) @@ -123,7 +129,7 @@ pub(crate) fn replace_output(path: &Path) { stdout: "{\"command\": \"build\"}\n".to_owned(), ..stored_output() }; - write_entry(&other, &stored_inputs(), &output, usize::MAX).expect("other compile"); + write_entry(&other, (&stored_inputs(), &output, None), usize::MAX).expect("other compile"); std::fs::rename(output_file(&other), output_file(path)).expect("swapped output"); } @@ -164,3 +170,16 @@ pub(crate) fn damage(path: &Path) { bytes[last] ^= 0xff; std::fs::write(path, bytes).expect("damaged slot"); } + +/// A small project image of two sections. +pub(crate) fn image() -> ImageSections { + vec![ + ("facts".to_owned(), b"{\"models\": [\"orders\"]}".to_vec()), + ("rules".to_owned(), b"{}".to_vec()), + ] +} + +/// The image file beside the slot at `path`. +pub(crate) fn image_file(path: &Path) -> PathBuf { + path.with_extension("image") +} diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs new file mode 100644 index 0000000000..cdabaf37dc --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_image.rs @@ -0,0 +1,268 @@ +use std::collections::{HashMap, HashSet}; + +use crate::project_reuse::_helpers::entry::{read_delta_base, read_delta_output, write_entry}; +use crate::project_reuse::_helpers::files::{project_file_changes, unverified_paths}; +use crate::project_reuse::models::StoredProjectFile; +use crate::project_reuse::tests::helpers::{ + image, image_file, stamp, stored_inputs, stored_output, +}; +use crate::project_reuse::tests::test_types::{ + ChangedFileTestCase, ImagePruneTestCase, ImageReadTestCase, ImageRestoreTestCase, +}; +use crate::project_reuse::types::ImageSections; + +#[test] +fn given_stored_compile_with_image_when_reading_delta_base_then_only_an_intact_image_is_read() { + let test_cases = [ + ImageReadTestCase { + description: "intact image", + edit: |_| {}, + expected_read: true, + }, + ImageReadTestCase { + description: "missing image", + edit: |path| std::fs::remove_file(image_file(path)).expect("removed"), + expected_read: false, + }, + ImageReadTestCase { + description: "damaged image", + edit: |path| { + let mut bytes = std::fs::read(image_file(path)).expect("image"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + std::fs::write(image_file(path), bytes).expect("damaged"); + }, + expected_read: false, + }, + ImageReadTestCase { + description: "image of another compile", + edit: |path| { + let other = path.with_file_name("other.store"); + let mut sections = image(); + sections[0].1 = b"{\"models\": [\"customers\"]}".to_vec(); + write_entry( + &other, + (&stored_inputs(), &stored_output(), Some(§ions)), + 8, + ) + .expect("other compile"); + std::fs::copy(image_file(&other), image_file(path)).expect("replaced"); + }, + expected_read: false, + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("reuse/slot.store"); + write_entry( + &path, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored compile"); + (test_case.edit)(&path); + + let read = read_delta_base(&path).and_then(|base| { + let output = read_delta_output(&path, &base)?; + Some((base.inputs, output, base.image)) + }); + + assert_eq!( + read, + test_case + .expected_read + .then(|| (stored_inputs(), stored_output(), image())), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_compile_with_image_when_storing_again_then_only_a_new_image_is_kept() { + let test_cases = [ + ImageRestoreTestCase { + description: "storing without an image removes the stale one", + stored_again: None, + expected_image_kept: false, + }, + ImageRestoreTestCase { + description: "storing with an image keeps it", + stored_again: Some(image), + expected_image_kept: true, + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("slot.store"); + write_entry( + &path, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored"); + let again: Option = test_case.stored_again.map(|image| image()); + + write_entry( + &path, + (&stored_inputs(), &stored_output(), again.as_ref()), + 8, + ) + .expect("stored again"); + + assert_eq!( + image_file(&path).exists(), + test_case.expected_image_kept, + "{}", + test_case.description + ); + assert_eq!( + read_delta_base(&path).is_some(), + test_case.expected_image_kept, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_images_of_many_slots_when_storing_then_only_the_most_recently_used_two_are_kept() { + let test_cases = [ + ImagePruneTestCase { + description: "the two newest images are kept", + slots: 4, + read: &[], + expected_images: &["slot3.image", "slot4.image"], + }, + ImagePruneTestCase { + description: "a read image counts as used", + slots: 3, + read: &[1], + expected_images: &["slot1.image", "slot3.image"], + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + for slot in 0..test_case.slots { + let path = folder.path().join(format!("slot{slot}.store")); + write_entry( + &path, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored"); + let aged = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(slot + 1); + std::fs::File::options() + .write(true) + .open(image_file(&path)) + .and_then(|file| file.set_modified(aged)) + .expect("aged image"); + } + for slot in test_case.read { + assert!(read_delta_base(&folder.path().join(format!("slot{slot}.store"))).is_some()); + } + let newest = folder.path().join(format!("slot{}.store", test_case.slots)); + write_entry( + &newest, + (&stored_inputs(), &stored_output(), Some(&image())), + 8, + ) + .expect("stored again"); + + let mut kept: Vec = std::fs::read_dir(folder.path()) + .expect("listed") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.ends_with(".image")) + .collect(); + kept.sort(); + + assert_eq!(kept, test_case.expected_images, "{}", test_case.description); + } +} + +#[test] +fn given_project_file_when_computing_delta_changes_then_only_content_changes_count() { + let test_cases = [ + ChangedFileTestCase { + description: "settled equal stamp is unchanged without reading", + stored_digest: None, + racy: false, + mtime_shift_ns: 0, + current_digest: None, + expected_changed: &[], + }, + ChangedFileTestCase { + description: "moved stamp with equal bytes is unchanged", + stored_digest: Some("b3:one"), + racy: false, + mtime_shift_ns: 1, + current_digest: Some("b3:one"), + expected_changed: &[], + }, + ChangedFileTestCase { + description: "moved stamp with new bytes is changed", + stored_digest: Some("b3:one"), + racy: false, + mtime_shift_ns: 1, + current_digest: Some("b3:two"), + expected_changed: &["models/orders.sql"], + }, + ChangedFileTestCase { + description: "racy equal stamp with new bytes is changed", + stored_digest: Some("b3:one"), + racy: true, + mtime_shift_ns: 0, + current_digest: Some("b3:two"), + expected_changed: &["models/orders.sql"], + }, + ChangedFileTestCase { + description: "moved stamp without a stored digest is changed", + stored_digest: None, + racy: false, + mtime_shift_ns: 1, + current_digest: Some("b3:one"), + expected_changed: &["models/orders.sql"], + }, + ChangedFileTestCase { + description: "unreadable moved file is changed", + stored_digest: Some("b3:one"), + racy: false, + mtime_shift_ns: 1, + current_digest: None, + expected_changed: &["models/orders.sql"], + }, + ]; + for test_case in test_cases { + let stored = vec![StoredProjectFile { + stamp: stamp("models/orders.sql", "f", None), + digest: test_case.stored_digest.map(str::to_owned), + racy: test_case.racy, + }]; + let mut current = stamp("models/orders.sql", "f", None); + current.mtime_ns += test_case.mtime_shift_ns; + let current = vec![current, stamp("models/customers.sql", "f", None)]; + let digests: HashMap = test_case + .current_digest + .map(|digest| HashMap::from([("models/orders.sql".to_owned(), digest.to_owned())])) + .unwrap_or_default(); + let unverified: HashSet = unverified_paths(&stored, ¤t); + + let changes = project_file_changes(&stored, ¤t, (&digests, &unverified)); + + assert_eq!( + changes.changed, test_case.expected_changed, + "{}", + test_case.description + ); + assert_eq!( + (changes.added, changes.removed), + ( + vec!["models/customers.sql".to_owned()], + Vec::::new() + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs index ff85acfbe6..e7598675a7 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs @@ -69,7 +69,7 @@ fn given_stored_compile_when_reading_then_only_an_intact_slot_round_trips() { for test_case in test_cases { let folder = tempfile::tempdir().expect("store folder"); let path = folder.path().join("reuse/slot.store"); - write_entry(&path, &stored_inputs(), &stored_output(), 8).expect("stored compile"); + write_entry(&path, (&stored_inputs(), &stored_output(), None), 8).expect("stored compile"); (test_case.edit)(&path); let read = read_entry(&path); @@ -269,7 +269,7 @@ fn given_stored_compile_when_refreshing_inputs_then_only_the_slot_file_is_rewrit for test_case in test_cases { let folder = tempfile::tempdir().expect("store folder"); let path = folder.path().join("reuse/slot.store"); - write_entry(&path, &stored_inputs(), &stored_output(), 8).expect("stored compile"); + write_entry(&path, (&stored_inputs(), &stored_output(), None), 8).expect("stored compile"); let (name, bytes) = test_case.extra_section; add_slot_section(&path, name, bytes); let output_before = std::fs::read(output_file(&path)).expect("output bytes"); diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs index 9e2f58b920..6c2416836d 100644 --- a/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs @@ -59,3 +59,36 @@ pub(crate) struct LinkCycleTestCase { pub(crate) description: &'static str, pub(crate) expected_paths: &'static [&'static str], } + +/// A stored compile with an image, edited before the delta base is read back. +pub(crate) struct ImageReadTestCase { + pub(crate) description: &'static str, + pub(crate) edit: fn(&Path), + pub(crate) expected_read: bool, +} + +/// Slots stored oldest first, each with an image, then some read as delta bases. +pub(crate) struct ImagePruneTestCase { + pub(crate) description: &'static str, + pub(crate) slots: u64, + /// Slots read as delta bases before one more slot is stored. + pub(crate) read: &'static [u64], + pub(crate) expected_images: &'static [&'static str], +} + +/// A stored and a current project file and how the delta changed set sees them. +pub(crate) struct ChangedFileTestCase { + pub(crate) description: &'static str, + pub(crate) stored_digest: Option<&'static str>, + pub(crate) racy: bool, + pub(crate) mtime_shift_ns: i64, + pub(crate) current_digest: Option<&'static str>, + pub(crate) expected_changed: &'static [&'static str], +} + +/// A stored compile with an image, stored again with or without one. +pub(crate) struct ImageRestoreTestCase { + pub(crate) description: &'static str, + pub(crate) stored_again: Option crate::project_reuse::types::ImageSections>, + pub(crate) expected_image_kept: bool, +} diff --git a/crates/sqlbuild-cache/src/project_reuse/types.rs b/crates/sqlbuild-cache/src/project_reuse/types.rs new file mode 100644 index 0000000000..476b4b4da0 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/types.rs @@ -0,0 +1,4 @@ +//! Type aliases of the project reuse store. + +/// Named sections of one project image, in stored order. +pub type ImageSections = Vec<(String, Vec)>; diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index ecb135feaf..021b10dbc3 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -21,8 +21,8 @@ use crate::project::constants::{ USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ - DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, - TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, + DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, + SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; @@ -242,6 +242,14 @@ pub(crate) fn load_settings(payload: Option<&ConfigValue>, file: ConfigFile) -> UNIT_TEST_MAX_REAL_MODELS_KEY, Built::Value(real_model_limit(mapping)?), ), + ( + DELTA_COMPILE_KEY, + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + DELTA_COMPILE_KEY, + true, + )?)), + ), ]; let mut fields = fields; for (key, seconds_key, default) in [ diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index dc1c783554..d225cbba22 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -40,9 +40,10 @@ pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; pub(crate) const UNIT_TEST_MAX_REAL_MODELS_KEY: &str = "unit_test_max_real_models"; pub(crate) const DEFAULT_UNIT_TEST_MAX_REAL_MODELS: i64 = 10; pub(crate) const TEST_TIMEOUT_KEY: &str = "test_timeout"; +pub(crate) const DELTA_COMPILE_KEY: &str = "delta_compile"; pub(crate) const SCENARIO_TIMEOUT_KEY: &str = "scenario_timeout"; pub(crate) const DEFAULT_TEST_TIMEOUT: &str = "30s"; -pub(crate) const SETTINGS_KEYS: [&str; 14] = [ +pub(crate) const SETTINGS_KEYS: [&str; 15] = [ "sql_analysis", "require_sql_analysis", "query_change_tracking", @@ -57,6 +58,7 @@ pub(crate) const SETTINGS_KEYS: [&str; 14] = [ "unit_test_max_real_models", "test_timeout", "scenario_timeout", + "delta_compile", ]; pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "append_cursor_inclusive", diff --git a/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs b/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs index a365eddd22..aba0c8a20f 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs @@ -25,6 +25,12 @@ pub fn discover_model_files( Ok(paths .into_par_iter() .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) + .filter(|relative_path| { + options + .path_filter + .as_ref() + .is_none_or(|paths| paths.contains(relative_path)) + }) .map(|relative_path| DiscoveredFile { outcome: discover_one(root, tree, &relative_path, options), relative_path, diff --git a/crates/sqlbuild-discovery/src/model_files/models.rs b/crates/sqlbuild-discovery/src/model_files/models.rs index 3bbc25da51..e618f6aa7a 100644 --- a/crates/sqlbuild-discovery/src/model_files/models.rs +++ b/crates/sqlbuild-discovery/src/model_files/models.rs @@ -12,6 +12,8 @@ pub struct ModelFileOptions { pub removed_keys: Vec, pub extract_implicit_alias_columns: bool, pub extract_output_column_locations: bool, + /// Discover only these project-relative paths, when given. + pub path_filter: Option>>, } /// One parsed model file; values mirror the fields of Python's `DiscoveredSqlModelFile`. diff --git a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs index 6b8beef554..084d8f1953 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs @@ -14,6 +14,7 @@ const FILE_PATH: &str = "/project/models/orders.sql"; fn options() -> ModelFileOptions { ModelFileOptions { + path_filter: None, supported_keys: [ "audits", "columns", diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs index 0568696b6a..a18f0b2287 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs @@ -13,11 +13,18 @@ pub(crate) fn validate_path_defaults(facts: &ProjectFacts) -> Result<(), Validat if facts.path_default_keys.is_empty() { return Ok(()); } - let model_paths: Vec = facts - .models - .iter() - .map(|model| { - let path: String = model.file.path.replace('\\', "/"); + let discovered: Vec<&str> = match &facts.project_model_paths { + Some(paths) => paths.iter().map(String::as_str).collect(), + None => facts + .models + .iter() + .map(|model| model.file.path.as_str()) + .collect(), + }; + let model_paths: Vec = discovered + .into_iter() + .map(|path| { + let path: String = path.replace('\\', "/"); path.strip_prefix(MODELS_PREFIX) .map_or(path.clone(), str::to_owned) }) diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index 1a0f4fdb09..58c65bb869 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -174,6 +174,8 @@ pub struct ProjectFacts { pub event_exporters: Vec, pub providers: Vec, pub path_default_keys: Vec, + /// Every model path of the whole project when discovery read only some of its models. + pub project_model_paths: Option>, /// Python's `csv.field_size_limit()`, or `None` for its default. pub csv_field_limit: Option, } diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs index c6a9d1bd52..f07daa347f 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs @@ -118,6 +118,19 @@ fn given_discovered_facts_when_validating_then_reports_python_first_conflict() { include: marts", ), }, + ValidationTestCase { + description: "a path default matching only undiscovered project models passes", + facts: ProjectFacts { + models: vec![model_at("orders", "models/marts/orders.sql")], + path_default_keys: vec!["staging".to_owned()], + project_model_paths: Some(vec![ + "models/marts/orders.sql".to_owned(), + "models/staging/customers.sql".to_owned(), + ]), + ..ProjectFacts::default() + }, + expected_failure: Ok(()), + }, ValidationTestCase { description: "accepts Windows-style model paths that match path defaults", facts: path_default_facts(&["models\\staging\\orders.sql"], &["staging"]), diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs index caf1f77e30..eb10a6a622 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs @@ -13,7 +13,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; pub(crate) fn discover_sql_files( root: &ProjectRoot, tree: &ProjectTree, - directory: &str, + (directory, path_filter): (&str, Option<&std::collections::BTreeSet>), parse: impl Fn(&str, String) -> Result + Sync, ) -> Result>, StageFailure> { let pool = discovery_pool().map_err(StageFailure::Internal)?; @@ -24,6 +24,7 @@ pub(crate) fn discover_sql_files( Ok(paths .into_par_iter() .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) + .filter(|relative_path| path_filter.is_none_or(|paths| paths.contains(relative_path))) .map(|relative_path| DiscoveredFile { outcome: read_and_parse(root, tree, &relative_path, &parse), relative_path, diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs index ccbf905907..3b175335b6 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs @@ -13,7 +13,10 @@ pub fn discover_scenario_files( tree: &ProjectTree, options: &SqlTestFileOptions, ) -> Result>, StageFailure> { - discover_sql_files(root, tree, SQL_SCENARIOS_ROOT, |file_path, contents| { - parse_scenario_file(file_path, contents, options) - }) + discover_sql_files( + root, + tree, + (SQL_SCENARIOS_ROOT, None), + |file_path, contents| parse_scenario_file(file_path, contents, options), + ) } diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs index 908475a3a0..a7286dff1a 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs @@ -16,7 +16,7 @@ pub fn discover_sql_test_files( discover_sql_files( root, tree, - &SQL_TESTS_ROOT.join("/"), + (&SQL_TESTS_ROOT.join("/"), options.path_filter.as_deref()), |file_path, contents| parse_sql_test_file(file_path, contents, options), ) } diff --git a/crates/sqlbuild-discovery/src/sql_tests/models.rs b/crates/sqlbuild-discovery/src/sql_tests/models.rs index 23a0099f72..6aa183b923 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/models.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/models.rs @@ -8,6 +8,8 @@ use sqlbuild_sqltext::compiler::models::AuthoredValue; pub struct SqlTestFileOptions { pub test_keys: Vec, pub scenario_keys: Vec, + /// Discover only these project-relative paths, when given. + pub path_filter: Option>>, } /// One `TEST(...)` block whose header parsed with only supported keys. diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs index 20ea382c79..b860bd6e41 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs @@ -13,6 +13,7 @@ fn keys(test_keys: &[&str]) -> Vec { fn options() -> SqlTestFileOptions { SqlTestFileOptions { + path_filter: None, test_keys: keys(&["name", "mode", "parameters", "cases"]), scenario_keys: keys(&["description", "tags"]), } diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 077bf8b416..8741c66d01 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -20,6 +20,7 @@ fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true rayon.workspace = true +rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 81960c08d6..2610876a8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -206,6 +206,15 @@ fn run_session<'py>( nullability.call_method1("setdefault", (&name, defaults))?; } let (additions, analysis_names, succeeded) = session.borrow_mut().finish(py)?; + if let Some(tables) = session + .borrow() + .finished_session() + .and_then(|finished| finished.relation_tables()) + { + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.relations = Some(tables.clone()); + }); + } record_catalog_changes(catalog, additions, &analysis_names, (&types, &nullability))?; record_cache(py, python, session.borrow().cache_stats())?; let positions: Bound<'py, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 6d7a78e232..daffde72a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -274,7 +274,10 @@ pub(crate) fn start_model_analysis_session( adapter_rules: Option<(Py, Py)>, ) -> PyResult { let py: Python<'_> = catalog.py(); - let mut request: SessionRequest = session_request(request); + let mut request: SessionRequest = + crate::bindings::_helpers::compile_delta::context::inject_relations(session_request( + request, + ))?; let rule_failure: RuleFailure = RuleFailure::default(); request.nullability_callback = adapter_rules .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); @@ -282,6 +285,8 @@ pub(crate) fn start_model_analysis_session( let mut session: AnalysisSession = py .compiler_detach(|| start_analysis_session(request, catalog)) .map_err(|reason| internal_error(&reason))?; + let cache: Option<(PathBuf, String)> = + cache.filter(|_| crate::bindings::_helpers::compile_delta::context::injection().is_none()); let mut cache_path: Option = None; let mut cache_stats: Option = None; if let Some((path, environment)) = cache { @@ -368,6 +373,8 @@ fn session_request(request: RequestRow) -> SessionRequest { catalog_schemas, dynamic_families_by_table: families_by_table(dynamic_families_by_table), models: models.into_iter().map(model_request).collect(), + forced_order: None, + retain_relations: crate::bindings::_helpers::compile_delta::context::capturing(), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index 5878950bff..9e4501a2cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -1,6 +1,6 @@ //! The invocation-scoped native CLI projection of canonical lifecycle facts. -use std::sync::{Mutex, MutexGuard}; +use std::sync::{Mutex, MutexGuard, PoisonError}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -43,6 +43,8 @@ pub(crate) struct NativeProgressProjector { state: Mutex, stream: Py, coordinator: Py, + /// The lines held back instead of written, while something holds them. + held: Mutex>>, } fn payload_value<'py>(payload: &Bound<'py, PyAny>, key: &str) -> PyResult> { @@ -169,6 +171,37 @@ pub(crate) fn terminal_stream(stream: &Bound<'_, PyAny>) -> PyResult { } impl NativeProgressProjector { + fn held(&self) -> MutexGuard<'_, Option>> { + self.held.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Hold back every line from now on instead of writing it. + pub(crate) fn hold_lines(&self) { + *self.held() = Some(Vec::new()); + } + + /// Replace the lines held back so far with `lines`, returning them; nothing while not holding. + pub(crate) fn exchange_held_lines(&self, lines: Vec) -> Vec { + match self.held().as_mut() { + Some(held) => std::mem::replace(held, lines), + None => Vec::new(), + } + } + + /// Write lines as they come again, returning those held back since `hold_lines`. + pub(crate) fn release_lines(&self) -> Vec { + self.held().take().unwrap_or_default() + } + + /// Write lines held back earlier, as they would have been written. + pub(crate) fn write_held(&self, py: Python<'_>, lines: &[String]) -> PyResult<()> { + let coordinator: &Bound<'_, PyAny> = self.coordinator.bind(py); + for line in lines { + persist(coordinator, self.stream.bind(py), line)?; + } + Ok(()) + } + fn locked(&self) -> PyResult> { self.state .lock() @@ -192,7 +225,12 @@ impl NativeProgressProjector { (lines, value) }; for line in lines { - persist(coordinator, self.stream.bind(py), &format!("{line}\n"))?; + let text: String = format!("{line}\n"); + if let Some(held) = self.held().as_mut() { + held.push(text); + continue; + } + persist(coordinator, self.stream.bind(py), &text)?; } Ok(value) })(); @@ -236,6 +274,7 @@ impl NativeProgressProjector { })), stream: stream.unbind(), coordinator: coordinator.unbind(), + held: Mutex::new(None), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 929f554d35..d06d17d092 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -163,6 +163,9 @@ impl<'py> ReportInputs<'py> { return Ok(PyDict::new(py)); } let reasons: Bound<'py, PyDict> = semantic_coverage(&self.project)?; + if crate::bindings::_helpers::compile_delta::context::injection().is_some() { + return Ok(reasons); + } let Some(names) = self.selected_names(MODEL_TYPE)? else { return Ok(reasons); }; @@ -296,6 +299,11 @@ fn shown_diagnostics( return Ok(Vec::new()); } let texts: Bound<'_, PyDict> = source_texts(&inputs.project)?; + crate::bindings::_helpers::compile_delta::report_hooks::complete_source_texts( + &inputs.project, + &inputs.diagnostics, + &texts, + )?; let warning_counts: Bound<'_, PyDict> = PyDict::new(py); for diagnostic in &inputs.diagnostics { if native_field(diagnostic, "severity")?.eq(WARNING_SEVERITY)? { @@ -358,6 +366,8 @@ pub(crate) fn format_compile_text<'py>( column_count: native_field(&model, "output_column_count")?.extract()?, }); } + let models: Vec = + crate::bindings::_helpers::compile_delta::report_hooks::text_model_lines(models); let lineage: LineageNote = if !inputs.lineage.is_none() && inputs.lineage_mode == RICH_LINEAGE { LineageNote::Rich } else if inputs.lineage.is_none() @@ -451,6 +461,11 @@ fn location_json<'py>(location: &Bound<'py, PyAny>) -> PyResult(diagnostic: &Bound<'py, PyAny>) -> PyResult> { + diagnostic_json(diagnostic) +} + /// `_diagnostic_to_json`: one diagnostic's JSON fields, omitting the unset ones. fn diagnostic_json<'py>(diagnostic: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = diagnostic.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 38fff9bfc4..1544d28982 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -14,6 +14,7 @@ use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FORMAT}; use crate::bindings::_helpers::cli::launch_state::launch_record_path; +use crate::bindings::_helpers::cli::progress::{NativeProgressProjector, current_projector}; use crate::bindings::_helpers::compile_command::constants::{ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, COMPILE_COMMAND_NAME, COMPILED_DIRECTORY_NAME, COMPUTE_LOG_DIRECTORY_NAME, EXCLUDED_DIRECTORIES, @@ -30,12 +31,17 @@ use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::runtime_identity::{ invocation_identity, redirected_output_files, runtime_identity, }; +use crate::bindings::_helpers::compile_delta::driver::{ + self, DELTA_VERIFY_ENV_VAR, DeltaOutcome, delta_enabled, env_enabled, +}; +use crate::bindings::_helpers::compile_delta::{context, verify}; use crate::bindings::_helpers::compile_driver::command::run_compile_command; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; use crate::bindings::_helpers::compile_outputs::reuse::{ReuseCheckPy, check_compile_reuse}; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; +use crate::bindings::_helpers::macro_bridge::first_compile::keep_delta_claim; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; @@ -49,6 +55,8 @@ const HIT: &str = "hit"; const CHECKED_HIT: &str = "hit"; const CHECKED_BYPASS: &str = "bypass"; const MISS: &str = "miss"; +/// The logging format that prints its one argument as given. +const LOG_TEXT: &str = "%s"; const BYPASS: &str = "bypass"; fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { @@ -59,6 +67,21 @@ fn reuse_logger(py: Python<'_>) -> PyResult> { imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) } +/// Log one message at debug level on the compile reuse logger. +pub(crate) fn reuse_debug(py: Python<'_>, message: &str) -> PyResult<()> { + reuse_debug_with(py, LOG_TEXT, message) +} + +/// Log `argument` through the logging `format` at debug level on the compile reuse logger. +fn reuse_debug_with<'py>( + py: Python<'py>, + format: &str, + argument: impl pyo3::IntoPyObject<'py>, +) -> PyResult<()> { + let _ = reuse_logger(py)?.call_method1("debug", (format, argument))?; + Ok(()) +} + /// The reuse constants the native check, the stored compile and the launch record share. pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult> { let rules: Bound<'_, PyDict> = PyDict::new(py); @@ -403,12 +426,10 @@ fn attempt_compile_reuse<'py>(request: &Bound<'py, PyAny>) -> PyResult(request: &Bound<'py, PyAny>) -> PyResult = if missed && delta_enabled() { + compiled_with_delta(request, &attempt) + } else { + run_compile_command(py, request.clone(), Some(attempt)) + }; + context::reset(); + verify::disarm(); + code.and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py)) +} + +/// Compile the changed files as a delta when it can, otherwise (or to verify it) in full. +fn compiled_with_delta<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = request.py(); + let verifying: bool = env_enabled(DELTA_VERIFY_ENV_VAR); + let started: f64 = monotonic(py)?; + let projector: Option> = current_projector(py)?; + if let Some(projector) = &projector { + projector.get().hold_lines(); + } + let ran: PyResult> = driver::run(request, attempt); + let held: Vec = projector + .as_ref() + .map(|projector| projector.get().release_lines()) + .unwrap_or_default(); + let delta: DeltaOutcome<'py> = match ran { + Ok(delta) => delta, + Err(error) => DeltaOutcome::Fallback(format!("the delta compile failed: {error}")), + }; + let write_held = || -> PyResult<()> { + match &projector { + Some(projector) => projector.get().write_held(py, &held), + None => Ok(()), + } + }; + context::reset(); + let project_dir: Bound<'py, PyAny> = native_field(attempt, "project_dir")?; + let cache: Bound<'py, PyAny> = cache_directory(&project_dir)?; + let compiled_root: PathBuf = std::path::absolute( + project_dir + .extract::()? + .join(TARGET_DIRECTORY_NAME) + .join(COMPILED_DIRECTORY_NAME), + )?; + match delta { + DeltaOutcome::Finished(finished) if !verifying => { + keep_delta_claim(); + write_held()?; + let (files, models) = (finished.changed_files, finished.rerun_models); + let committing: std::time::Instant = std::time::Instant::now(); + let code: i64 = driver::commit(*finished, (request, attempt))?; + driver::trace(py, "delta record and print", committing)?; + let seconds: f64 = elapsed_ms(py, started)? as f64 / MILLISECONDS_PER_SECOND; + print_stderr( + py, + &format!( + "Compiled {files} changed file(s) against the previous compile ({models} model(s), {seconds:.1}s)" + ) + .into_bound_py_any(py)?, + )?; + return Ok(code); + } + DeltaOutcome::Raised(error) if !verifying => { + keep_delta_claim(); + write_held()?; + return Err(error); + } + DeltaOutcome::Fallback(reason) => { + reuse_debug(py, &format!("Delta compile fell back: {reason}"))?; + if verifying { + print_stderr( + py, + &format!("delta compile fell back: {reason}").into_bound_py_any(py)?, + )?; + } + context::start_capture(); + return run_compile_command(py, request.clone(), Some(attempt.clone())); + } + DeltaOutcome::Finished(finished) => verify::hold_expected(verify::Expected { + stdout: Some(finished.stdout.clone()), + exit_code: Some(finished.phases.0.exit_code), + artifacts_written: finished.phases.0.artifacts_written, + raised: None, + compiled: verify::compiled_digests(&compiled_root), + compiled_root: compiled_root.to_string_lossy().into_owned(), + }), + DeltaOutcome::Raised(error) => verify::hold_expected(verify::Expected { + raised: Some(error.to_string()), + compiled_root: compiled_root.to_string_lossy().into_owned(), + ..verify::Expected::default() + }), + } + context::start_capture(); + let full: PyResult = run_compile_command(py, request.clone(), Some(attempt.clone())); + let raised: Option = full.as_ref().err().map(ToString::to_string); + let differences: Vec = verify::differences(raised); + if differences.is_empty() { + print_stderr(py, &"delta compile verified".into_bound_py_any(py)?)?; + return full; + } + let cache_dir: PathBuf = cache.extract()?; + verify::write_mismatch(&cache_dir, &differences); + for difference in &differences { + print_stderr( + py, + &format!("delta compile mismatch: {difference}").into_bound_py_any(py)?, + )?; + } + full?; + Ok(verify::MISMATCH_EXIT_CODE) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 01eab6546c..7b1471283a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -1,12 +1,13 @@ //! What one full compile printed and left in each artifact, and storing it for compile reuse. +use sqlbuild_cache::project_reuse::types::ImageSections; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Mutex, PoisonError}; use pyo3::exceptions::{PyException, PyOSError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_core::posix_path::main::posix_abspath::posix_abspath; @@ -81,6 +82,22 @@ impl CompileArtifactWrites { } } +impl CompileArtifactWrites { + /// Every recorded artifact's absolute path, with its digest when this compile wrote it. + pub(crate) fn written_digests(&self) -> HashMap> { + self.locked() + .iter() + .map(|(path, artifact)| { + let digest: Option = match artifact { + RecordedArtifact::Written(digest) => Some(digest.clone()), + RecordedArtifact::Kept { .. } => None, + }; + (path.clone(), digest) + }) + .collect() + } +} + #[pymethods] impl CompileArtifactWrites { /// Every recorded artifact by absolute path, as `(digest, size, mtime_ns)`. @@ -235,6 +252,32 @@ impl RecordedCompileOutput { fn locked(&self) -> std::sync::MutexGuard<'_, PrintedOutput> { self.output.lock().unwrap_or_else(PoisonError::into_inner) } + + /// Output already printed or recorded elsewhere: its stderr lines and its report. + pub(crate) fn recorded(py: Python<'_>, stderr_lines: &[String], stdout: &str) -> Self { + let lines: Vec> = stderr_lines + .iter() + .map(|line| PyString::new(py, line).unbind()) + .collect(); + Self { + output: Mutex::new((lines, Some(PyString::new(py, stdout).unbind()))), + } + } + + /// The recorded stderr lines and report, as text. + pub(crate) fn texts(&self, py: Python<'_>) -> PyResult<(Vec, Option)> { + let output = self.locked(); + let lines: Vec = output + .0 + .iter() + .map(|line| Ok(line.bind(py).to_str()?.to_owned())) + .collect::>()?; + let stdout: Option = match &output.1 { + Some(text) => Some(text.bind(py).to_str()?.to_owned()), + None => None, + }; + Ok((lines, stdout)) + } } /// `print(*values, file=sys.)`. @@ -254,9 +297,11 @@ pub(crate) fn print_to<'py>( #[pymethods] impl RecordedCompileOutput { - /// Print one stderr result line, such as a note, and record it. + /// Print one stderr result line, such as a note, and record it; a delta only records it. pub(crate) fn print_stderr_line(&self, line: Bound<'_, PyString>) -> PyResult<()> { - print_to(line.py(), "stderr", &line)?; + if crate::bindings::_helpers::compile_delta::context::injection().is_none() { + print_to(line.py(), "stderr", &line)?; + } self.locked().0.push(line.unbind()); Ok(()) } @@ -480,6 +525,29 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { ("rules", reuse_rules(py, true)?.into_any()), ], )?; + if let Some(mut capture) = crate::bindings::_helpers::compile_delta::context::take_capture() { + capture.environment_names = + native_field(&facts.input_reads, "environment_names")?.extract()?; + let json_output: bool = facts.json_output.is_truthy()?; + match crate::bindings::_helpers::compile_delta::image::ProjectImage::from_capture( + capture, + json_output, + ) { + Ok(image) => { + let mut pending: Option = None; + if let Ok(sections) = image.encode() { + pending = Some(sections); + } + crate::bindings::_helpers::compile_delta::context::set_pending_image(pending); + } + Err(reason) => { + crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug( + py, + &format!("Compile reuse kept no project image: {reason}"), + )?; + } + } + } let (outcome, digested): (String, Bound<'_, PyAny>) = facts.native.call_method1("record", (payload,))?.extract()?; let _ = reuse_logger(py)?.call_method1( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 20a731b1df..25c5bdb758 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -467,6 +467,9 @@ fn plan_tests<'py>( count_stage(py, SQL_TEST_MODEL_CHAINS, 1)?; let chains: Bound<'py, PyAny> = sql_test_model_chain_names_by_key(project, &tests)?.into_any(); + crate::bindings::_helpers::compile_delta::report_hooks::record_test_chains( + &tests, &chains, + )?; Some((context, chains)) }; let tests: Vec> = tests.try_iter()?.collect::>()?; @@ -613,9 +616,16 @@ fn write_static_tests<'py>( } } } - record_compile_timing("cache_publication_ms", || { - write_cache(&native_field(project, "compile_cache_dir")?, &records) - })?; + let cache_dir: Bound<'py, PyAny> = native_field(project, "compile_cache_dir")?; + if crate::bindings::_helpers::compile_delta::context::injection().is_some() { + let mut kept: HashMap = read_cache(&cache_dir)?; + for (key, record) in records.drain(..) { + kept.insert(key, record); + } + records = kept.into_iter().collect(); + records.sort_by(|left, right| left.0.cmp(&right.0)); + } + record_compile_timing("cache_publication_ms", || write_cache(&cache_dir, &records))?; Ok((managed, diagnostics)) } @@ -627,7 +637,8 @@ pub(crate) fn write_static_target<'py>( prepared: Prepared<'_, 'py>, ) -> PyResult> { let py: Python<'py> = target_dir.py(); - let remove_stale_files: bool = prepared_target(target_dir)?; + let remove_stale_files: bool = prepared_target(target_dir)? + && crate::bindings::_helpers::compile_delta::context::injection().is_none(); let mut batch: ArtifactBatch = ArtifactBatch::new(remove_stale_files, prepared.recorder); let mut managed: Vec> = Vec::new(); let mut managed_models: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs new file mode 100644 index 0000000000..1f2d422c3d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/context.rs @@ -0,0 +1,344 @@ +//! Process-wide slots for what a full compile captures and what a delta compile injects. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use sqlbuild_analysis::assembly::analysis_session::models::{RelationTables, SessionRequest}; +use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_rules::models::RetainedRows; + +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; +use crate::bindings::_helpers::compile_delta::image::PendingRules; + +/// The facts a full compile leaves for its project image beyond its analysis record. +#[derive(Debug, Default)] +pub(crate) struct ImageCapture { + /// The sources functions and audits read, which a delta compile always keeps. + pub(crate) pinned_sources: BTreeSet, + /// The sources each SQL test file reads, which a delta compile keeps when it reruns it. + pub(crate) test_sources: BTreeMap>, + /// The relation facts the model analysis session ended with. + pub(crate) relations: Option, + /// The built-in rules request and its raw findings, before suppressions. + pub(crate) rules: Option, + /// Each custom-rule fact digest the custom rule cache compared, by fact key. + pub(crate) fact_digests: BTreeMap, String>, + /// Whether the JSON report fell back to the standard library's serializer. + pub(crate) stdlib_json: bool, + /// Each model's scope usage digest, by model file path. + pub(crate) usages: BTreeMap, + /// Each SQL test's compiled-artifact chain, by test key text. + pub(crate) test_chains: Vec, + /// Each project model's `(path, name, materialized)`, when custom rules ran. + pub(crate) model_subjects: Option)>>, + /// Each compiled model's `(name, relative path, upstream (kind, name) pairs)`. + pub(crate) models: Vec, + /// Each report diagnostic as its JSON item text, with what it names. + pub(crate) diagnostics: Vec, + /// The text report's model lines `(name, failed, column count)`, when it rendered one. + pub(crate) text_models: Option>, + /// The template environment names the compile read. + pub(crate) environment_names: Vec, + /// Each discovered model's header keys, by model file path. + pub(crate) header_keys: + BTreeMap, + /// Each model file's SQL lint cache key, in project model order. + pub(crate) lint_model_keys: Option>, + /// Each other SQL file's lint cache key, in walk order. + pub(crate) lint_file_keys: Option)>>, + /// The closed shape each model published for semantic checks, in publication order. + pub(crate) model_shapes: Option>, + /// The notable semantic coverage facts of every model, when semantic checks reported them. + pub(crate) coverage: Option>, + /// Why this compile cannot be the base of a delta compile, when something ruled it out. + pub(crate) unsupported: Option, +} + +/// The built-in rules request a compile evaluated and its raw findings. +#[derive(Debug)] +pub(crate) struct CapturedRules { + pub(crate) retained: RetainedRows, +} + +/// One model's published `(column, type)` shape. +pub(crate) type ModelShape = (String, Vec<(String, String)>); + +/// One compiled model's identity and references. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct CapturedModel { + pub(crate) name: String, + pub(crate) relative_path: String, + pub(crate) upstream: Vec<(String, String)>, +} + +/// One report diagnostic: its JSON text and the resource and file it names. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct CapturedDiagnostic { + pub(crate) json: String, + pub(crate) resource_name: Option, + pub(crate) paths: Vec, + /// Whether the diagnostic is an error. + #[serde(default)] + pub(crate) error: bool, +} + +/// One SQL test's file and the models its compiled artifact is keyed by. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct CapturedTest { + pub(crate) key: String, + pub(crate) relative_path: String, + pub(crate) chain: Vec, +} + +/// What a delta compile hands the stages it reruns for the changed models. +#[derive(Debug, Default)] +pub(crate) struct DeltaInjection { + /// Model files to discover, as project-relative POSIX paths. + pub(crate) model_paths: BTreeSet, + /// SQL test files to discover, as project-relative POSIX paths. + pub(crate) test_paths: BTreeSet, + /// The paths and names of every model the delta compile reruns. + pub(crate) rerun_names: BTreeSet, + /// The project directory the stored compile and the delta compile both read. + pub(crate) project_dir: std::path::PathBuf, + /// Every model and SQL test file of the whole project, as project-relative POSIX paths. + pub(crate) source_paths: BTreeSet, + /// Every model file of the whole project, which path defaults are matched against. + pub(crate) project_model_paths: Vec, + /// Every model name of the whole project, which references resolve against. + pub(crate) model_names: Vec, + /// The whole project's relation facts, injected for relations the delta does not analyse. + pub(crate) relations: RelationTables, + /// The whole project's built-in rules request, taken by the round that evaluates it. + pub(crate) rules: Mutex>, + /// The whole project's custom-rule fact digests. + pub(crate) fact_digests: BTreeMap, String>, + /// Each project model's `(path, name, materialized)`, the custom-rule subjects. + pub(crate) model_subjects: Option)>>, + /// The files that changed since the stored compile, as project-relative POSIX paths. + pub(crate) changed_paths: BTreeSet, + /// Where loaded YAML documents are kept for later delta compiles. + pub(crate) yaml_store: Option, + /// The only sources discovery keeps, those the rerun models read; `None` keeps every one. + pub(crate) source_names: Option>, + /// The whole project's SQL lint cache keys, as the image recorded them. + pub(crate) lint_model_keys: Option>, + pub(crate) lint_file_keys: Option)>>, + /// The closed shapes the whole project's models published. + pub(crate) model_shapes: Vec, + /// The whole project's text report model lines, with the rerun models' lines to replace. + pub(crate) text_models: Vec<(String, bool, i64)>, + /// The whole project's notable semantic coverage facts, as the image recorded them. + pub(crate) coverage: Option>, + /// Every project model in project order, with the models it reads. + pub(crate) coverage_order: Vec<(String, Vec)>, +} + +static CAPTURE: Mutex> = Mutex::new(None); +static INJECTION: Mutex>> = Mutex::new(None); + +/// Why the running delta compile cannot continue, once something ruled it out. +static UNSUPPORTED: Mutex> = Mutex::new(None); +static PENDING_IMAGE: Mutex> = Mutex::new(None); +static RECORD_MERGE: Mutex> = Mutex::new(None); + +fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Start capturing image facts for the compile about to run. +pub(crate) fn start_capture() { + *locked(&CAPTURE) = Some(ImageCapture::default()); +} + +/// Stop capturing without keeping anything, for a project that turns delta compiles off. +pub(crate) fn stop_capture() { + *locked(&CAPTURE) = None; +} + +/// Stop capturing and take what was captured. +pub(crate) fn take_capture() -> Option { + locked(&CAPTURE).take() +} + +/// Whether the running compile captures image facts. +pub(crate) fn capturing() -> bool { + locked(&CAPTURE).is_some() +} + +/// Record into the capture, when one is running. +pub(crate) fn capture(record: impl FnOnce(&mut ImageCapture)) { + if let Some(capture) = locked(&CAPTURE).as_mut() { + record(capture); + } +} + +/// Record sources a resource other than a model reads, when a capture is running. +pub(crate) fn record_source_reads<'a>(names: impl IntoIterator) { + capture(|capture| { + capture + .pinned_sources + .extend(names.into_iter().map(str::to_owned)); + }); +} + +/// Record sources an SQL test file reads, when a capture is running. +pub(crate) fn record_test_source_reads<'a>( + relative_path: &str, + names: impl IntoIterator, +) { + capture(|capture| { + capture + .test_sources + .entry(relative_path.replace('\\', "/")) + .or_default() + .extend(names.into_iter().map(str::to_owned)); + }); +} + +/// Mark the running capture as unusable as a delta base. +pub(crate) fn capture_unsupported(reason: &str) { + capture(|capture| { + if capture.unsupported.is_none() { + capture.unsupported = Some(reason.to_owned()); + } + }); +} + +/// Run `body` with `injection` visible to the rerun stages; returns it unless still held. +pub(crate) fn with_injection( + injection: DeltaInjection, + body: impl FnOnce() -> T, +) -> (T, Option) { + *locked(&INJECTION) = Some(Arc::new(injection)); + let result: T = body(); + let injection: Option> = locked(&INJECTION).take(); + ( + result, + injection.and_then(|injection| { + let Ok(injection) = Arc::try_unwrap(injection) else { + return None; + }; + Some(injection) + }), + ) +} + +/// The running delta compile's injection, if any. +pub(crate) fn injection() -> Option> { + locked(&INJECTION).clone() +} + +/// Rule out the running delta compile: returns the error that unwinds to its fallback. +pub(crate) fn delta_unsupported(reason: &str) -> pyo3::PyErr { + let mut slot = locked(&UNSUPPORTED); + if slot.is_none() { + *slot = Some(reason.to_owned()); + } + pyo3::exceptions::PyRuntimeError::new_err(format!("delta compile unsupported: {reason}")) +} + +/// In a delta compile that leaves errors to full compiles, stop once `failed` reports one. +pub(crate) fn stop_on_errors(failed: bool) -> pyo3::PyResult<()> { + let errors_on_delta_path: bool = + crate::bindings::_helpers::compile_delta::driver::ERRORS_ON_DELTA_PATH; + if failed && !errors_on_delta_path && injection().is_some() { + return Err(delta_unsupported("the edit reports errors")); + } + Ok(()) +} + +/// Take the reason the delta compile was ruled out, if any. +pub(crate) fn take_delta_unsupported() -> Option { + locked(&UNSUPPORTED).take() +} + +/// The previous compile's digest of a custom-rule fact about a resource the delta did not rerun. +pub(crate) fn injected_fact_digest(key: &[String]) -> Option { + let injection: Arc = injection()?; + if key + .iter() + .skip(1) + .any(|part| injection.rerun_names.contains(part)) + { + return None; + } + injection.fact_digests.get(key).cloned() +} + +/// Give a delta's analysis session the whole project's relation facts and analysis order. +pub(crate) fn inject_relations(mut request: SessionRequest) -> pyo3::PyResult { + if !request.dynamic_families_by_table.is_empty() { + capture_unsupported("a model declares dynamic column families"); + } + let Some(injection) = injection() else { + return Ok(request); + }; + if !request.dynamic_families_by_table.is_empty() { + return Err(delta_unsupported( + "a model declares dynamic column families", + )); + } + let relations: &RelationTables = &injection.relations; + request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); + request.retain_relations = true; + let analysed: BTreeSet<&str> = request + .models + .iter() + .map(|model| model.name.as_str()) + .collect(); + for (table, injected) in [ + (&mut request.column_types, &relations.types), + (&mut request.column_nullability, &relations.nullability), + (&mut request.complete_schemas, &relations.complete), + ] { + let present: BTreeSet = table.iter().map(|(name, _)| name.clone()).collect(); + table.extend( + injected + .iter() + .filter(|(name, _)| !analysed.contains(name.as_str()) && !present.contains(name)) + .cloned(), + ); + } + Ok(request) +} + +/// What a delta compile's reuse record keeps of the stored compile it patched. +#[derive(Debug, Default)] +pub(crate) struct RecordMerge { + /// Each stored artifact's stamp, verified by the reuse check, kept unless the delta wrote it. + pub(crate) kept: Vec, + /// The template environment names the stored compile read. + pub(crate) environment_names: Vec, + /// The module files the stored compile depended on, such as those behind stored macro calls. + pub(crate) module_paths: Vec, +} + +/// Hand the next reuse record the image to keep beside it. +pub(crate) fn set_pending_image(image: Option) { + *locked(&PENDING_IMAGE) = image; +} + +/// The image the next reuse record keeps, taken. +pub(crate) fn take_pending_image() -> Option { + locked(&PENDING_IMAGE).take() +} + +/// Hand the next reuse record what it keeps of the patched compile. +pub(crate) fn set_record_merge(merge: Option) { + *locked(&RECORD_MERGE) = merge; +} + +/// What the next reuse record keeps of the patched compile, taken. +pub(crate) fn take_record_merge() -> Option { + locked(&RECORD_MERGE).take() +} + +/// Forget every capture, injection and pending record fact, after a compile ends either way. +pub(crate) fn reset() { + let _ = take_capture(); + *locked(&INJECTION) = None; + let _ = take_delta_unsupported(); + set_pending_image(None); + set_record_merge(None); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/coverage.rs new file mode 100644 index 0000000000..0b40ab3fd1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/coverage.rs @@ -0,0 +1,124 @@ +//! Partial semantic-check reasons from per-model facts, shared by full and delta compiles. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +/// The reason a model that opts out of SQL analysis is partial. +pub(crate) const ANALYSIS_DISABLED: &str = "SQL analysis disabled"; +const INVALID_DEPENDENCY: &str = "depends on invalid model"; + +/// What one model contributes to the partial-check reasons, apart from its dependencies. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct ModelCoverage { + pub(crate) name: String, + /// The model opts out of SQL analysis. + pub(crate) disabled: bool, + /// The model has a binding error, which makes every model reading it partial. + pub(crate) blocked: bool, + /// Its own reasons, sorted; those from invalid dependencies are derived. + pub(crate) partial: Vec, +} + +impl ModelCoverage { + /// Whether the model adds anything to the reasons. + pub(crate) fn notable(&self) -> bool { + self.disabled || self.blocked || !self.partial.is_empty() + } +} + +/// Each model's partial-check reasons, in `order`, which pairs each model with its upstream. +pub(crate) fn coverage_reasons( + order: &[(String, Vec)], + facts: &[ModelCoverage], +) -> Vec<(String, Vec)> { + let facts: HashMap<&str, &ModelCoverage> = facts + .iter() + .map(|model| (model.name.as_str(), model)) + .collect(); + let mut blocked_by: HashMap<&str, HashSet<&str>> = facts + .values() + .filter(|model| model.blocked) + .map(|model| (model.name.as_str(), HashSet::from([model.name.as_str()]))) + .collect(); + if !blocked_by.is_empty() { + for _ in order { + let mut changed: bool = false; + for (name, upstream) in order { + let name: &str = name.as_str(); + let mut causes: HashSet<&str> = blocked_by.get(name).cloned().unwrap_or_default(); + for upstream in upstream { + causes.extend( + blocked_by + .get(upstream.as_str()) + .cloned() + .unwrap_or_default(), + ); + } + if !causes.is_empty() && blocked_by.get(name) != Some(&causes) { + blocked_by.insert(name, causes); + changed = true; + } + } + if !changed { + break; + } + } + } + let mut reasons: Vec<(String, Vec)> = Vec::new(); + for (name, _) in order { + let name: &str = name.as_str(); + let model: Option<&&ModelCoverage> = facts.get(name); + if model.is_some_and(|model| model.disabled) { + reasons.push((name.to_owned(), vec![ANALYSIS_DISABLED.to_owned()])); + continue; + } + let mut partial: BTreeSet = model + .map(|model| model.partial.iter().cloned().collect()) + .unwrap_or_default(); + if let Some(causes) = blocked_by.get(name) { + for cause in causes.iter().filter(|cause| **cause != name) { + partial.insert(format!("{INVALID_DEPENDENCY} {cause}")); + } + } + if !partial.is_empty() { + reasons.push((name.to_owned(), partial.into_iter().collect())); + } + } + reasons +} + +/// The whole project's notable facts: `stored` with the fresh models' replaced by `fresh`. +pub(crate) fn merged_coverage( + stored: &[ModelCoverage], + fresh: &[ModelCoverage], +) -> Vec { + let rerun: HashSet<&str> = fresh.iter().map(|model| model.name.as_str()).collect(); + stored + .iter() + .filter(|model| !rerun.contains(model.name.as_str())) + .chain(fresh.iter().filter(|model| model.notable())) + .cloned() + .collect() +} + +/// Every project model with its upstream names, the fresh models' replacing the stored. +pub(crate) fn merged_order( + stored: &[(String, Vec)], + fresh: &[(String, Vec)], +) -> Vec<(String, Vec)> { + let fresh: HashMap<&str, &Vec> = fresh + .iter() + .map(|(name, upstream)| (name.as_str(), upstream)) + .collect(); + stored + .iter() + .map(|(name, upstream)| { + ( + name.clone(), + fresh + .get(name.as_str()) + .map_or(upstream, |fresh| *fresh) + .clone(), + ) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs new file mode 100644 index 0000000000..6051989506 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -0,0 +1,1206 @@ +//! Compile only changed models and SQL tests against the stored compile, or fall back. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyStringMethods; +use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; +use sqlbuild_cache::project_reuse::main::read_delta_base::read_delta_base; +use sqlbuild_cache::project_reuse::main::read_delta_output::read_delta_output; +use sqlbuild_cache::project_reuse::models::{DeltaBase, DeltaCandidate, ProjectFileChanges}; +use sqlbuild_cache::project_reuse::types::ImageSections; + +use crate::bindings::_helpers::cli::progress::{NativeProgressProjector, current_projector}; +use crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug; +use crate::bindings::_helpers::compile_command::reuse_recording::{ + RecordedCompileOutput, print_to, +}; +use crate::bindings::_helpers::compile_delta::context::{ + CapturedDiagnostic, CapturedModel, CapturedTest, DeltaInjection, ImageCapture, RecordMerge, + start_capture, take_capture, take_delta_unsupported, with_injection, +}; +use crate::bindings::_helpers::compile_delta::header_keys::{ + HeaderChange, custom_rule_reads, header_change, +}; +use crate::bindings::_helpers::compile_delta::image::{ + BaseRules, ImageGate, ImageRelations, ImageRules, PendingRules, ProjectImage, +}; +use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; +use crate::bindings::_helpers::compile_driver::command::{ + PhasesOutcome, resolved_request, unrecorded_compile, +}; +use crate::bindings::_helpers::compile_outputs::reuse::NativeReuseAttempt; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Set to `1` to always compile the whole project. +pub(crate) const DELTA_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_DELTA_COMPILE"; +/// Set to `1` to run each delta compile beside a full compile and fail on any difference. +pub(crate) const DELTA_VERIFY_ENV_VAR: &str = "SQLBUILD_VERIFY_DELTA_COMPILE"; +const ENABLED_VALUE: &str = "1"; +/// Rerunning more models than this, or this share of the project, compiles the whole project. +const MAX_RERUN_MODELS: usize = 500; +const MAX_RERUN_PERCENT: usize = 5; +/// Whether edits that report or raise errors compile as deltas; until then they fall back. +pub(crate) const ERRORS_ON_DELTA_PATH: bool = true; +/// How many times a changed relation shape may pull its dependents into the delta. +const MAX_ROUNDS: usize = 4; +const MODELS_ROOT: &str = "models/"; +/// How a rules diagnostic's report JSON names its phase; rules run over the whole project. +const RULE_PHASE: &str = "\"phase\": \"rule\""; +/// What every error for a reference to an undiscovered source says. +const UNKNOWN_SOURCE: &str = "unknown source"; +/// The upstream kind of a source a model reads. +const SOURCE_KIND: &str = "source"; +/// The upstream kind of a model a model reads. +const MODEL_KIND: &str = "model"; +const TESTS_ROOT: &str = "tests/"; +const STDERR: &str = "stderr"; +const SQL_SUFFIX: &str = ".sql"; +/// How generated SQL test CTE names mark their prefix, as in `__ref__orders`. +const CTE_PREFIX_MARK: &str = "__"; +const QUOTE: char = '\''; +/// What errors for a reference to a model the project lacks say. +const MISSING_MODEL_PHRASES: [&str; 2] = ["unknown model", "which does not exist"]; +const UNCOMPILED_MODEL: &str = "the delta compile reported a model it did not compile as missing"; +/// How many changed, added and removed paths the candidates debug line names. +const CANDIDATES_LOGGED: usize = 5; +const UNREADABLE_IMAGE: &str = "the project image cannot be read"; +const TOO_MANY_MODELS: &str = "too many models would compile again"; +/// The attribute holding a Python enum member's value. +const ENUM_VALUE: &str = "value"; +/// The lineage mode a delta compile reproduces; richer modes compile in full. +const FAST_LINEAGE: &str = "fast"; +/// Under verification, the delta's own report is kept here for inspection. +const VERIFY_REPORT_DIRECTORY: &str = "target/cache/compiler-native-v1"; +const VERIFY_REPORT_FILE: &str = "delta-report.out"; + +/// Whether the environment turns `name` on. +pub(crate) fn env_enabled(name: &str) -> bool { + std::env::var(name).is_ok_and(|value| value == ENABLED_VALUE) +} + +/// Whether delta compiles, and so project images, are on for this process. +pub(crate) fn delta_enabled() -> bool { + !env_enabled(DELTA_DISABLE_ENV_VAR) +} + +/// A delta compile that finished, ready to be stored and printed or compared. +pub(crate) struct Finished<'py> { + pub(crate) stdout: String, + pub(crate) stderr_lines: Vec, + pub(crate) phases: PhasesOutcome<'py>, + pub(crate) image: Option, + pub(crate) merge: RecordMerge, + pub(crate) changed_files: usize, + pub(crate) rerun_models: usize, +} + +/// What one delta attempt came to. +pub(crate) enum DeltaOutcome<'py> { + /// The delta cannot reproduce this compile; the reason is for diagnostics only. + Fallback(String), + /// The compile stopped on an error from a changed file, exactly as a full compile would. + Raised(PyErr), + Finished(Box>), +} + +fn fallback<'py>(reason: impl Into) -> PyResult> { + Ok(DeltaOutcome::Fallback(reason.into())) +} + +/// Whether the invocation asks for anything a delta compile does not reproduce. +fn ineligible(request: &Bound<'_, PyAny>) -> PyResult> { + let empty = |name: &str| -> PyResult { + let value: Bound<'_, PyAny> = native_field(request, name)?; + Ok(value.is_none() || !value.is_truthy()?) + }; + for (name, reason) in [ + ("select", "a selection was given"), + ("exclude", "an exclusion was given"), + ("dag_path", "a DAG artifact was requested"), + ("manifest", "a manifest was requested"), + ("defer_to", "deferral was requested"), + ("no_sql_validation", "SQL validation is off"), + ] { + if !empty(name)? { + return Ok(Some(reason)); + } + } + let lineage: Bound<'_, PyAny> = native_field(request, "lineage_mode")?; + if !lineage.is_none() && native_field(&lineage, ENUM_VALUE)?.str()?.to_str()? != FAST_LINEAGE { + return Ok(Some("rich lineage was requested")); + } + let flags: Bound<'_, PyAny> = native_field(request, "profile_flags")?; + for name in [ + "skip_discovery_sql_analysis", + "skip_column_inference", + "skip_contracts", + "skip_write", + ] { + if native_field(&flags, name)?.is_truthy()? { + return Ok(Some("a profiling flag was given")); + } + } + Ok(None) +} + +/// A path the delta may recompile: an SQL file in a resource tree, not a declaration tree. +fn resource_sql(path: &str, root: &str) -> bool { + path.starts_with(root) + && path.ends_with(SQL_SUFFIX) + && !path.split('/').any(|part| part.starts_with('_')) +} + +/// The stored compile's inputs and image; its report is read only after the delta runs. +struct Base { + store_path: PathBuf, + stored: DeltaBase, + image: ProjectImage, + project_dir: PathBuf, +} + +/// The project files the reuse check found changed, and the slot it checked. +fn changes_of( + attempt: &Bound<'_, PyAny>, +) -> PyResult> { + let native: Bound<'_, PyAny> = native_field(attempt, "native")?; + let Ok(native) = native.cast::() else { + return Ok(Err("the reuse check did not run".to_owned())); + }; + let (delta, store_path): (DeltaCandidate, PathBuf) = native + .get() + .with_attempt(|attempt| (attempt.delta.clone(), attempt.store_path.clone()))?; + Ok(match delta { + DeltaCandidate::Files(changes) => Ok((changes, store_path)), + DeltaCandidate::Unavailable(reason) => Err(reason.to_owned()), + }) +} + +/// The rerun set: models and SQL tests, with every path and name the delta recompiles. +#[derive(Clone, Debug, Default)] +struct Rerun { + changed: BTreeSet, + /// The sources the rerun models read, the only ones discovery keeps; `None` keeps all. + sources: Option>, + models: BTreeSet, + model_paths: BTreeSet, + test_paths: BTreeSet, + tests: BTreeSet, +} + +impl Rerun { + /// The rerun set grown by every SQL test it touches and every model those tests run. + fn with_tests(mut self, image: &ProjectImage, changed_tests: &BTreeSet) -> Self { + self.test_paths = changed_tests.clone(); + let test_files: BTreeSet<&str> = image + .tests + .iter() + .map(|test| test.relative_path.as_str()) + .collect(); + for diagnostic in &image.diagnostics { + for path in &diagnostic.paths { + if test_files.contains(path.as_str()) { + self.test_paths.insert(path.clone()); + } + } + } + let model_paths: HashMap<&str, &str> = image + .models + .iter() + .map(|model| (model.name.as_str(), model.relative_path.as_str())) + .collect(); + let mut growing: bool = true; + while growing { + growing = false; + for test in &image.tests { + if !self.test_paths.contains(&test.relative_path) + && !test.chain.iter().any(|name| self.models.contains(name)) + { + continue; + } + self.test_paths.insert(test.relative_path.clone()); + for name in &test.chain { + if let Some(path) = model_paths.get(name.as_str()) + && self.models.insert(name.clone()) + { + self.model_paths.insert((*path).to_owned()); + growing = true; + } + } + } + } + self.tests = image + .tests + .iter() + .filter(|test| self.test_paths.contains(&test.relative_path)) + .map(|test| test.key.clone()) + .collect(); + self + } + + fn with_models(mut self, image: &ProjectImage, added: Vec) -> Self { + let paths: HashMap<&str, &str> = image + .models + .iter() + .map(|model| (model.name.as_str(), model.relative_path.as_str())) + .collect(); + for name in added { + if let Some(path) = paths.get(name.as_str()) { + self.model_paths.insert((*path).to_owned()); + } + self.models.insert(name); + } + self + } + + fn names(&self) -> BTreeSet { + let mut names: BTreeSet = self.models.clone(); + names.extend(self.model_paths.iter().cloned()); + names.extend(self.tests.iter().cloned()); + names.extend(self.test_paths.iter().cloned()); + names + } + + fn owns(&self, diagnostic: &CapturedDiagnostic) -> bool { + diagnostic + .resource_name + .as_ref() + .is_some_and(|name| self.models.contains(name) || self.tests.contains(name)) + || diagnostic + .paths + .iter() + .any(|path| self.model_paths.contains(path) || self.test_paths.contains(path)) + } +} + +/// The injection of the image's whole-project facts, moved out of `image` while the delta runs. +fn lent_injection( + mut image: ProjectImage, + project_dir: PathBuf, + store_path: &std::path::Path, +) -> (ProjectImage, DeltaInjection) { + let injection: DeltaInjection = DeltaInjection { + yaml_store: store_path + .parent() + .and_then(std::path::Path::parent) + .map(crate::bindings::_helpers::compile_delta::yaml_cache::yaml_store_path), + project_dir, + source_paths: image + .models + .iter() + .map(|model| model.relative_path.clone()) + .chain(image.tests.iter().map(|test| test.relative_path.clone())) + .collect(), + project_model_paths: image + .models + .iter() + .map(|model| model.relative_path.clone()) + .collect(), + model_names: image + .models + .iter() + .map(|model| model.name.clone()) + .collect(), + relations: std::mem::take(&mut image.relations).into(), + rules: std::sync::Mutex::new(image.rules.take().and_then(ImageRules::pending)), + fact_digests: std::mem::take(&mut image.fact_digests) + .into_iter() + .collect(), + model_subjects: image.model_subjects.take(), + text_models: image.text_models.clone().unwrap_or_default(), + model_shapes: std::mem::take(&mut image.model_shapes), + coverage: image.coverage.take(), + coverage_order: image + .models + .iter() + .map(|model| (model.name.clone(), upstream_models(&model.upstream))) + .collect(), + lint_model_keys: image.lint_model_keys.take(), + lint_file_keys: image.lint_file_keys.take(), + ..DeltaInjection::default() + }; + (image, injection) +} + +/// Return the facts `lent_injection` moved out of `image`. +fn returned(mut image: ProjectImage, injection: DeltaInjection) -> ProjectImage { + image.relations = injection.relations.into(); + image.fact_digests = injection.fact_digests.into_iter().collect(); + image.model_subjects = injection.model_subjects; + image.model_shapes = injection.model_shapes; + image.coverage = injection.coverage; + image.lint_model_keys = injection.lint_model_keys; + image.lint_file_keys = injection.lint_file_keys; + image +} + +/// `injection` set up to rerun `rerun`. +fn for_rerun( + mut injection: DeltaInjection, + rerun: &Rerun, + sources_read: Option>, +) -> DeltaInjection { + injection.model_paths = rerun.model_paths.clone(); + injection.test_paths = rerun.test_paths.clone(); + injection.rerun_names = rerun.names(); + injection.changed_paths = rerun.changed.clone(); + injection.source_names = sources_read; + injection +} + +/// The relation facts of `name` that its dependents' analysis reads. +fn shape_of(relations: &RelationTables, name: &str) -> [Option>; 3] { + let find = |table: &[(String, Vec<(String, String)>)]| { + table + .iter() + .find(|(relation, _)| relation == name) + .map(|(_, shape)| shape.clone()) + }; + [ + find(&relations.types), + find(&relations.nullability), + find(&relations.complete), + ] +} + +/// `base`'s rows with each of `fresh`'s rows replacing the one of its name, new names appended. +fn merged_rows( + mut base: Vec<(String, Vec<(String, String)>)>, + fresh: Vec<(String, Vec<(String, String)>)>, +) -> Vec<(String, Vec<(String, String)>)> { + let mut positions: HashMap = base + .iter() + .enumerate() + .map(|(index, (name, _))| (name.clone(), index)) + .collect(); + for (name, shape) in fresh { + match positions.get(&name) { + Some(index) => base[*index].1 = shape, + None => { + positions.insert(name.clone(), base.len()); + base.push((name, shape)); + } + } + } + base +} + +/// One restricted compile of `rerun`, with what it captured. +struct Round<'py> { + stdout: Option, + stderr_lines: Vec, + phases: PhasesOutcome<'py>, + capture: ImageCapture, +} + +enum RoundOutcome<'py> { + Ran(Box>), + Stop(DeltaOutcome<'py>), +} + +/// Compile the rerun set once, keeping the longest round's lines: every phase a full compile runs. +fn round<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, + (injection, rerun): (DeltaInjection, &Rerun), +) -> PyResult<(RoundOutcome<'py>, Option)> { + start_capture(); + let projector: Option> = current_projector(request.py())?; + let earlier: Vec = projector + .as_ref() + .map(|projector| projector.get().exchange_held_lines(Vec::new())) + .unwrap_or_default(); + let (ran, injection) = with_injection(injection, || unrecorded_compile(request, attempt)); + if let Some(projector) = &projector { + let latest: Vec = projector.get().exchange_held_lines(Vec::new()); + let _ = projector + .get() + .exchange_held_lines(if latest.len() < earlier.len() { + earlier + } else { + latest + }); + } + Ok((ran_round(ran, rerun)?, injection)) +} + +fn ran_round<'py>( + ran: PyResult<( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + PyResult>, + )>, + rerun: &Rerun, +) -> PyResult> { + let unsupported: Option = take_delta_unsupported(); + let capture: Option = take_capture(); + if let Some(reason) = unsupported { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(reason))); + } + let (_, output, phases) = match ran { + Ok(ran) => ran, + Err(error) => { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(format!( + "the delta compile failed: {error}" + )))); + } + }; + let (stderr_lines, stdout): (Vec, Option) = output + .cast::()? + .get() + .texts(output.py())?; + let phases: PhasesOutcome<'py> = match phases { + Ok(phases) => phases, + Err(error) => { + let message: String = error.to_string(); + let owned: bool = rerun + .model_paths + .iter() + .chain(&rerun.test_paths) + .any(|path| message.contains(path.as_str())); + if rerun.sources.is_some() && message.contains(UNKNOWN_SOURCE) { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(format!( + "the delta compile read a source it did not discover: {message}" + )))); + } + return Ok(RoundOutcome::Stop(if owned && ERRORS_ON_DELTA_PATH { + DeltaOutcome::Raised(error) + } else { + DeltaOutcome::Fallback(format!("the delta compile raised: {message}")) + })); + } + }; + let Some(capture) = capture else { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback( + "the delta compile captured nothing".to_owned(), + ))); + }; + if let Some(reason) = &capture.unsupported { + return Ok(RoundOutcome::Stop(DeltaOutcome::Fallback(reason.clone()))); + } + Ok(RoundOutcome::Ran(Box::new(Round { + stdout, + stderr_lines, + phases, + capture, + }))) +} + +/// A delta compile ready to run its rounds: the stored compile, the changes and the first rerun. +struct Prepared { + base: Base, + changed_files: usize, + rerun: Rerun, + changed_tests: BTreeSet, + limit: usize, +} + +/// Compile only the changed files against the stored compile, or say why it cannot. +pub(crate) fn run<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, +) -> PyResult> { + if !delta_enabled() { + return fallback("delta compiles are off"); + } + if let Some(reason) = ineligible(request)? { + return fallback(reason); + } + let py: Python<'py> = request.py(); + let started: std::time::Instant = std::time::Instant::now(); + let prepared: Prepared = match prepared(request, attempt)? { + Ok(prepared) => prepared, + Err(reason) => return fallback(reason), + }; + trace(py, "delta base read", started)?; + let changed_files: usize = prepared.changed_files; + let (ran, rerun, base): (Round, Rerun, Base) = match rounds(request, attempt, prepared)? { + Ok(found) => found, + Err(outcome) => return Ok(outcome), + }; + trace(py, "delta rounds", started)?; + let outcome: PyResult> = finished(ran, base, &rerun, (py, changed_files)); + trace(py, "delta report", started)?; + outcome +} + +/// The first `CANDIDATES_LOGGED` of `paths`. +fn first_paths(paths: &[String]) -> &[String] { + &paths[..paths.len().min(CANDIDATES_LOGGED)] +} + +/// Why `changes` rule a delta out before anything is read, if they do. +fn changes_ruled_out(changes: &ProjectFileChanges) -> Option { + if !changes.added.is_empty() || !changes.removed.is_empty() { + return Some("files were added or removed".to_owned()); + } + if changes.changed.is_empty() { + return Some("no project file changed".to_owned()); + } + changes + .changed + .iter() + .find(|path| !resource_sql(path, MODELS_ROOT) && !resource_sql(path, TESTS_ROOT)) + .map(|path| format!("{path} is not a model or SQL test file")) +} + +/// The stored compile, its image and the first rerun set, or why the gate rules a delta out. +fn prepared( + request: &Bound<'_, PyAny>, + attempt: &Bound<'_, PyAny>, +) -> PyResult> { + let (changes, store_path): (ProjectFileChanges, PathBuf) = match changes_of(attempt)? { + Ok(found) => found, + Err(reason) => return Ok(Err(reason)), + }; + reuse_debug( + request.py(), + &format!( + "Delta compile candidates: changed {:?}, added {:?}, removed {:?}", + first_paths(&changes.changed), + first_paths(&changes.added), + first_paths(&changes.removed), + ), + )?; + if let Some(reason) = changes_ruled_out(&changes) { + return Ok(Err(reason)); + } + let Some(mut stored) = read_delta_base(&store_path) else { + return Ok(Err("the stored compile has no project image".to_owned())); + }; + let Some(gate) = ImageGate::decode(&stored.image) else { + return Ok(Err(UNREADABLE_IMAGE.to_owned())); + }; + if !ERRORS_ON_DELTA_PATH && gate.has_errors { + return Ok(Err("the stored compile reported errors".to_owned())); + } + if gate.json_output != native_field(request, "json_output")?.is_truthy()? { + return Ok(Err( + "the stored compile has the other output format".to_owned() + )); + } + let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; + let (rerun, changed_tests) = match first_rerun(&gate, &changes, &project_dir) { + Ok(found) => found, + Err(reason) => return Ok(Err(reason)), + }; + let limit: usize = MAX_RERUN_MODELS.max(gate.models.len() * MAX_RERUN_PERCENT / 100); + if rerun.models.len() > limit { + return Ok(Err(TOO_MANY_MODELS.to_owned())); + } + let Some(image) = ProjectImage::decode(std::mem::take(&mut stored.image), gate) else { + return Ok(Err(UNREADABLE_IMAGE.to_owned())); + }; + Ok(Ok(Prepared { + base: Base { + store_path, + stored, + image, + project_dir, + }, + changed_files: changes.changed.len(), + rerun, + changed_tests, + limit, + })) +} + +/// A fallback that stops the rounds. +fn stopped<'py, T>(reason: impl Into) -> PyResult>> { + Ok(Err(DeltaOutcome::Fallback(reason.into()))) +} + +/// Run rounds until no rerun model changes the shape a model outside the rerun set reads. +fn rounds<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, + prepared: Prepared, +) -> PyResult, Rerun, Base), DeltaOutcome<'py>>> { + let Prepared { + base, + mut rerun, + changed_tests, + limit, + .. + } = prepared; + let Base { + store_path, + stored, + image, + project_dir, + } = base; + let (image, mut injection): (ProjectImage, DeltaInjection) = + lent_injection(image, project_dir.clone(), &store_path); + for _ in 0..MAX_ROUNDS { + if rerun.models.len() > limit { + return stopped(TOO_MANY_MODELS); + } + rerun = rerun.with_tests(&image, &changed_tests); + if image + .diagnostics + .iter() + .any(|diagnostic| kept_only_by_full_compiles(&rerun, diagnostic)) + { + return stopped("the stored compile reports problems in files the edit leaves alone"); + } + rerun.sources = sources_read(&image, &rerun); + let injection_for_round: DeltaInjection = + for_rerun(injection, &rerun, rerun.sources.clone()); + let (outcome, back) = round(request, attempt, (injection_for_round, &rerun))?; + let model_names: &[String] = back + .as_ref() + .map_or(&[], |injection| injection.model_names.as_slice()); + let ran: Round<'py> = match outcome { + RoundOutcome::Ran(ran) => *ran, + RoundOutcome::Stop(DeltaOutcome::Raised(error)) + if names_uncompiled_model(&error.to_string(), model_names, &rerun.models) => + { + return stopped(UNCOMPILED_MODEL); + } + RoundOutcome::Stop(outcome) => return Ok(Err(outcome)), + }; + if ran + .capture + .diagnostics + .iter() + .any(|diagnostic| names_uncompiled_model(&diagnostic.json, model_names, &rerun.models)) + { + return stopped(UNCOMPILED_MODEL); + } + if let Some(path) = rechained_test(&ran.capture.test_chains, &image.tests, &changed_tests) { + return stopped(format!("{path} now covers other models")); + } + let Some(back) = back else { + return stopped("the delta compile's injection is still in use"); + }; + injection = back; + let added: Vec = match reshaped_dependents(&ran, &image, &injection, &rerun) { + Ok(added) => added, + Err(reason) => return stopped(reason), + }; + if added.is_empty() { + let base: Base = Base { + store_path, + stored, + image: returned(image, injection), + project_dir, + }; + return Ok(Ok((ran, rerun, base))); + } + let rules: Option = ran + .capture + .rules + .as_ref() + .and_then(|rules| BaseRules::decoded(&rules.retained)); + let Some(rules) = rules else { + return stopped("the delta compile retained no rules request"); + }; + injection.rules = std::sync::Mutex::new(Some(PendingRules::Ready(Box::new(rules)))); + rerun = rerun.with_models(&image, added); + } + stopped("changed shapes reach too far") +} + +/// The sources the rerun models, functions, audits and rerun SQL tests read; `None` for all. +fn sources_read(image: &ProjectImage, rerun: &Rerun) -> Option> { + if image.diagnostics.iter().any(outside_resources) { + return None; + } + Some( + image + .models + .iter() + .filter(|model| rerun.models.contains(&model.name)) + .flat_map(|model| &model.upstream) + .filter(|(kind, _)| kind == SOURCE_KIND) + .map(|(_, name)| name.clone()) + .chain(image.pinned_sources.iter().cloned()) + .chain( + rerun + .test_paths + .iter() + .filter_map(|path| image.test_sources.get(path)) + .flatten() + .cloned(), + ) + .collect(), + ) +} + +/// The models outside `rerun` whose inputs' shapes the round changed, or why it cannot go on. +fn reshaped_dependents( + ran: &Round<'_>, + image: &ProjectImage, + injection: &DeltaInjection, + rerun: &Rerun, +) -> Result, String> { + if rerun.models.is_empty() { + return Ok(Vec::new()); + } + let Some(relations) = ran.capture.relations.as_ref() else { + return Err("the delta compile analysed no models".to_owned()); + }; + let stored_relations: &RelationTables = &injection.relations; + if relations.cyclic != stored_relations.cyclic + || relations.dependency_ordered != stored_relations.dependency_ordered + { + return Err("the project's analysis order changed".to_owned()); + } + let stored_models: HashMap<&str, &CapturedModel> = image + .models + .iter() + .map(|model| (model.name.as_str(), model)) + .collect(); + if let Some(model) = ran + .capture + .models + .iter() + .find(|model| stored_models.get(model.name.as_str()) != Some(model)) + { + return Err(format!("{} changed its references", model.name)); + } + if let Some(path) = rerun + .model_paths + .iter() + .find(|path| ran.capture.usages.get(*path) != image.usages.get(*path)) + { + return Err(format!("{path} changed the declarations it uses")); + } + let reshaped: BTreeSet = rerun + .models + .iter() + .filter(|name| shape_of(relations, name) != shape_of(stored_relations, name)) + .cloned() + .collect(); + Ok(image + .dependents(&reshaped) + .into_iter() + .filter(|name| !rerun.models.contains(name)) + .collect()) +} + +/// The models and SQL tests the changed files rerun, or why the gate rules a delta out. +fn first_rerun( + gate: &ImageGate, + changes: &ProjectFileChanges, + project_dir: &std::path::Path, +) -> Result<(Rerun, BTreeSet), String> { + let model_by_path: HashMap<&str, &str> = gate + .models + .iter() + .map(|model| (model.relative_path.as_str(), model.name.as_str())) + .collect(); + let test_files: BTreeSet<&str> = gate + .tests + .iter() + .map(|test| test.relative_path.as_str()) + .collect(); + let mut rerun: Rerun = Rerun { + changed: changes.changed.iter().cloned().collect(), + ..Rerun::default() + }; + let mut changed_tests: BTreeSet = BTreeSet::new(); + for path in &changes.changed { + if resource_sql(path, MODELS_ROOT) + && let Some(name) = model_by_path.get(path.as_str()) + { + rerun.models.insert((*name).to_owned()); + rerun.model_paths.insert(path.clone()); + } else if resource_sql(path, TESTS_ROOT) && test_files.contains(path.as_str()) { + changed_tests.insert(path.clone()); + } else { + return Err(format!("{path} is not a model or SQL test file")); + } + } + let upstream_by_path: HashMap<&str, &[(String, String)]> = gate + .models + .iter() + .map(|model| (model.relative_path.as_str(), model.upstream.as_slice())) + .collect(); + for path in &rerun.model_paths { + let name: &str = model_by_path + .get(path.as_str()) + .copied() + .unwrap_or_default(); + let upstream: &[(String, String)] = upstream_by_path + .get(path.as_str()) + .copied() + .unwrap_or_default(); + match header_change(project_dir, path, (gate.header_keys.get(path), upstream)) { + HeaderChange::Unchanged => {} + HeaderChange::Unparsed if !ERRORS_ON_DELTA_PATH => { + return Err(format!("{path} does not parse")); + } + HeaderChange::Unparsed => {} + HeaderChange::Unsupported(reason) => return Err(reason), + HeaderChange::Patched(keys) => { + if gate.has_model_subjects && keys.contains("materialized") { + return Err("custom rules select models by their materialization".to_owned()); + } + if let Some(reason) = custom_rule_reads(&gate.fact_digests, (path, name), &keys) { + return Err(reason); + } + } + } + } + let path_by_model: HashMap<&str, &str> = model_by_path + .iter() + .map(|(path, name)| (*name, *path)) + .collect(); + let upstream_by_name: HashMap<&str, &[(String, String)]> = gate + .models + .iter() + .map(|model| (model.name.as_str(), model.upstream.as_slice())) + .collect(); + for test in &gate.tests { + if !changed_tests.contains(&test.relative_path) { + continue; + } + let ring = test.chain.iter().flat_map(|name| { + upstream_models( + upstream_by_name + .get(name.as_str()) + .copied() + .unwrap_or_default(), + ) + }); + let named: BTreeSet = + test_model_references(project_dir, &test.relative_path, &path_by_model)?; + for name in test.chain.iter().cloned().chain(ring).chain(named) { + let Some(path) = path_by_model.get(name.as_str()) else { + return Err(format!("{} tests a model the image lacks", test.key)); + }; + rerun.model_paths.insert((*path).to_owned()); + rerun.models.insert(name); + } + } + Ok((rerun, changed_tests)) +} + +/// The first edited SQL test whose model chain the delta resolved differently than stored. +fn rechained_test<'a>( + fresh: &[CapturedTest], + stored: &'a [CapturedTest], + changed_tests: &BTreeSet, +) -> Option<&'a str> { + let sorted = |chain: &[String]| -> BTreeSet { chain.iter().cloned().collect() }; + let fresh: HashMap<&str, BTreeSet> = fresh + .iter() + .map(|test| (test.key.as_str(), sorted(&test.chain))) + .collect(); + stored + .iter() + .filter(|test| changed_tests.contains(&test.relative_path)) + .find(|test| fresh.get(test.key.as_str()) != Some(&sorted(&test.chain))) + .map(|test| test.relative_path.as_str()) +} + +/// Every project model the SQL test file at `path` names, a harmless superset of its references. +fn test_model_references( + project_dir: &std::path::Path, + path: &str, + path_by_model: &HashMap<&str, &str>, +) -> Result, String> { + let Ok(text) = std::fs::read_to_string(project_dir.join(path)) else { + return Err(format!("{path} cannot be read")); + }; + let mut names: BTreeSet = BTreeSet::new(); + for word in text.split(|character: char| !(character.is_alphanumeric() || character == '_')) { + let unprefixed: &str = word + .strip_prefix(CTE_PREFIX_MARK) + .and_then(|rest| rest.split_once(CTE_PREFIX_MARK)) + .map_or(word, |(_, name)| name); + for candidate in [word, unprefixed] { + if path_by_model.contains_key(candidate) { + names.insert(candidate.to_owned()); + } + } + } + Ok(names) +} + +/// Whether `message` reports as missing a project model the delta did not compile. +pub(crate) fn names_uncompiled_model( + message: &str, + model_names: &[String], + rerun_models: &BTreeSet, +) -> bool { + MISSING_MODEL_PHRASES + .iter() + .any(|phrase| message.contains(phrase)) + && message + .split(QUOTE) + .skip(1) + .step_by(2) + .filter(|name| !rerun_models.contains(*name)) + .any(|name| project_model(model_names, name)) +} + +fn project_model(model_names: &[String], name: &str) -> bool { + model_names.iter().any(|model| model == name) +} + +/// The names of the models among `upstream`. +fn upstream_models(upstream: &[(String, String)]) -> Vec { + upstream + .iter() + .filter(|(kind, _)| kind == MODEL_KIND) + .map(|(_, name)| name.clone()) + .collect() +} + +/// Whether only a full compile would repeat a stored non-rule diagnostic outside the rerun. +fn kept_only_by_full_compiles(rerun: &Rerun, diagnostic: &CapturedDiagnostic) -> bool { + !rerun.owns(diagnostic) + && !diagnostic.json.contains(RULE_PHASE) + && diagnostic + .paths + .iter() + .any(|path| path.starts_with(MODELS_ROOT) || path.starts_with(TESTS_ROOT)) +} + +/// Whether a stored diagnostic concerns a file other than a model or SQL test. +fn outside_resources(diagnostic: &CapturedDiagnostic) -> bool { + diagnostic + .paths + .iter() + .any(|path| !path.starts_with(MODELS_ROOT) && !path.starts_with(TESTS_ROOT)) + || diagnostic.paths.is_empty() && diagnostic.resource_name.is_none() +} + +/// Log how long the delta has run, for profiling. +pub(crate) fn trace(py: Python<'_>, phase: &str, started: std::time::Instant) -> PyResult<()> { + let milliseconds: u128 = started.elapsed().as_millis(); + reuse_debug( + py, + &format!("Delta compile {phase} after {milliseconds} ms"), + ) +} + +/// Every diagnostic not owned by the rerun set, which the delta must leave as it was. +fn unowned<'a>(rerun: &Rerun, diagnostics: &'a [CapturedDiagnostic]) -> Vec<&'a str> { + diagnostics + .iter() + .filter(|diagnostic| !rerun.owns(diagnostic)) + .map(|diagnostic| diagnostic.json.as_str()) + .collect() +} + +fn finished<'py>( + ran: Round<'py>, + base: Base, + rerun: &Rerun, + (py, changed_files): (Python<'py>, usize), +) -> PyResult> { + if !ERRORS_ON_DELTA_PATH && ran.phases.0.exit_code != 0 { + let first: &str = ran + .capture + .diagnostics + .iter() + .find(|diagnostic| diagnostic.error) + .map_or("no error diagnostic", |diagnostic| diagnostic.json.as_str()); + return fallback(format!("the edit reports errors: {first}")); + } + if rerun.sources.is_some() + && ran + .capture + .diagnostics + .iter() + .any(|diagnostic| diagnostic.json.contains(UNKNOWN_SOURCE)) + { + return fallback("the delta compile read a source it did not discover"); + } + let (stored, fresh) = ( + unowned(rerun, &base.image.diagnostics), + unowned(rerun, &ran.capture.diagnostics), + ); + if stored != fresh { + let only = |left: &[&str], right: &[&str]| { + left.iter() + .find(|item| !right.contains(item)) + .map_or(String::new(), |item| (*item).to_owned()) + }; + return fallback(format!( + "diagnostics outside the changed files differ: stored {} / delta {}", + only(&stored, &fresh), + only(&fresh, &stored) + )); + } + let Some(delta_stdout) = ran.stdout else { + return fallback("the delta compile recorded no report"); + }; + if env_enabled(DELTA_VERIFY_ENV_VAR) { + let directory: PathBuf = base.project_dir.join(VERIFY_REPORT_DIRECTORY); + let _ = std::fs::create_dir_all(&directory); + let _ = std::fs::write(directory.join(VERIFY_REPORT_FILE), &delta_stdout); + } + let stderr_lines: Vec = ran.stderr_lines; + let merge: RecordMerge = RecordMerge { + kept: base.stored.inputs.target_files.clone(), + environment_names: base.image.environment_names.clone(), + module_paths: base + .stored + .inputs + .modules + .iter() + .map(|(path, _, _)| path.clone()) + .collect(), + }; + let json_output: bool = base.image.json_output; + let Base { + store_path, + stored, + image, + .. + } = base; + let capture: ImageCapture = ran.capture; + let (image, stdout): (Option, Result) = py.detach(|| { + rayon::join( + || { + let Ok(sections) = patched_image(image, rerun, capture).encode() else { + return None; + }; + Some(sections) + }, + || { + if !json_output { + return Ok(delta_stdout); + } + let Some(stored_output) = read_delta_output(&store_path, &stored) else { + return Err("the stored report changed".to_owned()); + }; + let scope: SpliceScope<'_> = SpliceScope { + models: &rerun.models, + tests: &rerun.tests, + sources: rerun.sources.as_ref(), + }; + spliced_report(&stored_output.stdout, &delta_stdout, &scope) + }, + ) + }); + let stdout: String = match stdout { + Ok(stdout) => stdout, + Err(reason) => return fallback(reason), + }; + let _ = py; + Ok(DeltaOutcome::Finished(Box::new(Finished { + stdout, + stderr_lines, + phases: ran.phases, + image, + merge, + changed_files, + rerun_models: rerun.models.len(), + }))) +} + +/// The stored image with the rerun models' and tests' facts replaced by the delta's. +fn patched_image(base: ProjectImage, rerun: &Rerun, capture: ImageCapture) -> ProjectImage { + let mut image: ProjectImage = base; + let fresh_models: HashMap = capture + .models + .into_iter() + .map(|model| (model.name.clone(), model)) + .collect(); + for model in &mut image.models { + if let Some(fresh) = fresh_models.get(&model.name) { + *model = fresh.clone(); + } + } + image.usages.extend(capture.usages); + let mut fresh_tests: HashMap = capture + .test_chains + .into_iter() + .map(|test| (test.key.clone(), test)) + .collect(); + for test in &mut image.tests { + if rerun.tests.contains(&test.key) + && let Some(fresh) = fresh_tests.remove(&test.key) + { + *test = fresh; + } + } + if let Some(relations) = capture.relations { + let fresh: ImageRelations = relations.into(); + image.relations.types = + merged_rows(std::mem::take(&mut image.relations.types), fresh.types); + image.relations.nullability = merged_rows( + std::mem::take(&mut image.relations.nullability), + fresh.nullability, + ); + image.relations.complete = merged_rows( + std::mem::take(&mut image.relations.complete), + fresh.complete, + ); + image.relations.schemas = + merged_rows(std::mem::take(&mut image.relations.schemas), fresh.schemas); + let mut legacy: BTreeSet = image.relations.legacy_models.iter().cloned().collect(); + for name in &rerun.models { + legacy.remove(name); + } + legacy.extend(fresh.legacy_models); + image.relations.legacy_models = legacy.into_iter().collect(); + } + let mut digests: BTreeMap, String> = image.fact_digests.into_iter().collect(); + digests.extend(capture.fact_digests); + image.fact_digests = digests.into_iter().collect(); + image.diagnostics = capture.diagnostics; + if capture.text_models.is_some() { + image.text_models = capture.text_models; + } + image.rules = capture + .rules + .map(|rules| ImageRules::Retained(rules.retained)); + if let Some(shapes) = capture.model_shapes { + image.model_shapes = shapes; + } + if capture.coverage.is_some() { + image.coverage = capture.coverage; + } + image.header_keys.extend(capture.header_keys); + let mut pinned: BTreeSet = std::mem::take(&mut image.pinned_sources) + .into_iter() + .collect(); + pinned.extend(capture.pinned_sources); + image.pinned_sources = pinned.into_iter().collect(); + for (path, names) in capture.test_sources { + image.test_sources.insert(path, names.into_iter().collect()); + } + image.lint_model_keys = capture.lint_model_keys; + image.lint_file_keys = capture.lint_file_keys; + image +} + +/// Store and print a finished delta compile; returns its exit code. +pub(crate) fn commit( + finished: Finished<'_>, + (request, attempt): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = attempt.py(); + let request: Bound<'_, PyAny> = resolved_request(request)?; + let Finished { + stdout, + stderr_lines, + phases, + image, + merge, + .. + } = finished; + let report: &str = stdout.strip_suffix('\n').unwrap_or(&stdout); + let output: Bound<'_, PyAny> = Bound::new( + py, + RecordedCompileOutput::recorded(py, &stderr_lines, &format!("{report}\n")), + )? + .into_any(); + for line in &stderr_lines { + print_to(py, STDERR, line.as_str())?; + } + crate::bindings::_helpers::compile_delta::context::set_pending_image(image); + crate::bindings::_helpers::compile_delta::context::set_record_merge(Some(merge)); + crate::bindings::_helpers::compile_driver::command::store_and_print( + (&request, attempt), + &output, + phases, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs new file mode 100644 index 0000000000..9cc07cefd5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/header_keys.rs @@ -0,0 +1,272 @@ +//! Each model header's keys as digests, so a delta compile can tell which keys an edit changed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::compile_delta::context::{capture, capturing}; +use crate::bindings::_helpers::discovery::model_records::model_file_options; + +/// One model's header keys with the digest of each authored value, sorted by key. +pub(crate) type HeaderKeys = Vec<(String, String)>; + +/// Header keys a delta compile recompiles the model for; any other changed key falls back. +const PATCHED_KEYS: [&str; 6] = [ + "description", + "columns", + "dynamic_columns", + "model_schema", + "audits", + "contract", +]; +/// `materialized` values a delta compile handles; others name materializations it cannot. +const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"]; +const MATERIALIZED_KEY: &str = "materialized"; +/// Plain `--` comments, which every adapter's SQL reads as comments. +const LINE_COMMENT: &str = "--"; +/// Reference call kinds and the upstream kinds they resolve to. +const REF_CALL: &str = "ref"; +const SOURCE_CALL: &str = "source"; +const SEED_CALL: &str = "seed"; +const MODEL_KIND: &str = "model"; +const SOURCE_KIND: &str = "source"; +const SEED_KIND: &str = "seed"; + +/// Bytes of a digest kept per header value; a collision would need 2^64 work to find. +const KEPT_DIGEST_BYTES: usize = 16; +const HEX: &[u8; 16] = b"0123456789abcdef"; + +/// An unambiguous encoding of header values: a tag byte, then length-prefixed parts. +#[derive(Default)] +struct ValueEncoding { + bytes: Vec, +} + +impl ValueEncoding { + fn text(&mut self, tag: u8, text: &str) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(text.len() as u64).to_le_bytes()); + self.bytes.extend_from_slice(text.as_bytes()); + } + + fn list(&mut self, tag: u8, items: &[AuthoredValue]) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(items.len() as u64).to_le_bytes()); + for item in items { + self.value(item); + } + } + + fn pairs(&mut self, tag: u8, items: &[(String, AuthoredValue)]) { + self.bytes.push(tag); + self.bytes + .extend_from_slice(&(items.len() as u64).to_le_bytes()); + for (key, item) in items { + self.text(b'k', key); + self.value(item); + } + } + + fn value(&mut self, value: &AuthoredValue) { + match value { + AuthoredValue::Null => self.bytes.push(b'n'), + AuthoredValue::Boolean(flag) => self.bytes.extend_from_slice(&[b'b', u8::from(*flag)]), + AuthoredValue::BareWord(word) => self.text(b'w', word), + AuthoredValue::String(string) => self.text(b's', string), + AuthoredValue::List(items) => self.list(b'l', items), + AuthoredValue::Map(items) => self.pairs(b'm', items), + AuthoredValue::Set(items) => self.list(b'e', items), + AuthoredValue::Tuple(items) => self.list(b't', items), + AuthoredValue::TypedConstant(items) => self.pairs(b'c', items), + AuthoredValue::InlineSqlHook(sql) => self.text(b'i', sql), + AuthoredValue::NamedSqlHook(name, items) => { + self.text(b'h', name); + self.pairs(b'a', items); + } + AuthoredValue::PythonHook(name, items) => { + self.text(b'p', name); + self.pairs(b'a', items); + } + } + } +} + +/// Whether every model, source and seed `query_sql` names was among the stored `upstream`. +fn references_read(query_sql: &str, upstream: &[(String, String)]) -> bool { + let syntax: LexicalSyntax = LexicalSyntax { + line_comment_prefixes: vec![LINE_COMMENT.to_owned()], + ..LexicalSyntax::default() + }; + let ReferenceExtraction::Extracted(scan) = extract_sql_references(query_sql, &syntax) else { + return false; + }; + scan.references.iter().all(|reference| { + let kind: Option<&str> = match reference.kind { + REF_CALL => Some(MODEL_KIND), + SOURCE_CALL => Some(SOURCE_KIND), + SEED_CALL => Some(SEED_KIND), + _ => None, + }; + match (kind, &reference.package) { + (Some(kind), None) => stored_upstream(upstream, kind, &reference.name), + _ => true, + } + }) +} + +/// Whether `upstream` holds the `kind` resource `name`. +fn stored_upstream(upstream: &[(String, String)], kind: &str, name: &str) -> bool { + upstream + .iter() + .any(|(stored_kind, stored_name)| stored_kind == kind && stored_name == name) +} + +fn short_hex(digest: &[u8]) -> String { + let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2); + for byte in &digest[..KEPT_DIGEST_BYTES] { + hex.push(char::from(HEX[usize::from(byte >> 4)])); + hex.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + hex +} + +fn header_keys(values: &[(String, AuthoredValue)]) -> HeaderKeys { + let mut keys: HeaderKeys = values + .iter() + .map(|(key, value)| { + let mut encoding: ValueEncoding = ValueEncoding::default(); + encoding.value(value); + let hex: String = short_hex(&bytes_digest(&encoding.bytes)); + (key.clone(), hex) + }) + .collect(); + keys.sort(); + keys +} + +/// Record one discovered model's header keys for the image. +pub(crate) fn record_header_keys(relative_path: &str, values: &[(String, AuthoredValue)]) { + if !capturing() { + return; + } + let keys: HeaderKeys = header_keys(values); + let path: String = relative_path.replace('\\', "/"); + capture(|capture| { + capture.header_keys.insert(path, keys); + }); +} + +/// What an edit did to one model's header. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum HeaderChange { + /// The header's keys and values are unchanged. + Unchanged, + /// These keys changed, every one of which the delta recompiles. + Patched(BTreeSet), + /// Why the delta cannot recompile this header change. + Unsupported(String), + /// The file does not parse; the delta reports its error as a full compile would. + Unparsed, +} + +/// How the header of the model file at `path` changed against its recorded keys. +pub(crate) fn header_change( + project_dir: &Path, + relative_path: &str, + (stored, upstream): (Option<&HeaderKeys>, &[(String, String)]), +) -> HeaderChange { + let Some(stored) = stored else { + return HeaderChange::Unsupported(format!("{relative_path} has no recorded header")); + }; + let Ok(contents) = std::fs::read_to_string(project_dir.join(relative_path)) else { + return HeaderChange::Unparsed; + }; + let file_path: String = project_dir + .join(relative_path) + .to_string_lossy() + .into_owned(); + let Ok(parsed) = parse_model_text(&file_path, contents, &model_file_options()) else { + return HeaderChange::Unparsed; + }; + if !references_read(&parsed.query_sql, upstream) { + return HeaderChange::Unsupported(format!("{relative_path} changed what it references")); + } + let current: HeaderKeys = header_keys(&parsed.header_values); + let stored_map: BTreeMap<&str, &str> = stored + .iter() + .map(|(key, digest)| (key.as_str(), digest.as_str())) + .collect(); + let current_map: BTreeMap<&str, &str> = current + .iter() + .map(|(key, digest)| (key.as_str(), digest.as_str())) + .collect(); + let changed: BTreeSet = stored_map + .keys() + .chain(current_map.keys()) + .filter(|key| stored_map.get(*key) != current_map.get(*key)) + .map(|key| (*key).to_owned()) + .collect(); + if changed.is_empty() { + return HeaderChange::Unchanged; + } + for key in &changed { + if key == MATERIALIZED_KEY { + let value = parsed.header_values.iter().find(|(name, _)| name == key); + let supported: bool = matches!( + value, + Some((_, AuthoredValue::BareWord(word) | AuthoredValue::String(word))) + if PATCHED_MATERIALIZATIONS.contains(&word.as_str()) + ); + if !supported { + return HeaderChange::Unsupported(format!( + "{relative_path} changed its materialization beyond views and tables" + )); + } + } else if !PATCHED_KEYS.contains(&key.as_str()) { + return HeaderChange::Unsupported(format!( + "{relative_path} changed its {key} header key" + )); + } + } + HeaderChange::Patched(changed) +} + +/// Whether a custom rule reads a fact of this model that the changed header keys decide. +pub(crate) fn custom_rule_reads( + fact_digests: &[(Vec, String)], + model: (&str, &str), + changed: &BTreeSet, +) -> Option { + let (path, name) = model; + for (key, _) in fact_digests { + let Some((fact, arguments)) = key.split_first() else { + continue; + }; + if !arguments + .iter() + .any(|argument| argument == path || argument == name) + { + continue; + } + let reads: &[&str] = match fact.as_str() { + "contracts.enforced" | "contract" => &["contract"], + "contracts.grain" | "grain" => &["contract", "columns"], + "columns.declared" | "columns.names" => &["columns", "dynamic_columns"], + "materialized" => &["materialized"], + "tests.for_model" | "audits.for_model" => &["audits", "columns"], + _ => &[], + }; + if let Some(key) = reads.iter().find(|key| changed.contains(**key)) { + return Some(format!("a custom rule reads {name}'s {key}")); + } + } + None +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs new file mode 100644 index 0000000000..dd32ec6f5e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -0,0 +1,679 @@ +//! The project image a full or delta compile stores beside its reuse slot. + +use std::collections::{BTreeMap, BTreeSet}; + +use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_rules::engine::main::retained_request::retained_request; +use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; + +use crate::bindings::_helpers::compile_delta::context::{ + CapturedDiagnostic, CapturedModel, CapturedTest, ImageCapture, ModelShape, +}; +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; +use crate::bindings::_helpers::compile_delta::header_keys::HeaderKeys; +use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; + +/// Changes whenever the image's meaning changes, so an older image is never read. +pub(crate) const IMAGE_FORMAT: &str = "sqlbuild-project-image-v8"; +pub(crate) const GATE_SECTION: &str = "gate"; +/// The upstream kind of a model another model reads. +const MODEL_KIND: &str = "model"; +pub(crate) const FACTS_SECTION: &str = "facts"; +const RULES_SECTION: &str = "rules"; +pub(crate) const RULE_DIGESTS_SECTION: &str = "rule-digests"; +const RULE_ROWS_SECTION: &str = "rule-rows"; + +/// Everything a delta compile reads of the compile it patches. +#[derive(Debug, Default)] +pub(crate) struct ProjectImage { + pub(crate) format: String, + pub(crate) json_output: bool, + pub(crate) models: Vec, + pub(crate) usages: BTreeMap, + pub(crate) tests: Vec, + pub(crate) relations: ImageRelations, + pub(crate) fact_digests: Vec<(Vec, String)>, + pub(crate) model_subjects: Option)>>, + pub(crate) diagnostics: Vec, + pub(crate) text_models: Option>, + pub(crate) environment_names: Vec, + pub(crate) pinned_sources: Vec, + pub(crate) test_sources: BTreeMap>, + pub(crate) model_shapes: Vec, + pub(crate) coverage: Option>, + pub(crate) lint_model_keys: Option>, + pub(crate) lint_file_keys: Option)>>, + pub(crate) header_keys: BTreeMap, + pub(crate) rules: Option, +} + +/// A stored compile's built-in rules request, as decoded for a delta compile. +#[derive(Clone, Debug)] +pub(crate) struct BaseRules { + pub(crate) request: EvaluateRequest, + /// Each model's payload digest, which keys its cached findings, in model order. + pub(crate) digests: Vec, +} + +impl BaseRules { + /// The rows a compile retained, decoded. + pub(crate) fn decoded(retained: &RetainedRows) -> Option { + Some(Self { + request: retained_request(retained)?, + digests: retained.digests.clone(), + }) + } +} + +/// An image's rules request: decoding from a stored image, or as a compile just retained it. +#[derive(Debug)] +pub(crate) enum ImageRules { + Decoding(PendingRules), + Retained(RetainedRows), +} + +/// A stored rules request, decoding on its own thread or decoded already. +#[derive(Debug)] +pub(crate) enum PendingRules { + Decoding(std::thread::JoinHandle>), + Ready(Box), +} + +impl ImageRules { + /// The rules request still decoding from a stored image, if this is one. + pub(crate) fn pending(self) -> Option { + match self { + Self::Decoding(rules) => Some(rules), + Self::Retained(_) => None, + } + } + + /// The rows a compile just retained, if this is them. + pub(crate) fn retained(self) -> Option { + match self { + Self::Retained(rules) => Some(rules), + Self::Decoding(_) => None, + } + } +} + +impl PendingRules { + /// The decoded request, or `None` when it could not be decoded. + pub(crate) fn resolved(self) -> Option { + match self { + Self::Decoding(handle) => { + let Ok(rules) = handle.join() else { + return None; + }; + rules + } + Self::Ready(rules) => Some(*rules), + } + } +} + +/// The value `bytes` hold as JSON, or `None` when they are malformed. +fn from_json(bytes: &[u8]) -> Option { + let Ok(value) = serde_json::from_slice(bytes) else { + return None; + }; + Some(value) +} + +fn decoded_rules(request_json: Vec, rows: Vec, digests: &[u8]) -> Option { + let retained: RetainedRows = RetainedRows { + request_json, + rows, + digests: decoded_strings(digests)?, + }; + BaseRules::decoded(&retained) +} + +/// The analysis session's final relation facts, as stored. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct ImageRelations { + pub(crate) types: Shapes, + pub(crate) nullability: Shapes, + pub(crate) complete: Shapes, + pub(crate) schemas: Shapes, + pub(crate) dependency_ordered: bool, + pub(crate) cyclic: bool, + pub(crate) legacy_models: Vec, +} + +impl From for ImageRelations { + fn from(tables: RelationTables) -> Self { + Self { + types: tables.types, + nullability: tables.nullability, + complete: tables.complete, + schemas: tables.schemas, + dependency_ordered: tables.dependency_ordered, + cyclic: tables.cyclic, + legacy_models: tables.legacy_models, + } + } +} + +impl From for RelationTables { + fn from(relations: ImageRelations) -> Self { + Self { + types: relations.types, + nullability: relations.nullability, + complete: relations.complete, + schemas: relations.schemas, + dependency_ordered: relations.dependency_ordered, + cyclic: relations.cyclic, + legacy_models: relations.legacy_models, + } + } +} + +/// Rows of `table` that differ from the same relation's `types` row; `None` where equal. +fn differing(table: &Shapes, types: &Shapes) -> StoredShapes { + let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types + .iter() + .map(|(name, shape)| (name.as_str(), shape)) + .collect(); + table + .iter() + .map(|(name, shape)| { + let same: bool = types + .get(name.as_str()) + .is_some_and(|typed| *typed == shape); + (name.clone(), (!same).then(|| shape.clone())) + }) + .collect() +} + +/// `differing`'s rows restored against `types`. +fn restored(rows: StoredShapes, types: &Shapes) -> Option { + let types: std::collections::HashMap<&str, &Vec<(String, String)>> = types + .iter() + .map(|(name, shape)| (name.as_str(), shape)) + .collect(); + rows.into_iter() + .map(|(name, shape)| { + let shape: Vec<(String, String)> = match shape { + Some(shape) => shape, + None => (*types.get(name.as_str())?).clone(), + }; + Some((name, shape)) + }) + .collect() +} + +/// A shape table as stored: `None` where a relation's row equals its `types` row. +type StoredShapes = Vec<(String, Option>)>; + +/// What a delta compile reads first to decide, cheaply, whether it can patch the image. +#[derive(Clone, Debug, Default)] +pub(crate) struct ImageGate { + pub(crate) format: String, + pub(crate) json_output: bool, + /// Whether the stored compile reported any error diagnostic. + pub(crate) has_errors: bool, + /// Whether custom rules recorded which models they select. + pub(crate) has_model_subjects: bool, + pub(crate) models: Vec, + pub(crate) tests: Vec, + pub(crate) fact_digests: Vec<(Vec, String)>, + pub(crate) header_keys: BTreeMap, +} + +/// The rest of the image's facts as stored; shapes equal to a relation's types are stored once. +struct StoredFacts { + usages: BTreeMap, + types: Shapes, + nullability: Shapes, + complete: StoredShapes, + schemas: StoredShapes, + dependency_ordered: bool, + cyclic: bool, + legacy_models: Vec, + model_subjects: Option)>>, + diagnostics: Vec, + text_models: Option>, + environment_names: Vec, + pinned_sources: Vec, + test_sources: BTreeMap>, + model_shapes: StoredShapes, + coverage: Option>, + lint_model_keys: Option>, + lint_file_keys: Option)>>, +} + +fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { + sections + .iter() + .find(|(section, _)| section == name) + .map(|(_, bytes)| bytes.as_slice()) +} + +/// The diagnostics and report lines an image keeps as JSON, being small and varied. +type StoredMisc = ( + Vec, + Option>, + Vec, + Option>, +); + +fn put_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a Shapes) { + encoder.count(shapes.len()); + for (name, shape) in shapes { + encoder.string(name); + encoder.pairs(shape); + } +} + +fn put_stored_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a StoredShapes) { + encoder.count(shapes.len()); + for (name, shape) in shapes { + encoder.string(name); + encoder.flag(shape.is_some()); + if let Some(shape) = shape { + encoder.pairs(shape); + } + } +} + +fn shapes(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| Some((decoder.string()?, decoder.pairs()?))) +} + +fn stored_shapes(decoder: &mut Decoder<'_>) -> Option { + decoder.list(|decoder| { + let name: String = decoder.string()?; + let shape: Option> = if decoder.flag()? { + Some(decoder.pairs()?) + } else { + None + }; + Some((name, shape)) + }) +} + +impl ImageGate { + fn encode(&self) -> Vec { + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.string(&self.format); + encoder.flag(self.json_output); + encoder.flag(self.has_errors); + encoder.flag(self.has_model_subjects); + encoder.count(self.models.len()); + for model in &self.models { + encoder.string(&model.name); + encoder.string(&model.relative_path); + encoder.pairs(&model.upstream); + } + encoder.count(self.tests.len()); + for test in &self.tests { + encoder.string(&test.key); + encoder.string(&test.relative_path); + encoder.strings(test.chain.iter().map(String::as_str)); + } + encoder.count(self.fact_digests.len()); + for (key, digest) in &self.fact_digests { + encoder.strings(key.iter().map(String::as_str)); + encoder.string(digest); + } + encoder.count(self.header_keys.len()); + for (path, keys) in &self.header_keys { + encoder.string(path); + encoder.pairs(keys); + } + encoder.finish() + } + + /// The gate stored in `sections`, or `None` when it is unreadable or of another format. + pub(crate) fn decode(sections: &ImageSections) -> Option { + let mut decoder: Decoder<'_> = Decoder::new(section(sections, GATE_SECTION)?)?; + let format: String = decoder.string()?; + if format != IMAGE_FORMAT { + return None; + } + let gate: Self = Self { + format, + json_output: decoder.flag()?, + has_errors: decoder.flag()?, + has_model_subjects: decoder.flag()?, + models: decoder.list(|decoder| { + Some(CapturedModel { + name: decoder.string()?, + relative_path: decoder.string()?, + upstream: decoder.pairs()?, + }) + })?, + tests: decoder.list(|decoder| { + Some(CapturedTest { + key: decoder.string()?, + relative_path: decoder.string()?, + chain: decoder.strings()?, + }) + })?, + fact_digests: decoder.list(|decoder| Some((decoder.strings()?, decoder.string()?)))?, + header_keys: decoder + .list(|decoder| Some((decoder.string()?, decoder.pairs()?)))? + .into_iter() + .collect(), + }; + decoder.finish()?; + Some(gate) + } +} + +impl StoredFacts { + fn encode(&self) -> Result, String> { + let misc: Vec = serde_json::to_vec(&( + &self.diagnostics, + &self.text_models, + &self.environment_names, + &self.coverage, + )) + .map_err(|error| error.to_string())?; + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.count(self.usages.len()); + for (path, digest) in &self.usages { + encoder.string(path); + encoder.string(digest); + } + put_shapes(&mut encoder, &self.types); + put_shapes(&mut encoder, &self.nullability); + put_stored_shapes(&mut encoder, &self.complete); + put_stored_shapes(&mut encoder, &self.schemas); + put_stored_shapes(&mut encoder, &self.model_shapes); + encoder.flag(self.dependency_ordered); + encoder.flag(self.cyclic); + encoder.strings(self.legacy_models.iter().map(String::as_str)); + encoder.flag(self.model_subjects.is_some()); + if let Some(subjects) = &self.model_subjects { + encoder.count(subjects.len()); + for (path, name, materialized) in subjects { + encoder.string(path); + encoder.string(name); + encoder.optional_string(materialized.as_deref()); + } + } + encoder.flag(self.lint_model_keys.is_some()); + if let Some(keys) = &self.lint_model_keys { + encoder.pairs(keys); + } + encoder.flag(self.lint_file_keys.is_some()); + if let Some(keys) = &self.lint_file_keys { + encoder.count(keys.len()); + for (path, key) in keys { + encoder.string(path); + encoder.optional_string(key.as_deref()); + } + } + encoder.strings(self.pinned_sources.iter().map(String::as_str)); + encoder.count(self.test_sources.len()); + for (path, names) in &self.test_sources { + encoder.string(path); + encoder.strings(names.iter().map(String::as_str)); + } + encoder.bytes(&misc); + Ok(encoder.finish()) + } + + fn decode(section: &[u8]) -> Option { + let mut decoder: Decoder<'_> = Decoder::new(section)?; + let usages: BTreeMap = decoder + .list(|decoder| Some((decoder.string()?, decoder.string()?)))? + .into_iter() + .collect(); + let types: Shapes = shapes(&mut decoder)?; + let nullability: Shapes = shapes(&mut decoder)?; + let complete: StoredShapes = stored_shapes(&mut decoder)?; + let schemas: StoredShapes = stored_shapes(&mut decoder)?; + let model_shapes: StoredShapes = stored_shapes(&mut decoder)?; + let dependency_ordered: bool = decoder.flag()?; + let cyclic: bool = decoder.flag()?; + let legacy_models: Vec = decoder.strings()?; + let model_subjects = if decoder.flag()? { + Some(decoder.list(|decoder| { + Some(( + decoder.string()?, + decoder.string()?, + decoder.optional_string()?, + )) + })?) + } else { + None + }; + let lint_model_keys = if decoder.flag()? { + Some(decoder.pairs()?) + } else { + None + }; + let lint_file_keys = if decoder.flag()? { + Some(decoder.list(|decoder| Some((decoder.string()?, decoder.optional_string()?)))?) + } else { + None + }; + let pinned_sources: Vec = decoder.strings()?; + let test_sources: BTreeMap> = decoder + .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? + .into_iter() + .collect(); + let (diagnostics, text_models, environment_names, coverage): StoredMisc = + from_json(decoder.bytes()?)?; + decoder.finish()?; + Some(Self { + usages, + types, + nullability, + complete, + schemas, + dependency_ordered, + cyclic, + legacy_models, + model_subjects, + diagnostics, + text_models, + environment_names, + pinned_sources, + test_sources, + model_shapes, + coverage, + lint_model_keys, + lint_file_keys, + }) + } +} + +fn encoded_strings(values: &[String]) -> Vec { + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.strings(values.iter().map(String::as_str)); + encoder.finish() +} + +fn decoded_strings(section: &[u8]) -> Option> { + let mut decoder: Decoder<'_> = Decoder::new(section)?; + let values: Vec = decoder.strings()?; + decoder.finish()?; + Some(values) +} + +impl ProjectImage { + /// The image of a finished compile's capture, or why it cannot be a delta base. + pub(crate) fn from_capture(capture: ImageCapture, json_output: bool) -> Result { + if let Some(reason) = capture.unsupported { + return Err(reason); + } + if capture.stdlib_json { + return Err("the report fell back to the standard library serializer".to_owned()); + } + let Some(relations) = capture.relations else { + return Err("no model analysis session ran".to_owned()); + }; + if !json_output && capture.text_models.is_none() { + return Err("no text report was rendered".to_owned()); + } + let Some(model_shapes) = capture.model_shapes else { + return Err("no semantic shapes were published".to_owned()); + }; + let mut environment_names: Vec = capture.environment_names; + environment_names.sort(); + environment_names.dedup(); + Ok(Self { + format: IMAGE_FORMAT.to_owned(), + json_output, + models: capture.models, + usages: capture.usages, + tests: capture.test_chains, + relations: relations.into(), + fact_digests: capture.fact_digests.into_iter().collect(), + model_subjects: capture.model_subjects, + diagnostics: capture.diagnostics, + text_models: capture.text_models, + environment_names, + pinned_sources: capture.pinned_sources.into_iter().collect(), + test_sources: capture + .test_sources + .into_iter() + .map(|(path, names)| (path, names.into_iter().collect())) + .collect(), + model_shapes, + coverage: capture.coverage, + lint_model_keys: capture.lint_model_keys, + lint_file_keys: capture.lint_file_keys, + header_keys: capture.header_keys, + rules: capture + .rules + .map(|rules| ImageRules::Retained(rules.retained)), + }) + } + + /// The image as the sections of its file. + pub(crate) fn encode(self) -> Result { + let gate: ImageGate = ImageGate { + format: self.format, + json_output: self.json_output, + has_errors: self.diagnostics.iter().any(|diagnostic| diagnostic.error), + has_model_subjects: self.model_subjects.is_some(), + models: self.models, + tests: self.tests, + fact_digests: self.fact_digests, + header_keys: self.header_keys, + }; + let relations: ImageRelations = self.relations; + let types: &Shapes = &relations.types; + let facts: StoredFacts = StoredFacts { + usages: self.usages, + complete: differing(&relations.complete, types), + schemas: differing(&relations.schemas, types), + model_shapes: differing(&self.model_shapes, types), + types: relations.types, + nullability: relations.nullability, + dependency_ordered: relations.dependency_ordered, + cyclic: relations.cyclic, + legacy_models: relations.legacy_models, + model_subjects: self.model_subjects, + diagnostics: self.diagnostics, + text_models: self.text_models, + environment_names: self.environment_names, + coverage: self.coverage, + pinned_sources: self.pinned_sources, + test_sources: self.test_sources, + lint_model_keys: self.lint_model_keys, + lint_file_keys: self.lint_file_keys, + }; + let rules: Option = self.rules.and_then(ImageRules::retained); + let (gate, facts) = rayon::join(|| gate.encode(), || facts.encode()); + let rules = rules.map(|rules| { + ( + rules.request_json, + rules.rows, + encoded_strings(&rules.digests), + ) + }); + let mut sections: ImageSections = vec![ + (GATE_SECTION.to_owned(), gate), + (FACTS_SECTION.to_owned(), facts?), + ]; + if let Some((request, rows, digests)) = rules { + sections.push((RULES_SECTION.to_owned(), request)); + sections.push((RULE_ROWS_SECTION.to_owned(), rows)); + sections.push((RULE_DIGESTS_SECTION.to_owned(), digests)); + } + Ok(sections) + } + + /// The image behind `gate`, its rules request decoding on its own thread; `None` if unreadable. + pub(crate) fn decode(mut sections: ImageSections, gate: ImageGate) -> Option { + let mut taken = |name: &str| { + sections + .iter_mut() + .find(|(section, _)| section == name) + .map(|(_, bytes)| std::mem::take(bytes)) + }; + let stored_rules = ( + taken(RULES_SECTION), + taken(RULE_ROWS_SECTION), + taken(RULE_DIGESTS_SECTION), + ); + let rules: Option = match stored_rules { + (Some(request), Some(rows), Some(digests)) => { + Some(ImageRules::Decoding(PendingRules::Decoding( + std::thread::spawn(move || decoded_rules(request, rows, &digests)), + ))) + } + (None, None, None) => None, + _ => return None, + }; + let facts: StoredFacts = StoredFacts::decode(section(§ions, FACTS_SECTION)?)?; + let types: Shapes = facts.types; + let relations: ImageRelations = ImageRelations { + complete: restored(facts.complete, &types)?, + schemas: restored(facts.schemas, &types)?, + nullability: facts.nullability, + dependency_ordered: facts.dependency_ordered, + cyclic: facts.cyclic, + legacy_models: facts.legacy_models, + types: Shapes::new(), + }; + let model_shapes: Shapes = restored(facts.model_shapes, &types)?; + Some(Self { + format: gate.format, + json_output: gate.json_output, + models: gate.models, + usages: facts.usages, + tests: gate.tests, + relations: ImageRelations { types, ..relations }, + fact_digests: gate.fact_digests, + model_subjects: facts.model_subjects, + diagnostics: facts.diagnostics, + text_models: facts.text_models, + environment_names: facts.environment_names, + pinned_sources: facts.pinned_sources, + test_sources: facts.test_sources, + model_shapes, + coverage: facts.coverage, + lint_model_keys: facts.lint_model_keys, + lint_file_keys: facts.lint_file_keys, + header_keys: gate.header_keys, + rules, + }) + } + + /// Every model whose reads include a relation of `names`, outside `names`. + pub(crate) fn dependents(&self, names: &BTreeSet) -> BTreeSet { + self.models + .iter() + .filter(|model| !names.contains(&model.name)) + .filter(|model| reads_any_model(model, names)) + .map(|model| model.name.clone()) + .collect() + } +} + +/// Whether `model` reads any of the models `names`. +fn reads_any_model(model: &CapturedModel, names: &BTreeSet) -> bool { + model + .upstream + .iter() + .any(|(kind, name)| kind == MODEL_KIND && names.contains(name)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs new file mode 100644 index 0000000000..3b9378d1c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_codec.rs @@ -0,0 +1,165 @@ +//! Image sections as length-prefixed records over a table of each distinct string. + +use rustc_hash::FxHashMap; +use sqlbuild_cache::store::errors::StoreDecodeError; +use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; + +/// The most items a list reserves room for before reading them, against corrupt counts. +const MAX_RESERVED_ITEMS: usize = 1 << 16; + +/// A stored value, or `None` when the section is damaged. +fn read(value: Result) -> Option { + let Ok(value) = value else { + return None; + }; + Some(value) +} + +/// A stored count or index, or `None` when it does not fit this platform. +fn length(value: u64) -> Option { + let Ok(length) = usize::try_from(value) else { + return None; + }; + Some(length) +} + +/// Writes one section: its values refer to a table of the distinct strings they hold. +#[derive(Default)] +pub(crate) struct Encoder<'a> { + ids: FxHashMap<&'a str, u64>, + strings: Vec<&'a str>, + body: RecordWriter, +} + +impl<'a> Encoder<'a> { + pub(crate) fn string(&mut self, value: &'a str) { + let next: u64 = self.strings.len() as u64; + let id: u64 = *self.ids.entry(value).or_insert(next); + if id == next { + self.strings.push(value); + } + self.body.put_u64(id); + } + + pub(crate) fn optional_string(&mut self, value: Option<&'a str>) { + self.flag(value.is_some()); + if let Some(value) = value { + self.string(value); + } + } + + pub(crate) fn count(&mut self, count: usize) { + self.body.put_u64(count as u64); + } + + pub(crate) fn flag(&mut self, value: bool) { + self.body.put_u64(u64::from(value)); + } + + pub(crate) fn bytes(&mut self, value: &[u8]) { + self.body.put_bytes(value); + } + + pub(crate) fn strings(&mut self, values: impl ExactSizeIterator) { + self.count(values.len()); + for value in values { + self.string(value); + } + } + + pub(crate) fn pairs(&mut self, pairs: &'a [(String, String)]) { + self.count(pairs.len()); + for (left, right) in pairs { + self.string(left); + self.string(right); + } + } + + /// The section: the string table, then the values. + pub(crate) fn finish(self) -> Vec { + let mut section: RecordWriter = RecordWriter::default(); + section.put_u64(self.strings.len() as u64); + for value in &self.strings { + section.put_str(value); + } + section.put_bytes(&self.body.into_bytes()); + section.into_bytes() + } +} + +/// Reads a section an [`Encoder`] wrote; any malformed value fails the whole read. +pub(crate) struct Decoder<'a> { + strings: Vec, + body: RecordReader<'a>, +} + +impl<'a> Decoder<'a> { + pub(crate) fn new(section: &'a [u8]) -> Option { + let mut reader: RecordReader<'a> = RecordReader::new(section); + let count: usize = length(read(reader.u64())?)?; + let mut strings: Vec = Vec::with_capacity(count.min(section.len())); + for _ in 0..count { + strings.push(read(reader.string())?); + } + let body: &'a [u8] = read(reader.bytes())?; + read(reader.finish())?; + Some(Self { + strings, + body: RecordReader::new(body), + }) + } + + pub(crate) fn string(&mut self) -> Option { + let id: usize = length(read(self.body.u64())?)?; + self.strings.get(id).cloned() + } + + pub(crate) fn optional_string(&mut self) -> Option> { + Some(if self.flag()? { + Some(self.string()?) + } else { + None + }) + } + + pub(crate) fn count(&mut self) -> Option { + length(read(self.body.u64())?) + } + + pub(crate) fn flag(&mut self) -> Option { + match read(self.body.u64())? { + 0 => Some(false), + 1 => Some(true), + _ => None, + } + } + + pub(crate) fn bytes(&mut self) -> Option<&'a [u8]> { + read(self.body.bytes()) + } + + pub(crate) fn list( + &mut self, + mut item: impl FnMut(&mut Self) -> Option, + ) -> Option> { + let count: usize = self.count()?; + let mut items: Vec = Vec::with_capacity(count.min(MAX_RESERVED_ITEMS)); + for _ in 0..count { + items.push(item(self)?); + } + Some(items) + } + + pub(crate) fn strings(&mut self) -> Option> { + self.list(Self::string) + } + + pub(crate) fn pairs(&mut self) -> Option> { + self.list(|decoder| Some((decoder.string()?, decoder.string()?))) + } + + /// Whether every value was read. + pub(crate) fn finish(self) -> Option<()> { + read(self.body.finish()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_facts.rs new file mode 100644 index 0000000000..301c9528db --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image_facts.rs @@ -0,0 +1,49 @@ +//! Facts a full compile's stages leave for its project image as they run. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyStringMethods; +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; + +use crate::bindings::_helpers::compile_delta::context::{capture, capture_unsupported}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Record the digest of one model's scope usages, which decide declaration placement. +pub(crate) fn record_model_usages( + model: &Bound<'_, PyAny>, + usages: (&[Bound<'_, PyAny>], &[Bound<'_, PyAny>]), +) -> PyResult<()> { + if let Err(error) = model_usages(model, usages) { + capture_unsupported(&format!( + "a model's scope usages could not be recorded: {error}" + )); + } + Ok(()) +} + +fn model_usages( + model: &Bound<'_, PyAny>, + (macro_usages, declaration_usages): (&[Bound<'_, PyAny>], &[Bound<'_, PyAny>]), +) -> PyResult<()> { + let Ok(path) = native_field(model, "model_file") + .and_then(|file| native_field(&file, "relative_path")) + .and_then(|path| Ok(path.str()?.to_str()?.replace('\\', "/"))) + else { + capture_unsupported("a model input has no file"); + return Ok(()); + }; + let mut text: String = String::new(); + for (group, usages) in [("macro", macro_usages), ("declaration", declaration_usages)] { + text.push_str(group); + text.push('\0'); + for usage in usages { + text.push_str(usage.repr()?.to_str()?); + text.push('\0'); + } + } + let digest: String = hex_digest(&bytes_digest(text.as_bytes())); + capture(|capture| { + capture.usages.insert(path, digest); + }); + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs new file mode 100644 index 0000000000..95d66e6339 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/lint_hooks.rs @@ -0,0 +1,106 @@ +//! The SQL lint keys a full compile leaves for its image, and a delta compile's use of them. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use pyo3::PyResult; + +use crate::bindings::_helpers::compile_delta::context::{ + capture, capturing, delta_unsupported, injection, +}; +use sqlbuild_core::format_command::constants::LINT_DIRECTORY_NAMES; + +use crate::bindings::_helpers::rules::lint_pipeline::LintFile; + +/// Each model file's lint key, recorded for the image or, in a delta, patched for reruns. +pub(crate) fn model_identities(fresh: Vec<(String, String)>) -> PyResult> { + let Some(injection) = injection() else { + if capturing() { + let keys: Vec<(String, String)> = fresh.clone(); + capture(|capture| capture.lint_model_keys = Some(keys)); + } + return Ok(fresh); + }; + let Some(stored) = injection.lint_model_keys.as_ref() else { + return Err(delta_unsupported( + "the lint keys of the models were not recorded", + )); + }; + let mut fresh: HashMap = fresh.into_iter().collect(); + let whole: Vec<(String, String)> = stored + .iter() + .map(|(path, key)| match fresh.remove(path) { + Some(key) => (path.clone(), key), + None => (path.clone(), key.clone()), + }) + .collect(); + if !fresh.is_empty() { + return Err(delta_unsupported( + "a recompiled model has no recorded lint key", + )); + } + capture(|capture| capture.lint_model_keys = Some(whole.clone())); + Ok(whole) +} + +/// The changed files a delta compile lints, read as a full lint walk reads them. +pub(crate) fn changed_files(project_dir: &Path) -> PyResult> { + let Some(injection) = injection() else { + return Ok(Vec::new()); + }; + let mut files: Vec = Vec::new(); + for path in &injection.changed_paths { + let linted: bool = path + .split_once('/') + .is_some_and(|(root, _)| LINT_DIRECTORY_NAMES.contains(&root)); + if !linted { + continue; + } + let path: PathBuf = project_dir.join(path); + let contents: String = std::fs::read_to_string(&path) + .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.to_string()))?; + files.push(LintFile { path, contents }); + } + Ok(files) +} + +/// Each non-model file's lint key in walk order, recorded or, in a delta, patched. +pub(crate) fn file_keys( + project_dir: &Path, + fresh: Vec<(PathBuf, String, Option)>, +) -> PyResult)>> { + let Some(injection) = injection() else { + if capturing() { + let keys: Vec<(String, Option)> = fresh + .iter() + .map(|(_, path, key)| (path.clone(), key.clone())) + .collect(); + capture(|capture| capture.lint_file_keys = Some(keys)); + } + return Ok(fresh); + }; + let Some(stored) = injection.lint_file_keys.as_ref() else { + return Err(delta_unsupported( + "the lint keys of the project files were not recorded", + )); + }; + let mut fresh: HashMap)> = + fresh.into_iter().map(|row| (row.1.clone(), row)).collect(); + let whole: Vec<(PathBuf, String, Option)> = stored + .iter() + .map(|(path, key)| { + fresh + .remove(path) + .unwrap_or_else(|| (project_dir.join(path), path.clone(), key.clone())) + }) + .collect(); + if !fresh.is_empty() { + return Err(delta_unsupported("a changed file has no recorded lint key")); + } + let keys: Vec<(String, Option)> = whole + .iter() + .map(|(_, path, key)| (path.clone(), key.clone())) + .collect(); + capture(|capture| capture.lint_file_keys = Some(keys)); + Ok(whole) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs new file mode 100644 index 0000000000..50b7b9c359 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/mod.rs @@ -0,0 +1,13 @@ +pub(crate) mod context; +pub(crate) mod coverage; +pub(crate) mod driver; +pub(crate) mod header_keys; +pub(crate) mod image; +pub(crate) mod image_codec; +pub(crate) mod image_facts; +pub(crate) mod lint_hooks; +pub(crate) mod report_hooks; +pub(crate) mod rules_rows; +pub(crate) mod splice; +pub(crate) mod verify; +pub(crate) mod yaml_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs new file mode 100644 index 0000000000..8ea8654232 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -0,0 +1,364 @@ +//! Where the compile's report and output stages leave image facts or take a delta's injection. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple, PyTupleMethods}; +use sqlbuild_core::cli_text::models::ModelLine; + +use crate::bindings::_helpers::compile_command::report::diagnostic_item; +use crate::bindings::_helpers::compile_delta::context::{ + CapturedDiagnostic, CapturedModel, CapturedTest, ModelShape, capture, capture_unsupported, + capturing, delta_unsupported, injection, +}; +use crate::bindings::_helpers::compile_delta::coverage::{ + ModelCoverage, coverage_reasons, merged_coverage, merged_order, +}; +use crate::bindings::_helpers::compile_outputs::json_report::plain_json; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::records::fields::field as native_field; + +const SOURCES_KEY: &str = "sources"; +const NAME_KEY: &str = "name"; +/// The graph key kind of a model. +const MODEL_KIND: &str = "model"; + +/// The text report's model lines, captured or, in a delta, patched for the rerun models. +pub(crate) fn text_model_lines(models: Vec) -> Vec { + let models: Vec = match injection() { + None => models, + Some(injection) => { + let mut fresh: HashMap = models + .into_iter() + .map(|line| (line.name.clone(), line)) + .collect(); + injection + .text_models + .iter() + .map(|(name, failed, column_count)| { + fresh.remove(name).unwrap_or_else(|| ModelLine { + name: name.clone(), + failed: *failed, + column_count: *column_count, + }) + }) + .collect() + } + }; + capture(|capture| { + capture.text_models = Some( + models + .iter() + .map(|line| (line.name.clone(), line.failed, line.column_count)) + .collect(), + ); + }); + models +} + +/// Record which models each SQL test's compiled artifact is keyed by. +fn record_test_chains_inner(tests: &Bound<'_, PyAny>, chains: &Bound<'_, PyAny>) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + let mut captured: Vec = Vec::new(); + for test in tests.try_iter()? { + let test: Bound<'_, PyAny> = test?; + let key: Bound<'_, PyAny> = native_field(&test, "key")?; + let chain: Vec = match chains.get_item(&key) { + Ok(chain) => chain.extract()?, + Err(_) => Vec::new(), + }; + captured.push(CapturedTest { + key: native_field(&test, "name")?.str()?.to_str()?.to_owned(), + relative_path: native_field(&native_field(&test, "test_file")?, "relative_path")? + .str()? + .to_str()? + .replace('\\', "/"), + chain, + }); + } + capture(|capture| capture.test_chains = captured); + Ok(()) +} + +/// Record the compiled models with their references, and the report's diagnostics. +fn record_compile_facts_inner( + graph: &Bound<'_, PyAny>, + diagnostics: &Bound<'_, PyTuple>, +) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + let native: Bound<'_, PyAny> = native_field(graph, "native")?; + let graph_native: &NativeProjectGraph = native.cast::()?.get(); + let mut models: Vec = Vec::new(); + for model in native_field(&native_field(graph, "project")?, "models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let name: String = native_field(&model, "name")?.extract()?; + let upstream: Vec<(String, String)> = + graph_native.upstream((MODEL_KIND.to_owned(), name.clone())); + models.push(CapturedModel { + name, + relative_path: native_field(&model, "relative_path")? + .str()? + .to_str()? + .replace('\\', "/"), + upstream, + }); + } + let mut captured: Vec = Vec::new(); + for diagnostic in diagnostics.iter() { + captured.push(captured_diagnostic(&diagnostic)?); + } + let test_count: usize = native_field(&native_field(graph, "project")?, "sql_tests")?.len()?; + capture(|capture| { + capture.models = models; + capture.diagnostics = captured; + if capture.test_chains.len() != test_count { + capture + .unsupported + .get_or_insert_with(|| "the SQL tests' model chains were not recorded".to_owned()); + } + }); + Ok(()) +} + +/// One diagnostic as its report JSON, with the resource and paths it names. +pub(crate) fn captured_diagnostic(diagnostic: &Bound<'_, PyAny>) -> PyResult { + let item = diagnostic_item(diagnostic)?; + let json: String = plain_json(item.as_any())?; + let value: serde_json::Value = match serde_json::from_str(&json) { + Ok(value) => value, + Err(_malformed) => serde_json::Value::Null, + }; + let text = |value: Option<&serde_json::Value>| -> Option { + value.and_then(serde_json::Value::as_str).map(str::to_owned) + }; + let mut paths: Vec = Vec::new(); + paths.extend(text(value.get("path"))); + paths.extend(text( + value + .get("location") + .and_then(|location| location.get("path")), + )); + Ok(CapturedDiagnostic { + json, + resource_name: text(value.get("resource_name")), + paths: paths + .into_iter() + .map(|path| path.replace('\\', "/")) + .collect(), + error: value.get("severity").and_then(serde_json::Value::as_str) == Some("error"), + }) +} + +/// The partial-check reasons of the `fresh` models, merged in a delta with the stored models'. +pub(crate) fn semantic_reasons( + order: Vec<(String, Vec)>, + fresh: Vec, +) -> PyResult)>> { + let (order, facts): (Vec<(String, Vec)>, Vec) = match injection() { + None => ( + order, + fresh.into_iter().filter(ModelCoverage::notable).collect(), + ), + Some(injection) => { + let Some(stored) = &injection.coverage else { + return Err(delta_unsupported( + "the stored compile recorded no semantic coverage", + )); + }; + ( + merged_order(&injection.coverage_order, &order), + merged_coverage(stored, &fresh), + ) + } + }; + let reasons: Vec<(String, Vec)> = coverage_reasons(&order, &facts); + if capturing() { + capture(|capture| { + if capture.coverage.is_none() { + capture.coverage = Some(facts); + } + }); + } + Ok(reasons) +} + +/// In a delta compile, publish the shapes of every model it does not compile again first. +pub(crate) fn inject_model_shapes( + models: &[Bound<'_, PyAny>], + shapes: &Bound<'_, PyDict>, +) -> PyResult<()> { + let Some(injection) = injection() else { + return Ok(()); + }; + let mut rerun: HashSet = HashSet::with_capacity(models.len()); + for model in models { + rerun.insert(native_field(model, "name")?.extract()?); + } + let py = shapes.py(); + for (name, columns) in &injection.model_shapes { + if rerun.contains(name) { + continue; + } + let shape: Bound<'_, PyDict> = PyDict::new(py); + for (column, column_type) in columns { + shape.set_item(column, column_type)?; + } + shapes.set_item(name, shape)?; + } + Ok(()) +} + +/// Record the model shapes the first semantic shape pass published. +fn record_model_shapes_inner( + project: &Bound<'_, PyAny>, + shapes: &Bound<'_, PyDict>, +) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + let mut names: HashSet = HashSet::new(); + for model in native_field(project, "models")?.try_iter()? { + names.insert(native_field(&model?, "name")?.extract()?); + } + if let Some(injection) = injection() { + names.extend(injection.model_shapes.iter().map(|(name, _)| name.clone())); + } + let mut published: Vec = Vec::new(); + for (name, shape) in shapes.iter() { + let name: String = name.extract()?; + if !names.contains(&name) { + continue; + } + let shape: Bound<'_, PyDict> = shape.cast_into()?; + let mut columns: Vec<(String, String)> = Vec::with_capacity(shape.len()); + for (column, column_type) in shape.iter() { + columns.push((column.extract()?, column_type.extract()?)); + } + published.push((name, columns)); + } + capture(|capture| { + if capture.model_shapes.is_none() { + capture.model_shapes = Some(published); + } + }); + Ok(()) +} + +/// Run a capture hook; a failure only rules the compile out as a delta base. +fn guarded(hook: impl FnOnce() -> PyResult<()>) -> PyResult<()> { + if !capturing() { + return Ok(()); + } + if let Err(error) = hook() { + capture_unsupported(&format!("an image fact could not be recorded: {error}")); + } + Ok(()) +} + +/// Record which models each SQL test's compiled artifact is keyed by. +pub(crate) fn record_test_chains( + tests: &Bound<'_, PyAny>, + chains: &Bound<'_, PyAny>, +) -> PyResult<()> { + guarded(|| record_test_chains_inner(tests, chains)) +} + +/// Record the compiled models with their references, and the report's diagnostics. +pub(crate) fn record_compile_facts( + graph: &Bound<'_, PyAny>, + diagnostics: &Bound<'_, PyTuple>, +) -> PyResult<()> { + guarded(|| record_compile_facts_inner(graph, diagnostics)) +} + +/// Record the model shapes the first semantic shape pass published. +pub(crate) fn record_model_shapes( + project: &Bound<'_, PyAny>, + shapes: &Bound<'_, PyDict>, +) -> PyResult<()> { + guarded(|| record_model_shapes_inner(project, shapes)) +} + +/// In a delta, add the text of files that stored diagnostics point into but were not rerun. +pub(crate) fn complete_source_texts( + project: &Bound<'_, PyAny>, + diagnostics: &[Bound<'_, PyAny>], + texts: &Bound<'_, PyDict>, +) -> PyResult<()> { + let Some(injection) = injection() else { + return Ok(()); + }; + let _ = project; + let project_dir: &std::path::Path = &injection.project_dir; + let mut locations: Vec> = Vec::new(); + for diagnostic in diagnostics { + locations.push(native_field(diagnostic, "location")?); + for item in native_field(diagnostic, "related_locations")?.try_iter()? { + locations.push(native_field(&item?, "location")?); + } + } + for location in locations.into_iter().filter(|location| !location.is_none()) { + let path: Bound<'_, PyAny> = native_field(&location, "path")?; + if texts.contains(&path)? { + continue; + } + let relative: String = path.str()?.to_str()?.replace('\\', "/"); + if !injection.source_paths.contains(&relative) { + continue; + } + if let Ok(text) = std::fs::read_to_string(project_dir.join(&relative)) { + texts.set_item(&path, text)?; + } + } + Ok(()) +} + +/// In a delta reading only some sources, drop every other loaded source YAML entry. +pub(crate) fn retain_read_sources( + outcomes: &mut [sqlbuild_discovery::yaml_files::models::YamlFileOutcome], +) { + use sqlbuild_config::models::ConfigValue; + use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; + let Some(injection) = injection() else { + return; + }; + let Some(read) = injection.source_names.as_ref() else { + return; + }; + fn text(value: &ConfigValue) -> Option<&str> { + match value { + ConfigValue::String(text) => Some(text.as_str()), + _ => None, + } + } + for outcome in outcomes { + let YamlFileOutcome::Loaded { + value: ConfigValue::Map(entries), + .. + } = outcome + else { + continue; + }; + for (key, value) in entries { + if text(key) != Some(SOURCES_KEY) { + continue; + } + let ConfigValue::List(sources) = value else { + continue; + }; + sources.retain(|source| { + let name: Option<&str> = source.as_map().and_then(|entries| { + entries + .iter() + .find(|(key, _)| text(key) == Some(NAME_KEY)) + .and_then(|(_, name)| text(name)) + }); + name.is_none_or(|name| read.contains(name)) + }); + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs new file mode 100644 index 0000000000..fa160532fd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/rules_rows.rs @@ -0,0 +1,93 @@ +//! A delta compile's built-in rules request: the previous compile's, with the rerun rows fresh. + +use std::collections::HashMap; + +use pyo3::PyResult; +use sqlbuild_rules::models::{EvaluateRequest, Model, SqlTestFact}; + +use crate::bindings::_helpers::compile_delta::context::{delta_unsupported, injection}; +use crate::bindings::_helpers::compile_delta::image::{BaseRules, PendingRules}; + +/// In a delta, the whole project's rules request with the rerun rows taken from `request`. +pub(crate) fn whole_project_request(request: EvaluateRequest) -> PyResult { + let Some(injection) = injection() else { + return Ok(request); + }; + let taken: Option = injection + .rules + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + .and_then(PendingRules::resolved); + let Some(base) = taken else { + return Err(delta_unsupported( + "the previous compile did not run the built-in rules", + )); + }; + if !request.initial_findings.is_empty() || !base.request.initial_findings.is_empty() { + return Err(delta_unsupported( + "the rules request carries initial findings", + )); + } + let mut whole: EvaluateRequest = restored(base); + let mut fresh: HashMap = request + .models + .into_iter() + .map(|model| (model.name.clone(), model)) + .collect(); + for model in &mut whole.models { + if let Some(replacement) = fresh.remove(&model.name) { + *model = replacement; + } + } + if !fresh.is_empty() { + return Err(delta_unsupported( + "a rerun model is new to the rules request", + )); + } + whole.sql_tests = replaced_tests(&whole.sql_tests, request.sql_tests)?; + Ok(whole) +} + +/// The retained request with each row's payload digest restored. +fn restored(base: BaseRules) -> EvaluateRequest { + let mut request: EvaluateRequest = base.request; + for (model, digest) in request.models.iter_mut().zip(base.digests) { + model.payload_digest = Some(digest); + model.stripped = true; + } + request +} + +/// `base` with the facts of every file `fresh` covers replaced, in place and in order. +fn replaced_tests(base: &[SqlTestFact], fresh: Vec) -> PyResult> { + let mut by_file: HashMap> = HashMap::new(); + let mut grouped: HashMap> = HashMap::new(); + for fact in fresh { + grouped + .entry(fact.source_path.clone()) + .or_default() + .push(fact); + } + let mut counts: HashMap<&str, usize> = HashMap::new(); + for fact in base { + *counts.entry(fact.source_path.as_str()).or_default() += 1; + } + for (path, facts) in grouped { + if counts.get(path.as_str()).copied().unwrap_or_default() != facts.len() { + return Err(delta_unsupported( + "a rerun SQL test file changed its test count", + )); + } + by_file.insert(path, facts.into_iter()); + } + Ok(base + .iter() + .map(|fact| { + by_file + .get_mut(&fact.source_path) + .and_then(Iterator::next) + .unwrap_or_else(|| fact.clone()) + }) + .collect()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs new file mode 100644 index 0000000000..bd71d20971 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs @@ -0,0 +1,354 @@ +//! Splice a delta compile's JSON report into the stored report it patches. + +use std::collections::{BTreeSet, HashMap}; + +use serde_json::Value; + +/// One JSON value's byte span in its document. +type Span = (usize, usize); + +/// Bytes per word `string_stop` checks at once. +const WORD: usize = 8; +const LOW_BITS: u64 = 0x0101_0101_0101_0101; +const HIGH_BITS: u64 = 0x8080_8080_8080_8080; + +const DELTA_SUMMARY_KEYS: [&str; 2] = ["errors", "warnings"]; + +/// Whether any byte of `word` equals `byte`. +fn has_byte(word: u64, byte: u8) -> bool { + let matched: u64 = word ^ (LOW_BITS * u64::from(byte)); + matched.wrapping_sub(LOW_BITS) & !matched & HIGH_BITS != 0 +} + +/// The first quote or backslash at or after `index`, or the end, skipping whole clean words. +fn string_stop(text: &[u8], mut index: usize) -> usize { + while index + WORD <= text.len() { + let mut word: [u8; WORD] = [0; WORD]; + word.copy_from_slice(&text[index..index + WORD]); + let word: u64 = u64::from_le_bytes(word); + if has_byte(word, b'"') || has_byte(word, b'\\') { + break; + } + index += WORD; + } + while index < text.len() && !matches!(text[index], b'"' | b'\\') { + index += 1; + } + index +} + +/// The end of the JSON value starting at `start`, skipping strings and nested containers. +fn value_end(text: &[u8], start: usize) -> Option { + let mut depth: usize = 0; + let mut index: usize = start; + let mut in_string: bool = false; + while index < text.len() { + let byte: u8 = text[index]; + if in_string { + index = string_stop(text, index); + let Some(&byte) = text.get(index) else { + break; + }; + match byte { + b'\\' => index += 1, + b'"' => { + in_string = false; + if depth == 0 { + return Some(index + 1); + } + } + _ => {} + } + } else { + match byte { + b'"' => in_string = true, + b'{' | b'[' => depth += 1, + b'}' | b']' => { + if depth == 0 { + return Some(index); + } + depth -= 1; + if depth == 0 { + return Some(index + 1); + } + } + b',' | b'\n' if depth == 0 => return Some(index), + _ => {} + } + } + index += 1; + } + (depth == 0 && !in_string).then_some(index) +} + +fn skip_space(text: &[u8], mut index: usize) -> usize { + while index < text.len() && matches!(text[index], b' ' | b'\n' | b'\r' | b'\t') { + index += 1; + } + index +} + +/// The value `text` holds as JSON, or `None` when it is malformed. +fn from_json(text: &str) -> Option { + let Ok(value) = serde_json::from_str(text) else { + return None; + }; + Some(value) +} + +/// The `(key, value span)` members of the object spanning `span`. +fn members(text: &str, span: Span) -> Option> { + let bytes: &[u8] = text.as_bytes(); + if bytes.get(span.0) != Some(&b'{') { + return None; + } + let mut found: Vec<(String, Span)> = Vec::new(); + let mut index: usize = skip_space(bytes, span.0 + 1); + while index < span.1 && bytes[index] != b'}' { + let key_end: usize = value_end(bytes, index)?; + let key: String = from_json(&text[index..key_end])?; + index = skip_space(bytes, key_end); + if bytes.get(index) != Some(&b':') { + return None; + } + let start: usize = skip_space(bytes, index + 1); + let end: usize = value_end(bytes, start)?; + found.push((key, (start, end))); + index = skip_space(bytes, end); + if bytes.get(index) == Some(&b',') { + index = skip_space(bytes, index + 1); + } + } + Some(found) +} + +/// The element spans of the array spanning `span`. +fn elements(text: &str, span: Span) -> Option> { + let bytes: &[u8] = text.as_bytes(); + if bytes.get(span.0) != Some(&b'[') { + return None; + } + let mut found: Vec = Vec::new(); + let mut index: usize = skip_space(bytes, span.0 + 1); + while index < span.1 && bytes[index] != b']' { + let end: usize = value_end(bytes, index)?; + found.push((index, end)); + index = skip_space(bytes, end); + if bytes.get(index) == Some(&b',') { + index = skip_space(bytes, index + 1); + } + } + Some(found) +} + +fn text<'a>(document: &'a str, span: &Span) -> &'a str { + &document[span.0..span.1] +} + +fn whole(text: &str) -> Span { + let end: usize = text.trim_end().len(); + (skip_space(text.as_bytes(), 0), end) +} + +fn member<'a>(found: &'a [(String, Span)], key: &str) -> Option<&'a Span> { + found + .iter() + .find(|(name, _)| name == key) + .map(|(_, span)| span) +} + +/// What the splice replaces: the rerun models, their audits and the rerun SQL tests. +pub(crate) struct SpliceScope<'a> { + pub(crate) models: &'a BTreeSet, + pub(crate) tests: &'a BTreeSet, + /// The sources the delta discovered, when it discovered only some; `None` when all. + pub(crate) sources: Option<&'a BTreeSet>, +} + +/// The string at `path` in the item at `span`, items keeping own keys before nested objects. +fn item_field(text: &str, span: Span, path: &[&str]) -> Option { + let bytes: &[u8] = text.as_bytes(); + let mut start: usize = span.0; + for (index, key) in path.iter().enumerate() { + let needle: String = format!("\"{key}\": "); + let found: usize = text[start..span.1].find(&needle)?; + start += found + needle.len(); + if index + 1 < path.len() && bytes.get(start) != Some(&b'{') { + return None; + } + } + if bytes.get(start) != Some(&b'"') { + return slow_item_field(text, span, path); + } + let end: usize = value_end(bytes, start)?; + from_json(&text[start..end]) +} + +fn slow_item_field(text: &str, span: Span, path: &[&str]) -> Option { + let mut value: Value = from_json(&text[span.0..span.1])?; + for key in path { + value = value.get(*key)?.clone(); + } + value.as_str().map(str::to_owned) +} + +/// The stored report patched with the delta's rerun items, diagnostics, counts and timings. +pub(crate) fn spliced_report( + base: &str, + delta: &str, + scope: &SpliceScope<'_>, +) -> Result { + let base_members: Vec<(String, Span)> = + members(base, whole(base)).ok_or("the stored report cannot be read")?; + let delta_members: Vec<(String, Span)> = + members(delta, whole(delta)).ok_or("the delta report cannot be read")?; + let keys = + |found: &[(String, Span)]| found.iter().map(|(key, _)| key.clone()).collect::>(); + if keys(&base_members) != keys(&delta_members) { + return Err("the reports have different sections".to_owned()); + } + let mut output: String = String::with_capacity(base.len() + delta.len()); + let mut cursor: usize = 0; + for ((key, base_span), (_, delta_span)) in base_members.iter().zip(&delta_members) { + let replacement: String = match key.as_str() { + "has_errors" | "diagnostics" | "semantic_checks_partial" | "compile_timings" => { + text(delta, delta_span).to_owned() + } + "summary" => spliced_summary(base, *base_span, delta, *delta_span) + .ok_or("the summaries do not line up")?, + "resources" => spliced_resources((base, *base_span), (delta, *delta_span), scope) + .map_err(|kind| format!("the {kind} do not line up"))?, + _ => { + if text(base, base_span) != text(delta, delta_span) { + return Err(format!("the reports' {key} differ")); + } + continue; + } + }; + output.push_str(&base[cursor..base_span.0]); + output.push_str(&replacement); + cursor = base_span.1; + } + output.push_str(&base[cursor..]); + Ok(output) +} + +fn spliced_summary(base: &str, base_span: Span, delta: &str, delta_span: Span) -> Option { + let base_members: Vec<(String, Span)> = members(base, base_span)?; + let delta_members: Vec<(String, Span)> = members(delta, delta_span)?; + let mut output: String = String::new(); + let mut cursor: usize = base_span.0; + for (key, span) in &base_members { + if !DELTA_SUMMARY_KEYS.contains(&key.as_str()) { + continue; + } + let replacement: &Span = member(&delta_members, key)?; + output.push_str(&base[cursor..span.0]); + output.push_str(&delta[replacement.0..replacement.1]); + cursor = span.1; + } + output.push_str(&base[cursor..base_span.1]); + Some(output) +} + +fn spliced_resources( + (base, base_span): (&str, Span), + (delta, delta_span): (&str, Span), + scope: &SpliceScope<'_>, +) -> Result { + let resources: String = "resources".to_owned(); + let base_members: Vec<(String, Span)> = members(base, base_span).ok_or(resources.clone())?; + let delta_members: Vec<(String, Span)> = members(delta, delta_span).ok_or(resources)?; + let mut output: String = String::new(); + let mut cursor: usize = base_span.0; + for (key, span) in &base_members { + let delta_list: &Span = member(&delta_members, key).ok_or(key.clone())?; + let replacement: Option> = match (key.as_str(), scope.sources) { + ("sources", Some(sources)) => { + spliced_items((base, *span), (delta, *delta_list), |document, item| { + item_field(document, item, &["name"]).filter(|name| sources.contains(name)) + }) + } + ("models", _) => { + spliced_items((base, *span), (delta, *delta_list), |document, item| { + item_field(document, item, &["name"]).filter(|name| scope.models.contains(name)) + }) + } + ("tests", _) => spliced_items((base, *span), (delta, *delta_list), |document, item| { + item_field(document, item, &["name"]).filter(|name| scope.tests.contains(name)) + }), + ("audits", _) => { + spliced_items((base, *span), (delta, *delta_list), |document, item| { + let owner: Option = + item_field(document, item, &["attached_to", "resource_type"]); + let name: Option = item_field(document, item, &["attached_to", "name"]); + (owner.as_deref() == Some("model")) + .then_some(name) + .flatten() + .filter(|name| scope.models.contains(name)) + }) + } + _ => { + if base[span.0..span.1] != delta[delta_list.0..delta_list.1] { + return Err(key.clone()); + } + Some(None) + } + }; + let replacement: Option = replacement.ok_or(key.clone())?; + if let Some(replacement) = replacement { + output.push_str(&base[cursor..span.0]); + output.push_str(&replacement); + cursor = span.1; + } + } + output.push_str(&base[cursor..base_span.1]); + Ok(output) +} + +/// The base list with owned items replaced by the delta's items of the same owner. +fn spliced_items( + (base, base_span): (&str, Span), + (delta, delta_span): (&str, Span), + owned: impl Fn(&str, Span) -> Option, +) -> Option> { + let base_items: Vec = elements(base, base_span)?; + let delta_items: Vec = elements(delta, delta_span)?; + let mut fresh: HashMap> = HashMap::new(); + let mut unmatched_delta: std::collections::HashSet<&str> = std::collections::HashSet::new(); + for item in &delta_items { + match owned(delta, *item) { + Some(owner) => fresh.entry(owner).or_default().push_back(*item), + None => { + unmatched_delta.insert(&delta[item.0..item.1]); + } + } + } + let mut output: String = String::new(); + let mut cursor: usize = base_span.0; + let mut replaced: bool = false; + for item in &base_items { + let Some(owner) = owned(base, *item) else { + if !unmatched_delta.is_empty() { + unmatched_delta.remove(&base[item.0..item.1]); + } + continue; + }; + let replacement: Span = fresh.get_mut(&owner)?.pop_front()?; + output.push_str(&base[cursor..item.0]); + output.push_str(&delta[replacement.0..replacement.1]); + cursor = item.1; + replaced = true; + } + if fresh.values().any(|items| !items.is_empty()) { + return None; + } + if !unmatched_delta.is_empty() { + return None; + } + if !replaced { + return Some(None); + } + output.push_str(&base[cursor..base_span.1]); + Some(Some(output)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs new file mode 100644 index 0000000000..5ab8ccb6e9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs @@ -0,0 +1,220 @@ +//! The hidden check that each delta compile matches the full compile it replaces. + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_cache::digest::main::digest_files::digest_files; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_cache::project_reuse::constants::NATIVE_DIGEST_PREFIX; + +use crate::bindings::_helpers::compile_command::reuse_recording::{ + CompileArtifactWrites, RecordedCompileOutput, +}; + +/// The exit code of a compile whose delta differed from its full compile. +pub(crate) const MISMATCH_EXIT_CODE: i64 = 70; +const MISMATCH_FILE: &str = "delta-mismatch.json"; + +/// What the delta compile produced, held until the full compile finishes. +#[derive(Debug, Default)] +pub(crate) struct Expected { + pub(crate) stdout: Option, + pub(crate) exit_code: Option, + pub(crate) raised: Option, + /// Every file under the compiled root after the delta, with its native digest. + pub(crate) compiled: BTreeMap, + pub(crate) compiled_root: String, + pub(crate) artifacts_written: bool, +} + +/// What the full compile produced. +#[derive(Debug, Default)] +struct Actual { + stdout: Option, + exit_code: Option, + artifacts_written: bool, + written: HashMap>, +} + +static EXPECTED: Mutex> = Mutex::new(None); +static ACTUAL: Mutex> = Mutex::new(None); + +fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Every file under `directory`, depth first. +fn files_under(directory: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(directory) else { + return Vec::new(); + }; + let mut found: Vec = Vec::new(); + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + match entry.file_type() { + Ok(kind) if kind.is_dir() => found.extend(files_under(&path)), + Ok(kind) if kind.is_file() => found.push(path), + _ => {} + } + } + found +} + +/// Every file under `root` with its native digest, keyed by absolute path text. +pub(crate) fn compiled_digests(root: &Path) -> BTreeMap { + let paths: Vec = files_under(root); + let digests = digest_files(&paths); + paths + .into_iter() + .zip(digests) + .filter_map(|(path, digest)| { + let Ok(digest) = digest else { + return None; + }; + Some(( + path.to_string_lossy().into_owned(), + format!("{NATIVE_DIGEST_PREFIX}{}", hex_digest(&digest)), + )) + }) + .collect() +} + +/// Hold the delta's result for comparison once the full compile finishes. +pub(crate) fn hold_expected(expected: Expected) { + *locked(&EXPECTED) = Some(expected); + *locked(&ACTUAL) = None; +} + +/// Note the full compile's report, exit code and artifact writes, when a check is armed. +pub(crate) fn full_compile_finished( + output: &Bound<'_, PyAny>, + (exit_code, artifacts_written): (i64, bool), + artifact_writes: &Bound<'_, PyAny>, +) -> PyResult<()> { + if locked(&EXPECTED).is_none() { + return Ok(()); + } + let (_, stdout): (Vec, Option) = output + .cast::()? + .get() + .texts(output.py())?; + let written: HashMap> = artifact_writes + .cast::() + .map(|writes| writes.get().written_digests()) + .unwrap_or_default(); + *locked(&ACTUAL) = Some(Actual { + stdout, + exit_code: Some(exit_code), + artifacts_written, + written, + }); + Ok(()) +} + +/// The report with its `compile_timings` value blanked. +fn without_timings(stdout: &str) -> String { + let Some(start) = stdout.find("\n \"compile_timings\": {") else { + return stdout.to_owned(); + }; + let Some(length) = stdout[start..].find("\n },\n") else { + return stdout.to_owned(); + }; + format!("{}{}", &stdout[..start], &stdout[start + length + 5..]) +} + +/// Compare the held delta result with the full compile's; every difference found. +pub(crate) fn differences(raised: Option) -> Vec { + let Some(expected) = locked(&EXPECTED).take() else { + return Vec::new(); + }; + let actual: Actual = locked(&ACTUAL).take().unwrap_or_default(); + let mut found: Vec = Vec::new(); + if expected.raised != raised { + found.push(format!( + "raised error differs: delta {:?}, full {:?}", + expected.raised, raised + )); + } + if expected.raised.is_some() || raised.is_some() { + return found; + } + if expected.exit_code != actual.exit_code { + found.push(format!( + "exit code differs: delta {:?}, full {:?}", + expected.exit_code, actual.exit_code + )); + } + let delta_stdout: Option = expected.stdout.as_deref().map(without_timings); + let full_stdout: Option = actual.stdout.as_deref().map(without_timings); + if delta_stdout != full_stdout { + let (delta, full) = ( + delta_stdout.unwrap_or_default(), + full_stdout.unwrap_or_default(), + ); + let common: usize = delta + .bytes() + .zip(full.bytes()) + .take_while(|(left, right)| left == right) + .count(); + let start: usize = common.saturating_sub(200); + let excerpt = |text: &str| -> String { + text.get(start..(common + 300).min(text.len())) + .unwrap_or_default() + .to_owned() + }; + found.push(format!( + "report differs at byte {common}:\n--- delta\n{}\n--- full\n{}", + excerpt(&delta), + excerpt(&full) + )); + } + if expected.artifacts_written != actual.artifacts_written { + found.push(format!( + "artifact writing differs: delta {}, full {}", + expected.artifacts_written, actual.artifacts_written + )); + } + if !actual.artifacts_written { + return found; + } + let prefix: &str = expected.compiled_root.as_str(); + let full_paths: BTreeMap<&str, &Option> = actual + .written + .iter() + .filter(|(path, _)| path.starts_with(prefix)) + .map(|(path, digest)| (path.as_str(), digest)) + .collect(); + for (path, digest) in &full_paths { + match (expected.compiled.get(*path), digest) { + (None, _) => found.push(format!("the delta left no artifact at {path}")), + (Some(left), Some(right)) if left != right => { + found.push(format!("artifact differs: {path}")); + } + _ => {} + } + } + for path in expected.compiled.keys() { + if !full_paths.contains_key(path.as_str()) { + found.push(format!( + "the delta left an artifact the full compile does not: {path}" + )); + } + } + found.truncate(50); + found +} + +/// Write the differences beside the compiler cache for inspection. +pub(crate) fn write_mismatch(cache_dir: &Path, differences: &[String]) { + let _ = std::fs::create_dir_all(cache_dir); + let text: String = serde_json::to_string_pretty(differences).unwrap_or_default(); + let _ = std::fs::write(cache_dir.join(MISMATCH_FILE), text); +} + +/// Forget any held result. +pub(crate) fn disarm() { + *locked(&EXPECTED) = None; + *locked(&ACTUAL) = None; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs new file mode 100644 index 0000000000..01fa3cb588 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/yaml_cache.rs @@ -0,0 +1,284 @@ +//! Loaded YAML documents a delta compile keeps by content digest instead of parsing again. + +use std::path::PathBuf; + +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_cache::digest::types::ContentDigest; +use sqlbuild_cache::store::models::NativeStore; +use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use sqlbuild_core::text::main::decode_text::decode_text; +use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; + +use crate::bindings::_helpers::compile_delta::image_codec::{Decoder, Encoder}; + +/// The native store kind holding loaded YAML documents. +pub(crate) const YAML_STORE_KIND: &str = "yaml-documents"; +/// The store file beside the compile reuse slots. +pub(crate) const YAML_STORE_FILE_NAME: &str = "yaml-documents.bin"; +/// Bumped whenever the stored encoding changes; older stores open empty. +pub(crate) const YAML_STORE_ENVIRONMENT: &str = "yaml-documents-v1"; + +const NULL: usize = 0; +const BOOL: usize = 1; +const INTEGER: usize = 2; +const BIG_INTEGER: usize = 3; +const FLOAT: usize = 4; +const STRING: usize = 5; +const DATE: usize = 6; +const DATE_TIME: usize = 7; +const TIME: usize = 8; +const LIST: usize = 9; +const MAP: usize = 10; +const NO_OFFSET: usize = 0; +const OFFSET: usize = 1; + +/// Where a delta compile keeps loaded YAML documents. +pub(crate) fn yaml_store_path(cache_dir: &std::path::Path) -> PathBuf { + cache_dir.join(YAML_STORE_FILE_NAME) +} + +/// The store key of a document's text. +pub(crate) fn document_key(contents: &str) -> ContentDigest { + bytes_digest(contents.as_bytes()) +} + +impl<'a> Encoder<'a> { + fn date(&mut self, date: &ConfigDate) { + self.count(usize::from(date.year)); + self.count(usize::from(date.month)); + self.count(usize::from(date.day)); + } + + fn time(&mut self, time: &ConfigTime) { + self.count(usize::from(time.hour)); + self.count(usize::from(time.minute)); + self.count(usize::from(time.second)); + self.bytes(&time.microsecond.to_le_bytes()); + } + + fn document(&mut self, value: &'a ConfigValue) { + match value { + ConfigValue::Null => self.count(NULL), + ConfigValue::Bool(flag) => { + self.count(BOOL); + self.flag(*flag); + } + ConfigValue::Integer(number) => { + self.count(INTEGER); + self.bytes(&number.to_le_bytes()); + } + ConfigValue::BigInteger(digits) => { + self.count(BIG_INTEGER); + self.string(digits); + } + ConfigValue::Float(number) => { + self.count(FLOAT); + self.bytes(&number.to_bits().to_le_bytes()); + } + ConfigValue::String(text) => { + self.count(STRING); + self.string(text); + } + ConfigValue::Date(date) => { + self.count(DATE); + self.date(date); + } + ConfigValue::DateTime(moment) => { + self.count(DATE_TIME); + self.date(&moment.date); + self.time(&moment.time); + match moment.utc_offset_seconds { + Some(offset) => { + self.count(OFFSET); + self.bytes(&offset.to_le_bytes()); + } + None => self.count(NO_OFFSET), + } + } + ConfigValue::Time(time) => { + self.count(TIME); + self.time(time); + } + ConfigValue::List(items) => { + self.count(LIST); + self.count(items.len()); + for item in items { + self.document(item); + } + } + ConfigValue::Map(entries) => { + self.count(MAP); + self.count(entries.len()); + for (key, item) in entries { + self.document(key); + self.document(item); + } + } + } + } +} + +/// A loaded document as stored. +pub(crate) fn encoded_document(value: &ConfigValue) -> Vec { + let mut encoder: Encoder<'_> = Encoder::default(); + encoder.document(value); + encoder.finish() +} + +impl Decoder<'_> { + fn fixed(&mut self) -> Option<[u8; N]> { + let Ok(bytes) = self.bytes()?.try_into() else { + return None; + }; + Some(bytes) + } + + fn small>(&mut self) -> Option { + let Ok(value) = T::try_from(self.count()?) else { + return None; + }; + Some(value) + } + + fn date(&mut self) -> Option { + Some(ConfigDate { + year: self.small()?, + month: self.small()?, + day: self.small()?, + }) + } + + fn time(&mut self) -> Option { + Some(ConfigTime { + hour: self.small()?, + minute: self.small()?, + second: self.small()?, + microsecond: u32::from_le_bytes(self.fixed()?), + }) + } + + fn document(&mut self) -> Option { + Some(match self.count()? { + NULL => ConfigValue::Null, + BOOL => ConfigValue::Bool(self.flag()?), + INTEGER => ConfigValue::Integer(i64::from_le_bytes(self.fixed()?)), + BIG_INTEGER => ConfigValue::BigInteger(self.string()?), + FLOAT => ConfigValue::Float(f64::from_bits(u64::from_le_bytes(self.fixed()?))), + STRING => ConfigValue::String(self.string()?), + DATE => ConfigValue::Date(self.date()?), + DATE_TIME => ConfigValue::DateTime(ConfigDateTime { + date: self.date()?, + time: self.time()?, + utc_offset_seconds: match self.count()? { + NO_OFFSET => None, + OFFSET => Some(i32::from_le_bytes(self.fixed()?)), + _ => return None, + }, + }), + TIME => ConfigValue::Time(self.time()?), + LIST => ConfigValue::List(self.list(Self::document)?), + MAP => ConfigValue::Map( + self.list(|decoder| Some((decoder.document()?, decoder.document()?)))?, + ), + _ => return None, + }) + } +} + +/// A stored document, or `None` when it cannot be decoded. +pub(crate) fn decoded_document(stored: &[u8]) -> Option { + let mut decoder: Decoder<'_> = Decoder::new(stored)?; + let document: ConfigValue = decoder.document()?; + decoder.finish()?; + Some(document) +} + +/// The store a delta compile reads and fills, opened beside its reuse slots. +pub(crate) struct DocumentStore { + pub(crate) store: NativeStore, + pub(crate) path: PathBuf, +} + +impl DocumentStore { + /// Each file's document, decoded from the store or loaded by `load` and stored. + pub(crate) fn load_with( + &mut self, + paths: &[String], + absolute: impl Fn(&str) -> PathBuf, + load: impl FnOnce(&[String]) -> Result, E>, + ) -> Result, E> { + let mut outcomes: Vec> = Vec::with_capacity(paths.len()); + let mut missed: Vec = Vec::new(); + for path in paths { + let stored: Option = stored_outcome(&mut self.store, &absolute(path)); + if stored.is_none() { + missed.push(path.clone()); + } + outcomes.push(stored); + } + let mut loaded = if missed.is_empty() { + Vec::new() + } else { + load(&missed)? + } + .into_iter(); + let mut complete: Vec = Vec::with_capacity(paths.len()); + for outcome in outcomes { + let outcome: YamlFileOutcome = match outcome { + Some(outcome) => outcome, + None => { + let Some(outcome) = loaded.next() else { + break; + }; + if let YamlFileOutcome::Loaded { contents, value } = &outcome { + self.store + .put(document_key(contents), encoded_document(value)); + } + outcome + } + }; + complete.push(outcome); + } + Ok(complete) + } +} + +/// In a delta compile, the document store beside its reuse slots, when it opens. +pub(crate) fn open_documents(py: pyo3::Python<'_>) -> Option { + let path: PathBuf = crate::bindings::_helpers::compile_delta::context::injection()? + .yaml_store + .clone()?; + match crate::bindings::_helpers::cache::native_store::open_store( + py, + &path, + YAML_STORE_KIND, + YAML_STORE_ENVIRONMENT, + ) { + Ok(store) => Some(DocumentStore { store, path }), + Err(_unopened) => None, + } +} + +/// The stored document of the file at `path` when its current text is stored. +fn stored_outcome(store: &mut NativeStore, path: &std::path::Path) -> Option { + let bytes: Vec = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(_unreadable) => return None, + }; + let contents: String = match decode_text(&bytes) { + Ok(contents) => contents, + Err(_undecodable) => return None, + }; + let value: ConfigValue = decoded_document(store.get(&document_key(&contents))?)?; + Some(YamlFileOutcome::Loaded { contents, value }) +} + +/// Save the documents this compile loaded; failing to only means parsing them again. +pub(crate) fn save_documents(py: pyo3::Python<'_>, documents: &DocumentStore) { + let _ = crate::bindings::_helpers::cache::native_store::save_store( + py, + &documents.store, + &documents.path, + &[], + ); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 23f91269da..fad8c21398 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -149,7 +149,8 @@ pub(crate) fn analyze<'py>( let options: AnalysisOptions<'py> = AnalysisOptions::read(py, options)?; let selection: &Bound<'py, PyAny> = &options.analysis_selection; let enabled: bool = !(native_field(selection, "select")?.is_truthy()? - || native_field(selection, "exclude")?.is_truthy()?); + || native_field(selection, "exclude")?.is_truthy()?) + && crate::bindings::_helpers::compile_delta::context::injection().is_none(); let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; let early_rules: EarlyRulesScope = EarlyRulesScope::new(); let analysis: PyResult> = @@ -209,6 +210,7 @@ fn analyzed<'py>( driver.graph((&discovered, &adapter), &lint_dialect)?; let project: Bound<'py, PyAny> = native_field(&graph, "project")?; let graph_failed: bool = any_error(&native_field(&project, "diagnostics")?)?; + crate::bindings::_helpers::compile_delta::context::stop_on_errors(graph_failed)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { if let Some(early_rules) = early_rules { @@ -218,6 +220,10 @@ fn analyzed<'py>( } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, &dialect)?; + crate::bindings::_helpers::compile_delta::context::stop_on_errors(any_error(&native_field( + &contracts.result, + "diagnostics", + )?)?)?; let selected_keys: Bound<'py, PyAny> = select_project_graph(&graph, &driver.select, &driver.exclude)?.into_any(); let core_diagnostics: Bound<'py, PyTuple> = PyTuple::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 0e03181f2e..85444d1942 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -108,7 +108,9 @@ pub(crate) fn assembled_project<'py>( let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; let resources: Bound<'py, PyAny> = native_field(&native, "resources")?; let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; - report_scope_index_errors(&scope_index)?; + if crate::bindings::_helpers::compile_delta::context::injection().is_none() { + report_scope_index_errors(&scope_index)?; + } let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; let models: Bound<'py, PyTuple> = compiled_models(inputs, &context, (&native_field(&native, "models")?, names))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 5942689a4c..cc4d880e2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -57,13 +57,16 @@ fn retain_until_exit(value: &Bound<'_, PyAny>) -> PyResult<()> { } /// What one full compile reports for reuse recording. -struct CommandOutcome<'py> { - exit_code: i64, - compile_cache_enabled: bool, - artifacts_written: bool, - dag_artifact_path: Bound<'py, PyAny>, +pub(crate) struct CommandOutcome<'py> { + pub(crate) exit_code: i64, + pub(crate) compile_cache_enabled: bool, + pub(crate) artifacts_written: bool, + pub(crate) dag_artifact_path: Bound<'py, PyAny>, } +/// One compile's outcome, input reads and artifact writes, before it is stored and printed. +pub(crate) type PhasesOutcome<'py> = (CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>); + /// The per-invocation recorders a compile runs inside. struct Recorders<'py> { detailed_timings: CompileTimingsHandle, @@ -84,34 +87,66 @@ pub(crate) fn run_compile_command<'py>( Some(attempt) => native_field(attempt, "started")?.extract()?, None => monotonic(py)?, }; - let path_class: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?; - let effective_request: Bound<'py, PyAny> = if native_field(&request, "project_dir")?.is_none() { - replaced( - &request.clone(), - &[("project_dir", path_class.call_method0("cwd")?)], - )? - } else { - request.clone() - }; + let effective_request: Bound<'py, PyAny> = resolved_request(&request)?; let attempt: Bound<'py, PyAny> = match reuse_attempt { Some(attempt) => attempt, None => bypass_attempt(py, &effective_request, total_start)?, }; let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); let status: Status<'py> = Status(compile_status(&request)?); - let gc: Bound<'py, PyModule> = py.import("gc")?; + collecting_later(py, || { + recorded_compile( + (&effective_request, &attempt), + (&output, &status), + total_start, + ) + })? +} + +/// The resolved request, output recorder and phases of a compile neither stored nor printed. +pub(crate) fn unrecorded_compile<'py>( + request: &Bound<'py, PyAny>, + attempt: &Bound<'py, PyAny>, +) -> PyResult<( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + PyResult>, +)> { + let py: Python<'py> = request.py(); + let total_start: f64 = native_field(attempt, "started")?.extract()?; + let effective_request: Bound<'py, PyAny> = resolved_request(request)?; + let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); + let status: Status<'py> = Status(py.None().into_bound(py)); + let phases: PyResult> = collecting_later(py, || { + compile_phases( + (&effective_request, attempt), + (&output, &status), + total_start, + ) + })?; + Ok((effective_request, output, phases)) +} + +/// `request` with its project directory resolved to the working directory when unset. +pub(crate) fn resolved_request<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + if !native_field(request, "project_dir")?.is_none() { + return Ok(request.clone()); + } + let cwd: Bound<'py, PyAny> = imported(request.py(), "pathlib", "Path")?.call_method0("cwd")?; + replaced(&request.clone(), &[("project_dir", cwd)]) +} + +/// Run `body` with Python's garbage collector paused, then resume or freeze it as compiles do. +fn collecting_later(py: Python<'_>, body: impl FnOnce() -> T) -> PyResult { + let gc: Bound<'_, PyModule> = py.import("gc")?; let was_enabled: bool = gc.call_method0("isenabled")?.is_truthy()?; gc.call_method0("disable")?; - let exit_code: PyResult = recorded_compile( - (&effective_request, &attempt), - (&output, &status), - total_start, - ); + let result: T = body(); let deferred: bool = RETAINED_UNTIL_EXIT.get(py).is_some(); if deferred || was_enabled { gc.call_method0(if deferred { "freeze" } else { "enable" })?; } - exit_code + Ok(result) } /// `CompileReuseAttempt` of a compile that did not consult the reuse record. @@ -166,55 +201,26 @@ fn recorded_compile<'py>( (output, status): (&Bound<'py, PyAny>, &Status<'py>), total_start: f64, ) -> PyResult { - let py: Python<'py> = request.py(); - let skip_write: bool = native_field(request, "profile_flags")? - .getattr("skip_write")? - .is_truthy()?; - let compiled: PyResult<(CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = - collect_compile_timings(|detailed_timings| { - within( - &Bound::new(py, record_compile_input_reads())?.into_any(), - |input_reads| { - let artifact_writes: Py = new_artifact_recorder(py)?; - let manager: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::PreparedCompileArtifacts)?, - &[ - ( - "enabled", - PyBool::new(py, !skip_write).to_owned().into_any(), - ), - ( - "artifact_writes", - artifact_writes.bind(py).clone().into_any(), - ), - ], - )?; - retain_until_exit(&manager)?; - within(&manager, |prepared_artifacts| { - let recorders: Recorders<'py> = Recorders { - detailed_timings: Rc::clone(detailed_timings), - input_reads: input_reads.clone(), - artifact_writes: artifact_writes.clone_ref(py), - prepared_artifacts: prepared_artifacts.clone(), - }; - let reuse_timings: Bound<'py, PyAny> = - compile_reuse_timings(attempt)?.into_any(); - let outcome: CommandOutcome<'py> = compiled_with_status( - (request, &recorders), - (status, output), - (total_start, &reuse_timings), - )?; - Ok(( - outcome, - recorders.input_reads, - recorders.artifact_writes.into_bound(py).into_any(), - )) - }) - }, - ) - }); + let phases: PyResult> = + compile_phases((request, attempt), (output, status), total_start); close_status(status)?; - let (outcome, input_reads, artifact_writes) = compiled?; + let phases: PhasesOutcome<'py> = phases?; + crate::bindings::_helpers::compile_delta::verify::full_compile_finished( + output, + (phases.0.exit_code, phases.0.artifacts_written), + &phases.2, + )?; + store_and_print((request, attempt), output, phases) +} + +/// Store a finished compile for reuse, then print its recorded output. +pub(crate) fn store_and_print<'py>( + (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + output: &Bound<'py, PyAny>, + phases: PhasesOutcome<'py>, +) -> PyResult { + let py: Python<'py> = request.py(); + let (outcome, input_reads, artifact_writes): PhasesOutcome<'py> = phases; let stored: PyResult<()> = write_reusable_compile( py, Some(keywords( @@ -248,6 +254,64 @@ fn recorded_compile<'py>( Ok(outcome.exit_code) } +/// Run the compile's phases inside its recorders, without storing or printing it. +pub(crate) fn compile_phases<'py>( + (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (output, status): (&Bound<'py, PyAny>, &Status<'py>), + total_start: f64, +) -> PyResult> { + let py: Python<'py> = request.py(); + let skip_write: bool = native_field(request, "profile_flags")? + .getattr("skip_write")? + .is_truthy()?; + let staging: bool = + !skip_write && crate::bindings::_helpers::compile_delta::context::injection().is_none(); + let (outcome, input_reads, artifact_writes): ( + CommandOutcome<'py>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = collect_compile_timings(|detailed_timings| { + within( + &Bound::new(py, record_compile_input_reads())?.into_any(), + |input_reads| { + let artifact_writes: Py = new_artifact_recorder(py)?; + let manager: Bound<'py, PyAny> = call_with( + ®istry::native(py, NativeAttr::PreparedCompileArtifacts)?, + &[ + ("enabled", PyBool::new(py, staging).to_owned().into_any()), + ( + "artifact_writes", + artifact_writes.bind(py).clone().into_any(), + ), + ], + )?; + retain_until_exit(&manager)?; + within(&manager, |prepared_artifacts| { + let recorders: Recorders<'py> = Recorders { + detailed_timings: Rc::clone(detailed_timings), + input_reads: input_reads.clone(), + artifact_writes: artifact_writes.clone_ref(py), + prepared_artifacts: prepared_artifacts.clone(), + }; + let reuse_timings: Bound<'py, PyAny> = + compile_reuse_timings(attempt)?.into_any(); + let outcome: CommandOutcome<'py> = compiled_with_status( + (request, &recorders), + (status, output), + (total_start, &reuse_timings), + )?; + Ok(( + outcome, + recorders.input_reads, + recorders.artifact_writes.into_bound(py).into_any(), + )) + }) + }, + ) + })?; + Ok((outcome, input_reads, artifact_writes)) +} + /// `_run_compile_with_status`: analyze, run the output phases and record the command output. fn compiled_with_status<'py>( (request, recorders): (&Bound<'py, PyAny>, &Recorders<'py>), @@ -309,6 +373,10 @@ fn compiled_with_status<'py>( .chain(withheld.try_iter()?) .collect::>>()?, )?; + crate::bindings::_helpers::compile_delta::report_hooks::record_compile_facts( + &native_field(&analysis, "graph")?, + &diagnostics, + )?; let mut exit_code: i64 = 0; for diagnostic in diagnostics.as_any().try_iter()? { if native_field(&diagnostic?, "is_error")?.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index d747ec27e6..f116a462bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -49,6 +49,8 @@ use crate::bindings::_helpers::records::ids::RecordId; use pyo3::IntoPyObjectExt; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; +/// The setting that turns delta compiles, and the project image they read, off. +const DELTA_COMPILE_SETTING_KEY: &str = "delta_compile"; /// Declaration file collections counted into the declaration-parse lifecycle, in field order. const DECLARATION_COLLECTIONS: [&str; 14] = [ @@ -236,6 +238,20 @@ fn assembled_inputs<'py>( local_config: &local_config, })?; reject_unsupported_python_paths(project_dir)?; + let delta_settings: &Bound<'py, PyAny> = + if native_field(&local_config, "setting_overrides")?.contains(DELTA_COMPILE_SETTING_KEY)? { + &local_config + } else { + &project_config + }; + if !native_field( + &native_field(delta_settings, "settings")?, + DELTA_COMPILE_SETTING_KEY, + )? + .is_truthy()? + { + crate::bindings::_helpers::compile_delta::context::stop_capture(); + } let sql_analysis_override: &Bound<'py, PyAny> = request.sql_analysis_enabled_override.bind(py); let sql_analysis_enabled: Bound<'py, PyAny> = if !sql_analysis_override.is_none() { sql_analysis_override.clone() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs index b1280c561e..96f3d65137 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -68,6 +68,9 @@ pub(crate) fn emit_json_report(report: &Bound<'_, PyAny>) -> PyResult { if let Some(error) = conversion.stdlib_error { return Err(error); } + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.stdlib_json = true; + }); emitted( &value, &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index ee68ca5a9e..2c61cac5ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -123,6 +123,14 @@ fn poisoned(_: T) -> pyo3::PyErr { PyValueError::new_err("compile reuse attempt is poisoned") } +impl NativeReuseAttempt { + /// Run `body` on the native attempt. + pub(crate) fn with_attempt(&self, body: impl FnOnce(&ReuseAttempt) -> T) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + Ok(body(&attempt)) + } +} + #[pymethods] impl NativeReuseAttempt { /// `"hit"`, `"miss"` or `"bypass"`. @@ -187,7 +195,7 @@ impl NativeReuseAttempt { fn record(&self, py: Python<'_>, record: CompileRecordPy) -> PyResult<(&'static str, usize)> { compiler_guard(|| { let rules: ReuseRules = record.rules.into(); - let compile: CompileRecord = CompileRecord { + let mut compile: CompileRecord = CompileRecord { stdout: record.stdout, stderr_lines: record.stderr_lines, exit_code: record.exit_code, @@ -207,7 +215,29 @@ impl NativeReuseAttempt { artifacts_written: record.artifacts_written, dag_path: record.dag_path, json_output: record.json_output, + image: crate::bindings::_helpers::compile_delta::context::take_pending_image(), + vouched_target_stamps: None, }; + if let Some(merge) = + crate::bindings::_helpers::compile_delta::context::take_record_merge() + { + for stamp in &merge.kept { + compile + .artifacts + .entry(stamp.relative_path.clone()) + .or_insert(RecordedArtifact::Kept { + size: stamp.size, + mtime_ns: stamp.mtime_ns, + }); + } + compile.vouched_target_stamps = Some(merge.kept); + compile + .template_environment_names + .extend(merge.environment_names); + compile.template_environment_names.sort(); + compile.template_environment_names.dedup(); + compile.module_paths.extend(merge.module_paths); + } let attempt = self.attempt.lock().map_err(poisoned)?; let result: RecordResult = py .detach(|| record_reuse(&attempt, &compile, &rules)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs index 20869e5c8f..f748dffb8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -1,5 +1,6 @@ //! `build_project_audit_inputs`: singular and attached generic audits attached natively. +use crate::bindings::_helpers::compile_delta::context::record_source_reads; use std::collections::{BTreeMap, HashMap}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; @@ -573,6 +574,12 @@ impl<'py> AuditScope<'_, 'py> { .downcast()? .clone(); let known: KnownResources<'_> = validator.get().known_resources(); + record_source_reads( + calls + .iter() + .filter(|(kind, _)| *kind == SOURCE_TARGET) + .map(|(_, name)| *name), + ); match audit_reference_error(relative_path, calls, &known) { Some(message) => Err(compile_error(self.context.classes, &message, None)), None => Ok(()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs index fe97660e20..ea653300d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs @@ -1,5 +1,6 @@ //! `build_sql_function_inputs`: every SQL and Python function attached natively. +use crate::bindings::_helpers::compile_delta::context::record_source_reads; use std::collections::{HashMap, HashSet}; use pyo3::exceptions::PyValueError; @@ -51,6 +52,9 @@ const TABLE_RETURNS: &str = "TABLE"; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; +/// The reference kind of a source. +const SOURCE_KIND: &str = "source"; + /// A function's paths, stem, discovered file, header values, body and SQL file contents. pub(crate) type FunctionRow = ( String, @@ -518,6 +522,12 @@ fn function_references( .iter() .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) .collect(); + record_source_reads( + calls + .iter() + .filter(|(kind, _)| *kind == SOURCE_KIND) + .map(|(_, name)| *name), + ); if let Some(message) = function_reference_error(relative_path, &calls, &known) { return Err(compile_error(context.classes, &message, None)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs index 2771b1321c..11af682292 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs @@ -1,5 +1,6 @@ //! `build_scenario_inputs`: SQL scenarios expanded, classified and checked natively. +use crate::bindings::_helpers::compile_delta::context::record_test_source_reads; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_helper_scope::scenario_helper_scope; @@ -154,6 +155,12 @@ impl<'py> TestScope<'_, 'py> { sources: source_names(&references), }); } + record_test_source_reads( + origin.1, + sources + .iter() + .flat_map(|cte| cte.sources.iter().map(String::as_str)), + ); if let Some(message) = scenario_source_violation(origin.1, &sources, &self.catalog.sources) { return Err(compile_error(self.context.classes, &message, None)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs index c181628576..07563842eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs @@ -1,5 +1,6 @@ //! `build_test_inputs`: SQL test blocks expanded, classified and checked natively. +use crate::bindings::_helpers::compile_delta::context::record_test_source_reads; use std::collections::{HashMap, HashSet}; use std::convert::Infallible; @@ -748,6 +749,7 @@ impl<'py> TestScope<'_, 'py> { .into_iter() .filter(|name| !mock_models.contains(name)) .collect(); + record_test_source_reads(relative_path, mock_sources.iter().map(String::as_str)); if let Some(message) = unknown_test_target( relative_path, &self.catalog, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 159055abea..8df9e95178 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -212,6 +212,7 @@ const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("test_timeout_seconds", FieldDefault::Int(30)), FieldSpec::new("scenario_timeout", FieldDefault::None), FieldSpec::new("scenario_timeout_seconds", FieldDefault::None), + FieldSpec::new("delta_compile", FieldDefault::True), ], }; const R_SCOPES_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index 1515583bcb..b42b199fd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -69,6 +69,10 @@ fn model_items<'py>( for file in files { let raw: bool = tree.get().inner.raw_relative(&file.relative_path).is_some(); if !raw && let FileOutcome::Parsed(model) = file.outcome { + crate::bindings::_helpers::compile_delta::header_keys::record_header_keys( + &file.relative_path, + &model.header_values, + ); let file_path: String = root.display_path(&file.relative_path); let paths = DiscoveredPaths::Native(file.relative_path.clone(), file_path); let built = model_record(py, model, paths, options.extract_implicit_alias_columns)? @@ -102,6 +106,17 @@ pub(crate) struct ModelFileSelection<'py> { pub(crate) on_fault: Option>, } +/// The options model discovery parses headers with. +pub(crate) fn model_file_options() -> ModelFileOptions { + ModelFileOptions { + supported_keys: sorted_keys(&constants::SQL_MODEL_HEADER_KEYS), + removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), + extract_implicit_alias_columns: false, + extract_output_column_locations: false, + path_filter: None, + } +} + /// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. pub(crate) fn discovered_model_files<'py>( project_dir: &Bound<'py, PyAny>, @@ -115,6 +130,8 @@ pub(crate) fn discovered_model_files<'py>( removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), extract_implicit_alias_columns, extract_output_column_locations, + path_filter: crate::bindings::_helpers::compile_delta::context::injection() + .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), }; let items: Vec> = model_items(project_dir, &options)?; let faults = FileFaults { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index fe0176e6ef..4a8806e1bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -1,5 +1,6 @@ //! The facts cross-file validation reads, taken from Python's `DiscoveredProjectInputs`. +use crate::bindings::_helpers::compile_delta::context::injection; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use pyo3::FromPyObject; @@ -69,6 +70,7 @@ pub(crate) fn project_facts( .try_iter()? .map(|key| key?.extract()) .collect::>()?, + project_model_paths: injection().map(|injection| injection.project_model_paths.clone()), csv_field_limit: Some(csv_field_limit(py)?), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index c7d5e780bf..0850dd66a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -105,6 +105,7 @@ fn parse_model_contents( removed_keys: request.removed_keys, extract_implicit_alias_columns: request.extract_implicit_alias_columns, extract_output_column_locations: request.extract_output_column_locations, + path_filter: None, }; let parsed = py .compiler_detach(|| Ok(parse_model_text(&request.file_path, contents, &options))) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index c5abe9872c..5ef3604e1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -57,6 +57,8 @@ fn options() -> PyResult { Ok(SqlTestFileOptions { test_keys: sorted_keys(&constants::SQL_TEST_HEADER_KEYS), scenario_keys: sorted_keys(&constants::SQL_SCENARIO_HEADER_KEYS), + path_filter: crate::bindings::_helpers::compile_delta::context::injection() + .map(|injection| std::sync::Arc::new(injection.test_paths.clone())), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index b2b19a4c73..9734f1a56b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -348,13 +348,41 @@ pub(crate) fn parse_sources_document<'py>( .map_err(|error| { parse_error(py, error, SOURCE_PARSE_ERROR).unwrap_or_else(|error| error) })?; + let referenced: Option> = + crate::bindings::_helpers::compile_delta::context::injection() + .and_then(|injection| injection.source_names.clone()); + crate::bindings::_helpers::compile_delta::context::record_source_reads( + loaded_sources(&sources).iter().map(String::as_str), + ); let mut built: Vec> = Vec::with_capacity(sources.len()); for source in &sources { + if let Some(referenced) = &referenced + && let Ok(name) = source.name.0.extract::() + && !referenced.contains(&name) + { + continue; + } built.push(source_entry(py, &classes, source)?); } PyTuple::new(py, built) } +/// The sources a loader fills, which own that loader wherever the project is compiled. +fn loaded_sources(sources: &[ParsedSourceEntry>]) -> Vec { + sources + .iter() + .filter(|source| { + source.managed || source.loader.is_some() || source.integration_loader.is_some() + }) + .filter_map(|source| { + let Ok(name) = source.name.0.extract::() else { + return None; + }; + Some(name) + }) + .collect() +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_schema_document, module)?)?; module.add_function(wrap_pyfunction!(parse_sources_document, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index 6349270505..4fcf280d58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -155,6 +155,12 @@ fn load_files<'py>( let py: Python<'py> = project_dir.py(); let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; let root: ProjectRoot = project_root(project_dir)?; + let mut documents: Option = + if matches!(kind, YamlKind::Source) { + crate::bindings::_helpers::compile_delta::yaml_cache::open_documents(py) + } else { + None + }; for batch in batches(&tree.get().inner, paths, YAML_BATCH_BYTES) { let named: Vec = batch .iter() @@ -164,11 +170,25 @@ fn load_files<'py>( let outcomes: Result, StageFailure> = { let inner = &tree.get().inner; let failure_kind: FailureKind = kind.failure_kind(); - py.detach(|| load_yaml(&root, inner, &named, failure_kind)) + match documents.as_mut() { + Some(documents) => py.detach(|| { + documents.load_with( + &named, + |path| inner.absolute(path), + |missed| load_yaml(&root, inner, missed, failure_kind), + ) + }), + None => py.detach(|| load_yaml(&root, inner, &named, failure_kind)), + } }; - let mut outcomes = outcomes - .map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))? - .into_iter(); + let mut outcomes: Vec = + outcomes.map_err(|failure| stage_error(&tree.get().inner, project_dir, failure))?; + if matches!(kind, YamlKind::Source) { + crate::bindings::_helpers::compile_delta::report_hooks::retain_read_sources( + &mut outcomes, + ); + } + let mut outcomes = outcomes.into_iter(); for path in batch { let relative: Bound<'py, PyAny> = path_object(py, &tree.get().inner, path.clone())?.into_bound(py); @@ -190,6 +210,9 @@ fn load_files<'py>( on_file((relative_path, file_path), built)?; } } + if let Some(documents) = &documents { + crate::bindings::_helpers::compile_delta::yaml_cache::save_documents(py, documents); + } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index c02be409fd..044bc2611c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -41,6 +41,9 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::{ use crate::bindings::_helpers::compile_command::constants::{ REUSE_DISABLE_ENV_VAR, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, }; +use crate::bindings::_helpers::compile_delta::driver::{ + DELTA_DISABLE_ENV_VAR, DELTA_VERIFY_ENV_VAR, +}; use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; use crate::bindings::_helpers::native_host::constants::CUSTOM_HOST_LAUNCH_MODULE; use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; @@ -69,6 +72,8 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("STAGED_PROMOTION_MODE", STAGED_PROMOTION_MODE), ("STAGE_CAPTURE_DIR_ENV_VAR", CAPTURE_DIR_ENV_VAR), ("COMPILE_REUSE_DISABLE_ENV_VAR", REUSE_DISABLE_ENV_VAR), + ("DELTA_COMPILE_DISABLE_ENV_VAR", DELTA_DISABLE_ENV_VAR), + ("DELTA_COMPILE_VERIFY_ENV_VAR", DELTA_VERIFY_ENV_VAR), ("SQLBUILD_QUERY_TAG_APP", QUERY_TAG_APP), ("COMMAND_OUTPUT_RECORD_TYPE", COMMAND_OUTPUT_RECORD_TYPE), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index 6a951e4a9b..62e7a80269 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -214,7 +214,7 @@ impl NativeProjectGraph { } /// The key's direct upstream lineage keys, in authored dependency order. - fn upstream(&self, key: ObjectKey) -> Vec { + pub(crate) fn upstream(&self, key: ObjectKey) -> Vec { edge_keys(&self.graph, &key, EdgeDirection::Upstream).to_vec() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs new file mode 100644 index 0000000000..69314bf43e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/first_compile.rs @@ -0,0 +1,52 @@ +//! Which compile of this process may reuse stored macro calls: only its first. + +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_delta::context::injection; + +/// Whether a compile claimed the first-compile slot, and whether a delta compile only borrowed it. +#[derive(Debug)] +struct FirstCompile { + claimed: bool, + lent_to_delta: bool, +} + +static FIRST_COMPILE: Mutex = Mutex::new(FirstCompile { + claimed: false, + lent_to_delta: false, +}); + +fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Record that a compile started; true only for this process's first compile invocation. +#[pyfunction] +pub(crate) fn claim_first_compile() -> bool { + let mut first: MutexGuard<'_, FirstCompile> = locked(&FIRST_COMPILE); + if injection().is_some() { + let granted: bool = !first.claimed || first.lent_to_delta; + first.claimed = true; + first.lent_to_delta = granted; + return granted; + } + if first.lent_to_delta { + first.lent_to_delta = false; + return true; + } + let granted: bool = !first.claimed; + first.claimed = true; + granted +} + +/// Keep the delta compile's claim: it finished, so no full compile of this invocation follows. +pub(crate) fn keep_delta_claim() { + locked(&FIRST_COMPILE).lent_to_delta = false; +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(claim_first_compile, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs index 008b35b5c2..8623ff9f42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod first_compile; pub(crate) mod module_digests; pub(crate) mod registration; pub(crate) mod store_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs index b483dec47b..eddce4b97a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::macro_bridge::{module_digests, store_keys}; +use crate::bindings::_helpers::macro_bridge::{first_compile, module_digests, store_keys}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + first_compile::register(module)?; module_digests::register(module)?; store_keys::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 1ad13fd9e6..d9d2208ac8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -14,6 +14,7 @@ pub(crate) mod clone; pub(crate) mod command_runtime; pub(crate) mod compile_attachment; pub(crate) mod compile_command; +pub(crate) mod compile_delta; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compile_records; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index 8e23c3f973..68a0ba0b93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -58,9 +58,12 @@ impl NativeModelValidator { /// Model, seed, source, function and table function names discovered in `inputs`. fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { let model_stem = |file: &Bound<'_, PyAny>| stem(file, FILE_PATH_ATTRIBUTE); - let models: HashSet = each(inputs, "model_files", model_stem)? + let mut models: HashSet = each(inputs, "model_files", model_stem)? .into_iter() .collect(); + if let Some(injection) = crate::bindings::_helpers::compile_delta::context::injection() { + models.extend(injection.model_names.iter().cloned()); + } let mut seeds: HashSet = HashSet::new(); for schema in native_field(inputs, "schema_files")?.try_iter()? { seeds.extend(each(&schema?, "seed_entries", column_name)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs index 6e4cd932ca..33c0d0fcb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -62,6 +62,11 @@ impl MacroCallMemo { path: PathBuf, environment: &str, ) -> PyResult<(Vec, usize)> { + if crate::bindings::_helpers::compile_delta::context::injection().is_some() { + return Err(pyo3::exceptions::PyRuntimeError::new_err( + "a delta compile leaves the macro call store to full compiles", + )); + } let store: NativeStore = open_store(py, &path, MACRO_CALL_STORE_KIND, environment)?; let opened = (store.metadata().to_vec(), store.loaded_entries()); self.inner.lock().map_err(poisoned)?.attach_store(store); @@ -76,6 +81,11 @@ impl MacroCallMemo { Ok(()) } + /// Whether this compile may use the call store; a delta compile leaves it to full compiles. + fn store_allowed(&self) -> bool { + crate::bindings::_helpers::compile_delta::context::injection().is_none() + } + /// Whether this compile recorded calls the attached store does not hold yet. fn store_needs_save(&self) -> PyResult { Ok(self diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index f5169cf5c5..06199b4ab1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -97,7 +97,32 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( }; let project_dir: PathBuf = run.project_dir.extract()?; let mut digests: FactDigests<'py> = FactDigests::new(run.project, &project_dir)?; - let models: Vec<(String, String, Option)> = digests.replay.model_rows()?; + let models: Vec<(String, String, Option)> = + match crate::bindings::_helpers::compile_delta::context::injection() { + Some(injection) => { + let Some(mut rows) = injection.model_subjects.clone() else { + return Err( + crate::bindings::_helpers::compile_delta::context::delta_unsupported( + "the custom rules' subjects were not recorded", + ), + ); + }; + let fresh: Vec<(String, String, Option)> = digests.replay.model_rows()?; + for row in &mut rows { + if let Some(found) = fresh.iter().find(|candidate| candidate.0 == row.0) { + row.clone_from(found); + } + } + rows + } + None => { + let rows: Vec<(String, String, Option)> = digests.replay.model_rows()?; + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.model_subjects = Some(rows.clone()); + }); + rows + } + }; let mut model_paths: Vec = models.iter().map(|(path, _, _)| path.clone()).collect(); model_paths.sort(); let mut cache: CustomRuleCache = CustomRuleCache::new( @@ -112,6 +137,14 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( readset_digests.push(reproducible(digests.combined(readset))?); } let plan: HostPlan = cache.plan(readset_digests); + if !plan.is_empty() && crate::bindings::_helpers::compile_delta::context::injection().is_some() + { + return Err( + crate::bindings::_helpers::compile_delta::context::delta_unsupported( + "a custom rule must run again", + ), + ); + } if !plan.is_empty() { let rules: Vec<(String, HostRuleSubject)> = custom .iter() @@ -135,6 +168,15 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( let evaluations: Vec = evaluations(host_run, &mut digests, enabled)?; cache.record(py, evaluations)?; } + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.fact_digests.extend( + digests + .digests + .iter() + .filter(|(key, _)| !is_tree_fact(key)) + .map(|(key, digest)| (key.clone(), digest.clone())), + ); + }); let elapsed_ms: i64 = (started.elapsed().as_secs_f64() * 1000.0).round_ties_even() as i64; let cpu_ms: i64 = ((cpu_seconds(py)? - started_cpu) * 1000.0) as i64; outcome( @@ -232,6 +274,12 @@ impl<'py> FactDigests<'py> { if let Some(cached) = self.digests.get(key) { return Ok(cached.clone()); } + if let Some(injected) = + crate::bindings::_helpers::compile_delta::context::injected_fact_digest(key) + { + self.digests.insert(key.clone(), injected.clone()); + return Ok(injected); + } let digest: String = self.replay.digest(key)?; self.digests.insert(key.clone(), digest.clone()); Ok(digest) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index cef85ca350..94c3848ffd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -160,6 +160,7 @@ pub(crate) fn selectionless_rules_request( request: Mutex::new(Some(RowsRequest { request, memo: None, + retain: false, })), }) }) @@ -293,11 +294,19 @@ fn built_request( let model_rows: Vec = project_model_rows(facts.borrow().facts(), &names).map_err(value_error)?; request.models = built_models(&rows, &request, model_rows)?; + let request: EvaluateRequest = + crate::bindings::_helpers::compile_delta::rules_rows::whole_project_request(request)?; + let delta: bool = crate::bindings::_helpers::compile_delta::context::injection().is_some(); let memo: Option<(String, PathBuf)> = rows .memo .take() + .filter(|_| !delta) .map(|(identity, path)| (identity, PathBuf::from(path))); - Ok(RowsRequest { request, memo }) + Ok(RowsRequest { + request, + memo, + retain: crate::bindings::_helpers::compile_delta::context::capturing(), + }) }) } @@ -332,10 +341,16 @@ fn evaluated<'py>( .map_err(value_error)?, Some((early, interrupted)) => early.answer(py, request, interrupted)?, }; - let evaluation: RowsEvaluation = evaluation.map_err(|error| match error { + let mut evaluation: RowsEvaluation = evaluation.map_err(|error| match error { RowsError::Rules(message) => value_error(message), RowsError::Io(error) => PyOSError::new_err(error.to_string()), })?; + if let Some(retained) = evaluation.retained.take() { + crate::bindings::_helpers::compile_delta::context::capture(|capture| { + capture.rules = + Some(crate::bindings::_helpers::compile_delta::context::CapturedRules { retained }); + }); + } let classes = FindingClasses::new(py)?; let findings: Vec> = evaluation .faults diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index d3124c629b..ed839bc92a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -148,12 +148,15 @@ pub(crate) fn run_planned_sql_rules<'py>( }; let lint_key: String = settings_lint_key(&run.settings); let identities: Vec<(String, String)> = model_rule_identities(&run, &lint_key)?; - let project_files: Option> = - if !identities.is_empty() && run.prepared().is_none() && !run.model_selection { - Some(lint_sources(&run.project_dir, None, None)?) - } else { - None - }; + let project_files: Option> = if !identities.is_empty() + && run.prepared().is_none() + && !run.model_selection + && crate::bindings::_helpers::compile_delta::context::injection().is_none() + { + Some(lint_sources(&run.project_dir, None, None)?) + } else { + None + }; let started: Instant = Instant::now(); let (findings, hits, misses) = evaluated_sql_rules(&run, (&lint_key, identities), project_files)?; @@ -298,13 +301,14 @@ fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult = model_rule_keys(run.project.py(), lint_key, run.codes, &models); - Ok(models + let fresh: Vec<(String, String)> = models .into_iter() .map(|(path, ..)| { let key: String = keys[&path].clone(); (path, key) }) - .collect()) + .collect(); + crate::bindings::_helpers::compile_delta::lint_hooks::model_identities(fresh) } /// Canonical text of a hand-built model's dynamic column contract that SQL rule findings depend on. @@ -342,6 +346,29 @@ fn prepared_result<'a>( Some(prepared.result(py)) } +/// The files a whole-project lint reads: prepared, walked, or in a delta only the changed ones. +fn linted_files( + run: &SqlRun<'_, '_>, + prepared_files: Option>, +) -> PyResult> { + let py: Python<'_> = run.project.py(); + let project_dir: &Path = &run.project_dir; + let prepared = prepared_result(run, py); + if crate::bindings::_helpers::compile_delta::context::injection().is_some() { + return crate::bindings::_helpers::compile_delta::lint_hooks::changed_files(project_dir); + } + Ok(match prepared { + Some(Ok(result)) => result.outcome.files.clone(), + Some(Err(error)) if !error.is_instance_of::(py) => { + return Err(error.clone_ref(py)); + } + Some(Err(_)) | None => match prepared_files { + Some(files) => files, + None => lint_sources(project_dir, None, None)?, + }, + }) +} + /// Each analysed model's findings, cached or linted, then cache accounting. fn evaluated_sql_rules( run: &SqlRun<'_, '_>, @@ -382,16 +409,7 @@ fn evaluated_sql_rules( for model in project_attribute(&run.project, "models")?.try_iter()? { model_paths.insert(project_dir.join(model_posix_path(&model?)?)); } - let files: Vec = match prepared_result(run, py) { - Some(Ok(result)) => result.outcome.files.clone(), - Some(Err(error)) if !error.is_instance_of::(py) => { - return Err(error.clone_ref(py)); - } - Some(Err(_)) | None => match prepared_files { - Some(files) => files, - None => lint_sources(project_dir, None, None)?, - }, - }; + let files: Vec = linted_files(run, prepared_files)?; let mut file_keys: Vec<(PathBuf, String, Option)> = Vec::new(); for file in &files { if model_paths.contains(&file.path) { @@ -409,6 +427,11 @@ fn evaluated_sql_rules( ); file_keys.push((file.path.clone(), relative_path, key)); } + let file_keys: Vec<(PathBuf, String, Option)> = + crate::bindings::_helpers::compile_delta::lint_hooks::file_keys( + project_dir, + file_keys, + )?; let mut keyed: Vec<(String, String)> = Vec::new(); for (_, path, key) in &file_keys { if let Some(key) = key { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index f1672c0545..8338ebd9bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -355,10 +355,21 @@ pub(crate) fn scope_index_with_compile_usages<'py>( usages.extend(usage_records( model_schema_scope_usages(&schema_files, &resolver)?.as_any(), )?); + let digests: bool = crate::bindings::_helpers::compile_delta::context::capturing(); for model in native_field(inputs, "model_inputs")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - usages.extend(usage_records(&native_field(&model, "macro_usages")?)?); - usages.extend(usage_records(&native_field(&model, "declaration_usages")?)?); + let macro_usages: Vec> = + usage_records(&native_field(&model, "macro_usages")?)?; + let declaration_usages: Vec> = + usage_records(&native_field(&model, "declaration_usages")?)?; + if digests { + crate::bindings::_helpers::compile_delta::image_facts::record_model_usages( + &model, + (¯o_usages, &declaration_usages), + )?; + } + usages.extend(macro_usages); + usages.extend(declaration_usages); } for group in COMPILE_USAGE_GROUPS { for item in native_field(inputs, group)?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs index d756c0ffa4..d3dc4ed58f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs @@ -1,12 +1,14 @@ //! Per-model reasons a semantic check was only partial, for machine reports. -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::BTreeSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::analysis::model_reads::{inferred_rows, star_flags}; +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; +use crate::bindings::_helpers::compile_delta::report_hooks::semantic_reasons; use crate::bindings::_helpers::compiled_project::compiled_model_view::InferredRow; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; @@ -24,97 +26,89 @@ pub(crate) fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult = project_shapes(project, None)?; - let models: Vec> = native_field(project, "models")? - .try_iter()? - .collect::>()?; - let mut names: Vec = Vec::with_capacity(models.len()); - let mut references: Vec> = Vec::with_capacity(models.len()); - let mut blocked_by: HashMap> = HashMap::new(); - for model in &models { - let name: String = native_field(model, "name")?.extract()?; + let dialect: Bound<'py, PyAny> = native_field(project, "sql_analysis_dialect")?; + let dialect: String = if dialect.is_truthy()? { + dialect.extract()? + } else { + "generic".to_owned() + }; + let mut order: Vec<(String, Vec)> = Vec::new(); + let mut facts: Vec = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let name: String = native_field(&model, "name")?.extract()?; let mut rows: Vec<(String, String)> = Vec::new(); - for row in native_field(model, "reference_rows")?.try_iter()? { + for row in native_field(&model, "reference_rows")?.try_iter()? { let row: Bound<'py, PyAny> = row?; rows.push(( row.get_item(0)?.str()?.extract()?, row.get_item(1)?.extract()?, )); } - for item in native_field(model, "binding_diagnostics")?.try_iter()? { + let mut blocked: bool = false; + for item in native_field(&model, "binding_diagnostics")?.try_iter()? { if native_field(&item?, "is_error")?.is_truthy()? { - blocked_by.insert(name.clone(), HashSet::from([name.clone()])); + blocked = true; break; } } - names.push(name); - references.push(rows); + let opted: Bound<'py, PyAny> = native_field(&native_field(&model, "config")?, "values")? + .call_method1("get", ("sql_analysis",))?; + let disabled: bool = opted.is(PyBool::new(py, false)); + let partial: BTreeSet = if disabled { + BTreeSet::new() + } else { + own_reasons((&model, &name), &rows, (&shapes, &dialect))? + }; + facts.push(ModelCoverage { + name: name.clone(), + disabled, + blocked, + partial: partial.into_iter().collect(), + }); + order.push(( + name, + rows.into_iter().map(|(_, upstream)| upstream).collect(), + )); } - if !blocked_by.is_empty() { - for _ in &names { - let mut changed: bool = false; - for (name, rows) in names.iter().zip(&references) { - let mut causes: HashSet = blocked_by.get(name).cloned().unwrap_or_default(); - for (_, upstream) in rows { - causes.extend(blocked_by.get(upstream).cloned().unwrap_or_default()); - } - if !causes.is_empty() && blocked_by.get(name) != Some(&causes) { - blocked_by.insert(name.clone(), causes); - changed = true; - } - } - if !changed { - break; - } - } + for (name, partial) in semantic_reasons(order, facts)? { + reasons.set_item(name, PyTuple::new(py, partial)?)?; } - let dialect: Bound<'py, PyAny> = native_field(project, "sql_analysis_dialect")?; - let dialect: String = if dialect.is_truthy()? { - dialect.extract()? - } else { - "generic".to_owned() - }; - for ((model, name), rows) in models.iter().zip(&names).zip(&references) { - let opted: Bound<'py, PyAny> = native_field(&native_field(model, "config")?, "values")? - .call_method1("get", ("sql_analysis",))?; - if opted.is(PyBool::new(py, false)) { - reasons.set_item(name, PyTuple::new(py, ["SQL analysis disabled"])?)?; - continue; - } - let mut partial: BTreeSet = BTreeSet::new(); - for (kind, upstream) in rows { - if OPEN_KINDS.contains(&kind.as_str()) && !shapes.contains(upstream)? { - partial.insert(if kind == SOURCE_KIND { - format!("open source: {upstream}") - } else { - format!("open input: {upstream}") - }); - } - } - if let Some(causes) = blocked_by.get(name) { - for cause in causes.iter().filter(|cause| *cause != name) { - partial.insert(format!("depends on invalid model {cause}")); - } - } - let inferred: Vec = inferred_rows(model)?.unwrap_or_default(); - if inferred.is_empty() { - partial.insert("output shape could not be inferred".to_owned()); - } else if star_flags(model)?.0 && !shapes.contains(name)? { - partial.insert("unresolved star over open inputs".to_owned()); - } - if inferred - .iter() - .any(|(_, column_type, _)| column_type.is_none()) - { - partial.insert("some output types are unknown".to_owned()); - } - if !TYPE_CHECKED_DIALECTS.contains(&dialect.as_str()) { - partial.insert("type checking awaits dialect coercion support".to_owned()); - } - if !partial.is_empty() { - reasons.set_item(name, PyTuple::new(py, partial)?)?; + Ok(reasons) +} + +/// A model's partial-check reasons apart from those its invalid dependencies give it. +fn own_reasons<'py>( + (model, name): (&Bound<'py, PyAny>, &str), + rows: &[(String, String)], + (shapes, dialect): (&Bound<'py, PyDict>, &str), +) -> PyResult> { + let mut partial: BTreeSet = BTreeSet::new(); + for (kind, upstream) in rows { + if OPEN_KINDS.contains(&kind.as_str()) && !shapes.contains(upstream)? { + partial.insert(if kind == SOURCE_KIND { + format!("open source: {upstream}") + } else { + format!("open input: {upstream}") + }); } } - Ok(reasons) + let inferred: Vec = inferred_rows(model)?.unwrap_or_default(); + if inferred.is_empty() { + partial.insert("output shape could not be inferred".to_owned()); + } else if star_flags(model)?.0 && !shapes.contains(name)? { + partial.insert("unresolved star over open inputs".to_owned()); + } + if inferred + .iter() + .any(|(_, column_type, _)| column_type.is_none()) + { + partial.insert("some output types are unknown".to_owned()); + } + if !TYPE_CHECKED_DIALECTS.contains(&dialect) { + partial.insert("type checking awaits dialect coercion support".to_owned()); + } + Ok(partial) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 6ea7f7db18..2973fa30de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -317,6 +317,7 @@ fn model_shapes<'py>( let mut pending: Vec> = native_field(project, "models")? .try_iter()? .collect::>()?; + crate::bindings::_helpers::compile_delta::report_hooks::inject_model_shapes(&pending, shapes)?; while !pending.is_empty() { let mut remaining: Vec> = Vec::new(); for model in &pending { @@ -362,7 +363,7 @@ fn model_shapes<'py>( } pending = remaining; } - Ok(()) + crate::bindings::_helpers::compile_delta::report_hooks::record_model_shapes(project, shapes) } fn all_known(names: &[String], shapes: &Bound<'_, PyDict>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/mod.rs b/crates/sqlbuild-python/src/bindings/mod.rs index d887a75d22..6a1afca4b9 100644 --- a/crates/sqlbuild-python/src/bindings/mod.rs +++ b/crates/sqlbuild-python/src/bindings/mod.rs @@ -2,3 +2,6 @@ pub(crate) mod _helpers; pub mod main; pub(crate) mod models; pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs new file mode 100644 index 0000000000..44f522ea77 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -0,0 +1,12 @@ +#[path = "tests/test_coverage.rs"] +mod coverage; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_image.rs"] +mod image; +#[path = "tests/test_splice.rs"] +mod splice; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_yaml_cache.rs"] +mod yaml_cache; diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs new file mode 100644 index 0000000000..f4fb041c20 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -0,0 +1,246 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; + +use crate::bindings::_helpers::compile_delta::context::{ + CapturedDiagnostic, CapturedModel, CapturedTest, +}; +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; +use crate::bindings::_helpers::compile_delta::image::{ + BaseRules, FACTS_SECTION, GATE_SECTION, IMAGE_FORMAT, ImageRelations, ImageRules, PendingRules, + ProjectImage, RULE_DIGESTS_SECTION, +}; + +/// The rows of a request without models: a zero model count. +const RULE_ROWS: [u8; 8] = [0; 8]; +const OTHER_FORMAT: &str = "sqlbuild-project-image-v0"; + +pub(crate) fn pairs(values: &[(&str, &str)]) -> Vec<(String, String)> { + values + .iter() + .map(|(left, right)| ((*left).to_owned(), (*right).to_owned())) + .collect() +} + +pub(crate) fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// An image holding at least one value of every kind it stores. +pub(crate) fn image() -> ProjectImage { + let order_types: Vec<(String, String)> = pairs(&[("order_id", "BIGINT"), ("amount", "DOUBLE")]); + let customer_types: Vec<(String, String)> = pairs(&[("customer_id", "BIGINT")]); + let request: Vec = serde_json::to_vec(&EvaluateRequest::default()).unwrap_or_default(); + ProjectImage { + format: String::new(), + json_output: true, + models: vec![ + CapturedModel { + name: "orders".to_owned(), + relative_path: "models/orders.sql".to_owned(), + upstream: pairs(&[("source", "raw_orders"), ("model", "customers")]), + }, + CapturedModel { + name: "customers".to_owned(), + relative_path: "models/customers.sql".to_owned(), + upstream: Vec::new(), + }, + ], + usages: BTreeMap::from([("models/orders.sql".to_owned(), "digest-1".to_owned())]), + tests: vec![CapturedTest { + key: "orders_test".to_owned(), + relative_path: "tests/unit/orders.sql".to_owned(), + chain: strings(&["orders", "customers"]), + }], + relations: ImageRelations { + types: vec![ + ("orders".to_owned(), order_types.clone()), + ("customers".to_owned(), customer_types.clone()), + ], + nullability: vec![("orders".to_owned(), pairs(&[("order_id", "never")]))], + complete: vec![("orders".to_owned(), order_types.clone())], + schemas: vec![("customers".to_owned(), pairs(&[("customer_id", "INTEGER")]))], + dependency_ordered: true, + cyclic: false, + legacy_models: strings(&["customers"]), + }, + fact_digests: vec![(strings(&["model", "orders"]), "fact-digest".to_owned())], + model_subjects: Some(vec![ + ( + "models/orders.sql".to_owned(), + "orders".to_owned(), + Some("table".to_owned()), + ), + ( + "models/customers.sql".to_owned(), + "customers".to_owned(), + None, + ), + ]), + diagnostics: vec![CapturedDiagnostic { + json: "{\"code\": \"W1\"}".to_owned(), + resource_name: Some("orders".to_owned()), + paths: strings(&["models/orders.sql"]), + error: false, + }], + text_models: Some(vec![("orders".to_owned(), true, 3)]), + environment_names: strings(&["SQLBUILD_TARGET"]), + pinned_sources: strings(&["raw_customers"]), + test_sources: BTreeMap::from([( + "tests/unit/orders.sql".to_owned(), + strings(&["raw_orders"]), + )]), + model_shapes: vec![ + ("orders".to_owned(), order_types), + ("customers".to_owned(), pairs(&[("customer_id", "VARCHAR")])), + ], + coverage: Some(vec![ModelCoverage { + name: "orders".to_owned(), + disabled: false, + blocked: true, + partial: strings(&["open source: raw_orders"]), + }]), + lint_model_keys: Some(pairs(&[("models/orders.sql", "lint-key")])), + lint_file_keys: Some(vec![ + ("models/orders.sql".to_owned(), Some("file-key".to_owned())), + ("models/customers.sql".to_owned(), None), + ]), + header_keys: BTreeMap::from([( + "models/orders.sql".to_owned(), + pairs(&[("description", "0123")]), + )]), + rules: Some(ImageRules::Retained(RetainedRows { + request_json: request, + rows: RULE_ROWS.to_vec(), + digests: Vec::new(), + })), + } +} + +/// Every stored fact of `image` but its rules, in a comparable form. +pub(crate) fn facts(image: &ProjectImage) -> String { + format!( + "{:?}", + ( + ( + &image.json_output, + &image.models, + &image.usages, + &image.tests, + &image.relations, + &image.fact_digests, + &image.model_subjects, + ), + ( + &image.diagnostics, + &image.text_models, + &image.environment_names, + &image.pinned_sources, + &image.test_sources, + &image.model_shapes, + &image.coverage, + &image.lint_model_keys, + &image.lint_file_keys, + &image.header_keys, + ), + ) + ) +} + +pub(crate) fn current_format() -> String { + IMAGE_FORMAT.to_owned() +} + +pub(crate) fn encoded(image: ProjectImage) -> ImageSections { + let stored: ProjectImage = ProjectImage { + format: current_format(), + ..image + }; + stored.encode().unwrap_or_default() +} + +/// The image `image()` without any of its optional facts. +pub(crate) fn sparse_image() -> ProjectImage { + ProjectImage { + model_subjects: None, + text_models: None, + lint_model_keys: None, + lint_file_keys: None, + rules: None, + ..image() + } +} + +/// The rules request a decoded image holds, resolved. +pub(crate) fn decoded_rules(image: ProjectImage) -> Option { + image + .rules + .and_then(ImageRules::pending) + .and_then(PendingRules::resolved) +} + +/// `sections` with the gate of an image of another format. +pub(crate) fn with_other_format(sections: &mut ImageSections) { + let other: ProjectImage = ProjectImage { + format: OTHER_FORMAT.to_owned(), + ..ProjectImage::default() + }; + let gate: Vec = other + .encode() + .unwrap_or_default() + .into_iter() + .find(|(name, _)| name == GATE_SECTION) + .map(|(_, bytes)| bytes) + .unwrap_or_default(); + sections + .iter_mut() + .filter(|(name, _)| name == GATE_SECTION) + .for_each(|section| section.1 = gate.clone()); +} + +/// `sections` with their facts cut in half. +pub(crate) fn with_truncated_facts(sections: &mut ImageSections) { + sections + .iter_mut() + .filter(|(name, _)| name == FACTS_SECTION) + .for_each(|(_, bytes)| bytes.truncate(bytes.len() / 2)); +} + +/// `sections` without the rules request's digests. +pub(crate) fn without_rule_digests(sections: &mut ImageSections) { + sections.retain(|(name, _)| name != RULE_DIGESTS_SECTION); +} + +/// The names in `values`, as a set. +pub(crate) fn names(values: &[&str]) -> BTreeSet { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// A YAML string value. +pub(crate) fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) +} + +/// One model's semantic coverage facts. +pub(crate) fn coverage( + name: &str, + (disabled, blocked): (bool, bool), + partial: &[&str], +) -> ModelCoverage { + ModelCoverage { + name: name.to_owned(), + disabled, + blocked, + partial: strings(partial), + } +} + +/// Models paired with their upstream names, or with their reasons. +pub(crate) fn order(models: &[(&str, &[&str])]) -> Vec<(String, Vec)> { + models + .iter() + .map(|(name, upstream)| ((*name).to_owned(), strings(upstream))) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs b/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs new file mode 100644 index 0000000000..6edbf806ef --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_coverage.rs @@ -0,0 +1,115 @@ +use std::collections::BTreeSet; + +use crate::bindings::_helpers::compile_delta::coverage::{ + ModelCoverage, coverage_reasons, merged_coverage, merged_order, +}; +use crate::bindings::_helpers::compile_delta::driver::names_uncompiled_model; +use crate::bindings::tests::helpers::{coverage, names, order, strings}; +use crate::bindings::tests::test_types::{CoverageTestCase, UncompiledModelTestCase}; + +const DISABLED: &str = "SQL analysis disabled"; +const OPEN_INPUT: &str = "open input: raw_orders"; + +#[test] +fn given_stored_and_fresh_coverage_when_merging_then_reasons_match_a_whole_project_compile() { + let chain: &[(&str, &[&str])] = &[ + ("orders", &[]), + ("payments", &["orders"]), + ("customers", &["payments"]), + ("returns", &[]), + ]; + let test_cases = [ + CoverageTestCase { + description: "a stored model without SQL analysis keeps its reason", + stored: vec![coverage("returns", (true, false), &[])], + fresh: vec![coverage("orders", (false, false), &[])], + stored_order: order(chain), + fresh_order: order(&[("orders", &[])]), + expected_reasons: order(&[("returns", &[DISABLED])]), + }, + CoverageTestCase { + description: "a rerun model turning invalid makes every stored dependent partial", + stored: vec![coverage("customers", (false, false), &[OPEN_INPUT])], + fresh: vec![coverage("orders", (false, true), &[])], + stored_order: order(chain), + fresh_order: order(&[("orders", &[])]), + expected_reasons: order(&[ + ("payments", &["depends on invalid model orders"]), + ( + "customers", + &["depends on invalid model orders", OPEN_INPUT], + ), + ]), + }, + CoverageTestCase { + description: "a rerun model fixed drops the reasons its dependents had from it", + stored: vec![coverage("orders", (false, true), &[])], + fresh: vec![coverage("orders", (false, false), &[])], + stored_order: order(chain), + fresh_order: order(&[("orders", &[])]), + expected_reasons: vec![], + }, + CoverageTestCase { + description: "a rerun model reading a stored invalid model becomes partial", + stored: vec![coverage("returns", (false, true), &[])], + fresh: vec![coverage("orders", (false, false), &[])], + stored_order: order(chain), + fresh_order: order(&[("orders", &["returns"])]), + expected_reasons: order(&[ + ("orders", &["depends on invalid model returns"]), + ("payments", &["depends on invalid model returns"]), + ("customers", &["depends on invalid model returns"]), + ]), + }, + ]; + + for test_case in test_cases { + let facts: Vec = merged_coverage(&test_case.stored, &test_case.fresh); + let project_order: Vec<(String, Vec)> = + merged_order(&test_case.stored_order, &test_case.fresh_order); + + assert_eq!( + coverage_reasons(&project_order, &facts), + test_case.expected_reasons, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_missing_model_errors_when_judging_a_delta_then_only_uncompiled_models_rule_it_out() { + let test_cases = [ + UncompiledModelTestCase { + description: "an expectation of a project model the delta did not compile", + message: "test 'orders_check' expects model 'payments' which does not exist", + expected_uncompiled: true, + }, + UncompiledModelTestCase { + description: "a reference to a model the project lacks is a real error", + message: "references unknown model 'refunds'", + expected_uncompiled: false, + }, + UncompiledModelTestCase { + description: "a reference to a model the delta compiled is a real error", + message: "references unknown model 'orders'", + expected_uncompiled: false, + }, + UncompiledModelTestCase { + description: "an unrelated error naming a project model", + message: "model 'payments' has a type error", + expected_uncompiled: false, + }, + ]; + let model_names: Vec = strings(&["orders", "payments"]); + let rerun: BTreeSet = names(&["orders"]); + + for test_case in test_cases { + assert_eq!( + names_uncompiled_model(test_case.message, &model_names, &rerun), + test_case.expected_uncompiled, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_image.rs b/crates/sqlbuild-python/src/bindings/tests/test_image.rs new file mode 100644 index 0000000000..16c9cbbe23 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_image.rs @@ -0,0 +1,111 @@ +use sqlbuild_cache::project_reuse::types::ImageSections; + +use crate::bindings::_helpers::compile_delta::image::{ImageGate, ProjectImage}; +use crate::bindings::tests::helpers::{ + current_format, decoded_rules, encoded, facts, image, sparse_image, with_other_format, + with_truncated_facts, without_rule_digests, +}; +use crate::bindings::tests::test_types::{DamagedImageTestCase, ImageRoundTripTestCase}; + +#[test] +fn given_project_image_when_encoded_and_decoded_then_every_fact_and_the_rules_survive() { + let test_cases = [ + ImageRoundTripTestCase { + description: "every fact and a retained rules request", + image, + expected_model_subjects: true, + expected_rule_models: Some(0), + }, + ImageRoundTripTestCase { + description: "no optional facts and no rules request", + image: sparse_image, + expected_model_subjects: false, + expected_rule_models: None, + }, + ]; + for test_case in test_cases { + let sections: ImageSections = encoded((test_case.image)()); + + let gate: Option = ImageGate::decode(§ions); + let gate_facts: Option<(bool, bool, bool)> = gate + .as_ref() + .map(|gate| (gate.json_output, gate.has_errors, gate.has_model_subjects)); + let decoded: Option = + gate.and_then(|gate| ProjectImage::decode(sections, gate)); + let decoded_facts: Option<(String, String)> = decoded + .as_ref() + .map(|image| (image.format.clone(), facts(image))); + let rule_models: Option = decoded + .and_then(decoded_rules) + .map(|rules| rules.request.models.len() + rules.digests.len()); + + assert_eq!( + gate_facts, + Some((true, false, test_case.expected_model_subjects)), + "{}", + test_case.description + ); + assert_eq!( + decoded_facts, + Some((current_format(), facts(&(test_case.image)()))), + "{}", + test_case.description + ); + assert_eq!( + rule_models, test_case.expected_rule_models, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_damaged_image_when_decoded_then_it_is_never_read() { + let test_cases = [ + DamagedImageTestCase { + description: "intact", + damage: |_| {}, + expected_gate: true, + expected_image: true, + }, + DamagedImageTestCase { + description: "truncated facts", + damage: with_truncated_facts, + expected_gate: true, + expected_image: false, + }, + DamagedImageTestCase { + description: "rules request without its digests", + damage: without_rule_digests, + expected_gate: true, + expected_image: false, + }, + DamagedImageTestCase { + description: "gate of another format", + damage: with_other_format, + expected_gate: false, + expected_image: false, + }, + ]; + for test_case in test_cases { + let mut sections: ImageSections = encoded(image()); + (test_case.damage)(&mut sections); + + let gate: Option = ImageGate::decode(§ions); + let gate_read: bool = gate.is_some(); + let decoded: Option = + gate.and_then(|gate| ProjectImage::decode(sections, gate)); + + assert_eq!( + gate_read, test_case.expected_gate, + "{}", + test_case.description + ); + assert_eq!( + decoded.is_some(), + test_case.expected_image, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_splice.rs b/crates/sqlbuild-python/src/bindings/tests/test_splice.rs new file mode 100644 index 0000000000..705202a2fe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_splice.rs @@ -0,0 +1,91 @@ +use std::collections::BTreeSet; + +use crate::bindings::_helpers::compile_delta::splice::{SpliceScope, spliced_report}; +use crate::bindings::tests::helpers::names; +use crate::bindings::tests::test_types::SpliceTestCase; + +const BASE: &str = r#"{"has_errors": false, "summary": {"models": 2, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 1}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "not_null", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "compiled"}]}, "diagnostics": [], "compile_timings": {"total_ms": 9}}"#; + +#[test] +fn given_stored_and_delta_reports_when_splicing_then_only_rerun_items_and_delta_sections_change() { + let test_cases = [ + SpliceTestCase { + description: "a rerun model, its audit and the delta's diagnostics replace the stored ones", + base: BASE, + delta: r#"{"has_errors": true, "summary": {"models": 1, "errors": 1, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 3}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "unique", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": []}, "diagnostics": [{"code": "E1"}], "compile_timings": {"total_ms": 1}}"#, + models: &["orders"], + tests: &[], + sources: None, + expected_report: Some( + r#"{"has_errors": true, "summary": {"models": 2, "errors": 1, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 3}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [{"name": "unique", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "compiled"}]}, "diagnostics": [{"code": "E1"}], "compile_timings": {"total_ms": 1}}"#, + ), + }, + SpliceTestCase { + description: "a rerun SQL test and the only discovered source replace the stored ones", + base: BASE, + delta: r#"{"has_errors": false, "summary": {"models": 0, "errors": 0, "warnings": 0}, "resources": {"models": [], "sources": [{"name": "raw_customers", "column_count": 5}], "audits": [], "tests": [{"name": "orders_test", "status": "changed"}]}, "diagnostics": [], "compile_timings": {"total_ms": 1}}"#, + models: &[], + tests: &["orders_test"], + sources: Some(&["raw_customers"]), + expected_report: Some( + r#"{"has_errors": false, "summary": {"models": 2, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "orders", "columns": 1}, {"name": "customers", "columns": 2}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 5}], "audits": [{"name": "not_null", "attached_to": {"resource_type": "model", "name": "orders"}}], "tests": [{"name": "orders_test", "status": "changed"}]}, "diagnostics": [], "compile_timings": {"total_ms": 1}}"#, + ), + }, + SpliceTestCase { + description: "a delta item for a resource outside the rerun set must already be stored", + base: BASE, + delta: r#"{"has_errors": false, "summary": {"models": 1, "errors": 0, "warnings": 0}, "resources": {"models": [{"name": "customers", "columns": 9}], "sources": [{"name": "raw_orders", "column_count": 1}, {"name": "raw_customers", "column_count": 2}], "audits": [], "tests": []}, "diagnostics": [], "compile_timings": {"total_ms": 1}}"#, + models: &["orders"], + tests: &[], + sources: None, + expected_report: None, + }, + SpliceTestCase { + description: "the delta's partial-check reasons, already the whole project's, replace the stored", + base: r#"{"has_errors": false, "semantic_checks_partial": {"customers": ["open input: orders"]}, "resources": {}, "diagnostics": []}"#, + delta: r#"{"has_errors": true, "semantic_checks_partial": {"customers": ["depends on invalid model orders"]}, "resources": {}, "diagnostics": [{"code": "B002"}]}"#, + models: &["orders"], + tests: &[], + sources: None, + expected_report: Some( + r#"{"has_errors": true, "semantic_checks_partial": {"customers": ["depends on invalid model orders"]}, "resources": {}, "diagnostics": [{"code": "B002"}]}"#, + ), + }, + SpliceTestCase { + description: "a whole-project section that differs cannot be spliced", + base: r#"{"has_errors": false, "target": "dev", "summary": {"errors": 0, "warnings": 0}, "resources": {}, "diagnostics": [], "compile_timings": {}}"#, + delta: r#"{"has_errors": false, "target": "prod", "summary": {"errors": 0, "warnings": 0}, "resources": {}, "diagnostics": [], "compile_timings": {}}"#, + models: &[], + tests: &[], + sources: None, + expected_report: None, + }, + SpliceTestCase { + description: "reports with different sections cannot be spliced", + base: r#"{"has_errors": false, "resources": {}}"#, + delta: r#"{"has_errors": false}"#, + models: &[], + tests: &[], + sources: None, + expected_report: None, + }, + ]; + for test_case in test_cases { + let models: BTreeSet = names(test_case.models); + let tests: BTreeSet = names(test_case.tests); + let sources: Option> = test_case.sources.map(names); + let scope: SpliceScope<'_> = SpliceScope { + models: &models, + tests: &tests, + sources: sources.as_ref(), + }; + let spliced: Result = + spliced_report(test_case.base, test_case.delta, &scope); + assert_eq!( + spliced.ok().as_deref(), + test_case.expected_report, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs new file mode 100644 index 0000000000..eb32f89cee --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -0,0 +1,59 @@ +use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_config::models::ConfigValue; + +use crate::bindings::_helpers::compile_delta::coverage::ModelCoverage; +use crate::bindings::_helpers::compile_delta::image::ProjectImage; + +/// A stored report, a delta report and what splicing the delta into the stored one gives. +pub(crate) struct SpliceTestCase { + pub(crate) description: &'static str, + pub(crate) base: &'static str, + pub(crate) delta: &'static str, + pub(crate) models: &'static [&'static str], + pub(crate) tests: &'static [&'static str], + pub(crate) sources: Option<&'static [&'static str]>, + /// The spliced report, or `None` when the reports must not line up. + pub(crate) expected_report: Option<&'static str>, +} + +/// A loaded YAML document kept by the delta compile's document store. +pub(crate) struct DocumentTestCase { + pub(crate) description: &'static str, + pub(crate) document: ConfigValue, + /// How many truncations of the stored document still read back as it. + pub(crate) expected_truncations_read: usize, +} + +/// Stored image bytes damaged before they are read back. +pub(crate) struct DamagedImageTestCase { + pub(crate) description: &'static str, + pub(crate) damage: fn(&mut ImageSections), + pub(crate) expected_gate: bool, + pub(crate) expected_image: bool, +} + +/// An image encoded and decoded again. +pub(crate) struct ImageRoundTripTestCase { + pub(crate) description: &'static str, + pub(crate) image: fn() -> ProjectImage, + pub(crate) expected_model_subjects: bool, + /// The retained rules request's model and digest count, or `None` without one. + pub(crate) expected_rule_models: Option, +} + +/// Stored and fresh semantic coverage facts, and the whole project's partial-check reasons. +pub(crate) struct CoverageTestCase { + pub(crate) description: &'static str, + pub(crate) stored: Vec, + pub(crate) fresh: Vec, + pub(crate) stored_order: Vec<(String, Vec)>, + pub(crate) fresh_order: Vec<(String, Vec)>, + pub(crate) expected_reasons: Vec<(String, Vec)>, +} + +/// An error message a delta round reported. +pub(crate) struct UncompiledModelTestCase { + pub(crate) description: &'static str, + pub(crate) message: &'static str, + pub(crate) expected_uncompiled: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs b/crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs new file mode 100644 index 0000000000..60ea19a5f8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_yaml_cache.rs @@ -0,0 +1,99 @@ +use sqlbuild_config::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; + +use crate::bindings::_helpers::compile_delta::yaml_cache::{decoded_document, encoded_document}; +use crate::bindings::tests::helpers::text; +use crate::bindings::tests::test_types::DocumentTestCase; + +#[test] +fn given_loaded_document_when_stored_and_read_back_then_it_is_unchanged() { + let date: ConfigDate = ConfigDate { + year: 2026, + month: 2, + day: 28, + }; + let time: ConfigTime = ConfigTime { + hour: 23, + minute: 59, + second: 58, + microsecond: 123_456, + }; + let test_cases = [ + DocumentTestCase { + description: "empty document", + expected_truncations_read: 0, + document: ConfigValue::Null, + }, + DocumentTestCase { + description: "sources document with repeated strings", + expected_truncations_read: 0, + document: ConfigValue::Map(vec![( + text("sources"), + ConfigValue::List(vec![ + ConfigValue::Map(vec![ + (text("name"), text("orders")), + (text("schema"), text("raw")), + ]), + ConfigValue::Map(vec![ + (text("name"), text("customers")), + (text("schema"), text("raw")), + ]), + ]), + )]), + }, + DocumentTestCase { + description: "every scalar kind", + expected_truncations_read: 0, + document: ConfigValue::List(vec![ + ConfigValue::Bool(true), + ConfigValue::Bool(false), + ConfigValue::Integer(-42), + ConfigValue::BigInteger("123456789012345678901234567890".to_owned()), + ConfigValue::Float(-0.5), + ConfigValue::Date(date), + ConfigValue::Time(time), + ConfigValue::DateTime(ConfigDateTime { + date, + time, + utc_offset_seconds: Some(-3600), + }), + ConfigValue::DateTime(ConfigDateTime { + date, + time, + utc_offset_seconds: None, + }), + text(""), + text("unicode ✓"), + ]), + }, + DocumentTestCase { + description: "mapping with non-string keys", + expected_truncations_read: 0, + document: ConfigValue::Map(vec![ + (ConfigValue::Integer(1), text("one")), + (ConfigValue::Null, ConfigValue::List(Vec::new())), + ]), + }, + ]; + for test_case in test_cases { + let stored: Vec = encoded_document(&test_case.document); + + let decoded: Option = decoded_document(&stored); + let truncations_read: usize = (0..stored.len()) + .filter(|length| { + decoded_document(&stored[..*length]).as_ref() == Some(&test_case.document) + }) + .count(); + + assert_eq!( + decoded.as_ref(), + Some(&test_case.document), + "{}", + test_case.description + ); + assert_eq!( + truncations_read, test_case.expected_truncations_read, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml index 6fb65866b2..559fa48c1a 100644 --- a/crates/sqlbuild-rules/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -20,6 +20,7 @@ serde_json.workspace = true sha2.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true +sqlbuild-cache.workspace = true sqlbuild-core.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs index 6430d93cea..5bf79b7bc4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs @@ -15,6 +15,18 @@ pub(crate) struct RuleCacheEntry { pub faults: Vec, } +const FIXTURE_SUMMARIES_FILE: &str = "sql-test-fixtures.json"; + +/// Where the SQL-test fixture summaries sit, beside the bulk native rules cache. +pub(crate) fn fixture_summaries_path(project_dir: &Path, rules_cache_path: &str) -> PathBuf { + let native: PathBuf = if rules_cache_path.is_empty() { + project_dir.join("target/rules-cache/bulk/native.json") + } else { + PathBuf::from(rules_cache_path) + }; + native.with_file_name(FIXTURE_SUMMARIES_FILE) +} + pub(crate) struct Cache { native_rules_path: PathBuf, } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 383296000a..f0f4e3c2ed 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -22,7 +22,7 @@ use rayon::iter::IndexedParallelIterator; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; @@ -60,6 +60,10 @@ struct PendingModelRule<'a> { identity: Option, } +/// The error a delta compile's rules request raises when a row without SQL needs evaluating. +pub(crate) const STRIPPED_ROW_MISSED: &str = + "delta compile unsupported: a model's rule findings are no longer cached"; + struct CompletedModelRule { index: usize, identity: Option, @@ -152,6 +156,9 @@ fn evaluate_native_models( identity, }); } + if pending_models.iter().any(|pending| pending.model.stripped) { + return Err(STRIPPED_ROW_MISSED.to_owned()); + } let project_dir = Path::new(&context.request.project_dir); let completed = evaluate_pending_models(pending_models, context)?; for completed_model in completed { @@ -327,6 +334,26 @@ pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result, +) -> Result { + let store: Option = request.config.cache.enabled.then(|| { + crate::engine::_helpers::cache::fixture_summaries_path( + Path::new(&request.project_dir), + &request.rules_cache_path, + ) + }); + crate::rules::_helpers::sql_test_rules::with_fixture_memo(store.as_deref(), || { + let evaluated: Result = evaluated_request(request, control); + let keep: bool = match &evaluated { + Ok(_) => !control.cancelled(), + Err(_failed) => false, + }; + (evaluated, keep) + }) +} + +fn evaluated_request( + request: EvaluateRequest, + control: &EvaluationControl<'_>, ) -> Result { if request.version != API_VERSION { return Err(format!( diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index b49f3ff516..d6e217a141 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -9,8 +9,13 @@ use serde_json::Value; use crate::constants::{API_VERSION, NATIVE_ROWS_MEMO_VERSION}; use crate::engine::_helpers::cache::PendingBucketWrite; use crate::engine::_helpers::evaluation::{EvaluationControl, evaluate_request_with}; +use crate::engine::_helpers::retained::encoded_rows; use crate::errors::RowsError; -use crate::models::{EvaluateRequest, Model, RowsEvaluation, RowsRequest}; +use crate::models::{EvaluateRequest, Model, RetainedRows, RowsEvaluation, RowsRequest}; + +/// The shared key prefix of fields a request serializes but evaluation derives again. +const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; +const DERIVED_FIELDS: [&str; 2] = ["_test_count\":", "\":"]; /// A rows request whose models are digested, ready to evaluate once. #[derive(Debug)] @@ -20,6 +25,8 @@ pub(crate) struct PreparedRows { memo: Option<(String, PathBuf)>, /// The digest of every evaluation input the request carries, when it was asked for. pub(crate) key: Option, + /// The request kept for the project image, when it was asked for. + pub(crate) retained: Option, } /// What a speculative evaluation found, with every write it would have made still pending. @@ -41,13 +48,21 @@ pub(crate) fn evaluate_rows(rows: RowsRequest) -> Result Result { - let RowsRequest { mut request, memo } = rows; + let RowsRequest { + mut request, + memo, + retain, + } = rows; let models: Vec = std::mem::take(&mut request.models) .into_iter() .map(digested) .collect::>()?; + let mut request_json: Option> = None; let memo: Option<(String, PathBuf)> = match memo { - Some((code_identity, path)) => Some((identity(&request, &models, &code_identity)?, path)), + Some((code_identity, path)) => { + let kept: Option<&mut Option>> = retain.then_some(&mut request_json); + Some((identity(&request, &models, &code_identity, kept)?, path)) + } None => None, }; let key: Option = if keyed { @@ -55,13 +70,80 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result = if retain { + retained_rows(&request, &models, request_json) + } else { + None + }; request.models = models; - Ok(PreparedRows { request, memo, key }) + Ok(PreparedRows { + request, + memo, + key, + retained, + }) +} + +/// `request`'s model-free JSON, each model's row without SQL text, and each model's digest. +fn retained_rows( + request: &EvaluateRequest, + models: &[Model], + request_json: Option>, +) -> Option { + let request_json: Vec = match request_json { + Some(json) => json, + None => { + let Ok(json) = serde_json::to_vec(request) else { + return None; + }; + json + } + }; + Some(RetainedRows { + request_json: without_derived_fields(request_json), + rows: encoded_rows(models), + digests: models + .iter() + .map(|model| model.payload_digest.clone().unwrap_or_default()) + .collect(), + }) +} + +/// `json` without the derived fields; compact JSON escapes quotes, so keys only match as keys. +fn without_derived_fields(json: Vec) -> Vec { + let Ok(text) = std::str::from_utf8(&json) else { + return json; + }; + let mut kept: Vec = Vec::with_capacity(json.len()); + let mut copied: usize = 0; + for (start, _) in text.match_indices(DERIVED_FIELD_PREFIX) { + if start < copied { + continue; + } + let rest: &str = &text[start + DERIVED_FIELD_PREFIX.len()..]; + let Some(field) = DERIVED_FIELDS + .iter() + .find(|field| rest.starts_with(**field)) + else { + continue; + }; + let value: &str = &rest[field.len()..]; + let length: usize = value.find([',', '}']).unwrap_or(value.len()); + kept.extend_from_slice(&json[copied..start]); + copied = start + DERIVED_FIELD_PREFIX.len() + field.len() + length; + } + kept.extend_from_slice(&json[copied..]); + kept } /// Answer a prepared request from the memo or evaluate it, writing the cache and memo at once. pub(crate) fn evaluate_prepared(prepared: PreparedRows) -> Result { - let PreparedRows { request, memo, .. } = prepared; + let PreparedRows { + request, + memo, + retained, + .. + } = prepared; let reused: Option = memo .as_ref() .and_then(|(identity, path)| read_response(path, identity)); @@ -76,7 +158,9 @@ pub(crate) fn evaluate_prepared(prepared: PreparedRows) -> Result { hasher.update(b"unmemoized\0"); - hasher.update(identity(request, models, "")?.as_bytes()); + hasher.update(identity(request, models, "", None)?.as_bytes()); } } Ok(hasher.finalize().to_hex().to_string()) @@ -162,24 +246,34 @@ fn speculation_key( /// `model` with the BLAKE3 digest of its exact facts, which keys its cached findings. fn digested(mut model: Model) -> Result { - let mut hasher: blake3::Hasher = blake3::Hasher::new(); - serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; - model.payload_digest = Some(hasher.finalize().to_hex().to_string()); + if model.payload_digest.is_none() { + let mut hasher: blake3::Hasher = blake3::Hasher::new(); + serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; + model.payload_digest = Some(hasher.finalize().to_hex().to_string()); + } Ok(model) } -/// The memo version, build, model-free request and every model's digest, in order. +/// The memo version, build, request and model digests, keeping the request's JSON in `kept`. fn identity( request: &EvaluateRequest, models: &[Model], code_identity: &str, + kept: Option<&mut Option>>, ) -> Result { let mut hasher: blake3::Hasher = blake3::Hasher::new(); for part in [NATIVE_ROWS_MEMO_VERSION, code_identity] { hasher.update(part.as_bytes()); hasher.update(b"\0"); } - serde_json::to_writer(&mut hasher, request).map_err(rules_error)?; + match kept { + Some(kept) => { + let json: Vec = serde_json::to_vec(request).map_err(rules_error)?; + hasher.update(&json); + *kept = Some(json); + } + None => serde_json::to_writer(&mut hasher, request).map_err(rules_error)?, + } hasher.update(b"\0"); hasher.update(&(models.len() as u64).to_le_bytes()); for model in models { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs index ec8f8aa27d..234f764470 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs @@ -3,5 +3,6 @@ pub mod custom_host; pub mod evaluation; pub(crate) mod memo; pub(crate) mod project_rows; +pub(crate) mod retained; pub(crate) mod rows; pub(crate) mod speculation; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs new file mode 100644 index 0000000000..15c69a0ea5 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -0,0 +1,408 @@ +//! Retained model rows in the native record format, without their SQL text. + +use std::collections::BTreeMap; +use std::str::FromStr; + +use serde_json::{Map, Number, Value}; +use sqlbuild_cache::store::errors::StoreDecodeError; +use sqlbuild_cache::store::models::{RecordReader, RecordWriter}; + +use crate::models::{ + Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Model, Reference, + RetainedRows, +}; + +const NULL: u64 = 0; +const FALSE: u64 = 1; +const TRUE: u64 = 2; +const NUMBER: u64 = 3; +const STRING: u64 = 4; +const ARRAY: u64 = 5; +const OBJECT: u64 = 6; +const ABSENT: u64 = 0; +const PRESENT: u64 = 1; +/// Items reserved up front per decoded list at most, whatever count a damaged row claims. +const MAX_RESERVED_ITEMS: usize = 1 << 12; + +/// Writes model rows as length-prefixed records. +#[derive(Default)] +struct RowWriter { + record: RecordWriter, +} + +impl RowWriter { + fn flag(&mut self, value: bool) { + self.record.put_u64(u64::from(value)); + } + + fn count(&mut self, value: usize) { + self.record.put_u64(value as u64); + } + + fn text(&mut self, value: &str) { + self.record.put_str(value); + } + + fn optional_text(&mut self, value: Option<&str>) { + match value { + Some(text) => { + self.record.put_u64(PRESENT); + self.text(text); + } + None => self.record.put_u64(ABSENT), + } + } + + fn texts(&mut self, values: &[String]) { + self.count(values.len()); + for value in values { + self.text(value); + } + } + + fn value(&mut self, value: &Value) { + match value { + Value::Null => self.record.put_u64(NULL), + Value::Bool(flag) => self.record.put_u64(if *flag { TRUE } else { FALSE }), + Value::Number(number) => { + self.record.put_u64(NUMBER); + self.text(&number.to_string()); + } + Value::String(text) => { + self.record.put_u64(STRING); + self.text(text); + } + Value::Array(items) => { + self.record.put_u64(ARRAY); + self.count(items.len()); + for item in items { + self.value(item); + } + } + Value::Object(entries) => { + self.record.put_u64(OBJECT); + self.count(entries.len()); + for (key, item) in entries { + self.text(key); + self.value(item); + } + } + } + } + + fn optional_value(&mut self, value: Option<&Value>) { + match value { + Some(value) => { + self.record.put_u64(PRESENT); + self.value(value); + } + None => self.record.put_u64(ABSENT), + } + } + + fn declarations(&mut self, declarations: &[Declaration]) { + self.count(declarations.len()); + for declaration in declarations { + let Declaration { + name, + relative_path, + members, + value, + value_type, + render_as, + } = declaration; + self.text(name); + self.text(relative_path); + self.count(members.len()); + for DeclarationMember { name, value } in members { + self.text(name); + self.value(value); + } + self.optional_value(value.as_ref()); + self.optional_text(value_type.as_deref()); + self.optional_text(render_as.as_deref()); + } + } + + /// One model's row; its SQL text, digest and derived counts are left out. + fn model(&mut self, model: &Model) { + let Model { + sql_analysis_disabled, + name, + relative_path, + query_sql: _, + authored_sql: _, + config, + authored_config_keys, + logical_schema, + references, + columns, + dynamic_columns, + dynamic_columns_proven, + bare_dynamic_pivot, + enum_columns, + enum_declarations, + constant_declarations, + declared_audit_count, + targeting_test_count, + empty_input_only_test_count: _, + payload_digest: _, + stripped: _, + } = model; + self.flag(*sql_analysis_disabled); + self.text(name); + self.text(relative_path); + self.count(config.len()); + for (key, value) in config { + self.text(key); + self.value(value); + } + self.texts(authored_config_keys); + self.optional_text(logical_schema.as_deref()); + self.count(references.len()); + for Reference { + ref_kind, + ref_name, + ref_package, + } in references + { + self.text(ref_kind); + self.text(ref_name); + self.optional_text(ref_package.as_deref()); + } + self.count(columns.len()); + for Column { + name, + data_type, + nullable, + audit_count, + type_proven, + } in columns + { + self.text(name); + self.text(data_type); + self.record.put_u64(match nullable { + None => NULL, + Some(false) => FALSE, + Some(true) => TRUE, + }); + self.record.put_u64(u64::from(*audit_count)); + self.flag(*type_proven); + } + self.count(dynamic_columns.len()); + for DynamicColumnFamily { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + type_proven, + } in dynamic_columns + { + for text in [name, pivot_column, value_column, aggregate, data_type] { + self.text(text); + } + self.optional_text(name_pattern.as_deref()); + self.flag(*type_proven); + } + self.flag(*dynamic_columns_proven); + self.flag(*bare_dynamic_pivot); + self.texts(enum_columns); + self.declarations(enum_declarations); + self.declarations(constant_declarations); + self.record.put_u64(u64::from(*declared_audit_count)); + self.record.put_u64(u64::from(*targeting_test_count)); + } +} + +/// Reads the rows a [`RowWriter`] wrote. +struct RowReader<'a> { + record: RecordReader<'a>, +} + +type Read = Result; + +impl RowReader<'_> { + fn count(&mut self) -> Read { + usize::try_from(self.record.u64()?).map_err(|_| StoreDecodeError) + } + + fn small(&mut self) -> Read { + u32::try_from(self.record.u64()?).map_err(|_| StoreDecodeError) + } + + fn flag(&mut self) -> Read { + match self.record.u64()? { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(StoreDecodeError), + } + } + + fn text(&mut self) -> Read { + self.record.string() + } + + fn present(&mut self) -> Read { + match self.record.u64()? { + ABSENT => Ok(false), + PRESENT => Ok(true), + _ => Err(StoreDecodeError), + } + } + + fn optional_text(&mut self) -> Read> { + Ok(if self.present()? { + Some(self.text()?) + } else { + None + }) + } + + fn list(&mut self, mut item: impl FnMut(&mut Self) -> Read) -> Read> { + let count: usize = self.count()?; + let mut items: Vec = Vec::with_capacity(count.min(MAX_RESERVED_ITEMS)); + for _ in 0..count { + items.push(item(self)?); + } + Ok(items) + } + + fn value(&mut self) -> Read { + Ok(match self.record.u64()? { + NULL => Value::Null, + FALSE => Value::Bool(false), + TRUE => Value::Bool(true), + NUMBER => Value::Number(Number::from_str(&self.text()?).map_err(|_| StoreDecodeError)?), + STRING => Value::String(self.text()?), + ARRAY => Value::Array(self.list(Self::value)?), + OBJECT => { + let count: usize = self.count()?; + let mut entries: Map = Map::new(); + for _ in 0..count { + let key: String = self.text()?; + entries.insert(key, self.value()?); + } + Value::Object(entries) + } + _ => return Err(StoreDecodeError), + }) + } + + fn optional_value(&mut self) -> Read> { + Ok(if self.present()? { + Some(self.value()?) + } else { + None + }) + } + + fn declaration(&mut self) -> Read { + Ok(Declaration { + name: self.text()?, + relative_path: self.text()?, + members: self.list(|reader| { + Ok(DeclarationMember { + name: reader.text()?, + value: reader.value()?, + }) + })?, + value: self.optional_value()?, + value_type: self.optional_text()?, + render_as: self.optional_text()?, + }) + } + + fn column(&mut self) -> Read { + Ok(Column { + name: self.text()?, + data_type: self.text()?, + nullable: match self.record.u64()? { + NULL => None, + FALSE => Some(false), + TRUE => Some(true), + _ => return Err(StoreDecodeError), + }, + audit_count: self.small()?, + type_proven: self.flag()?, + }) + } + + fn dynamic_column(&mut self) -> Read { + Ok(DynamicColumnFamily { + name: self.text()?, + pivot_column: self.text()?, + value_column: self.text()?, + aggregate: self.text()?, + data_type: self.text()?, + name_pattern: self.optional_text()?, + type_proven: self.flag()?, + }) + } + + fn model(&mut self) -> Read { + Ok(Model { + sql_analysis_disabled: self.flag()?, + name: self.text()?, + relative_path: self.text()?, + query_sql: String::new(), + authored_sql: String::new(), + config: { + let count: usize = self.count()?; + let mut config: BTreeMap = BTreeMap::new(); + for _ in 0..count { + let key: String = self.text()?; + config.insert(key, self.value()?); + } + config + }, + authored_config_keys: self.list(Self::text)?, + logical_schema: self.optional_text()?, + references: self.list(|reader| { + Ok(Reference { + ref_kind: reader.text()?, + ref_name: reader.text()?, + ref_package: reader.optional_text()?, + }) + })?, + columns: self.list(Self::column)?, + dynamic_columns: self.list(Self::dynamic_column)?, + dynamic_columns_proven: self.flag()?, + bare_dynamic_pivot: self.flag()?, + enum_columns: self.list(Self::text)?, + enum_declarations: self.list(Self::declaration)?, + constant_declarations: self.list(Self::declaration)?, + declared_audit_count: self.small()?, + targeting_test_count: self.small()?, + empty_input_only_test_count: 0, + payload_digest: None, + stripped: false, + }) + } +} + +/// Every model's row without its SQL text, in model order. +pub(crate) fn encoded_rows(models: &[Model]) -> Vec { + let mut writer: RowWriter = RowWriter::default(); + writer.count(models.len()); + for model in models { + writer.model(model); + } + writer.record.into_bytes() +} + +/// The request `retained` holds, each model without its SQL text, digest or derived counts. +pub(crate) fn retained_request(retained: &RetainedRows) -> Read { + let mut request: EvaluateRequest = + serde_json::from_slice(&retained.request_json).map_err(|_| StoreDecodeError)?; + let mut reader: RowReader<'_> = RowReader { + record: RecordReader::new(&retained.rows), + }; + request.models = reader.list(RowReader::model)?; + reader.record.finish()?; + if request.models.len() != retained.digests.len() { + return Err(StoreDecodeError); + } + Ok(request) +} diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index 4058e6b6f2..fc694bf75b 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -93,6 +93,7 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result SpeculativeAnswer { - let prepared: PreparedRows = match prepare(rows, true) { + let mut prepared: PreparedRows = match prepare(rows, true) { Ok(prepared) => prepared, Err(error) => { drop(speculative); @@ -77,7 +77,10 @@ pub(crate) fn answer( }; match outcome { Some(outcome) => SpeculativeAnswer::Answered { - evaluation: commit(outcome), + evaluation: commit(outcome).map(|mut evaluation| { + evaluation.retained = prepared.retained.take(); + evaluation + }), used: true, }, None => SpeculativeAnswer::Answered { diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index a20ecdbdd4..1237dba54e 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -5,4 +5,5 @@ pub mod discard_speculative_rows; pub mod evaluate_rows; pub mod finalize_rows; pub mod project_model_rows; +pub mod retained_request; pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/retained_request.rs b/crates/sqlbuild-rules/src/engine/main/retained_request.rs new file mode 100644 index 0000000000..1023e8ebc4 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/retained_request.rs @@ -0,0 +1,9 @@ +use crate::models::{EvaluateRequest, RetainedRows}; + +/// The request a compile retained, its models without SQL text; `None` when it is damaged. +pub fn retained_request(retained: &RetainedRows) -> Option { + let Ok(request) = crate::engine::_helpers::retained::retained_request(retained) else { + return None; + }; + Some(request) +} diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index ed6cc5ba16..5195d6dd8a 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -10,6 +10,8 @@ mod helpers; mod incremental_rules; #[path = "tests/test_query_facts.rs"] mod query_facts; +#[path = "tests/test_retained_rows.rs"] +mod retained_rows; #[path = "tests/test_scope_facts.rs"] mod scope_facts; #[path = "tests/test_speculative_rows.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index 93a3fee44b..d68e0cc521 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use serde_json::{Value, json}; @@ -8,8 +9,10 @@ use std::sync::atomic::AtomicBool; use crate::engine::main::answer_speculative_rows::answer_speculative_rows; use crate::engine::main::discard_speculative_rows::discard_speculative_rows; +use crate::engine::main::retained_request::retained_request; use crate::models::{ - EvaluateRequest, Fault, RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows, + Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Fault, Model, + Reference, RetainedRows, RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows, }; use crate::engine::_helpers::evaluation::evaluate_json; @@ -531,6 +534,7 @@ pub(crate) fn speculative_rows_request( project_dir.join(SPECULATIVE_MEMO_FILE), ) }), + retain: false, }) } @@ -637,3 +641,97 @@ pub(crate) fn query_facts_faults( }) .collect()) } + +/// The retained request with each row's digest restored, as a delta compile rebuilds it. +pub(crate) fn restored_rows(retained: &RetainedRows) -> Result { + let mut request: EvaluateRequest = + retained_request(retained).ok_or("the retained request does not decode")?; + for (model, digest) in request.models.iter_mut().zip(&retained.digests) { + model.payload_digest = Some(digest.clone()); + model.stripped = true; + } + Ok(request) +} + +/// A model with a value in every field the retained rows keep, and SQL text they drop. +pub(crate) fn every_field_model() -> Model { + let declaration: Declaration = Declaration { + name: "status".to_owned(), + relative_path: "enums/status.sql".to_owned(), + members: vec![DeclarationMember { + name: "open".to_owned(), + value: json!({"code": 1, "ratio": -0.25, "tags": ["a", null, true]}), + }], + value: Some(json!(12345678901234567890_u64)), + value_type: Some("BIGINT".to_owned()), + render_as: None, + }; + Model { + sql_analysis_disabled: true, + name: "orders".to_owned(), + relative_path: "models/orders.sql".to_owned(), + query_sql: "SELECT 1".to_owned(), + authored_sql: "SELECT 1 -- authored".to_owned(), + config: BTreeMap::from([ + ("materialized".to_owned(), json!("table")), + ("tags".to_owned(), json!(["finance", "daily"])), + ]), + authored_config_keys: vec!["materialized".to_owned()], + logical_schema: Some("analytics".to_owned()), + references: vec![Reference { + ref_kind: "source".to_owned(), + ref_name: "raw_orders".to_owned(), + ref_package: Some("shop".to_owned()), + }], + columns: vec![Column { + name: "amount ✓".to_owned(), + data_type: "DOUBLE".to_owned(), + nullable: Some(false), + audit_count: 2, + type_proven: true, + }], + dynamic_columns: vec![DynamicColumnFamily { + name: "amounts".to_owned(), + pivot_column: "region".to_owned(), + value_column: "amount".to_owned(), + aggregate: "sum".to_owned(), + data_type: "DOUBLE".to_owned(), + name_pattern: Some("amount_{region}".to_owned()), + type_proven: false, + }], + dynamic_columns_proven: true, + bare_dynamic_pivot: true, + enum_columns: vec!["status".to_owned()], + enum_declarations: vec![declaration.clone()], + constant_declarations: vec![declaration], + declared_audit_count: 3, + targeting_test_count: 4, + empty_input_only_test_count: 0, + payload_digest: Some("digest".to_owned()), + stripped: false, + } +} + +/// Two models: one with every field set, one with none. +pub(crate) fn coded_models() -> Vec { + vec![every_field_model(), Model::default()] +} + +/// No models. +pub(crate) fn no_models() -> Vec { + Vec::new() +} + +/// `models` as retained rows read back would hold them: SQL text and digests dropped. +pub(crate) fn without_sql(models: Vec) -> Value { + let stripped: Vec = models + .into_iter() + .map(|model| Model { + query_sql: String::new(), + authored_sql: String::new(), + payload_digest: None, + ..model + }) + .collect(); + serde_json::to_value(stripped).unwrap_or_default() +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs new file mode 100644 index 0000000000..eb8f0503f8 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_retained_rows.rs @@ -0,0 +1,137 @@ +use serde_json::Value; +use tempfile::TempDir; + +use crate::engine::_helpers::evaluation::STRIPPED_ROW_MISSED; +use crate::engine::_helpers::retained::{encoded_rows, retained_request}; +use crate::engine::main::evaluate_rows::evaluate_rows; +use crate::engine::tests::helpers::{coded_models, no_models, without_sql}; +use crate::engine::tests::helpers::{ + evaluation_summary, orders_models, restored_rows, speculative_rows_request, +}; +use crate::engine::tests::test_types::{RetainedRowsTestCase, RowCodecTestCase}; +use crate::errors::RowsError; +use crate::models::{EvaluateRequest, Fault, Model, RetainedRows, RowsEvaluation, RowsRequest}; + +#[test] +fn given_retained_rows_when_evaluating_without_sql_then_only_cached_findings_stand_in() +-> Result<(), String> { + let test_cases = [ + RetainedRowsTestCase { + description: "the findings cached by the retaining evaluation stand in", + fresh_cache: false, + expected_same_findings: true, + expected_missed: None, + }, + RetainedRowsTestCase { + description: "without cached findings a stripped row is refused", + fresh_cache: true, + expected_same_findings: false, + expected_missed: Some(STRIPPED_ROW_MISSED), + }, + ]; + for test_case in test_cases { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let mut request: RowsRequest = + speculative_rows_request(project_dir.path(), true, &orders_models())?; + request.memo = None; + request.retain = true; + let first = evaluate_rows(request).map_err(|_| "first evaluation failed".to_owned())?; + let retained: RetainedRows = first.retained.clone().ok_or("nothing was retained")?; + let kept: EvaluateRequest = restored_rows(&retained)?; + let other_dir = TempDir::new().map_err(|error| error.to_string())?; + let mut stripped: EvaluateRequest = restored_rows(&retained)?; + stripped.project_dir = [project_dir.path(), other_dir.path()] + [usize::from(test_case.fresh_cache)] + .to_string_lossy() + .into_owned(); + + let evaluated: Result = evaluate_rows(RowsRequest { + request: stripped, + memo: None, + retain: false, + }); + + assert_eq!( + retained.digests.len(), + kept.models.len(), + "{}", + test_case.description + ); + assert!( + kept.models + .iter() + .all(|model| model.query_sql.is_empty() && model.authored_sql.is_empty()), + "{}", + test_case.description + ); + let findings: Option> = evaluated + .as_ref() + .ok() + .map(|evaluation| evaluation_summary(evaluation).0); + assert_eq!( + findings == Some(evaluation_summary(&first).0), + test_case.expected_same_findings, + "{}", + test_case.description + ); + let missed: Option = evaluated.as_ref().err().map(|error| format!("{error:?}")); + assert_eq!( + missed, + test_case + .expected_missed + .map(|message| format!("{:?}", RowsError::Rules(message.to_owned()))), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_model_rows_when_encoded_and_decoded_then_every_fact_but_the_sql_survives() { + let test_cases = [ + RowCodecTestCase { + description: "a model with every field set and a default model", + models: coded_models, + expected_truncations_read: 0, + }, + RowCodecTestCase { + description: "no models", + models: no_models, + expected_truncations_read: 0, + }, + ]; + for test_case in test_cases { + let models: Vec = (test_case.models)(); + let retained: RetainedRows = RetainedRows { + request_json: serde_json::to_vec(&EvaluateRequest::default()).unwrap_or_default(), + rows: encoded_rows(&models), + digests: vec![String::new(); models.len()], + }; + + let decoded: Option = retained_request(&retained) + .ok() + .and_then(|request| serde_json::to_value(request.models).ok()); + let truncations_read: usize = (0..retained.rows.len()) + .filter(|length| { + retained_request(&RetainedRows { + rows: retained.rows[..*length].to_vec(), + ..retained.clone() + }) + .is_ok() + }) + .count(); + + assert_eq!( + decoded, + Some(without_sql(models)), + "{}", + test_case.description + ); + assert_eq!( + truncations_read, test_case.expected_truncations_read, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index a2501f8c01..c22ae94525 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -282,3 +282,19 @@ pub(crate) struct QueryFactsTestCase { pub(crate) columns: Value, pub(crate) expected_faults: &'static [(&'static str, u64, u64, &'static str)], } + +/// A retained request evaluated again without SQL, with or without the cached findings. +pub(crate) struct RetainedRowsTestCase { + pub(crate) description: &'static str, + pub(crate) fresh_cache: bool, + pub(crate) expected_same_findings: bool, + pub(crate) expected_missed: Option<&'static str>, +} + +/// Model rows written in the native record format and read back. +pub(crate) struct RowCodecTestCase { + pub(crate) description: &'static str, + pub(crate) models: fn() -> Vec, + /// How many truncations of the rows still read back. + pub(crate) expected_truncations_read: usize, +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index c98860e8ff..744677e574 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -571,6 +571,9 @@ pub struct Model { pub empty_input_only_test_count: u32, #[serde(skip)] pub payload_digest: Option, + /// Whether this row's SQL text was dropped, so only a cached result may stand for it. + #[serde(skip)] + pub stripped: bool, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -926,6 +929,18 @@ pub struct RowsRequest { pub request: EvaluateRequest, /// The compiled-code identity and memo path, when the rules cache is enabled. pub memo: Option<(String, std::path::PathBuf)>, + /// Keep the request, its models without SQL text, for the project image. + pub retain: bool, +} + +/// A request kept for the project image: its model-free JSON and its rows without SQL text. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RetainedRows { + pub request_json: Vec, + /// Each model's row without its SQL text, in the native record format, in model order. + pub rows: Vec, + /// Each model's payload digest, which keys its cached findings, in model order. + pub digests: Vec, } /// A built-in rules evaluation running speculatively on its own thread; dropping it cancels it. @@ -963,4 +978,6 @@ pub struct RowsEvaluation { pub built_in_ms: u64, #[serde(skip)] pub reused: bool, + #[serde(skip)] + pub retained: Option, } diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs index 678022978a..d9673c87f1 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs @@ -10,9 +10,14 @@ use crate::rules::_helpers::evaluation::path_fault; use crate::rules::models::ProjectEvaluationRequest; use polyglot_sql::Expression; use polyglot_sql::expressions::{Literal, Select}; +use serde::{Deserialize, Serialize}; use sqlbuild_analysis::compiler::main::sql_test_fixture_facts::fixture_facts; +use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; use std::path::Path; +use std::rc::Rc; + +use crate::constants::NATIVE_BUILD_IDENTITY; const UNIT_ROOT: &str = "tests/unit"; const SCENARIO_ROOT: &str = "tests/scenarios"; @@ -23,6 +28,11 @@ use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_PATH; pub(crate) fn evaluate_project( evaluation: ProjectEvaluationRequest<'_>, ) -> Result, String> { + let resources: &[ScopeResource] = &evaluation.request.scope_index.resources; + with_scope_lookup(resources, || evaluated_project(evaluation)) +} + +fn evaluated_project(evaluation: ProjectEvaluationRequest<'_>) -> Result, String> { let mut faults: Vec = Vec::new(); let mut tests: Vec<&SqlTestFact> = evaluation.request.sql_tests.iter().collect(); tests.sort_by(|left, right| { @@ -56,7 +66,13 @@ pub(crate) fn evaluate_project( continue; } if let Err(reason) = fixture_eligibility.entry(&cte.sql).or_insert_with(|| { - fixture_query(&cte.sql, &evaluation.request.dialect).map(|_| ()) + match fixture_summary(&cte.sql, &evaluation.request.dialect) + .error + .clone() + { + Some(error) => Err(error), + None => Ok(()), + } }) { let mut fault = path_fault( rule, @@ -381,17 +397,57 @@ fn allowed_subjects(evaluation: &ProjectEvaluationRequest<'_>, test: &SqlTestFac Vec::new() } +/// Each `(is a model, name)`'s first scope resource index, for the resources it was built from. +type ScopeLookup = (usize, HashMap<(bool, String), usize>); + +thread_local! { + static SCOPE_LOOKUP: RefCell> = const { RefCell::new(None) }; +} + +/// Run `body` with the scope resources indexed by kind and name on this thread. +fn with_scope_lookup(resources: &[ScopeResource], body: impl FnOnce() -> T) -> T { + let mut index: HashMap<(bool, String), usize> = HashMap::new(); + for (position, item) in resources.iter().enumerate() { + let key: Option = match item.kind { + ResourceKind::Model => Some(true), + ResourceKind::Function => Some(false), + _ => None, + }; + if let Some(model) = key { + index.entry((model, item.name.clone())).or_insert(position); + } + } + let outer = SCOPE_LOOKUP.with(|slot| slot.borrow_mut().replace((resources.len(), index))); + let result: T = body(); + SCOPE_LOOKUP.with(|slot| *slot.borrow_mut() = outer); + result +} + fn scope_resource<'a>( evaluation: &'a ProjectEvaluationRequest<'_>, kind: ResourceKind, name: &str, ) -> Option<&'a ScopeResource> { - evaluation - .request - .scope_index - .resources - .iter() - .find(|item| same_resource_kind(&item.kind, &kind) && item.name == name) + let resources: &'a [ScopeResource] = &evaluation.request.scope_index.resources; + let model: Option = match kind { + ResourceKind::Model => Some(true), + ResourceKind::Function => Some(false), + _ => None, + }; + let indexed: Option> = SCOPE_LOOKUP.with(|slot| { + let slot = slot.borrow(); + let (count, index) = slot.as_ref()?; + if *count != resources.len() { + return None; + } + Some(model.and_then(|model| index.get(&(model, name.to_owned())).copied())) + }); + match indexed { + Some(position) => position.map(|position| &resources[position]), + None => resources + .iter() + .find(|item| same_resource_kind(&item.kind, &kind) && item.name == name), + } } fn same_resource_kind(left: &ResourceKind, right: &ResourceKind) -> bool { @@ -690,21 +746,156 @@ fn empty_relation(cte: &SqlTestCteFact, dialect: &str) -> bool { if fixture_facts(&cte.name, &cte.sql).empty_fixture_marker { return true; } - let Ok(fixture) = fixture_query(&cte.sql, dialect) else { - return false; - }; + fixture_summary(&cte.sql, dialect).empty +} + +/// Whether a parsed fixture selects no rows. +fn selects_no_rows(fixture: &ParsedFixture) -> bool { let Expression::Select(select) = &fixture.statement else { return false; }; select.with.is_none() && (limited_to_zero_rows(select) - || (filtered_to_zero_rows(select, &fixture) && !fixture.order_by_calls_function())) + || (filtered_to_zero_rows(select, fixture) && !fixture.order_by_calls_function())) +} + +/// What the SQL-test rules read of one parsed fixture. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct FixtureSummary { + /// Why the fixture does not parse, when it does not. + error: Option, + /// Whether it selects no rows. + empty: bool, + /// The model a bare row-existence assertion reads, when the fixture is one. + row_existence_target: Option, +} + +/// Fixture summaries by digest of `(dialect, SQL)`, shared by the rules of one evaluation. +#[derive(Default)] +struct FixtureMemo { + summaries: HashMap>, + /// The digests this evaluation read, which are the ones saved again. + used: std::collections::HashSet, + /// Whether any summary was computed rather than loaded. + computed: bool, +} + +thread_local! { + static FIXTURE_MEMO: RefCell> = const { RefCell::new(None) }; +} + +/// The fixture summaries stored at `path`; missing or unreadable ones are summarised again. +fn stored_summaries(path: &Path) -> Option { + let bytes: Vec = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(_unreadable) => return None, + }; + let Ok(stored) = serde_json::from_slice::(&bytes) else { + return None; + }; + Some(stored) +} + +/// The stored fixture summaries, kept only for one native build. +#[derive(Default, Serialize, Deserialize)] +struct StoredSummaries { + build: String, + summaries: BTreeMap, +} + +/// Run `body` summarising each fixture once, from and, when `body` says so, back to `store`. +pub(crate) fn with_fixture_memo(store: Option<&Path>, body: impl FnOnce() -> (T, bool)) -> T { + let loaded: BTreeMap = store + .and_then(stored_summaries) + .filter(|stored| stored.build == NATIVE_BUILD_IDENTITY) + .map(|stored| stored.summaries) + .unwrap_or_default(); + let loaded_count: usize = loaded.len(); + let memo: FixtureMemo = FixtureMemo { + summaries: loaded + .into_iter() + .map(|(key, summary)| (key, Rc::new(summary))) + .collect(), + used: std::collections::HashSet::new(), + computed: false, + }; + let outer: Option = FIXTURE_MEMO.with(|slot| slot.borrow_mut().replace(memo)); + let (result, write): (T, bool) = body(); + let memo: Option = FIXTURE_MEMO.with(|slot| { + let mut slot = slot.borrow_mut(); + let memo = slot.take(); + *slot = outer; + memo + }); + if let (Some(path), Some(memo), true) = (store, memo, write) + && !memo.used.is_empty() + && (memo.computed || memo.used.len() != loaded_count) + { + let FixtureMemo { + summaries, used, .. + } = memo; + let stored: StoredSummaries = StoredSummaries { + build: NATIVE_BUILD_IDENTITY.to_owned(), + summaries: summaries + .into_iter() + .filter(|(key, _)| used.contains(key)) + .map(|(key, summary)| (key, summary.as_ref().clone())) + .collect(), + }; + let _ = write_summaries(path, &stored); + } + result +} + +fn write_summaries(path: &Path, stored: &StoredSummaries) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary = path.with_extension(format!("tmp-{}", std::process::id())); + std::fs::write(&temporary, serde_json::to_vec(stored)?)?; + std::fs::rename(&temporary, path) } -fn fixture_query(sql: &str, dialect: &str) -> Result { - model_shape(sql, dialect) +fn summarised(sql: &str, dialect: &str) -> FixtureSummary { + let parsed: Result = model_shape(sql, dialect) .and_then(|shape| parse_fixture(shape, dialect)) - .map_err(|error| format!("could not parse SQL-test fixture: {error}")) + .map_err(|error| format!("could not parse SQL-test fixture: {error}")); + match parsed { + Err(error) => FixtureSummary { + error: Some(error), + empty: false, + row_existence_target: None, + }, + Ok(fixture) => FixtureSummary { + error: None, + empty: selects_no_rows(&fixture), + row_existence_target: row_existence_target(&fixture), + }, + } +} + +fn fixture_summary(sql: &str, dialect: &str) -> Rc { + FIXTURE_MEMO.with(|memo| { + let mut memo = memo.borrow_mut(); + let Some(memo) = memo.as_mut() else { + return Rc::new(summarised(sql, dialect)); + }; + let mut hasher: blake3::Hasher = blake3::Hasher::new(); + hasher.update(dialect.as_bytes()); + hasher.update(b"\0"); + hasher.update(sql.as_bytes()); + let key: String = hasher.finalize().to_hex().to_string(); + if let Some(summary) = memo.summaries.get(&key) { + let summary: Rc = Rc::clone(summary); + memo.used.insert(key); + return summary; + } + let summary: Rc = Rc::new(summarised(sql, dialect)); + memo.summaries.insert(key.clone(), Rc::clone(&summary)); + memo.used.insert(key); + memo.computed = true; + summary + }) } fn limited_to_zero_rows(select: &Select) -> bool { @@ -755,18 +946,23 @@ fn numeric_literal(expression: &Expression) -> Option { } fn bare_row_existence(sql: &str, targets: &[String], dialect: &str) -> bool { - let Ok(fixture) = fixture_query(sql, dialect) else { - return false; - }; + fixture_summary(sql, dialect) + .row_existence_target + .as_ref() + .is_some_and(|target| targets.contains(target)) +} + +/// The model a fixture reads when it is a bare existence check of one model's rows. +fn row_existence_target(fixture: &ParsedFixture) -> Option { let Expression::Select(select) = &fixture.statement else { - return false; + return None; }; let [source] = select .from .as_ref() .map_or(&[][..], |from| from.expressions.as_slice()) else { - return false; + return None; }; let unfiltered = select.where_clause.is_none() && select.prewhere.is_none() @@ -775,16 +971,19 @@ fn bare_row_existence(sql: &str, targets: &[String], dialect: &str) -> bool { && select.connect.is_none() && select.lateral_views.is_empty() && select.group_by.is_none(); - select.with.is_none() + let bare: bool = select.with.is_none() && unfiltered && select.joins.is_empty() - && reference_target(source, &fixture).is_some_and(|target| targets.contains(&target)) && !fixture.projection_calls_function() && !fixture.order_by_calls_function() && select.limit.is_none() && select.offset.is_none() && select.fetch.is_none() - && fixture.query_count() == 1 + && fixture.query_count() == 1; + if !bare { + return None; + } + reference_target(source, fixture) } fn reference_target(source: &Expression, fixture: &ParsedFixture) -> Option { diff --git a/scripts/cold_compile_performance/_helpers/dense_project.py b/scripts/cold_compile_performance/_helpers/dense_project.py index 0d7ef833c5..28e8fe8d76 100644 --- a/scripts/cold_compile_performance/_helpers/dense_project.py +++ b/scripts/cold_compile_performance/_helpers/dense_project.py @@ -26,6 +26,27 @@ def dense_model_name(index: int) -> str: return f"sales{index // 1000:03d}__int_clean__orders_{index:05d}" +def dense_model_path(*, project_dir: Path, index: int) -> Path: + """The file of the generated dense model at `index`.""" + + return ( + project_dir + / "models" + / f"sales{index // 1000:03d}" + / "intermediate" + / "clean" + / f"batch{index // 5:05d}" + / f"{dense_model_name(index)}.sql" + ) + + +def dense_test_path(*, project_dir: Path, index: int) -> Path: + """The SQL test file of the generated dense model at `index`.""" + + name: str = dense_model_name(index).replace("__", "_") + return next((project_dir / "tests").rglob(f"test_{name}__*.sql")) + + def dense_column_count(index: int) -> int: rank: int = (index * 17) % 1000 for upper, width in ( diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 1c64a16985..32b4209acc 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -17,6 +17,7 @@ from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit from scripts.compile_performance_ratio.constants import ( + ANALYSED_MODEL_COUNTERS, ANALYSIS_CACHE_BYPASSES, ANALYSIS_CACHE_MISSES, BASE_GENERATOR_ENTRY, @@ -166,7 +167,7 @@ def compare_builds( mode=mode, compile_args=compile_args, ) - check_cache_use(run=run, mode=mode) + check_cache_use(run=run, mode=mode, models=models) results[label].append(run) if mode != COLD_MODE: _, head_python, head_dir = builds[1] @@ -213,13 +214,21 @@ def _comparison( ) -def check_cache_use(*, run: CompileRun, mode: str) -> None: - """Fail a bypassing run, a warm run with misses, or an edit run that analysed nothing.""" +def check_cache_use(*, run: CompileRun, mode: str, models: int) -> None: + """Fail a bypassing full compile, a warm run with misses, or an edit that analysed nothing.""" misses: int | None = run.analysis_cache_misses if misses is None or mode == COLD_MODE: return bypasses: int = run.analysis_cache_bypasses or 0 + delta: bool = run.analysed_models is not None and run.analysed_models < models + if mode == EDIT_MODE and delta: + if misses + bypasses == 0: + raise CompileComparisonError( + f"{run.label} one-model edit delta compile analysed no model afresh, so the " + "edit did not invalidate the edited model" + ) + return if bypasses != 0: raise CompileComparisonError( f"{run.label} {mode} compile bypassed the analysis cache for {bypasses} models, so " @@ -328,6 +337,11 @@ def _compile_once( timings_ms={name: value for name, value in numeric.items() if name in REPORTED_PHASES}, analysis_cache_misses=numeric.get(ANALYSIS_CACHE_MISSES), analysis_cache_bypasses=numeric.get(ANALYSIS_CACHE_BYPASSES), + analysed_models=( + sum(numeric[name] for name in ANALYSED_MODEL_COUNTERS) + if all(name in numeric for name in ANALYSED_MODEL_COUNTERS) + else None + ), report=json.dumps(payload, sort_keys=True), ) diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index 95054308b2..eb7a87d241 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -20,6 +20,12 @@ } ANALYSIS_CACHE_MISSES: str = "analysis_cache_misses" ANALYSIS_CACHE_BYPASSES: str = "analysis_cache_bypasses" +ANALYSED_MODEL_COUNTERS: tuple[str, ...] = ( + "analysis_batch_cache_hits", + "analysis_entry_cache_hits", + ANALYSIS_CACHE_MISSES, + ANALYSIS_CACHE_BYPASSES, +) MODELS_DIRECTORY: str = "models" MODEL_FILE_PATTERN: str = "*.sql" EDIT_COMMENT: str = "-- Benchmark edit {revision}.\n" diff --git a/scripts/compile_performance_ratio/models.py b/scripts/compile_performance_ratio/models.py index 88f83587ba..bfd5b230c4 100644 --- a/scripts/compile_performance_ratio/models.py +++ b/scripts/compile_performance_ratio/models.py @@ -13,6 +13,8 @@ class CompileRun: timings_ms: dict[str, int] analysis_cache_misses: int | None = None analysis_cache_bypasses: int | None = None + analysed_models: int | None = None + """How many models the compile analysed; a delta compile analyses only the models it reran.""" report: str = "" """The JSON report without its timings, which an uncached compile must reproduce.""" diff --git a/scripts/delta_compile_series/__init__.py b/scripts/delta_compile_series/__init__.py new file mode 100644 index 0000000000..2dd71392bd --- /dev/null +++ b/scripts/delta_compile_series/__init__.py @@ -0,0 +1 @@ +"""Seeded edit series that check delta compiles against full compiles.""" diff --git a/scripts/delta_compile_series/_helpers/__init__.py b/scripts/delta_compile_series/_helpers/__init__.py new file mode 100644 index 0000000000..0e330898e1 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/__init__.py @@ -0,0 +1 @@ +"""Edit and outcome helpers for delta compile series.""" diff --git a/scripts/delta_compile_series/_helpers/edits.py b/scripts/delta_compile_series/_helpers/edits.py new file mode 100644 index 0000000000..2f941de525 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/edits.py @@ -0,0 +1,55 @@ +"""Apply and revert the edits of a delta compile series.""" + +from __future__ import annotations + +import random +import re +from pathlib import Path + +from scripts.delta_compile_series.constants import ( + EDIT_ATTEMPTS, + PRIVATE_PATH_MARKER, + PROJECT_CONFIG, + RANDOM_VALUE_LIMIT, + SQL_GLOB, +) +from scripts.delta_compile_series.exceptions import DeltaCompileSeriesError +from scripts.delta_compile_series.models import DeltaCompileEdit +from scripts.delta_compile_series.types import EditTarget + + +def project_files(*, project_dir: Path, edit: DeltaCompileEdit) -> list[Path]: + """The files `edit` may change.""" + + if edit.target is EditTarget.PROJECT_CONFIG: + return [project_dir / PROJECT_CONFIG] + root: Path = project_dir / edit.target.value + return sorted( + path for path in root.rglob(SQL_GLOB) if PRIVATE_PATH_MARKER not in path.as_posix() + ) + + +def applied(*, path: Path, edit: DeltaCompileEdit, rng: random.Random) -> bool: + """Apply `edit` to `path`; return whether it changed the file.""" + + text: str = path.read_text(encoding="utf-8") + replacement: str = edit.replacement.replace("{value}", str(rng.randint(1, RANDOM_VALUE_LIMIT))) + edited, count = re.subn(edit.pattern, replacement, text, count=1) + if count == 0 or edited == text: + return False + path.write_text(edited, encoding="utf-8") + return True + + +def edited_file( + *, project_dir: Path, edit: DeltaCompileEdit, rng: random.Random +) -> tuple[Path, str]: + """Apply `edit` to a randomly chosen file it changes; return the file and its old text.""" + + candidates: list[Path] = project_files(project_dir=project_dir, edit=edit) + for _ in range(EDIT_ATTEMPTS): + path: Path = rng.choice(candidates) + original: str = path.read_text(encoding="utf-8") + if applied(path=path, edit=edit, rng=rng): + return path, original + raise DeltaCompileSeriesError(f"no file takes the {edit.kind} edit") diff --git a/scripts/delta_compile_series/_helpers/outcomes.py b/scripts/delta_compile_series/_helpers/outcomes.py new file mode 100644 index 0000000000..7f69434af1 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/outcomes.py @@ -0,0 +1,95 @@ +"""Judge what one compile of a delta compile series did.""" + +from __future__ import annotations + +import re +from itertools import zip_longest +from pathlib import Path + +from scripts.delta_compile_series.constants import ( + COMPILE_REUSE_HIT_LINE, + COMPILE_TIMINGS_PATTERN, + DELTA_COMPILE_LINE, + DELTA_FALLBACK_PATTERN, + DELTA_MISMATCH_PATTERN, + DELTA_VERIFIED_LINE, + DURATION_PATTERN, + MISMATCH_EXIT_CODE, + STALE_PROBLEMS, +) +from scripts.delta_compile_series.models import DeltaCompileEdit, DeltaCompileOutcome + +_STREAMS: tuple[str, ...] = ("exit code", "stdout", "stderr") + + +def fallback_reason(*, outcome: DeltaCompileOutcome) -> str | None: + """Why the compile fell back from a delta, when it says.""" + + reasons: list[str] = re.findall(DELTA_FALLBACK_PATTERN, outcome.stderr, flags=re.MULTILINE) + return reasons[0] if reasons else None + + +def judged( + *, + outcome: DeltaCompileOutcome, + edit: DeltaCompileEdit, + reverted: bool, + others_edited: bool, + after_failure: bool = False, +) -> tuple[bool, str | None]: + """Whether the compile was a verified delta, and what was unexpected about it, if anything.""" + + mismatches: list[str] = re.findall(DELTA_MISMATCH_PATTERN, outcome.stderr, flags=re.MULTILINE) + if outcome.returncode == MISMATCH_EXIT_CODE or mismatches: + return False, f"mismatch: {mismatches}" + if COMPILE_REUSE_HIT_LINE in outcome.stderr: + return False, None if reverted else "replayed a compile after an edit" + reason: str | None = fallback_reason(outcome=outcome) + if others_edited and reason is not None and STALE_PROBLEMS in reason: + return False, None + if edit.fallback_reason is None: + verified: bool = DELTA_VERIFIED_LINE in outcome.stderr + return verified, None if verified else f"fell back: {reason}" + if reverted and after_failure and reason is not None: + return False, None + if reason is None or edit.fallback_reason not in reason: + return False, f"expected a fallback for {edit.fallback_reason!r}, got {reason!r}" + return False, None + + +def _seen(*, outcome: DeltaCompileOutcome, project_dir: Path, other_dir: Path) -> tuple[str, ...]: + """What a user sees of a compile, without durations, timings and the delta's own line.""" + + def shown(text: str) -> str: + text = COMPILE_TIMINGS_PATTERN.sub("", text.replace(str(other_dir), str(project_dir))) + kept: list[str] = [line for line in text.splitlines() if DELTA_COMPILE_LINE not in line] + return DURATION_PATTERN.sub("", "\n".join(kept)) + + return (str(outcome.returncode), shown(outcome.stdout), shown(outcome.stderr)) + + +def full_compile_difference( + *, + outcome: DeltaCompileOutcome, + full: DeltaCompileOutcome, + project_dir: Path, + full_project_dir: Path, +) -> str | None: + """The first line a user sees differently than after a full compile of a copy, if any.""" + + if COMPILE_REUSE_HIT_LINE in outcome.stderr: + return None + + seen: tuple[str, ...] = _seen(outcome=outcome, project_dir=project_dir, other_dir=project_dir) + expected: tuple[str, ...] = _seen( + outcome=full, project_dir=project_dir, other_dir=full_project_dir + ) + for stream, left, right in zip(_STREAMS, seen, expected, strict=True): + if left != right: + first: tuple[str, str] = next( + pair + for pair in zip_longest(left.splitlines(), right.splitlines(), fillvalue="") + if pair[0] != pair[1] + ) + return f"{stream} differs from a full compile: {first[0]!r} vs {first[1]!r}" + return None diff --git a/scripts/delta_compile_series/_helpers/sequence.py b/scripts/delta_compile_series/_helpers/sequence.py new file mode 100644 index 0000000000..c89d857ef9 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/sequence.py @@ -0,0 +1,52 @@ +"""Apply the named edits of a fixed delta compile sequence.""" + +from __future__ import annotations + +import random +from pathlib import Path + +from scripts.delta_compile_series._helpers.edits import applied, project_files +from scripts.delta_compile_series.constants import ( + BODY_EDIT, + BODY_MODEL_INDEX, + BODY_SEED, + DOUBLE_CAST, + DOUBLE_COLUMN, + FLOAT_CAST, + FLOAT_COLUMN, +) +from scripts.delta_compile_series.exceptions import DeltaCompileSeriesError +from scripts.delta_compile_series.types import SequenceEdit + + +def reshaped(*, project_dir: Path) -> Path: + """Change the first model whose first quantity is a DOUBLE to output a FLOAT instead.""" + + models: list[Path] = project_files(project_dir=project_dir, edit=BODY_EDIT) + path: Path = next(path for path in models if DOUBLE_COLUMN in path.read_text("utf-8")) + path.write_text( + path.read_text(encoding="utf-8") + .replace(DOUBLE_COLUMN, FLOAT_COLUMN, 1) + .replace(DOUBLE_CAST, FLOAT_CAST), + encoding="utf-8", + ) + return path + + +def edited_body(*, project_dir: Path) -> Path: + """Change a constant in one model's query.""" + + model: Path = project_files(project_dir=project_dir, edit=BODY_EDIT)[BODY_MODEL_INDEX] + if not applied(path=model, edit=BODY_EDIT, rng=random.Random(BODY_SEED)): + raise DeltaCompileSeriesError(f"the body edit does not apply to {model.name}") + return model + + +def sequence_edited(*, project_dir: Path, edit: SequenceEdit) -> Path | None: + """Apply `edit`; return the file it changed, if any.""" + + if edit is SequenceEdit.BODY: + return edited_body(project_dir=project_dir) + if edit is SequenceEdit.SHAPE: + return reshaped(project_dir=project_dir) + return None diff --git a/scripts/delta_compile_series/constants.py b/scripts/delta_compile_series/constants.py new file mode 100644 index 0000000000..d82dbe34cc --- /dev/null +++ b/scripts/delta_compile_series/constants.py @@ -0,0 +1,89 @@ +"""Edits and markers of delta compile series.""" + +from __future__ import annotations + +import re + +from scripts.delta_compile_series.models import DeltaCompileEdit +from scripts.delta_compile_series.types import EditTarget + +DELTA_COMPILE_LINE: str = "against the previous compile" +DELTA_VERIFIED_LINE: str = "delta compile verified" +DELTA_FALLBACK_PATTERN: str = r"^delta compile fell back: (.*)$" +DELTA_MISMATCH_PATTERN: str = r"^delta compile mismatch: .*$" +COMPILE_REUSE_HIT_LINE: str = "Inputs unchanged; reused the previous compile" +MISMATCH_EXIT_CODE: int = 70 +REVERT_PROBABILITY: float = 0.5 +EDIT_ATTEMPTS: int = 50 +DELTA_WEIGHT: int = 3 +FALLBACK_WEIGHT: int = 1 +STALE_PROBLEMS: str = "reports problems in files the edit leaves alone" +PROJECT_CONFIG: str = "sqlbuild_project.toml" +RANDOM_VALUE_LIMIT: int = 10**6 +FULL_COPY_SUFFIX: str = "_full_copy" +DURATION_PATTERN: re.Pattern[str] = re.compile(r"\s*\(\d+(?:\.\d+)?s\)") +COMPILE_TIMINGS_PATTERN: re.Pattern[str] = re.compile(r'\n "compile_timings": \{[^{}]*\n \}') + +EDITS: tuple[DeltaCompileEdit, ...] = ( + DeltaCompileEdit("body_constant", r"COALESCE\(b\.amount, \d+\)", "COALESCE(b.amount, {value})"), + DeltaCompileEdit("body_comment", r"\Z", "\n-- reviewed {value}\n"), + DeltaCompileEdit( + "description", r'MODEL \(description "[^"]*"', 'MODEL (description "Edited {value}."' + ), + DeltaCompileEdit( + "column_description", + r'description "Order quantity measure"', + 'description "Edited measure {value}"', + ), + DeltaCompileEdit("type_error", r"quantity_000 \(type DOUBLE", "quantity_000 (type VARCHAR"), + DeltaCompileEdit("parse_error", r"projected AS \(SELECT", "projected AS (SELEC"), + DeltaCompileEdit( + "test_expectation", + r"(__expected__\w+ AS \(SELECT CAST\()\d+", + r"\g<1>{value}", + target=EditTarget.TESTS, + ), + DeltaCompileEdit( + "analysis_off", + r"MODEL \(description", + "MODEL (sql_analysis false, description", + fallback_reason="changed its sql_analysis header key", + ), + DeltaCompileEdit( + "test_retarget", + r"(__expected__\w+_\d{4})[1-9]( AS)", + r"\g<1>0\g<2>", + target=EditTarget.TESTS, + fallback_reason="now covers other models", + ), + DeltaCompileEdit( + "materialization", + r"materialized table", + "materialized view", + fallback_reason="custom rules select models by their materialization", + ), + DeltaCompileEdit("contract", r"contract enforced, ", "", fallback_reason="a custom rule reads"), + DeltaCompileEdit( + "unknown_reference", + r'__ref\("([^"]+)"\)', + r'__ref("\1_missing")', + fallback_reason="changed what it references", + ), + DeltaCompileEdit( + "project_config", + r"\Z", + "\n# reviewed {value}\n", + target=EditTarget.PROJECT_CONFIG, + fallback_reason="sqlbuild_project.toml is not a model or SQL test file", + ), +) +BODY_EDIT: DeltaCompileEdit = EDITS[0] +PRIVATE_PATH_MARKER: str = "/_" +SQL_GLOB: str = "*.sql" +FAILURE_TAIL_CHARACTERS: int = 2000 +BODY_MODEL_INDEX: int = 11 +BODY_SEED: int = 3 +DOUBLE_COLUMN: str = "quantity_000 (type DOUBLE" +FLOAT_COLUMN: str = "quantity_000 (type FLOAT" +DOUBLE_CAST: str = "AS DOUBLE) AS quantity_000" +FLOAT_CAST: str = "AS FLOAT) AS quantity_000" diff --git a/scripts/delta_compile_series/exceptions.py b/scripts/delta_compile_series/exceptions.py new file mode 100644 index 0000000000..219f801bd0 --- /dev/null +++ b/scripts/delta_compile_series/exceptions.py @@ -0,0 +1,5 @@ +"""Delta compile series errors.""" + + +class DeltaCompileSeriesError(RuntimeError): + """Raised when a series cannot apply an edit or its first compile fails.""" diff --git a/scripts/delta_compile_series/main/__init__.py b/scripts/delta_compile_series/main/__init__.py new file mode 100644 index 0000000000..0be72e24e5 --- /dev/null +++ b/scripts/delta_compile_series/main/__init__.py @@ -0,0 +1 @@ +"""Entry points for delta compile series.""" diff --git a/scripts/delta_compile_series/main/run_delta_compile_sequence.py b/scripts/delta_compile_series/main/run_delta_compile_sequence.py new file mode 100644 index 0000000000..4f32ae1559 --- /dev/null +++ b/scripts/delta_compile_series/main/run_delta_compile_sequence.py @@ -0,0 +1,41 @@ +"""Compile after each of a fixed sequence of named edits, then once in full.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +from scripts.delta_compile_series._helpers.sequence import sequence_edited +from scripts.delta_compile_series.constants import ( + COMPILE_REUSE_HIT_LINE, + DELTA_COMPILE_LINE, + FAILURE_TAIL_CHARACTERS, +) +from scripts.delta_compile_series.exceptions import DeltaCompileSeriesError +from scripts.delta_compile_series.models import DeltaCompileOutcome, DeltaCompileSequence +from scripts.delta_compile_series.types import SequenceEdit + + +def run_delta_compile_sequence( + *, + project_dir: Path, + edits: tuple[SequenceEdit, ...], + compile_project: Callable[[], DeltaCompileOutcome], + compile_in_full: Callable[[], DeltaCompileOutcome], +) -> DeltaCompileSequence: + """Compile once, then after each edit, then in full.""" + + first: DeltaCompileOutcome = compile_project() + if first.returncode != 0: + raise DeltaCompileSeriesError(first.stderr[-FAILURE_TAIL_CHARACTERS:]) + outcomes: list[DeltaCompileOutcome] = [] + for edit in edits: + _ = sequence_edited(project_dir=project_dir, edit=edit) + outcomes.append(compile_project()) + full: DeltaCompileOutcome = compile_in_full() + return DeltaCompileSequence( + delta_lines=tuple(DELTA_COMPILE_LINE in outcome.stderr for outcome in outcomes), + replayed_last=COMPILE_REUSE_HIT_LINE in outcomes[-1].stderr, + last_report=outcomes[-1].stdout, + full_report=full.stdout, + ) diff --git a/scripts/delta_compile_series/main/run_delta_compile_series.py b/scripts/delta_compile_series/main/run_delta_compile_series.py new file mode 100644 index 0000000000..ac38d9beba --- /dev/null +++ b/scripts/delta_compile_series/main/run_delta_compile_series.py @@ -0,0 +1,96 @@ +"""Run a seeded series of edits, compiling and judging after each one.""" + +from __future__ import annotations + +import random +import shutil +from collections.abc import Callable +from pathlib import Path + +from scripts.delta_compile_series._helpers.edits import edited_file +from scripts.delta_compile_series._helpers.outcomes import full_compile_difference, judged +from scripts.delta_compile_series.constants import ( + DELTA_COMPILE_LINE, + DELTA_WEIGHT, + EDITS, + FALLBACK_WEIGHT, + FULL_COPY_SUFFIX, + REVERT_PROBABILITY, +) +from scripts.delta_compile_series.models import ( + DeltaCompileEdit, + DeltaCompileOutcome, + DeltaCompileSeries, +) + + +def _compared( + *, + project_dir: Path, + compile_project: Callable[[], DeltaCompileOutcome], + compile_in_full: Callable[[Path], DeltaCompileOutcome], +) -> tuple[DeltaCompileOutcome, str | None]: + """Compile, and compile a copy of the project as it was in full; how what they print differs.""" + + copy: Path = project_dir.with_name(project_dir.name + FULL_COPY_SUFFIX) + shutil.rmtree(copy, ignore_errors=True) + _ = shutil.copytree(project_dir, copy, symlinks=True) + outcome: DeltaCompileOutcome = compile_project() + problem: str | None = full_compile_difference( + outcome=outcome, full=compile_in_full(copy), project_dir=project_dir, full_project_dir=copy + ) + return outcome, problem + + +def run_delta_compile_series( + *, + project_dir: Path, + seed: int, + steps: int, + compile_project: Callable[[], DeltaCompileOutcome], + compile_in_full: Callable[[Path], DeltaCompileOutcome] | None = None, +) -> DeltaCompileSeries: + """Edit or revert `steps` times, compiling after each, and judge or compare every compile.""" + + rng: random.Random = random.Random(seed) + weights: list[int] = [ + FALLBACK_WEIGHT if edit.fallback_reason else DELTA_WEIGHT for edit in EDITS + ] + pending: list[tuple[Path, str, DeltaCompileEdit]] = [] + described: list[str] = [] + unexpected: list[str] = [] + deltas: int = 0 + failing: bool = compile_project().returncode != 0 + for index in range(steps): + reverted: bool = bool(pending) and (failing or rng.random() < REVERT_PROBABILITY) + if reverted: + path, original, edit = pending.pop() + path.write_text(original, encoding="utf-8") + else: + edit: DeltaCompileEdit = rng.choices(EDITS, weights=weights)[0] + path, original = edited_file(project_dir=project_dir, edit=edit, rng=rng) + pending.append((path, original, edit)) + step: str = f"step {index}: {'revert ' if reverted else ''}{edit.kind} in {path.name}" + described.append(step) + if compile_in_full is None: + outcome: DeltaCompileOutcome = compile_project() + others_edited: bool = any(other != path for other, _, _ in pending) + delta, problem = judged( + outcome=outcome, + edit=edit, + reverted=reverted, + others_edited=others_edited, + after_failure=failing, + ) + else: + outcome, problem = _compared( + project_dir=project_dir, + compile_project=compile_project, + compile_in_full=compile_in_full, + ) + delta: bool = DELTA_COMPILE_LINE in outcome.stderr + deltas += int(delta) + if problem is not None: + unexpected.append(f"{step}: {problem}") + failing = outcome.returncode != 0 + return DeltaCompileSeries(steps=tuple(described), deltas=deltas, unexpected=tuple(unexpected)) diff --git a/scripts/delta_compile_series/main/write_delta_compile_project.py b/scripts/delta_compile_series/main/write_delta_compile_project.py new file mode 100644 index 0000000000..338b4c56bb --- /dev/null +++ b/scripts/delta_compile_series/main/write_delta_compile_project.py @@ -0,0 +1,23 @@ +"""Write the dense synthetic project for delta compile series, on DuckDB by default.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from scripts.delta_compile_series.constants import PROJECT_CONFIG + + +def write_delta_compile_project( + *, project_dir: Path, model_count: int, adapter: str = "duckdb" +) -> None: + """Write sources, seeds, functions, macros, models, SQL tests and custom rules on `adapter`.""" + + write_dense_compile_project(project_dir=project_dir, model_count=model_count) + config: Path = project_dir / PROJECT_CONFIG + config.write_text( + config.read_text(encoding="utf-8").replace( + 'adapter = "snowflake"', f'adapter = "{adapter}"' + ), + encoding="utf-8", + ) diff --git a/scripts/delta_compile_series/models.py b/scripts/delta_compile_series/models.py new file mode 100644 index 0000000000..26628e2dfc --- /dev/null +++ b/scripts/delta_compile_series/models.py @@ -0,0 +1,46 @@ +"""Edits, compile outcomes and series results of delta compile series.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.delta_compile_series.types import EditTarget + + +@dataclass(frozen=True) +class DeltaCompileEdit: + """One edit a series can make, and the fallback reason it must give when not a delta.""" + + kind: str + pattern: str + replacement: str + target: EditTarget = EditTarget.MODELS + fallback_reason: str | None = None + + +@dataclass(frozen=True) +class DeltaCompileOutcome: + """One compile's exit code and output.""" + + returncode: int + stdout: str + stderr: str + + +@dataclass(frozen=True) +class DeltaCompileSeries: + """What a seeded series found: each step, the deltas it took and anything unexpected.""" + + steps: tuple[str, ...] + deltas: int + unexpected: tuple[str, ...] + + +@dataclass(frozen=True) +class DeltaCompileSequence: + """What a fixed sequence of edits printed: which compiles were deltas and the last report.""" + + delta_lines: tuple[bool, ...] + replayed_last: bool + last_report: str + full_report: str diff --git a/scripts/delta_compile_series/types.py b/scripts/delta_compile_series/types.py new file mode 100644 index 0000000000..7b07a6c4b1 --- /dev/null +++ b/scripts/delta_compile_series/types.py @@ -0,0 +1,21 @@ +"""Enumerations of delta compile series.""" + +from __future__ import annotations + +from enum import StrEnum + + +class EditTarget(StrEnum): + """Which project files an edit changes.""" + + MODELS = "models" + TESTS = "tests" + PROJECT_CONFIG = "project_config" + + +class SequenceEdit(StrEnum): + """One step of a fixed edit sequence.""" + + BODY = "body" + SHAPE = "shape" + NONE = "none" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 7fec47b9f4..bfffb7f32d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -129,11 +129,33 @@ prints `Recording compile for reuse...` and a completion line to stderr before t ### Compiling after an edit -Any change runs the whole compile again: every model, test, and audit is rendered again, and -contracts and semantic validation run for the whole project. No render is carried over from the -previous compile; rendering runs natively and does not run unchanged macro calls again. Every -model's SQL references are scanned again too; no reference scan is kept between compiles. A few -caches under `target/cache/` shorten the rest: +When only model SQL files and SQL test files changed since the stored compile, `sqb compile` +compiles just those files against it. It renders, analyses and checks the changed models and SQL +tests, plus any model whose inputs' columns, types or nullability the edit changed, and takes +everything else from a project image stored beside the stored compile. It prints one line to +stderr, for example `Compiled 1 changed file(s) against the previous compile (1 model(s), 2.1s)`, +then the same report and exit code a full compile would produce, including any errors the edit +introduces or fixes. Project-wide rules still run over the whole project. + +The compile runs in full instead, decided before any compile work starts, when: + +- A file was added, removed or renamed, or any file other than a model or SQL test changed, such + as `sqlbuild_project.toml`, a source or schema YAML file, a macro, a seed or a function. +- An edited model changes what it references, or changes a header setting other than its + `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch + between `view` and `table` materialization. +- A custom rule reads something the edit changed, such as a model's contract or materialization. +- The stored compile reported errors or warnings other than rule findings in model or SQL test + files the edit did not change. +- The stored compile used the other output format (`--json` or text), or has no project image. + +A few edits are found to need a full compile only part-way through, for example when a macro's +arguments change which models or sources it reads, or when the edit changes the shape of so many +models that a full compile is cheaper. + +Otherwise, an edit runs the whole compile again, and a few caches under `target/cache/` shorten +it: every model, test, and audit is rendered again, and contracts and semantic validation run for +the whole project. Every model's SQL references are scanned again too. - An unchanged model takes its column analysis from `model-analyses.bin`. Entries are keyed by the model's query and settings, the inferred columns, types and nullability of every relation @@ -146,8 +168,9 @@ caches under `target/cache/` shorten the rest: - Macro call results are reused as described in [Macros must be deterministic](../concepts/macros.md#macros-must-be-deterministic). -An edit therefore costs a full render of the project plus the analysis, test scans, and rules of -what changed. The output is byte-identical to `sqb compile --no-cache`. +Either way, the output is byte-identical to `sqb compile --no-cache`. To turn compiling only the +changed files off, set `delta_compile = false` under `[settings]` or set +`SQLBUILD_DISABLE_DELTA_COMPILE=1`; compiles then no longer store a project image. Known limits: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index b47766fc67..b4a40fe677 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -652,6 +652,7 @@ default_audit_run_scope = "delta_and_final" unit_test_max_real_models = 10 test_timeout = "30s" scenario_timeout = "15m" +delta_compile = true ``` | Field | Default | Description | @@ -669,6 +670,7 @@ scenario_timeout = "15m" | `unit_test_max_real_models` | `10` | Most real (unmocked) models one SQL unit test may execute; a larger test fails compilation. See [Unit test and scenario limits](#unit-test-and-scenario-limits). | | `test_timeout` | `30s` | Time limit for each SQL unit test, or `"disabled"` | | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | +| `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](../cli/compile.md#compiling-after-an-edit). | ### Unit test and scenario limits diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bddfa169bb..1e057364aa 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -176,6 +176,8 @@ TABLE_PROMOTION_MODE_SETTING_KEY: str STAGED_PROMOTION_MODE: str STAGE_CAPTURE_DIR_ENV_VAR: str COMPILE_REUSE_DISABLE_ENV_VAR: str +DELTA_COMPILE_DISABLE_ENV_VAR: str +DELTA_COMPILE_VERIFY_ENV_VAR: str SQLBUILD_QUERY_TAG_APP: str STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] @@ -4066,6 +4068,7 @@ def compile_frontier[T]( native_stage: Callable[[], T] | None = None, ) -> T: ... def render_stage_capture(value: object) -> str: ... +def claim_first_compile() -> bool: ... def digest_module_files(paths: Iterable[str], /) -> dict[str, str] | None: ... def module_digests_metadata(digests: dict[str, str], /) -> bytes: ... def unchanged_module_digests( @@ -5292,6 +5295,7 @@ class MacroCallMemo: def stats(self) -> tuple[int, int, int]: ... def attach_store(self, path: str, environment: str) -> tuple[bytes, int]: ... def discard_store(self) -> None: ... + def store_allowed(self) -> bool: ... def store_needs_save(self) -> bool: ... def store_stats(self) -> tuple[int, int]: ... def save_store(self, path: str, metadata: bytes) -> int | None: ... @@ -13755,6 +13759,7 @@ class SettingsConfig(NativeRecord): test_timeout_seconds: int | None scenario_timeout: str | None scenario_timeout_seconds: int | None + delta_compile: bool def __init__( self, sql_analysis: bool = ..., @@ -13773,6 +13778,7 @@ class SettingsConfig(NativeRecord): test_timeout_seconds: int | None = ..., scenario_timeout: str | None = ..., scenario_timeout_seconds: int | None = ..., + delta_compile: bool = ..., ) -> None: ... @property def sql_validation(self) -> bool: ... diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py index 262903156b..6ff6923335 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -43,7 +43,7 @@ def __init__(self) -> None: def attach_store(self, *, cache_dir: Path, project_dir: Path, model_paths: list[str]) -> None: """Reuse call results stored by earlier compiles of the same code and project files.""" - if self._store_path is not None: + if self._store_path is not None or not self.memo.store_allowed(): return path: Path = cache_dir / MACRO_CALL_STORE_FILE_NAME metadata: bytes diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 0226aab70f..d3edb4ac37 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -2,13 +2,11 @@ from __future__ import annotations -import itertools from contextvars import ContextVar ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( "sqlbuild_active_macro_bridge", default=None ) -COMPILES_STARTED: itertools.count[int] = itertools.count() STORE_MODULE_PATHS: set[str] = set() MACRO_CALL_STORE_FILE_NAME: str = "macro-calls.bin" MACRO_CALL_STORE_ENVIRONMENT_VERSION: str = "macro-call-store-v2" diff --git a/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py b/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py index ef5d3668c6..1c28dabe64 100644 --- a/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py +++ b/src/sqlbuild/compiler/macro_bridge/main/_claim_first_compile.py @@ -1,11 +1,11 @@ -"""Count the compiles this process has started, so only its first one uses the macro call store.""" +"""Ask whether this compile is the process's first, so only that one uses the macro call store.""" from __future__ import annotations -from sqlbuild.compiler.macro_bridge.constants import COMPILES_STARTED +from sqlbuild._native import claim_first_compile as claim_native_first_compile def claim_first_compile() -> bool: - """Record that a compile started; True only for the first compile of this process.""" + """Record that a compile started; True only for the first compile invocation of this process.""" - return next(COMPILES_STARTED) == 0 + return claim_native_first_compile() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 0a68348ecf..a07b1346fa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -9,6 +9,7 @@ import pytest +from scripts.delta_compile_series.types import SequenceEdit from tests.integration.src.sqlbuild.compiler.pipeline._test_types import PreparedCompile if TYPE_CHECKING: @@ -275,6 +276,19 @@ class DenseWarmEditCompileGuardTestCase: expected_edit_rule_cache_misses: int +@dataclass(frozen=True) +class DenseDeltaEditLoopGuardTestCase: + description: str + model_count: int + edited_model_index: int + full_pairs: int + expected_max_full_seconds: float + expected_max_delta_seconds: float + expected_max_delta_cpu_ratio: float + expected_max_fallback_cpu_ratio: float + expected_max_rss_bytes: int + + @dataclass(frozen=True) class RuleGatedTestDiagnosticsTestCase: description: str @@ -803,6 +817,17 @@ class BrokenEditInvalidationTestCase: expected_matches_uncached: bool +@dataclass(frozen=True) +class StoredMacroDeltaFallbackTestCase: + """A project edit whose delta compile falls back after compiling its round.""" + + description: str + edited_path: str + old: str + new: str + expected_outcome: tuple[int, int, bool, int] + + @dataclass(frozen=True) class StoredMacroModuleEditTestCase: """An outside module behind a stored macro call, edited after a compile that reused the call.""" @@ -1103,3 +1128,43 @@ class EarlyBuiltinRulesEditTestCase: expected_diagnostics: tuple[tuple[str, str, int, int], ...] expected_early_outcomes: tuple[tuple[int, int], ...] expected_rule_cache_outcomes: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class DeltaCompileSeriesTestCase: + description: str + seed: int + json_output: bool + steps: int + adapter: str = "duckdb" + expected_unexpected_outcomes: tuple[str, ...] = () + expected_minimum_deltas: int = 3 + + +@dataclass(frozen=True) +class DeltaCompileFullCopyTestCase: + description: str + edits: tuple[tuple[str, str, str], ...] + expected_delta_line: bool + stored_edits: tuple[tuple[str, str, str], ...] = () + json_output: bool = False + adapter: str = "duckdb" + expected_difference: None = None + + +@dataclass(frozen=True) +class DeltaCompileSequenceTestCase: + description: str + edits: tuple[SequenceEdit, ...] + expected_delta_lines: tuple[bool, ...] + expected_replayed_last: bool + expected_same_as_full: bool + + +@dataclass(frozen=True) +class DeltaCompileOptOutTestCase: + description: str + settings: str + env: dict[str, str] + expected_delta_line: bool = False + expected_images: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index b852152b73..da20018bd0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -20,6 +20,7 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager, redirect_stdout from dataclasses import replace +from functools import partial from io import StringIO from pathlib import Path from types import FrameType, ModuleType @@ -38,8 +39,13 @@ semantic_compile_fingerprint, ) from scripts.compiler_differential.constants import FAILURE_BASE_FILES +from scripts.delta_compile_series.models import DeltaCompileOutcome from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR -from sqlbuild._native import ContractValidationResult, compiler_cache_directory +from sqlbuild._native import ( + DELTA_COMPILE_DISABLE_ENV_VAR, + ContractValidationResult, + compiler_cache_directory, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -661,12 +667,14 @@ def _run_fresh_process_compile_benchmark( expected_max_wall_seconds: float, compile_args: tuple[str, ...], retained_payload_keys: tuple[str, ...] = _RETAINED_BENCHMARK_PAYLOAD_KEYS, + env: dict[str, str] | None = None, ) -> FreshProcessCompileBenchmarkResult: run: _TimedSqbRun = _run_timed_sqb( project_dir=project_dir, label=label, sqb_args=("compile", "--json", *compile_args), expected_max_wall_seconds=expected_max_wall_seconds, + env=env, ) assert isinstance(run.payload, dict) payload: dict[str, object] = cast(dict[str, object], run.payload) @@ -702,6 +710,7 @@ def _run_timed_sqb( label: str, sqb_args: tuple[str, ...], expected_max_wall_seconds: float, + env: dict[str, str] | None = None, ) -> _TimedSqbRun: measurement_path: Path = project_dir.parent / f"{label}-measurement.txt" output_path: Path = project_dir.parent / f"{label}.out" @@ -732,6 +741,7 @@ def _run_timed_sqb( stdout=output_file, stderr=stderr_file, start_new_session=True, + env={**os.environ, **(env or {})}, ) as process: try: returncode: int = process.wait(timeout=expected_max_wall_seconds + 10.0) @@ -937,6 +947,136 @@ def run_dense_warm_edit_benchmark( return measurements +class DeltaEditLoopMeasurements(NamedTuple): + """Fresh-process compiles of an edit loop: deltas, delta fallbacks and delta-disabled ones.""" + + deltas: dict[str, FreshProcessCompileBenchmarkResult] + fallbacks: tuple[FreshProcessCompileBenchmarkResult, ...] + full: tuple[FreshProcessCompileBenchmarkResult, ...] + + +DELTA_EDIT_LOOP_ENV: dict[str, str] = {REUSE_DISABLE_ENV_VAR: "0"} +_DELTA_DISABLED_ENV: dict[str, str] = {**DELTA_EDIT_LOOP_ENV, DELTA_COMPILE_DISABLE_ENV_VAR: "1"} +_WARM_UP_PAIRS: int = 1 +# Each pair swaps which compile runs first, so neither always runs on a warmer system. +_ALTERNATING_CONFIG_RUNS: tuple[tuple[tuple[str, dict[str, str]], ...], ...] = ( + (("fallback", DELTA_EDIT_LOOP_ENV), ("full", _DELTA_DISABLED_ENV)), + (("full", _DELTA_DISABLED_ENV), ("fallback", DELTA_EDIT_LOOP_ENV)), +) +_DENSE_DESCRIPTION: str = 'description "Generated dense model."' +_DENSE_EXPECTED_VALUE: re.Pattern[str] = re.compile(r"(__expected__\w+ AS \(SELECT CAST\()\d+") +_ANALYSIS_COUNTS: tuple[str, ...] = ( + "analysis_batch_cache_hits", + "analysis_entry_cache_hits", + "analysis_cache_misses", + "analysis_cache_bypasses", +) + + +def analysed_model_count(result: FreshProcessCompileBenchmarkResult) -> int: + """How many models the compile analysed, from cache or not; a delta analyses only its own.""" + + timings: dict[str, int] = cast(dict[str, int], result.payload["compile_timings"]) + return sum(timings[name] for name in _ANALYSIS_COUNTS) + + +def _edit_loop_compile( + *, project_dir: Path, label: str, max_seconds: float, env: dict[str, str] +) -> FreshProcessCompileBenchmarkResult: + return _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label=label, + expected_max_wall_seconds=max_seconds, + compile_args=(), + env=env, + ) + + +def _edited_test_expectation(path: Path) -> None: + contents: str = path.read_text(encoding="utf-8") + edited, count = _DENSE_EXPECTED_VALUE.subn(r"\g<1>97", contents, count=1) + assert count == 1 and edited != contents, path + path.write_text(edited, encoding="utf-8") + + +def run_dense_delta_edit_loop_benchmark( + *, + project_dir: Path, + edited_files: tuple[Path, Path, Path], + full_pairs: int, + expected_max_full_seconds: float, + expected_max_delta_seconds: float, +) -> DeltaEditLoopMeasurements: + """Compile, edit a body, a header and a SQL test, then edit config with and without deltas.""" + + body_model, header_model, test_file = edited_files + _ = _edit_loop_compile( + project_dir=project_dir, + label="delta-loop-first", + max_seconds=2 * expected_max_full_seconds, + env=DELTA_EDIT_LOOP_ENV, + ) + deltas: dict[str, FreshProcessCompileBenchmarkResult] = {} + for label, edit in ( + ( + "body", + partial( + _replace_benchmark_text, + path=body_model, + old="COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)", + new="COALESCE(b.amount, 1000) + CAST(b.id AS DOUBLE)", + ), + ), + ( + "header", + partial( + _replace_benchmark_text, + path=header_model, + old=_DENSE_DESCRIPTION, + new='description "Edited dense model."', + ), + ), + ( + "sql_test", + partial(_edited_test_expectation, test_file), + ), + ): + edit() + deltas[label] = _edit_loop_compile( + project_dir=project_dir, + label=f"delta-loop-{label}", + max_seconds=expected_max_delta_seconds, + env=DELTA_EDIT_LOOP_ENV, + ) + runs: dict[str, list[FreshProcessCompileBenchmarkResult]] = {"fallback": [], "full": []} + for pair in range(_WARM_UP_PAIRS + full_pairs): + for kind, env in _ALTERNATING_CONFIG_RUNS[pair % len(_ALTERNATING_CONFIG_RUNS)]: + config: Path = project_dir / "sqlbuild_project.toml" + config.write_text( + config.read_text(encoding="utf-8") + f"# {kind} edit {pair}\n", + encoding="utf-8", + ) + runs[kind].append( + _edit_loop_compile( + project_dir=project_dir, + label=f"delta-loop-{kind}-{pair}", + max_seconds=expected_max_full_seconds, + env=env, + ) + ) + return DeltaEditLoopMeasurements( + deltas=deltas, + fallbacks=tuple(runs["fallback"][_WARM_UP_PAIRS:]), + full=tuple(runs["full"][_WARM_UP_PAIRS:]), + ) + + +def median_cpu_seconds(results: tuple[FreshProcessCompileBenchmarkResult, ...]) -> float: + """The median CPU seconds of same-runner compiles.""" + + return statistics.median(result.cpu_seconds for result in results) + + def _append_benchmark_edit(path: Path, label: str) -> None: path.write_text( path.read_text(encoding="utf-8") + f"\n-- one {label} edit\n", @@ -5095,3 +5235,30 @@ def json_diagnostic_codes(stdout: str) -> tuple[str, ...]: document: dict[str, Any] = json.loads(stdout or "{}") return tuple(str(diagnostic["code"]) for diagnostic in document.get("diagnostics", ())) + + +def run_delta_compile( + *, project_dir: Path, args: tuple[str, ...], env: dict[str, str] +) -> DeltaCompileOutcome: + """Compile in a fresh process with compile reuse enabled and the given extra environment.""" + + result: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=args, env={REUSE_DISABLE_ENV_VAR: "0", **env} + ) + return DeltaCompileOutcome( + returncode=result.returncode, stdout=result.stdout, stderr=result.stderr + ) + + +def run_full_compile(project_dir: Path, *, args: tuple[str, ...]) -> DeltaCompileOutcome: + """Compile the project at `project_dir` in full, with delta compiles off.""" + + return run_delta_compile( + project_dir=project_dir, args=args, env={DELTA_COMPILE_DISABLE_ENV_VAR: "1"} + ) + + +def without_compile_timings(*, report: str) -> str: + """A JSON report without its compile timings, which differ between compiles.""" + + return _COMPILE_TIMINGS_PATTERN.sub("", report) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py new file mode 100644 index 0000000000..55c6ad429a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -0,0 +1,329 @@ +"""Delta compiles must print exactly what a full compile of the same edit prints.""" + +from __future__ import annotations + +import shutil +from functools import partial +from pathlib import Path + +import pytest + +from scripts.delta_compile_series._helpers.outcomes import full_compile_difference +from scripts.delta_compile_series.constants import ( + DELTA_COMPILE_LINE, + DOUBLE_CAST, + DOUBLE_COLUMN, + FLOAT_CAST, + FLOAT_COLUMN, + PROJECT_CONFIG, +) +from scripts.delta_compile_series.main.run_delta_compile_sequence import ( + run_delta_compile_sequence, +) +from scripts.delta_compile_series.main.run_delta_compile_series import run_delta_compile_series +from scripts.delta_compile_series.main.write_delta_compile_project import ( + write_delta_compile_project, +) +from scripts.delta_compile_series.models import ( + DeltaCompileOutcome, + DeltaCompileSequence, + DeltaCompileSeries, +) +from scripts.delta_compile_series.types import SequenceEdit +from sqlbuild._native import DELTA_COMPILE_DISABLE_ENV_VAR, DELTA_COMPILE_VERIFY_ENV_VAR +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + DeltaCompileFullCopyTestCase, + DeltaCompileOptOutTestCase, + DeltaCompileSequenceTestCase, + DeltaCompileSeriesTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + replace_project_text, + run_delta_compile, + run_full_compile, + without_compile_timings, +) + +_MODEL_COUNT: int = 300 +_JSON_COMPILE: tuple[str, ...] = ("compile", "--json") +_TEXT_COMPILE: tuple[str, ...] = ("compile",) +_UNTYPED_ADAPTER: str = "sqlserver" +_SMALL_MODEL_COUNT: int = 30 +_SMALL_MODEL_ROOT: str = "models/sales000/intermediate/clean" +_ANALYSED_MODEL: str = f"{_SMALL_MODEL_ROOT}/batch00000/sales000__int_clean__orders_00001.sql" +_SMALL_EDITED_MODEL: str = f"{_SMALL_MODEL_ROOT}/batch00001/sales000__int_clean__orders_00005.sql" +_RESHAPED_MODEL: str = f"{_SMALL_MODEL_ROOT}/batch00002/sales000__int_clean__orders_00010.sql" +_SMALL_EDITED_TEST: str = ( + "tests/unit/sales000/intermediate/clean/batch00001/" + "test_sales000_int_clean_orders_00005__preserves_order_id.sql" +) +_EXPECTED_MODEL_CTE: str = "__expected__sales000__int_clean__orders_00005 AS" +_RETARGETED_MODEL_CTE: str = "__expected__sales000__int_clean__orders_00020 AS" +_BINDING_ERROR_EDIT: tuple[str, str, str] = ( + _SMALL_EDITED_MODEL, + "COALESCE(b.amount, ", + "COALESCE(b.missing_col, ", +) +_BODY_EDIT: tuple[str, str, str] = ( + _SMALL_EDITED_MODEL, + "COALESCE(b.amount, ", + "COALESCE(b.amount, 1 + ", +) +_PATH_DEFAULT_EDIT: tuple[str, str, str] = ( + "sqlbuild_project.toml", + "[defaults]", + '[path_defaults."sales000/intermediate/clean/batch00000"]\nmaterialized = "table"\n\n' + "[defaults]", +) +_REUSE_ROOT: str = "target/cache/compiler-native-v1/project-reuse-v3" +_EDITED_MODEL: str = ( + "models/sales000/intermediate/clean/batch00000/sales000__int_clean__orders_00003.sql" +) + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileSeriesTestCase(description="json_report", seed=11, json_output=True, steps=12), + DeltaCompileSeriesTestCase(description="text_report", seed=12, json_output=False, steps=12), + DeltaCompileSeriesTestCase( + description="long_json_series", seed=13, json_output=True, steps=20 + ), + DeltaCompileSeriesTestCase( + description="partial_checks_text_report", + seed=14, + json_output=False, + steps=12, + adapter=_UNTYPED_ADAPTER, + ), + ], + ids=lambda case: case.description, +) +def test_given_seeded_edit_series_when_compiling_each_edit_then_deltas_match_full_compiles( + test_case: DeltaCompileSeriesTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project( + project_dir=project_dir, model_count=_MODEL_COUNT, adapter=test_case.adapter + ) + args: tuple[str, ...] = (_TEXT_COMPILE, _JSON_COMPILE)[int(test_case.json_output)] + + series: DeltaCompileSeries = run_delta_compile_series( + project_dir=project_dir, + seed=test_case.seed, + steps=test_case.steps, + compile_project=partial( + run_delta_compile, + project_dir=project_dir, + args=args, + env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + ), + ) + + assert series.unexpected == test_case.expected_unexpected_outcomes, series.steps + assert series.deltas >= test_case.expected_minimum_deltas, series.steps + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileSeriesTestCase(description="text_report", seed=15, json_output=False, steps=12), + DeltaCompileSeriesTestCase( + description="partial_checks_json_report", + seed=16, + json_output=True, + steps=12, + adapter=_UNTYPED_ADAPTER, + ), + ], + ids=lambda case: case.description, +) +def test_given_seeded_edit_series_when_compiling_each_edit_then_output_matches_full_compiles_of_copies( + test_case: DeltaCompileSeriesTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project( + project_dir=project_dir, model_count=_MODEL_COUNT, adapter=test_case.adapter + ) + args: tuple[str, ...] = (_TEXT_COMPILE, _JSON_COMPILE)[int(test_case.json_output)] + + series: DeltaCompileSeries = run_delta_compile_series( + project_dir=project_dir, + seed=test_case.seed, + steps=test_case.steps, + compile_project=partial(run_delta_compile, project_dir=project_dir, args=args, env={}), + compile_in_full=partial(run_full_compile, args=args), + ) + + assert series.unexpected == test_case.expected_unexpected_outcomes, series.steps + assert series.deltas >= test_case.expected_minimum_deltas, series.steps + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileFullCopyTestCase( + description="a model without SQL analysis keeps its partial-check notice", + stored_edits=((_ANALYSED_MODEL, "MODEL (", "MODEL (sql_analysis false, "),), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileFullCopyTestCase( + description="an invalid model makes its unchanged dependents partial", + edits=(_BINDING_ERROR_EDIT,), + expected_delta_line=True, + ), + DeltaCompileFullCopyTestCase( + description="an invalid model makes its unchanged dependents partial in JSON", + edits=(_BINDING_ERROR_EDIT,), + expected_delta_line=True, + json_output=True, + ), + DeltaCompileFullCopyTestCase( + description="a dialect without type checks reports every model partial", + edits=(_BODY_EDIT,), + expected_delta_line=True, + json_output=True, + adapter=_UNTYPED_ADAPTER, + ), + DeltaCompileFullCopyTestCase( + description="a path default matching only unchanged models", + stored_edits=(_PATH_DEFAULT_EDIT,), + edits=(_BODY_EDIT,), + expected_delta_line=True, + ), + DeltaCompileFullCopyTestCase( + description="a SQL test expecting another model falls back after its round", + edits=((_SMALL_EDITED_TEST, _EXPECTED_MODEL_CTE, _RETARGETED_MODEL_CTE),), + expected_delta_line=False, + ), + DeltaCompileFullCopyTestCase( + description="a shape change printed once over several rounds", + edits=( + (_RESHAPED_MODEL, DOUBLE_COLUMN, FLOAT_COLUMN), + (_RESHAPED_MODEL, DOUBLE_CAST, FLOAT_CAST), + (_RESHAPED_MODEL, DOUBLE_CAST, FLOAT_CAST), + ), + expected_delta_line=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_compile_when_compiling_an_edit_then_it_prints_what_a_full_compile_prints( + test_case: DeltaCompileFullCopyTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project( + project_dir=project_dir, model_count=_SMALL_MODEL_COUNT, adapter=test_case.adapter + ) + args: tuple[str, ...] = (_TEXT_COMPILE, _JSON_COMPILE)[int(test_case.json_output)] + for relative_path, old, new in test_case.stored_edits: + replace_project_text(project_dir, relative_path, old, new) + _ = run_delta_compile(project_dir=project_dir, args=args, env={}) + for relative_path, old, new in test_case.edits: + replace_project_text(project_dir, relative_path, old, new) + copy: Path = tmp_path / "full_copy" + _ = shutil.copytree(project_dir, copy, symlinks=True) + + edited: DeltaCompileOutcome = run_delta_compile(project_dir=project_dir, args=args, env={}) + + assert (DELTA_COMPILE_LINE in edited.stderr) == test_case.expected_delta_line, edited.stderr + assert ( + full_compile_difference( + outcome=edited, + full=run_full_compile(copy, args=args), + project_dir=project_dir, + full_project_dir=copy, + ) + == test_case.expected_difference + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileSequenceTestCase( + description="an unchanged project replays the delta", + edits=(SequenceEdit.BODY, SequenceEdit.NONE), + expected_delta_lines=(True, False), + expected_replayed_last=True, + expected_same_as_full=True, + ), + DeltaCompileSequenceTestCase( + description="an edit after a multi-round delta is a delta too", + edits=(SequenceEdit.SHAPE, SequenceEdit.BODY), + expected_delta_lines=(True, True), + expected_replayed_last=False, + expected_same_as_full=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_edit_sequence_when_compiling_each_edit_then_deltas_store_and_replay_like_full_compiles( + test_case: DeltaCompileSequenceTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) + + sequence: DeltaCompileSequence = run_delta_compile_sequence( + project_dir=project_dir, + edits=test_case.edits, + compile_project=partial( + run_delta_compile, project_dir=project_dir, args=_JSON_COMPILE, env={} + ), + compile_in_full=partial( + run_delta_compile, + project_dir=project_dir, + args=_JSON_COMPILE, + env={DELTA_COMPILE_DISABLE_ENV_VAR: "1"}, + ), + ) + + assert sequence.delta_lines == test_case.expected_delta_lines + assert sequence.replayed_last == test_case.expected_replayed_last + assert ( + without_compile_timings(report=sequence.last_report) + == without_compile_timings(report=sequence.full_report) + ) == test_case.expected_same_as_full + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileOptOutTestCase( + description="project_setting", settings="\n[settings]\ndelta_compile = false\n", env={} + ), + DeltaCompileOptOutTestCase( + description="environment", settings="", env={DELTA_COMPILE_DISABLE_ENV_VAR: "1"} + ), + ], + ids=lambda case: case.description, +) +def test_given_delta_compiles_turned_off_when_compiling_an_edit_then_it_compiles_in_full( + test_case: DeltaCompileOptOutTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "dense_orders" + write_delta_compile_project(project_dir=project_dir, model_count=_MODEL_COUNT) + config: Path = project_dir / PROJECT_CONFIG + config.write_text(config.read_text(encoding="utf-8") + test_case.settings, encoding="utf-8") + first: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=_TEXT_COMPILE, env=test_case.env + ) + replace_project_text( + project_dir, _EDITED_MODEL, "COALESCE(b.amount, ", "COALESCE(b.amount, 1 + " + ) + + edited: DeltaCompileOutcome = run_delta_compile( + project_dir=project_dir, args=_TEXT_COMPILE, env=test_case.env + ) + + assert (first.returncode, edited.returncode) == (0, 0), edited.stderr[-2000:] + assert (DELTA_COMPILE_LINE in edited.stderr) == test_case.expected_delta_line + assert ( + tuple(sorted(path.name for path in (project_dir / _REUSE_ROOT).glob("*.image"))) + == test_case.expected_images + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index e5e44d18b7..c2c786313e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -8,7 +8,8 @@ import pytest from scripts.cold_compile_performance._helpers.dense_project import ( - dense_model_name, + dense_model_path, + dense_test_path, write_dense_compile_project, ) from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( @@ -23,18 +24,25 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, + DenseDeltaEditLoopGuardTestCase, DenseWarmEditCompileGuardTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + DeltaEditLoopMeasurements, FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, + analysed_model_count, fresh_process_compile_cache_metrics, measure_model_sql_bytes, + median_cpu_seconds, + run_dense_delta_edit_loop_benchmark, run_dense_warm_edit_benchmark, ) _MIB: int = 1024 * 1024 _GIB: int = 1024 * _MIB +# A delta reruns the edited model or the edited SQL test's few models; a full compile all of them. +_MAX_DELTA_ANALYSED_MODELS: int = 10 @pytest.mark.performance @@ -45,28 +53,28 @@ DenseCompileGuardTestCase( "dense_models_1000_all_rules", 1000, - 13.0, + 7.5, 800 * _MIB, "de23b14d87e954a65c2da91918fce062d158419edd63cc8cf3dc938b3e47cd44", ), DenseCompileGuardTestCase( "dense_models_3000_all_rules", 3000, - 25.0, + 16.5, 3 * _GIB // 2, "196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed", ), DenseCompileGuardTestCase( "dense_models_5000_all_rules", 5000, - 42.0, + 26.0, 9 * _GIB // 4, "3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22", ), DenseCompileGuardTestCase( "dense_models_10000_all_rules", 10000, - 77.0, + 51.0, 15 * _GIB // 4, "f2a574af6e8068a0634fa3c078b168dff9a475ada5d71502482ad2926c90e1c3", ), @@ -146,8 +154,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_3000_warm_and_one_edit", 3000, 1521, - 11.0, - 12.0, + 8.0, + 8.5, 3 * _GIB // 2, "196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed", "1afe754dd8c66bc36e38a40c66e9183ecbaf365fd8d87a784069cd615c3ce64a", @@ -157,8 +165,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_5000_warm_and_one_edit", 5000, 2521, - 19.0, - 21.0, + 13.0, + 13.5, 9 * _GIB // 4, "3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22", "dfbada4c59a3ab8ec7733784890d74c7b64e9b0adcb03364baaabef5bdba5574", @@ -174,15 +182,8 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches assert_required_cgroup_memory_limit() project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) - index: int = test_case.edited_model_index - edited_model_path: Path = ( - project_dir - / "models" - / f"sales{index // 1000:03d}" - / "intermediate" - / "clean" - / f"batch{index // 5:05d}" - / f"{dense_model_name(index)}.sql" + edited_model_path: Path = dense_model_path( + project_dir=project_dir, index=test_case.edited_model_index ) measurements: dict[str, FreshProcessCompileBenchmarkResult] = run_dense_warm_edit_benchmark( project_dir=project_dir, @@ -227,3 +228,81 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) + + +@pytest.mark.performance +@pytest.mark.cold_compile_performance +@pytest.mark.parametrize( + "test_case", + ( + DenseDeltaEditLoopGuardTestCase( + description="dense_models_3000_delta_edit_loop", + model_count=3000, + edited_model_index=1521, + full_pairs=4, + expected_max_full_seconds=15.0, + expected_max_delta_seconds=4.5, + expected_max_delta_cpu_ratio=0.375, + expected_max_fallback_cpu_ratio=1.05, + expected_max_rss_bytes=3 * _GIB // 2, + ), + DenseDeltaEditLoopGuardTestCase( + description="dense_models_10000_delta_edit_loop", + model_count=10000, + edited_model_index=5021, + full_pairs=3, + expected_max_full_seconds=60.0, + expected_max_delta_seconds=9.0, + expected_max_delta_cpu_ratio=0.30, + expected_max_fallback_cpu_ratio=1.05, + expected_max_rss_bytes=15 * _GIB // 4, + ), + ), + ids=lambda case: case.description, +) +def test_given_dense_project_when_editing_one_file_then_delta_compiles_stay_cheap( + test_case: DenseDeltaEditLoopGuardTestCase, + tmp_path: Path, +) -> None: + assert_required_cgroup_memory_limit() + project_dir: Path = tmp_path / "dense_orders" + write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) + index: int = test_case.edited_model_index + + loop: DeltaEditLoopMeasurements = run_dense_delta_edit_loop_benchmark( + project_dir=project_dir, + edited_files=( + dense_model_path(project_dir=project_dir, index=index), + dense_model_path(project_dir=project_dir, index=index + 1), + dense_test_path(project_dir=project_dir, index=index + 2), + ), + full_pairs=test_case.full_pairs, + expected_max_full_seconds=test_case.expected_max_full_seconds, + expected_max_delta_seconds=test_case.expected_max_delta_seconds, + ) + + full_cpu: float = median_cpu_seconds(loop.full) + fallback_cpu: float = median_cpu_seconds(loop.fallbacks) + for label, result in loop.deltas.items(): + print( + f"delta {label} models={test_case.model_count} wall={result.elapsed_seconds:.2f}s " + f"cpu={result.cpu_seconds:.2f}s cpu_ratio={result.cpu_seconds / full_cpu:.3f} " + f"analysed={analysed_model_count(result)}", + flush=True, + ) + print( + f"delta loop models={test_case.model_count} full_cpu={full_cpu:.2f}s " + f"fallback_cpu={fallback_cpu:.2f}s fallback_ratio={fallback_cpu / full_cpu:.3f}", + flush=True, + ) + for result in (*loop.deltas.values(), *loop.fallbacks, *loop.full): + assert result.payload["has_errors"] is False + assert result.payload["diagnostics"] == [] + assert result.peak_rss_bytes < test_case.expected_max_rss_bytes + for result in loop.deltas.values(): + assert 0 < analysed_model_count(result) <= _MAX_DELTA_ANALYSED_MODELS, result + assert result.elapsed_seconds < test_case.expected_max_delta_seconds, result + assert result.cpu_seconds <= test_case.expected_max_delta_cpu_ratio * full_cpu, result + for result in (*loop.fallbacks, *loop.full): + assert analysed_model_count(result) == test_case.model_count, result + assert fallback_cpu <= test_case.expected_max_fallback_cpu_ratio * full_cpu diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 84b298e45c..601275d93e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -50,9 +50,9 @@ macro_count=500, test_count=2_945, audit_count=5_056, - expected_cold_max_wall_seconds=22.0, - expected_warm_max_wall_seconds=11.0, - expected_edit_max_wall_seconds=11.5, + expected_cold_max_wall_seconds=10.5, + expected_warm_max_wall_seconds=6.0, + expected_edit_max_wall_seconds=6.0, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, @@ -84,9 +84,9 @@ macro_count=834, test_count=4_908, audit_count=8_427, - expected_cold_max_wall_seconds=35.0, - expected_warm_max_wall_seconds=18.0, - expected_edit_max_wall_seconds=18.0, + expected_cold_max_wall_seconds=18.0, + expected_warm_max_wall_seconds=10.5, + expected_edit_max_wall_seconds=10.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, @@ -118,9 +118,9 @@ macro_count=1_667, test_count=9_816, audit_count=16_855, - expected_cold_max_wall_seconds=68.0, - expected_warm_max_wall_seconds=37.5, - expected_edit_max_wall_seconds=37.5, + expected_cold_max_wall_seconds=34.5, + expected_warm_max_wall_seconds=19.5, + expected_edit_max_wall_seconds=19.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_LARGE_CACHE_WRITE_CPU_OVERHEAD_RATIO, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index 568b961a17..0431b89012 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -36,7 +36,7 @@ macro_count=37, test_count=2_945, audit_count=5_056, - expected_max_wall_seconds=17.0, + expected_max_wall_seconds=9.5, expected_max_rss_bytes=int(1.5 * _GIB), expected_semantic_fingerprint=( "d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543" @@ -51,7 +51,7 @@ macro_count=61, test_count=4_908, audit_count=8_427, - expected_max_wall_seconds=25.0, + expected_max_wall_seconds=16.0, expected_max_rss_bytes=int(1.75 * _GIB), expected_semantic_fingerprint=( "874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46" @@ -66,7 +66,7 @@ macro_count=123, test_count=9_816, audit_count=16_855, - expected_max_wall_seconds=50.0, + expected_max_wall_seconds=31.0, expected_max_rss_bytes=int(2.5 * _GIB), expected_semantic_fingerprint=( "5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_store_module_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_store_module_edits.py index a0d232512b..b0e0ac297f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_store_module_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_store_module_edits.py @@ -7,6 +7,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + StoredMacroDeltaFallbackTestCase, StoredMacroModuleEditTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( @@ -15,6 +16,7 @@ add_stored_flavor_macro, compiled_text, logged_calls, + replace_project_text, run_reuse_compile, staging_comment, write_external_modules, @@ -90,5 +92,40 @@ def test_given_stored_call_reused_when_its_lazily_imported_module_changes_then_o ) +@pytest.mark.parametrize( + "test_case", + [ + StoredMacroDeltaFallbackTestCase( + description="a model error found by the delta round", + edited_path="models/staging/stg_customers.sql", + old=" email,\n", + new=" email,\n unknown_column,\n", + expected_outcome=(0, 1, False, 1), + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_call_when_a_delta_falls_back_after_its_round_then_the_full_compile_reuses_it( + compile_reuse_project: Path, tmp_path: Path, test_case: StoredMacroDeltaFallbackTestCase +) -> None: + extlib: Path = tmp_path / "extlib" + log_path: Path = tmp_path / "flavor-calls.log" + write_external_modules(extlib=extlib, files=_MODULE) + env: dict[str, str] = add_stored_flavor_macro( + project_dir=compile_reuse_project, extlib=extlib, import_name="extflavor", log_path=log_path + ) + cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + replace_project_text(compile_reuse_project, test_case.edited_path, test_case.old, test_case.new) + + edited: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + + assert ( + cold.returncode, + edited.returncode, + edited.reused, + logged_calls(log_path), + ) == test_case.expected_outcome, edited.stderr + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/goldens/compiler/failure/local-config-unknown-key.json b/tests/goldens/compiler/failure/local-config-unknown-key.json index 5aea9e667d..d8b2adc939 100644 --- a/tests/goldens/compiler/failure/local-config-unknown-key.json +++ b/tests/goldens/compiler/failure/local-config-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: auto_load_sources, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", + "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: auto_load_sources, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", "help": "did you mean 'sql_analysis'?" } ] diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py index 26693a6d92..79cf282263 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py @@ -51,6 +51,7 @@ class CacheUseTestCase: description: str run: CompileRun mode: str + models: int = 3000 expected_result: None = None @@ -60,6 +61,7 @@ class CacheUseErrorTestCase: run: CompileRun mode: str expected_message: str + models: int = 3000 @dataclass(frozen=True) diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py b/tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py index 84886bf562..1a3c470cb6 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py @@ -78,6 +78,20 @@ def test_given_compile_runs_when_taking_phase_medians_then_reports_only_complete timings_ms={}, analysis_cache_misses=1, analysis_cache_bypasses=0, + analysed_models=3000, + ), + mode=EDIT_MODE, + ), + CacheUseTestCase( + description="a delta edit analyses only the edited model, afresh", + run=CompileRun( + label="head", + wall_seconds=1.0, + cpu_seconds=1.0, + timings_ms={}, + analysis_cache_misses=0, + analysis_cache_bypasses=1, + analysed_models=1, ), mode=EDIT_MODE, ), @@ -110,7 +124,10 @@ def test_given_compile_runs_when_taking_phase_medians_then_reports_only_complete def test_given_observed_cache_use_when_checking_then_accepts_the_run( test_case: CacheUseTestCase, ) -> None: - assert check_cache_use(run=test_case.run, mode=test_case.mode) is test_case.expected_result + assert ( + check_cache_use(run=test_case.run, mode=test_case.mode, models=test_case.models) + is test_case.expected_result + ) @pytest.mark.parametrize( @@ -138,10 +155,39 @@ def test_given_observed_cache_use_when_checking_then_accepts_the_run( timings_ms={}, analysis_cache_misses=0, analysis_cache_bypasses=3000, + analysed_models=3000, ), mode=EDIT_MODE, expected_message="bypassed the analysis cache for 3000 models", ), + CacheUseErrorTestCase( + description="a delta edit that analysed nothing afresh did not observe the edit", + run=CompileRun( + label="head", + wall_seconds=1.0, + cpu_seconds=1.0, + timings_ms={}, + analysis_cache_misses=0, + analysis_cache_bypasses=0, + analysed_models=1, + ), + mode=EDIT_MODE, + expected_message="delta compile analysed no model afresh", + ), + CacheUseErrorTestCase( + description="a warm compile that bypassed the cache, even for few models, is rejected", + run=CompileRun( + label="head", + wall_seconds=1.0, + cpu_seconds=1.0, + timings_ms={}, + analysis_cache_misses=0, + analysis_cache_bypasses=1, + analysed_models=1, + ), + mode=WARM_MODE, + expected_message="bypassed the analysis cache for 1 models", + ), CacheUseErrorTestCase( description="a warm compile that missed the cache is not an unchanged compile", run=CompileRun( @@ -162,7 +208,7 @@ def test_given_unobserved_cache_use_when_checking_then_rejects_the_run( test_case: CacheUseErrorTestCase, ) -> None: with pytest.raises(CompileComparisonError, match=test_case.expected_message): - check_cache_use(run=test_case.run, mode=test_case.mode) + check_cache_use(run=test_case.run, mode=test_case.mode, models=test_case.models) @pytest.mark.parametrize( diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 978f0493ee..0ded0472b2 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -120,11 +120,33 @@ prints `Recording compile for reuse...` and a completion line to stderr before t ### Compiling after an edit -Any change runs the whole compile again: every model, test, and audit is rendered again, and -contracts and semantic validation run for the whole project. No render is carried over from the -previous compile; rendering runs natively and does not run unchanged macro calls again. Every -model's SQL references are scanned again too; no reference scan is kept between compiles. A few -caches under `target/cache/` shorten the rest: +When only model SQL files and SQL test files changed since the stored compile, `sqb compile` +compiles just those files against it. It renders, analyses and checks the changed models and SQL +tests, plus any model whose inputs' columns, types or nullability the edit changed, and takes +everything else from a project image stored beside the stored compile. It prints one line to +stderr, for example `Compiled 1 changed file(s) against the previous compile (1 model(s), 2.1s)`, +then the same report and exit code a full compile would produce, including any errors the edit +introduces or fixes. Project-wide rules still run over the whole project. + +The compile runs in full instead, decided before any compile work starts, when: + +- A file was added, removed or renamed, or any file other than a model or SQL test changed, such + as `sqlbuild_project.toml`, a source or schema YAML file, a macro, a seed or a function. +- An edited model changes what it references, or changes a header setting other than its + `description`, `columns`, `dynamic_columns`, `model_schema`, `audits`, `contract`, or a switch + between `view` and `table` materialization. +- A custom rule reads something the edit changed, such as a model's contract or materialization. +- The stored compile reported errors or warnings other than rule findings in model or SQL test + files the edit did not change. +- The stored compile used the other output format (`--json` or text), or has no project image. + +A few edits are found to need a full compile only part-way through, for example when a macro's +arguments change which models or sources it reads, or when the edit changes the shape of so many +models that a full compile is cheaper. + +Otherwise, an edit runs the whole compile again, and a few caches under `target/cache/` shorten +it: every model, test, and audit is rendered again, and contracts and semantic validation run for +the whole project. Every model's SQL references are scanned again too. - An unchanged model takes its column analysis from `model-analyses.bin`. Entries are keyed by the model's query and settings, the inferred columns, types and nullability of every relation @@ -137,8 +159,9 @@ caches under `target/cache/` shorten the rest: - Macro call results are reused as described in [Macros must be deterministic](/docs/concepts/macros#macros-must-be-deterministic). -An edit therefore costs a full render of the project plus the analysis, test scans, and rules of -what changed. The output is byte-identical to `sqb compile --no-cache`. +Either way, the output is byte-identical to `sqb compile --no-cache`. To turn compiling only the +changed files off, set `delta_compile = false` under `[settings]` or set +`SQLBUILD_DISABLE_DELTA_COMPILE=1`; compiles then no longer store a project image. Known limits: diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 19e4b3324f..d2badeca2e 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -629,6 +629,7 @@ default_audit_run_scope = "delta_and_final" unit_test_max_real_models = 10 test_timeout = "30s" scenario_timeout = "15m" +delta_compile = true ``` | Field | Default | Description | @@ -646,6 +647,7 @@ scenario_timeout = "15m" | `unit_test_max_real_models` | `10` | Most real (unmocked) models one SQL unit test may execute; a larger test fails compilation. See [Unit test and scenario limits](#unit-test-and-scenario-limits). | | `test_timeout` | `30s` | Time limit for each SQL unit test, or `"disabled"` | | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | +| `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](/docs/cli/compile#compiling-after-an-edit). | ### Unit test and scenario limits