From 26a0cb4d7cd9ef7eefdc096eba43eebf71ef7655 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:04:05 +0100 Subject: [PATCH 1/5] perf(compile): reuse custom-rule results whose recorded fact reads are unchanged The isolated custom-rule host now records which compiler facts each rule invocation reads through its context. Results are cached per rule and subject and reused only while every recorded fact digest, the rule implementation and import closure, the rules configuration, the installed code identity, and the dialect are unchanged. Rules whose reachable module state changes between invocations, or that touch private view state, are treated as reading every fact. Custom rules no longer serialise the whole compiled project for a cache key: the project is pickled only when a host run is needed, and built-in rules key their project-level identity on request facts instead of walking project files. Refs CHI-443, CHI-444 --- .../src/bindings/_helpers/functions.rs | 7 + crates/sqlbuild-rules-native/src/constants.rs | 6 - .../src/engine/_helpers/cache.rs | 45 -- .../src/engine/_helpers/custom_host.rs | 29 + .../src/engine/_helpers/evaluation.rs | 518 ++--------------- .../src/engine/_helpers/mod.rs | 1 + .../src/engine/main/custom_host.rs | 3 + .../src/engine/main/mod.rs | 1 + .../src/engine/tests/helpers.rs | 1 - crates/sqlbuild-rules-native/src/models.rs | 8 +- src/sqlbuild/_native.pyi | 1 + .../compiler/fact_cache/main/code_identity.py | 14 + .../_helpers/engine/custom_rule_evidence.py | 35 -- .../_helpers/engine/custom_rules.py | 523 ++++++++++++++++++ .../_helpers/engine/fact_replay.py | 179 ++++++ .../rule_engine/_helpers/engine/native.py | 355 +++--------- .../_helpers/host/custom_evaluation.py | 260 ++++++--- .../rule_engine/_helpers/host/custom_host.py | 66 ++- .../rule_engine/_helpers/host/module_state.py | 218 ++++++++ .../_helpers/host/tracked_facts.py | 258 +++++++++ .../rule_engine/_helpers/run/rules.py | 2 - .../rule_engine/classes/fact_digests.py | 47 ++ .../rule_engine/classes/fact_reads.py | 50 ++ .../rule_engine/classes/project_tree.py | 8 +- .../rule_engine/classes/rule_context.py | 9 + src/sqlbuild/rule_engine/constants.py | 39 +- src/sqlbuild/rule_engine/exceptions.py | 8 + src/sqlbuild/rule_engine/models.py | 28 + src/sqlbuild/rule_engine/types.py | 2 + .../cli/commands/main/rules/_test_types.py | 11 + .../cli/commands/main/rules/helpers.py | 145 +++++ .../rules/test_rules_cache_invalidation.py | 140 +++++ .../sqlbuild/cli/commands/main/test_rules.py | 52 +- .../_helpers/engine/_test_types.py | 20 +- .../rule_engine/_helpers/engine/helpers.py | 40 +- .../_helpers/engine/test_custom_rule_reads.py | 122 ++++ .../_helpers/engine/test_custom_rules.py | 12 +- .../_helpers/engine/test_native_facts.py | 53 -- 38 files changed, 2275 insertions(+), 1041 deletions(-) create mode 100644 crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs create mode 100644 crates/sqlbuild-rules-native/src/engine/main/custom_host.rs create mode 100644 src/sqlbuild/compiler/fact_cache/main/code_identity.py create mode 100644 src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py create mode 100644 src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py create mode 100644 src/sqlbuild/rule_engine/_helpers/host/module_state.py create mode 100644 src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py create mode 100644 src/sqlbuild/rule_engine/classes/fact_digests.py create mode 100644 src/sqlbuild/rule_engine/classes/fact_reads.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py create mode 100644 tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 696e9578db..fdc5c9b548 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -73,6 +73,12 @@ fn evaluate_json(py: Python<'_>, request_json: &str) -> PyResult { .map_err(value_error) } +#[pyfunction] +fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { + py.compiler_detach(|| crate::engine::main::custom_host::run_custom_host_json(spec_json)) + .map_err(value_error) +} + #[pyfunction] fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| crate::sql_lint::main::engine::lint_json(request_json)) @@ -422,6 +428,7 @@ fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(evaluate_json, module)?)?; module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?; + module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; module.add( diff --git a/crates/sqlbuild-rules-native/src/constants.rs b/crates/sqlbuild-rules-native/src/constants.rs index 1d2d0a3c31..6e4aec2a8e 100644 --- a/crates/sqlbuild-rules-native/src/constants.rs +++ b/crates/sqlbuild-rules-native/src/constants.rs @@ -13,7 +13,6 @@ pub(crate) const GENERIC_BASIC_NAME: &str = "basic"; pub(crate) const GENERIC_SCENARIO_NAME: &str = "scenario"; pub(crate) const GENERIC_CASE_NAME: &str = "case"; pub(crate) const CUSTOM_RULE_NAMESPACE: &str = "XSQBR"; -pub(crate) const TARGET_DIRECTORY: &str = "target"; pub(crate) const REFERENCE_KIND: &str = "ref"; pub(crate) const MODEL_SCHEMA_CONFIG_KEY: &str = "schema"; pub(crate) const STAGING_LAYER_DIRECTORY: &str = "staging"; @@ -33,11 +32,6 @@ pub(crate) const NEGATION_OPERATOR: &str = "-"; pub(crate) const DECLARATION_DOMAIN_COMPONENTS: usize = 3; pub(crate) const DIRECT_DEPENDENCY_PATH_LENGTH: usize = 2; pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3; -pub(crate) const CUSTOM_HOST_REQUIRED_ERROR: &str = "selected custom rules require a custom host"; -pub(crate) const LOGS_DIRECTORY: &str = "logs"; -pub(crate) const GIT_DIRECTORY: &str = ".git"; -pub(crate) const RULES_DIRECTORY: &str = "rules"; -pub(crate) const PYTHON_EXTENSION: &str = "py"; pub(crate) const SQL_WILDCARD: &str = "*"; pub(crate) const UNKNOWN_SQL_TYPE: &str = "UNKNOWN"; pub(crate) const VARCHAR_SQL_TYPE: &str = "VARCHAR"; diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs index 07585624e0..ed1749bf96 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs @@ -1,6 +1,4 @@ use crate::models::Fault; -use fensu_policy::lifecycle::models::CacheRead; -use fensu_policy::{read_cache, write_cache}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -28,36 +26,6 @@ impl Cache { }) } - pub(crate) fn get(&self, path: &str, fingerprint: &str) -> Result>, String> { - match read_cache::>(&self.root, path, fingerprint) - .map_err(|error| format!("could not read rules cache: {error}"))? - { - CacheRead::Hit(faults) => Ok(Some(faults)), - CacheRead::Miss | CacheRead::Invalidated => Ok(None), - } - } - - pub(crate) fn put( - &self, - path: &str, - fingerprint: &str, - faults: &[Fault], - ) -> Result<(), String> { - write_cache(&self.root, path, fingerprint, &faults) - .map_err(|error| format!("could not write rules cache: {error}")) - } - - pub(crate) fn custom_rule_bucket( - &self, - code: &str, - rule_identity: &str, - ) -> Result { - read_bucket( - &self.root.join("bulk").join(format!("custom-{code}.json")), - rule_identity, - ) - } - pub(crate) fn native_rules_bucket( &self, ruleset_identity: &str, @@ -76,19 +44,6 @@ impl Cache { bucket, ) } - - pub(crate) fn put_custom_rule_bucket( - &self, - code: &str, - rule_identity: &str, - bucket: &RuleCacheBucket, - ) -> Result<(), String> { - write_bucket( - &self.root.join("bulk").join(format!("custom-{code}.json")), - rule_identity, - bucket, - ) - } } fn read_bucket(path: &Path, fingerprint: &str) -> Result { diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs new file mode 100644 index 0000000000..f46a484e0e --- /dev/null +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs @@ -0,0 +1,29 @@ +use crate::models::CustomHostSpec; +use fensu_policy::lifecycle::models::{ + CustomHostInvocation, CustomHostOutputLimits, CustomHostRequest, +}; +use fensu_policy::run_custom_host; +use std::path::Path; +use std::time::Duration; + +pub(crate) fn run_custom_host_json(spec_json: &str) -> Result { + let spec: CustomHostSpec = serde_json::from_str(spec_json) + .map_err(|error| format!("invalid custom host request: {error}"))?; + let request = CustomHostRequest { + protocol: fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION, + runtime_version: spec.runtime_version.clone(), + payload: spec.payload, + }; + let response = run_custom_host::<_, serde_json::Value>(CustomHostInvocation { + program: Path::new(&spec.program), + arguments: &spec.arguments, + timeout: Duration::from_millis(spec.timeout_millis), + output_limits: CustomHostOutputLimits::default(), + request: &request, + }) + .map_err(|error| format!("invalid rules lifecycle: {error}"))?; + let payload = response.payload.ok_or_else(|| { + "custom rule host returned no payload after successful validation".to_owned() + })?; + serde_json::to_string(&payload).map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs index 4fad5b6fc6..746de6cda6 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs @@ -1,9 +1,6 @@ use crate::configuration::main::validate as config; -use crate::constants::{ - API_VERSION, CUSTOM_HOST_REQUIRED_ERROR, GIT_DIRECTORY, LOGS_DIRECTORY, NATIVE_BUILD_IDENTITY, - PYTHON_EXTENSION, RULES_DIRECTORY, TARGET_DIRECTORY, -}; -use crate::engine::_helpers::cache::{Cache, RuleCacheBucket, RuleCacheEntry}; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; +use crate::engine::_helpers::cache::{Cache, RuleCacheEntry}; use crate::models::{ EvaluateRequest, EvaluateResponse, Fault, Model, RuleMetadata, RulesCodeGrammar, }; @@ -17,21 +14,21 @@ use crate::rules::models::{ use fensu_policy::lifecycle::constants::ANALYSIS_BATCH_SCHEMA_VERSION; use fensu_policy::lifecycle::errors::LifecycleError; use fensu_policy::lifecycle::models::{ - AnalysisBatchRequest, AnalysisInput, ApplySuppressionsRequest, CustomHostInvocation, - CustomHostOutputLimits, CustomHostRequest, ExactSuppression, Finding, FindingSeverity, - RuntimeIdentity, ScopedIgnore, + AnalysisBatchRequest, AnalysisInput, ApplySuppressionsRequest, ExactSuppression, Finding, + FindingSeverity, RuntimeIdentity, ScopedIgnore, }; -use fensu_policy::{apply_suppressions, evaluate_batch, run_custom_host}; +use fensu_policy::{apply_suppressions, evaluate_batch}; use rayon::iter::{ IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, }; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant}; +use std::path::Path; +use std::time::Instant; const NATIVE_RULE_WORKERS: usize = 4; const NATIVE_RULE_STACK_BYTES: usize = 16 * 1024 * 1024; +const COMPILER_FACTS_FINGERPRINT_SEED: &str = "compiler-owned-rules-facts-v2"; struct PendingModelRule<'a> { index: usize, @@ -225,103 +222,51 @@ pub(crate) fn evaluate_json(request_json: &str) -> Result { cache_hits: 0, cache_misses: 0, ruleset_fingerprint, + selected_codes: Vec::new(), built_in_ms: 0, - custom_ms: 0, }) .map_err(|error| error.to_string()); } - let custom_selected = selected.iter().any(|rule| rule.custom); let cache_enabled = request.config.cache.enabled; let project_aware = selected.iter().any(|rule| rule.project_wide || rule.custom); - let project_fingerprint = if project_aware { - Some(if custom_selected { - match &request.project_fingerprint { - Some(value) => fingerprint_request_facts(value.as_bytes(), &request)?, - None => fingerprint_project(Path::new(&request.project_dir), &request)?, - } - } else { - fingerprint_request_facts(b"compiler-owned-rules-facts-v1", &request)? + let project_fingerprint = project_aware + .then(|| { + fingerprint_request_facts( + request + .project_fingerprint + .as_deref() + .unwrap_or(COMPILER_FACTS_FINGERPRINT_SEED) + .as_bytes(), + &request, + ) }) - } else { - None - }; + .transpose()?; let cache = cache_enabled .then(|| Cache::open(Path::new(&request.project_dir))) .transpose()?; - let (cached_custom, custom_probe_ms) = probe_custom_rules_cache(CustomRulesCacheRequest { - request: &request, - selected: &selected_by_code, + let built_in_started = Instant::now(); + let mut raw_faults = request.initial_findings.clone(); + raw_faults.extend(evaluate_project::evaluate_project( + ProjectEvaluationRequest { + selected: &selected_by_code, + request: &request, + }, + )?); + let native = evaluate_native_models(NativeModelRuleRequest { + context: ModelRuleBatchContext { + request: &request, + selected: &selected_by_code, + threshold_overrides: &threshold_overrides, + }, cache: cache.as_ref(), + ruleset_fingerprint: &ruleset_fingerprint, project_fingerprint: project_fingerprint.as_deref(), })?; - let (raw_faults, cache_hits, cache_misses, built_in_ms, custom_ms) = - std::thread::scope(|scope| { - let custom_parallel_started = Instant::now(); - let custom_task = cached_custom.is_none().then(|| { - scope.spawn(|| { - evaluate_custom_rules_cached(CustomRulesCacheRequest { - request: &request, - selected: &selected_by_code, - cache: cache.as_ref(), - project_fingerprint: project_fingerprint.as_deref(), - }) - }) - }); - let built_in_started = Instant::now(); - let mut raw_faults = request.initial_findings.clone(); - raw_faults.extend(evaluate_project::evaluate_project( - ProjectEvaluationRequest { - selected: &selected_by_code, - request: &request, - }, - )?); - let native = evaluate_native_models(NativeModelRuleRequest { - context: ModelRuleBatchContext { - request: &request, - selected: &selected_by_code, - threshold_overrides: &threshold_overrides, - }, - cache: cache.as_ref(), - ruleset_fingerprint: &ruleset_fingerprint, - project_fingerprint: project_fingerprint.as_deref(), - })?; - let mut cache_hits = native.hits; - let mut cache_misses = native.misses; - raw_faults.extend(native.faults); - let built_in_ms = built_in_started.elapsed().as_millis() as u64; - let custom_started = if custom_task.is_some() { - custom_parallel_started - } else { - Instant::now() - }; - let custom_result = if let Some(task) = custom_task { - task.join() - .map_err(|_| "custom rule evaluation worker panicked".to_string())? - } else { - match cached_custom { - Some(custom) => Ok(custom), - None => evaluate_custom_rules_cached(CustomRulesCacheRequest { - request: &request, - selected: &selected_by_code, - cache: cache.as_ref(), - project_fingerprint: project_fingerprint.as_deref(), - }), - } - }; - let custom = match custom_result { - Ok(value) => value, - Err(error) if error == CUSTOM_HOST_REQUIRED_ERROR => { - return Err(format!("{error} [native_cache_misses={cache_misses}]")); - } - Err(error) => return Err(error), - }; - cache_hits += custom.hits; - cache_misses += custom.misses; - raw_faults.extend(custom.faults); - let custom_ms = custom_probe_ms + custom_started.elapsed().as_millis() as u64; - Ok::<_, String>((raw_faults, cache_hits, cache_misses, built_in_ms, custom_ms)) - })?; + let cache_hits = native.hits; + let cache_misses = native.misses; + raw_faults.extend(native.faults); + let built_in_ms = built_in_started.elapsed().as_millis() as u64; validate_exception_paths(&request)?; let faults = if request.defer_suppressions { raw_faults @@ -335,263 +280,12 @@ pub(crate) fn evaluate_json(request_json: &str) -> Result { cache_hits, cache_misses, ruleset_fingerprint, + selected_codes: selected.iter().map(|rule| rule.code.clone()).collect(), built_in_ms, - custom_ms, }) .map_err(|error| error.to_string()) } -struct CustomRulesCacheRequest<'a> { - request: &'a EvaluateRequest, - selected: &'a BTreeMap, - cache: Option<&'a Cache>, - project_fingerprint: Option<&'a str>, -} - -struct CustomEvaluation { - faults: Vec, - hits: usize, - misses: usize, -} - -fn probe_custom_rules_cache( - request: CustomRulesCacheRequest<'_>, -) -> Result<(Option, u64), String> { - let started = Instant::now(); - let cached = if request.request.custom_host.is_none() { - Some(evaluate_custom_rules_cached(request)?) - } else { - None - }; - Ok((cached, started.elapsed().as_millis() as u64)) -} - -type LocalRuleMiss<'a> = ( - &'a RuleMetadata, - String, - RuleCacheBucket, - BTreeMap, -); - -fn count_custom_rules(selected: &BTreeMap) -> usize { - let mut count = 0; - for rule in selected.values() { - if rule.custom { - count += 1; - } - } - count -} - -fn evaluate_custom_rules_cached( - request: CustomRulesCacheRequest<'_>, -) -> Result { - let CustomRulesCacheRequest { - request, - selected, - cache, - project_fingerprint, - } = request; - let custom_selected = selected.values().any(|rule| rule.custom); - if !custom_selected { - return Ok(CustomEvaluation { - faults: Vec::new(), - hits: 0, - misses: 0, - }); - } - let Some(cache) = cache else { - return evaluate_custom_rules(request, selected, None).map(|faults| CustomEvaluation { - faults, - hits: 0, - misses: count_custom_rules(selected), - }); - }; - let mut faults: Vec = Vec::new(); - let mut hits = 0; - let mut misses_count = 0; - let mut project_misses: BTreeMap = BTreeMap::new(); - let mut local_misses: BTreeMap> = BTreeMap::new(); - let mut selected_misses: BTreeMap = BTreeMap::new(); - let mut selected_model_paths: BTreeSet = BTreeSet::new(); - let model_fact_identities = custom_model_fact_identities(&request.models)?; - for (code, rule) in selected - .iter() - .filter(|(_, rule)| rule.custom && rule.project_wide) - { - let rule_identity = custom_rule_identity(rule, request)?; - let identity = custom_project_identity(&rule_identity, project_fingerprint); - let cache_path = format!("__custom_rules__/{code}/project"); - if let Some(cached) = cache.get(&cache_path, &identity)? { - faults.extend(cached); - hits += 1; - } else { - project_misses.insert(code.clone(), (*rule, identity)); - selected_misses.insert(code.clone(), *rule); - misses_count += 1; - } - } - for (code, rule) in selected - .iter() - .filter(|(_, rule)| rule.custom && !rule.project_wide) - { - let rule_identity = custom_rule_identity(rule, request)?; - let project_dependent = request - .custom_rules - .iter() - .find(|custom| custom.code == *code) - .is_some_and(|custom| custom.project_dependent); - let bucket = cache.custom_rule_bucket(code, &rule_identity)?; - let mut misses: BTreeMap = BTreeMap::new(); - for model in &request.models { - let model_identity = model_fact_identities - .get(&model.relative_path) - .ok_or_else(|| format!("missing custom rule model identity for {}", model.name))?; - let identity = custom_local_identity( - model_identity, - &rule_identity, - project_dependent.then_some(project_fingerprint).flatten(), - ); - if let Some(entry) = bucket - .entries - .get(&model.relative_path) - .filter(|entry| entry.identity == identity) - { - faults.extend(entry.faults.clone()); - hits += 1; - } else { - misses.insert(model.relative_path.clone(), identity); - selected_model_paths.insert(model.relative_path.clone()); - misses_count += 1; - } - } - if !misses.is_empty() { - selected_misses.insert(code.clone(), *rule); - local_misses.insert(code.clone(), (*rule, rule_identity, bucket, misses)); - } - } - if selected_misses.is_empty() { - return Ok(CustomEvaluation { - faults, - hits, - misses: misses_count, - }); - } - let selected_paths = (!selected_model_paths.is_empty()).then_some(&selected_model_paths); - let evaluated = evaluate_custom_rules(request, &selected_misses, selected_paths)?; - for (code, (_, _, _, _)) in &local_misses { - if let Some(fault) = evaluated - .iter() - .find(|fault| fault.code == *code && !selected_model_paths.contains(&fault.path)) - { - return Err(format!( - "model-subject custom rule returned a finding for unrelated path {}", - fault.path - )); - } - } - for (code, (_, rule_identity, mut bucket, misses)) in local_misses { - for (path, identity) in misses { - let model_faults: Vec = evaluated - .iter() - .filter(|fault| fault.code == code && fault.path == path) - .cloned() - .collect(); - let _ = bucket.entries.insert( - path, - RuleCacheEntry { - identity, - faults: model_faults.clone(), - }, - ); - faults.extend(model_faults); - } - cache.put_custom_rule_bucket(&code, &rule_identity, &bucket)?; - } - for (code, (_, identity)) in project_misses { - let project_faults: Vec = evaluated - .iter() - .filter(|fault| fault.code == code) - .cloned() - .collect(); - cache.put( - &format!("__custom_rules__/{code}/project"), - &identity, - &project_faults, - )?; - faults.extend(project_faults); - } - Ok(CustomEvaluation { - faults, - hits, - misses: misses_count, - }) -} - -fn custom_project_identity(rule: &str, project: Option<&str>) -> String { - let mut digest = Sha256::new(); - digest.update(b"custom-rule-project-v3"); - digest.update(rule.as_bytes()); - digest.update(project.unwrap_or("no-project-fingerprint").as_bytes()); - format!("{:x}", digest.finalize()) -} - -fn custom_local_identity(model_identity: &str, rule: &str, project: Option<&str>) -> String { - let mut digest = Sha256::new(); - digest.update(b"custom-rule-model-v3"); - digest.update(model_identity.as_bytes()); - digest.update(rule.as_bytes()); - digest.update(project.unwrap_or("model-local").as_bytes()); - format!("{:x}", digest.finalize()) -} - -fn custom_model_fact_identities( - models: &[crate::models::Model], -) -> Result, String> { - let identity = |model: &crate::models::Model| -> Result<(String, String), String> { - let mut digest = Sha256::new(); - digest.update(b"custom-rule-model-facts-v1"); - digest.update(ANALYSIS_BATCH_SCHEMA_VERSION.to_le_bytes()); - digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); - digest.update(serde_json::to_vec(model).map_err(|error| error.to_string())?); - Ok(( - model.relative_path.clone(), - format!("{:x}", digest.finalize()), - )) - }; - let results: Vec> = if models.len() <= 1 { - models.iter().map(identity).collect() - } else { - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(NATIVE_RULE_WORKERS.min(models.len())) - .stack_size(NATIVE_RULE_STACK_BYTES) - .build() - .map_err(|error| error.to_string())?; - pool.install(|| models.par_iter().map(identity).collect()) - }; - results.into_iter().collect() -} - -fn custom_rule_identity(rule: &RuleMetadata, request: &EvaluateRequest) -> Result { - let mut digest = Sha256::new(); - digest.update(b"custom-rule-identity-v1"); - digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); - digest.update(serde_json::to_vec(rule).map_err(|error| error.to_string())?); - digest.update( - serde_json::to_vec(&request.config.rule_options.get(&rule.code)) - .map_err(|error| error.to_string())?, - ); - digest.update(request.dialect.as_bytes()); - if let Some(custom) = request - .custom_rules - .iter() - .find(|custom| custom.code == rule.code) - { - digest.update(custom.fact_fingerprint.as_bytes()); - } - Ok(format!("{:x}", digest.finalize())) -} - fn model_cache_identities( models: &[&crate::models::Model], ruleset: &str, @@ -639,23 +333,6 @@ fn model_cache_identity( .map_err(lifecycle_error) } -fn fingerprint_project(project_dir: &Path, request: &EvaluateRequest) -> Result { - let mut files = collect_rule_input_files(project_dir, project_dir)?; - files.sort(); - let mut digest = Sha256::new(); - for path in files { - let relative = path.strip_prefix(project_dir).unwrap_or(&path); - digest.update(relative.to_string_lossy().as_bytes()); - digest.update(std::fs::read(&path).map_err(|error| { - format!( - "could not fingerprint rules project file {}: {error}", - path.display() - ) - })?); - } - fingerprint_request_facts(&digest.finalize(), request) -} - fn fingerprint_request_facts(seed: &[u8], request: &EvaluateRequest) -> Result { let mut digest = Sha256::new(); digest.update(seed); @@ -666,48 +343,14 @@ fn fingerprint_request_facts(seed: &[u8], request: &EvaluateRequest) -> Result Result, String> { - let mut output: Vec = Vec::new(); - for entry in std::fs::read_dir(directory) - .map_err(|error| format!("could not read rules project directory: {error}"))? - { - let entry = - entry.map_err(|error| format!("could not read rules project entry: {error}"))?; - let path = entry.path(); - if path.is_dir() { - if path.file_name().is_some_and(|name| { - name == TARGET_DIRECTORY || name == LOGS_DIRECTORY || name == GIT_DIRECTORY - }) { - continue; - } - output.extend(collect_rule_input_files(root, &path)?); - continue; - } - let supported = path - .extension() - .and_then(|value| value.to_str()) - .is_some_and(|value| { - matches!(value, PYTHON_EXTENSION | "sql" | "toml" | "yaml" | "yml") - }); - let is_rule_python = path.strip_prefix(root).is_ok_and(|relative| { - relative.starts_with(RULES_DIRECTORY) - && path - .extension() - .is_some_and(|value| value == PYTHON_EXTENSION) - }); - if supported && !is_rule_python && path.starts_with(root) { - output.push(path); - } - } - Ok(output) -} - fn validate_suppression_codes( request: &EvaluateRequest, catalogue: &[RuleMetadata], @@ -736,83 +379,6 @@ fn validate_suppression_codes( Ok(()) } -fn validate_custom_faults( - faults: &[Fault], - selected: &BTreeMap, -) -> Result<(), String> { - for fault in faults { - if !selected.get(&fault.code).is_some_and(|rule| rule.custom) { - return Err(format!( - "custom rule host returned a finding for unselected rule {}", - fault.code - )); - } - } - Ok(()) -} - -fn evaluate_custom_rules( - request: &EvaluateRequest, - selected: &BTreeMap, - model_paths: Option<&BTreeSet>, -) -> Result, String> { - let custom_selected = selected.values().any(|rule| rule.custom); - let Some(spec) = &request.custom_host else { - return if custom_selected { - Err("selected custom rules require a custom host".to_owned()) - } else { - Ok(Vec::new()) - }; - }; - if !custom_selected { - return Ok(Vec::new()); - } - let mut payload = spec.payload.clone(); - let payload_object = payload - .as_object_mut() - .ok_or_else(|| "custom rule host payload must be an object".to_owned())?; - payload_object.insert( - "selected_codes".to_owned(), - serde_json::Value::Array( - selected - .values() - .filter(|rule| rule.custom) - .map(|rule| serde_json::Value::String(rule.code.clone())) - .collect(), - ), - ); - payload_object.insert( - "selected_model_paths".to_owned(), - model_paths.map_or(serde_json::Value::Null, |paths| { - serde_json::Value::Array( - paths - .iter() - .cloned() - .map(serde_json::Value::String) - .collect(), - ) - }), - ); - let host_request = CustomHostRequest { - protocol: fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION, - runtime_version: spec.runtime_version.clone(), - payload, - }; - let response = run_custom_host::<_, Vec>(CustomHostInvocation { - program: Path::new(&spec.program), - arguments: &spec.arguments, - timeout: Duration::from_millis(spec.timeout_millis), - output_limits: CustomHostOutputLimits::default(), - request: &host_request, - }) - .map_err(lifecycle_error)?; - let faults = response.payload.ok_or_else(|| { - "custom rule host returned no finding payload after successful validation".to_owned() - })?; - validate_custom_faults(&faults, selected)?; - Ok(faults) -} - fn validate_exception_paths(request: &EvaluateRequest) -> Result<(), String> { let root = Path::new(&request.project_dir); for entry in &request.config.rule_exceptions { diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs index e45aa12145..d6c96b34b0 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod cache; +pub(crate) mod custom_host; pub(crate) mod evaluation; diff --git a/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs b/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs new file mode 100644 index 0000000000..b59958c6dd --- /dev/null +++ b/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs @@ -0,0 +1,3 @@ +pub(crate) fn run_custom_host_json(spec_json: &str) -> Result { + crate::engine::_helpers::custom_host::run_custom_host_json(spec_json) +} diff --git a/crates/sqlbuild-rules-native/src/engine/main/mod.rs b/crates/sqlbuild-rules-native/src/engine/main/mod.rs index 0dea504272..ae4f2a1b3e 100644 --- a/crates/sqlbuild-rules-native/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/engine/main/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod custom_host; pub(crate) mod evaluate; pub(crate) mod finalize; diff --git a/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs b/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs index 62d143eb92..ee51aec85c 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs @@ -212,7 +212,6 @@ pub(crate) fn request_with_scope(scope: Value) -> Value { "public_enums": [], "public_constants": [], "custom_rules": [], - "custom_host": null, "project_fingerprint": null, "scope_index": scope }) diff --git a/crates/sqlbuild-rules-native/src/models.rs b/crates/sqlbuild-rules-native/src/models.rs index b537e9d86d..e20ba27d73 100644 --- a/crates/sqlbuild-rules-native/src/models.rs +++ b/crates/sqlbuild-rules-native/src/models.rs @@ -691,10 +691,6 @@ pub(crate) struct CustomRule { #[serde(default)] pub project_wide: bool, #[serde(default)] - pub project_dependent: bool, - #[serde(default)] - pub fact_fingerprint: String, - #[serde(default)] pub check_name: String, #[serde(default)] pub test_case_count: u32, @@ -729,7 +725,6 @@ pub(crate) struct EvaluateRequest { pub initial_findings: Vec, pub defer_suppressions: bool, pub custom_rules: Vec, - pub custom_host: Option, pub project_fingerprint: Option, } @@ -767,7 +762,6 @@ impl Default for EvaluateRequest { initial_findings: vec![], defer_suppressions: false, custom_rules: vec![], - custom_host: None, project_fingerprint: None, } } @@ -791,8 +785,8 @@ pub(crate) struct EvaluateResponse { pub cache_hits: usize, pub cache_misses: usize, pub ruleset_fingerprint: String, + pub selected_codes: Vec, pub built_in_ms: u64, - pub custom_ms: u64, } #[derive(Clone, Debug, Serialize)] diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 118f3d2f5d..5c54c88e00 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -14,6 +14,7 @@ def analyze_column_references_json(request_json: str) -> str: ... def catalogue_json() -> str: ... def evaluate_json(request_json: str) -> str: ... def finalize_rule_findings_json(request_json: str) -> str: ... +def run_custom_host_json(spec_json: str) -> str: ... def format_sql_json(request_json: str) -> str: ... def format_sql_batch_json(request_json: str) -> str: ... def query_fingerprint(sql: str, dialect: str) -> str: ... diff --git a/src/sqlbuild/compiler/fact_cache/main/code_identity.py b/src/sqlbuild/compiler/fact_cache/main/code_identity.py new file mode 100644 index 0000000000..afdc7f4ea1 --- /dev/null +++ b/src/sqlbuild/compiler/fact_cache/main/code_identity.py @@ -0,0 +1,14 @@ +"""Public identity of the installed compiler code for derived caches.""" + +import hashlib + +import sqlbuild._native as _native +from sqlbuild.compiler.fact_cache._helpers.code_identity import installed_code_identity + + +def compiled_code_identity() -> str: + """Return the digest that changes with every released, locally edited, or rebuilt version.""" + + return hashlib.sha256( + f"{installed_code_identity()}\0{_native.BUILD_IDENTITY}".encode() + ).hexdigest() diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py index d049e0ee9d..e91ac52abc 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py @@ -19,9 +19,6 @@ _PARAMETRIZE_ATTRIBUTE: str = "parametrize" _PYTEST_MARK_ATTRIBUTE: str = "mark" _PYTEST_MODULE_NAME: str = "pytest" -_PROJECT_FACT_ATTRIBUTES: frozenset[str] = frozenset( - {"audits", "declarations", "graph", "project", "tests"} -) @dataclass(frozen=True, order=True) @@ -92,38 +89,6 @@ def _fingerprint_label(*, path: Path, root: Path) -> str: return path.relative_to(root).as_posix() if path.is_relative_to(root) else path.as_posix() -def custom_rule_project_fact_attributes( - *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] | None = None -) -> frozenset[str]: - """Return project-wide context attributes used by implementation or helper code.""" - - if rule.project_wide: - return frozenset({"project"}) - if rule.source is None: - sources: tuple[str, ...] = (inspect.getsource(rule.check),) - else: - helper_paths: tuple[Path, ...] = ( - _import_closure( - source_path=Path(rule.source).resolve(), project_dir=project_dir.resolve() - ) - if import_closure is None - else import_closure - ) - sources = ( - inspect.getsource(rule.check), - *(path.read_text(encoding="utf-8") for path in helper_paths), - ) - attributes: set[str] = set() - for source in sources: - tree: ast.Module = ast.parse(source) - attributes.update( - node.attr - for node in ast.walk(tree) - if isinstance(node, ast.Attribute) and node.attr in _PROJECT_FACT_ATTRIBUTES - ) - return frozenset(attributes) - - def _import_closure(*, source_path: Path, project_dir: Path) -> tuple[Path, ...]: rules_root: Path = (project_dir / _RULES_DIRECTORY_NAME).resolve() pending: list[Path] = [source_path] diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py new file mode 100644 index 0000000000..80c170899d --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -0,0 +1,523 @@ +"""Incremental custom-rule evaluation keyed on the compiler facts each invocation read.""" + +from __future__ import annotations + +import hashlib +import os +import pickle +import sys +import tempfile +import time +from collections.abc import Callable +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Any, cast + +import orjson + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( + custom_rule_implementation_fingerprint, + custom_rule_import_closure, +) +from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys +from sqlbuild.rule_engine.classes.fact_digests import FactDigests +from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views +from sqlbuild.rule_engine.constants import ( + CUSTOM_HOST_LAUNCH_MODULE, + CUSTOM_HOST_RUNTIME_VERSION, + CUSTOM_RULES_CACHE_FILE, + CUSTOM_RULES_CACHE_VERSION, +) +from sqlbuild.rule_engine.exceptions import FactDigestError, RulesError +from sqlbuild.rule_engine.models import CustomRulesOutcome, Rule, RulesConfig +from sqlbuild.rule_engine.types import FactKey, RuleSubject + +_HOST_TIMEOUT_MILLIS: int = 120_000 +_PROJECT_SUBJECT: str = "" + + +@dataclass(frozen=True) +class _Subject: + key: str + identity: str + + +@dataclass(frozen=True) +class _Evaluated: + findings: tuple[dict[str, object], ...] + reads: tuple[FactKey, ...] | None + + +@dataclass(frozen=True) +class _CachedEntry: + subject_identity: str + readset: int + reads_digest: str + findings: tuple[dict[str, object], ...] + + +@dataclass +class _RuleCache: + identity: str + readsets: list[tuple[FactKey, ...]] + entries: dict[str, _CachedEntry] + + +def evaluate_custom_rules_cached( + *, + project: CompiledProject, + config: RulesConfig, + project_dir: Path, + rules: tuple[Rule, ...], + dialect: str, + verify_determinism: bool = False, +) -> CustomRulesOutcome: + """Reuse results whose recorded fact reads are unchanged and evaluate only the rest.""" + + started: float = time.monotonic() + custom: tuple[Rule, ...] = tuple(sorted(rules, key=lambda rule: rule.code)) + if not custom: + return CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) + if not config.cache.enabled: + evaluated: dict[tuple[str, str], _Evaluated] = _run_host( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + plan={rule.code: None for rule in custom}, + track_reads=False, + verify_determinism=verify_determinism, + ) + return CustomRulesOutcome( + findings=_ordered_findings( + rules=custom, + subjects=_subjects(project=project, rules=custom), + results={key: value.findings for key, value in evaluated.items()}, + ), + cache_hits=0, + cache_misses=len(custom), + custom_ms=_elapsed_ms(started), + ) + digests: FactDigests = FactDigests( + views=_views_factory(project=project, project_dir=project_dir, dialect=dialect) + ) + subjects: dict[str, tuple[_Subject, ...]] = _subjects(project=project, rules=custom) + identities: dict[str, str] = { + rule.code: _rule_identity( + rule=rule, config=config, project_dir=project_dir, dialect=dialect + ) + for rule in custom + } + cache_path: Path = project_dir / CUSTOM_RULES_CACHE_FILE + stored: dict[str, _RuleCache] = _read_cache(cache_path) + results: dict[tuple[str, str], tuple[dict[str, object], ...]] = {} + plan: dict[str, list[str] | None] = {} + hits: int = 0 + for rule in custom: + cached: _RuleCache | None = stored.get(rule.code) + if cached is not None and cached.identity != identities[rule.code]: + cached = None + current_digests: dict[int, str | None] = {} + missing: list[str] = [] + for subject in subjects[rule.code]: + entry: _CachedEntry | None = None if cached is None else cached.entries.get(subject.key) + if ( + entry is not None + and cached is not None + and entry.subject_identity == subject.identity + and _readset_digest( + cached=cached, index=entry.readset, digests=digests, memo=current_digests + ) + == entry.reads_digest + ): + results[(rule.code, subject.key)] = entry.findings + hits += 1 + else: + missing.append(subject.key) + if missing: + plan[rule.code] = None if len(missing) == len(subjects[rule.code]) else sorted(missing) + misses: int = sum( + len(subjects[code]) if planned is None else len(planned) for code, planned in plan.items() + ) + if plan: + evaluated = _run_host( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + plan=plan, + track_reads=True, + verify_determinism=verify_determinism, + ) + _validate_paths(project=project, rules=custom, evaluated=evaluated) + stored = _store( + stored=stored, + identities=identities, + subjects=subjects, + evaluated=evaluated, + digests=digests, + ) + for key, value in evaluated.items(): + results[key] = value.findings + _write_cache(path=cache_path, stored=stored, subjects=subjects) + return CustomRulesOutcome( + findings=_ordered_findings( + rules=custom, + subjects=subjects, + results=results, + ), + cache_hits=hits, + cache_misses=misses, + custom_ms=_elapsed_ms(started), + ) + + +def _views_factory( + *, project: CompiledProject, project_dir: Path, dialect: str +) -> Callable[[], RuleFactViews]: + def build() -> RuleFactViews: + return build_rule_fact_views( + project=_host_project(project), project_dir=project_dir, dialect=dialect + ) + + return build + + +def _subjects( + *, project: CompiledProject, rules: tuple[Rule, ...] +) -> dict[str, tuple[_Subject, ...]]: + project_subject: tuple[_Subject, ...] = ( + _Subject( + key=_PROJECT_SUBJECT, + identity=_digest([len(project.models), project.effective_target_name]), + ), + ) + model_subjects: tuple[_Subject, ...] = tuple( + _Subject( + key=model.relative_path.as_posix(), + identity=_digest([model.name, model.relative_path.as_posix(), _materialization(model)]), + ) + for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) + ) + return { + rule.code: project_subject if rule.subject is RuleSubject.PROJECT else model_subjects + for rule in rules + } + + +def _materialization(model: Any) -> str | None: + value: object = model.config.values.get("materialized") + return value if isinstance(value, str) else None + + +def _rule_identity(*, rule: Rule, config: RulesConfig, project_dir: Path, dialect: str) -> str: + closure: tuple[Path, ...] = custom_rule_import_closure(rule=rule, project_dir=project_dir) + return _digest( + [ + CUSTOM_RULES_CACHE_VERSION, + CUSTOM_HOST_RUNTIME_VERSION, + compiled_code_identity(), + rule.code, + rule.family, + rule.slug, + rule.message, + rule.remediation, + None if rule.subject is None else rule.subject.value, + custom_rule_implementation_fingerprint( + rule=rule, project_dir=project_dir, import_closure=closure + ), + asdict(config), + dialect, + str(project_dir.resolve()), + ] + ) + + +def _readset_digest( + *, cached: _RuleCache, index: int, digests: FactDigests, memo: dict[int, str | None] +) -> str | None: + if index not in memo: + try: + memo[index] = ( + digests.combined(cached.readsets[index]) + if 0 <= index < len(cached.readsets) + else None + ) + except FactDigestError: + memo[index] = None + return memo[index] + + +def _run_host( + *, + project: CompiledProject, + config: RulesConfig, + project_dir: Path, + dialect: str, + plan: dict[str, list[str] | None], + track_reads: bool, + verify_determinism: bool, +) -> dict[tuple[str, str], _Evaluated]: + input_dir: Path = project_dir / "target" / "rules-cache" / "host-inputs" + input_dir.mkdir(parents=True, exist_ok=True) + input_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + dir=input_dir, prefix="project-", suffix=".pickle", delete=False + ) as handle: + input_path = Path(handle.name) + pickle.dump((_host_project(project), config), handle) + response_json: str = _native.run_custom_host_json( + orjson.dumps( + { + "program": sys.executable, + "arguments": ["-m", CUSTOM_HOST_LAUNCH_MODULE], + "timeout_millis": _HOST_TIMEOUT_MILLIS, + "runtime_version": CUSTOM_HOST_RUNTIME_VERSION, + "payload": { + "project_pickle_path": str(input_path.resolve()), + "project_dir": str(project_dir.resolve()), + "dialect": dialect, + "verify_determinism": verify_determinism, + "track_reads": track_reads, + "plan": plan, + }, + } + ).decode() + ) + except (ValueError, TypeError) as error: + raise RulesError(str(error)) from error + finally: + if input_path is not None: + input_path.unlink(missing_ok=True) + return _decode_host_response(orjson.loads(response_json)) + + +def _host_project(project: CompiledProject) -> CompiledProject: + return replace( + project, + binding_catalog=None, + sql_expansions={}, + loaded_macros={}, + loader_functions=(), + hook_functions=(), + materialization_files=(), + external_sql_reference_resolver=None, + ) + + +def _decode_host_response(response: object) -> dict[tuple[str, str], _Evaluated]: + if not isinstance(response, dict): + raise RulesError("custom rule host returned an invalid payload") + payload: dict[str, Any] = cast(dict[str, Any], response) + raw_readsets: Any = payload.get("readsets") + raw_evaluations: Any = payload.get("evaluations") + raw_untracked: Any = payload.get("untracked") + if ( + not isinstance(raw_readsets, list) + or not isinstance(raw_evaluations, list) + or not isinstance(raw_untracked, list) + ): + raise RulesError("custom rule host returned an invalid payload") + readsets: list[tuple[FactKey, ...]] = list(map(_decode_readset, raw_readsets)) + untracked: frozenset[str] = frozenset(str(code) for code in raw_untracked) + evaluated: dict[tuple[str, str], _Evaluated] = {} + for raw_item in raw_evaluations: + if not isinstance(raw_item, dict): + raise RulesError("custom rule host returned an invalid evaluation") + item: dict[str, Any] = cast(dict[str, Any], raw_item) + code: str = str(item["code"]) + subject: object = item.get("subject") + findings: Any = item.get("findings") + reads: object = item.get("reads") + if not isinstance(findings, list): + raise RulesError("custom rule host returned invalid findings") + evaluated[(code, _PROJECT_SUBJECT if subject is None else str(subject))] = _Evaluated( + findings=tuple(map(_finding_payload, findings)), + reads=(readsets[reads] if isinstance(reads, int) and code not in untracked else None), + ) + return evaluated + + +def _validate_paths( + *, + project: CompiledProject, + rules: tuple[Rule, ...], + evaluated: dict[tuple[str, str], _Evaluated], +) -> None: + codes: frozenset[str] = frozenset(rule.code for rule in rules) + model_rules: frozenset[str] = frozenset( + rule.code for rule in rules if rule.subject is RuleSubject.MODEL + ) + model_paths: frozenset[str] = frozenset( + model.relative_path.as_posix() for model in project.models + ) + for (code, _subject), result in evaluated.items(): + for finding in result.findings: + if finding.get("code") not in codes: + raise RulesError( + f"custom rule host returned a finding for unselected rule {finding.get('code')}" + ) + if code in model_rules and finding.get("path") not in model_paths: + raise RulesError( + "model-subject custom rule returned a finding for unrelated path " + f"{finding.get('path')}" + ) + + +def _store( + *, + stored: dict[str, _RuleCache], + identities: dict[str, str], + subjects: dict[str, tuple[_Subject, ...]], + evaluated: dict[tuple[str, str], _Evaluated], + digests: FactDigests, +) -> dict[str, _RuleCache]: + updated: dict[str, _RuleCache] = {} + full_keys: tuple[FactKey, ...] | None = None + for code, identity in identities.items(): + previous: _RuleCache | None = stored.get(code) + current: _RuleCache = ( + _RuleCache(identity=identity, readsets=[], entries={}) + if previous is None or previous.identity != identity + else _RuleCache( + identity=identity, + readsets=list(previous.readsets), + entries=dict(previous.entries), + ) + ) + updated[code] = current + by_subject: dict[str, _Subject] = {subject.key: subject for subject in subjects[code]} + positions: dict[tuple[FactKey, ...], int] = { + readset: index for index, readset in enumerate(current.readsets) + } + combined_by_readset: dict[tuple[FactKey, ...], str | None] = {} + for (evaluated_code, subject_key), result in evaluated.items(): + if evaluated_code != code: + continue + reads: tuple[FactKey, ...] | None = result.reads + if reads is None: + if full_keys is None: + full_keys = full_fact_keys(digests.views) + reads = full_keys + readset: tuple[FactKey, ...] = tuple(sorted(set(reads))) + if readset not in combined_by_readset: + try: + combined_by_readset[readset] = digests.combined(readset) + except FactDigestError: + combined_by_readset[readset] = None + combined: str | None = combined_by_readset[readset] + if combined is None: + current.entries.pop(subject_key, None) + continue + index: int | None = positions.get(readset) + if index is None: + index = len(current.readsets) + current.readsets.append(readset) + positions[readset] = index + current.entries[subject_key] = _CachedEntry( + subject_identity=by_subject[subject_key].identity, + readset=index, + reads_digest=combined, + findings=result.findings, + ) + return updated + + +def _ordered_findings( + *, + rules: tuple[Rule, ...], + subjects: dict[str, tuple[_Subject, ...]], + results: dict[tuple[str, str], tuple[dict[str, object], ...]], +) -> tuple[dict[str, object], ...]: + findings: list[dict[str, object]] = [] + for rule in rules: + for subject in subjects[rule.code]: + findings.extend(results.get((rule.code, subject.key), ())) + return tuple(findings) + + +def _read_cache(path: Path) -> dict[str, _RuleCache]: + try: + payload: object = orjson.loads(path.read_bytes()) + except (OSError, orjson.JSONDecodeError): + return {} + if not isinstance(payload, dict) or payload.get("version") != CUSTOM_RULES_CACHE_VERSION: + return {} + rules: object = payload.get("rules") + if not isinstance(rules, dict): + return {} + stored: dict[str, _RuleCache] = {} + for code, value in rules.items(): + try: + stored[str(code)] = _RuleCache( + identity=str(value["identity"]), + readsets=list(map(_decode_readset, value["readsets"])), + entries={ + str(subject): _decode_entry(entry) + for subject, entry in value["entries"].items() + }, + ) + except (KeyError, TypeError, ValueError, IndexError): + continue + return stored + + +def _write_cache( + *, path: Path, stored: dict[str, _RuleCache], subjects: dict[str, tuple[_Subject, ...]] +) -> None: + rules: dict[str, object] = {} + for code, cached in stored.items(): + if code not in subjects: + continue + live: frozenset[str] = frozenset(subject.key for subject in subjects[code]) + entries: dict[str, _CachedEntry] = { + key: entry for key, entry in cached.entries.items() if key in live + } + used: list[int] = sorted({entry.readset for entry in entries.values()}) + remap: dict[int, int] = {old: new for new, old in enumerate(used)} + rules[code] = { + "identity": cached.identity, + "readsets": [list(map(list, cached.readsets[index])) for index in used], + "entries": { + key: [ + entry.subject_identity, + remap[entry.readset], + entry.reads_digest, + list(entry.findings), + ] + for key, entry in entries.items() + }, + } + path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path = path.with_suffix(f".tmp-{os.getpid()}") + temporary.write_bytes(orjson.dumps({"version": CUSTOM_RULES_CACHE_VERSION, "rules": rules})) + temporary.replace(path) + + +def _decode_readset(value: Any) -> tuple[FactKey, ...]: + return tuple(tuple(map(str, key)) for key in value) + + +def _decode_entry(value: Any) -> _CachedEntry: + return _CachedEntry( + subject_identity=str(value[0]), + readset=int(value[1]), + reads_digest=str(value[2]), + findings=tuple(map(_finding_payload, value[3])), + ) + + +def _finding_payload(value: Any) -> dict[str, object]: + return {str(key): item for key, item in value.items()} + + +def _digest(value: object) -> str: + return hashlib.sha256(orjson.dumps(value, default=str)).hexdigest() + + +def _elapsed_ms(started: float) -> int: + return round((time.monotonic() - started) * 1000) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py new file mode 100644 index 0000000000..99c3f2d3f0 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py @@ -0,0 +1,179 @@ +"""Recompute and digest the compiler facts a custom-rule evaluation recorded.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable +from decimal import Decimal +from pathlib import Path, PurePath + +import orjson + +from sqlbuild.rule_engine.classes.sql_document import SqlDocument +from sqlbuild.rule_engine.constants import ( + FACT_AUDITS_ALL, + FACT_AUDITS_FOR_MODEL, + FACT_COLUMNS_DECLARED, + FACT_COLUMNS_INFERRED, + FACT_COLUMNS_NAMES, + FACT_CONTRACTS_ENFORCED, + FACT_CONTRACTS_GRAIN, + FACT_DECLARATIONS_CONSTANTS, + FACT_DECLARATIONS_ENUMS, + FACT_DECLARATIONS_PUBLIC_CONSTANTS, + FACT_DECLARATIONS_PUBLIC_ENUMS, + FACT_GRAPH_DEPENDENCIES, + FACT_GRAPH_DEPENDENTS, + FACT_PROJECT_AUDITS, + FACT_PROJECT_FUNCTIONS, + FACT_PROJECT_MODELS, + FACT_PROJECT_SEEDS, + FACT_PROJECT_SOURCES, + FACT_PROJECT_TESTS, + FACT_SQL_FOR_MODEL, + FACT_TESTS_ALL, + FACT_TESTS_FOR_MODEL, + FACT_TREE_CHILDREN, + FACT_TREE_DESCENDANTS, + FACT_TREE_GLOB, + FACT_TREE_PATHS, + FACT_TREE_READ_TEXT, + FACT_TREE_RELATIVE_PARTS, + FACT_TREE_RESOURCES_UNDER, +) +from sqlbuild.rule_engine.exceptions import FactDigestError +from sqlbuild.rule_engine.models import Model, RuleFactViews +from sqlbuild.rule_engine.types import FactKey + +_TRACKED_TEXT_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) +_ORJSON_OPTIONS: int = orjson.OPT_SORT_KEYS | orjson.OPT_NON_STR_KEYS + +_GLOBAL_FACTS: dict[str, Callable[[RuleFactViews], object]] = { + FACT_PROJECT_MODELS: lambda views: views.project.models, + FACT_PROJECT_SOURCES: lambda views: views.project.sources, + FACT_PROJECT_SEEDS: lambda views: views.project.seeds, + FACT_PROJECT_FUNCTIONS: lambda views: views.project.functions, + FACT_PROJECT_TESTS: lambda views: views.project.tests, + FACT_PROJECT_AUDITS: lambda views: views.project.audits, + FACT_TREE_PATHS: lambda views: views.project.tree.paths(), + FACT_TESTS_ALL: lambda views: views.tests.all(), + FACT_AUDITS_ALL: lambda views: views.audits.all(), + FACT_DECLARATIONS_PUBLIC_ENUMS: lambda views: views.declarations.public_enums, + FACT_DECLARATIONS_PUBLIC_CONSTANTS: lambda views: views.declarations.public_constants, + FACT_DECLARATIONS_ENUMS: lambda views: views.declarations.enums, + FACT_DECLARATIONS_CONSTANTS: lambda views: views.declarations.constants, +} +_MODEL_PATH_FACTS: dict[str, Callable[[RuleFactViews, Model], object]] = { + FACT_SQL_FOR_MODEL: lambda views, model: views.sql.for_model(model), + FACT_GRAPH_DEPENDENCIES: lambda views, model: views.graph.dependencies(model), + FACT_GRAPH_DEPENDENTS: lambda views, model: views.graph.dependents(model), + FACT_COLUMNS_DECLARED: lambda views, model: views.columns.declared(model), + FACT_COLUMNS_INFERRED: lambda views, model: views.columns.inferred(model), + FACT_COLUMNS_NAMES: lambda views, model: views.columns.names(model), + FACT_CONTRACTS_ENFORCED: lambda views, model: views.contracts.enforced(model), + FACT_CONTRACTS_GRAIN: lambda views, model: views.contracts.grain(model), +} +_MODEL_NAME_FACTS: dict[str, Callable[[RuleFactViews, Model], object]] = { + FACT_TESTS_FOR_MODEL: lambda views, model: views.tests.for_model(model), + FACT_AUDITS_FOR_MODEL: lambda views, model: views.audits.for_model(model), +} +_TEXT_FACTS: dict[str, Callable[[RuleFactViews, tuple[str, ...]], object]] = { + FACT_TREE_CHILDREN: lambda views, arguments: views.project.tree.children(*arguments), + FACT_TREE_DESCENDANTS: lambda views, arguments: views.project.tree.descendants(*arguments), + FACT_TREE_GLOB: lambda views, arguments: views.project.tree.glob(*arguments), + FACT_TREE_RELATIVE_PARTS: lambda views, arguments: views.project.tree.relative_parts( + path=arguments[0], under=arguments[1] + ), + FACT_TREE_RESOURCES_UNDER: lambda views, arguments: views.project.tree.resources_under( + *arguments + ), + FACT_TREE_READ_TEXT: lambda views, arguments: views.project.tree.read_text(*arguments), +} +_TEXT_FACT_ARITY: dict[str, int] = { + FACT_TREE_CHILDREN: 1, + FACT_TREE_DESCENDANTS: 1, + FACT_TREE_GLOB: 1, + FACT_TREE_RELATIVE_PARTS: 2, + FACT_TREE_RESOURCES_UNDER: 1, + FACT_TREE_READ_TEXT: 1, +} + + +def fact_key_payload(key: tuple[object, ...]) -> list[str]: + """Return the JSON form of one recorded fact key.""" + + return [value.as_posix() if isinstance(value, PurePath) else str(value) for value in key] + + +def full_fact_keys(views: RuleFactViews) -> tuple[FactKey, ...]: + """Return keys covering every fact a custom rule can observe through its context.""" + + keys: list[FactKey] = [(fact,) for fact in _GLOBAL_FACTS] + models: tuple[Model, ...] = views.project.models + for model in models: + path: str = model.path.as_posix() + keys.extend((fact, path) for fact in _MODEL_PATH_FACTS) + keys.extend((fact, model.name) for fact in _MODEL_NAME_FACTS) + keys.extend( + (FACT_TREE_READ_TEXT, node.value) + for node in views.project.tree.paths() + if node.is_file and Path(node.value).suffix.lower() in _TRACKED_TEXT_SUFFIXES + ) + return tuple(keys) + + +def fact_outcome_digest(*, views: RuleFactViews, key: FactKey) -> str: + """Recompute one recorded fact and digest its value or raised error.""" + + try: + value: object = _fact_value(views=views, key=key) + except FactDigestError: + raise + except Exception as error: + return _encoded_digest(("error", type(error).__name__, str(error))) + return _encoded_digest(("value", value)) + + +def _fact_value(*, views: RuleFactViews, key: FactKey) -> object: + if not key: + raise FactDigestError("empty fact key") + fact: str = key[0] + arguments: tuple[str, ...] = key[1:] + if fact in _GLOBAL_FACTS and not arguments: + return _GLOBAL_FACTS[fact](views) + if fact in _MODEL_PATH_FACTS and len(arguments) == 1: + return _MODEL_PATH_FACTS[fact]( + views, Model(name="", path=Path(arguments[0]), materialization=None) + ) + if fact in _MODEL_NAME_FACTS and len(arguments) == 1: + return _MODEL_NAME_FACTS[fact]( + views, Model(name=arguments[0], path=Path(), materialization=None) + ) + if fact in _TEXT_FACTS and len(arguments) == _TEXT_FACT_ARITY[fact]: + return _TEXT_FACTS[fact](views, arguments) + raise FactDigestError(f"unknown fact key: {fact}") + + +def _encoded_digest(value: object) -> str: + try: + encoded: bytes = orjson.dumps(value, default=_encode_default, option=_ORJSON_OPTIONS) + except (TypeError, FactDigestError) as error: + raise FactDigestError(str(error)) from error + return hashlib.sha256(encoded).hexdigest() + + +def _encode_default(value: object) -> object: + if isinstance(value, PurePath): + return value.as_posix() + if isinstance(value, SqlDocument): + return {"source": value.source} + if isinstance(value, Decimal): + return str(value) + if isinstance(value, (set, frozenset)): + return sorted( + orjson.dumps(item, default=_encode_default, option=_ORJSON_OPTIONS).decode() + for item in value + ) + if isinstance(value, tuple): + return list(value) + raise FactDigestError(f"fact value of type {type(value).__name__} is not reproducible") diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 57555ec24c..fe56e76ec2 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -2,15 +2,10 @@ from __future__ import annotations -import hashlib import inspect import json -import pickle -import re -import sys -import tempfile -from collections.abc import Sequence -from dataclasses import asdict, fields, replace +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import asdict, replace from decimal import Decimal from pathlib import Path from typing import Any, cast @@ -20,8 +15,6 @@ import sqlbuild._native as _native from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, CompiledModel, CompiledModelSqlTestPayload, CompiledProject, @@ -37,22 +30,16 @@ from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( custom_rule_implementation_fingerprint, custom_rule_import_closure, - custom_rule_project_fact_attributes, custom_rule_test_evidence, ) +from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine.constants import ( - CUSTOM_HOST_LAUNCH_MODULE, - CUSTOM_HOST_RUNTIME_VERSION, - RULE_CONTEXT_AUDITS_FACT, - RULE_CONTEXT_DECLARATIONS_FACT, - RULE_CONTEXT_GRAPH_FACT, - RULE_CONTEXT_PROJECT_FACT, - RULE_CONTEXT_TESTS_FACT, RULES_NATIVE_API_VERSION, TYPE_PROOF_RULE_CODES, ) from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import ( + CustomRulesOutcome, Finding, Rule, RulesConfig, @@ -61,9 +48,6 @@ from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind -_CUSTOM_HOST_REQUIRED: str = "selected custom rules require a custom host" -_NATIVE_CACHE_MISSES_PATTERN: re.Pattern[str] = re.compile(r"native_cache_misses=(\d+)") - def evaluate_native( *, @@ -75,15 +59,15 @@ def evaluate_native( initial_findings: tuple[Finding, ...] = (), defer_suppressions: bool = False, verify_determinism: bool = False, - rules_cache_present_at_start: bool | None = None, ) -> RulesResult: - """Evaluate one compiled model batch through the native engine.""" + """Evaluate built-in rules natively while custom rules run incrementally beside them.""" - selected_catalogue: list[Rule] = [] - for rule in catalogue: - selected_by_config: bool = any(rule.code.startswith(selector) for selector in config.select) - if rule.custom and selected_by_config: - selected_catalogue.append(rule) + custom_payloads: list[dict[str, object]] = _custom_rule_payloads( + catalogue=catalogue, project_dir=project_dir + ) + custom_rules: tuple[Rule, ...] = _selected_custom_rules( + config=config, catalogue=catalogue, custom_payloads=custom_payloads + ) request: dict[str, object] = { "version": RULES_NATIVE_API_VERSION, "project_dir": str(project_dir.resolve()), @@ -106,77 +90,73 @@ def evaluate_native( ], "scope_index": scope_metadata_projection(index=project.scope_index), "initial_findings": [_finding_payload(finding) for finding in initial_findings], - "defer_suppressions": defer_suppressions, - "custom_rules": _custom_rule_payloads( - catalogue=catalogue, - project=project, - project_dir=project_dir, - cache_enabled=config.cache.enabled, - ), + "defer_suppressions": True, + "custom_rules": custom_payloads, } - custom_host_input: Path | None = None - retry_native_misses: int = 0 - request["custom_host"] = None - try: - if not config.cache.enabled or not ( - rules_cache_exists(project_dir) - if rules_cache_present_at_start is None - else rules_cache_present_at_start - ): - custom_host, custom_host_input = _custom_host_payload( + with ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as pool: + custom_future: Future[CustomRulesOutcome] | None = ( + pool.submit( + evaluate_custom_rules_cached, project=project, config=config, - project_dir=project_dir, - catalogue=tuple(selected_catalogue), + project_dir=project_dir.resolve(), + rules=custom_rules, dialect=dialect, verify_determinism=verify_determinism, ) - request["custom_host"] = custom_host + if custom_rules + else None + ) try: - response_json: str = _evaluate_request(request) - except ValueError as error: - if _CUSTOM_HOST_REQUIRED not in str(error): - raise - match: re.Match[str] | None = _NATIVE_CACHE_MISSES_PATTERN.search(str(error)) - retry_native_misses = 0 if match is None else int(match.group(1)) - custom_host, custom_host_input = _custom_host_payload( - project=project, - config=config, - project_dir=project_dir, - catalogue=tuple(selected_catalogue), - dialect=dialect, - verify_determinism=verify_determinism, - ) - request["custom_host"] = custom_host - response_json = _evaluate_request(request) - response: object = orjson.loads(response_json) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - finally: - if custom_host_input is not None: - custom_host_input.unlink(missing_ok=True) + response: object = orjson.loads(_evaluate_request(request)) + except (ValueError, TypeError) as error: + raise RulesError(str(error)) from error + custom: CustomRulesOutcome = ( + CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) + if custom_future is None + else custom_future.result() + ) if not isinstance(response, dict): raise RulesError("native rules engine returned an invalid response") payload: dict[str, Any] = response if payload.get("version") != RULES_NATIVE_API_VERSION: raise RulesError("native rules engine returned an unsupported response version") raw_findings: object = payload.get("faults") - if not isinstance(raw_findings, list): + selected_codes: object = payload.get("selected_codes") + if not isinstance(raw_findings, list) or not isinstance(selected_codes, list): raise RulesError("native rules engine returned invalid findings") + findings: tuple[Finding, ...] = tuple( + decode_rule_finding(value) for value in (*raw_findings, *custom.findings) + ) + if not defer_suppressions: + findings = finalize_native_findings( + config=config, + project_dir=project_dir, + evaluated_codes=tuple(str(code) for code in selected_codes), + findings=findings, + ) return RulesResult( - findings=tuple(decode_rule_finding(value) for value in raw_findings), + findings=findings, evaluated_models=int(payload.get("evaluated_models", 0)), - cache_hits=max(0, int(payload.get("cache_hits", 0)) - retry_native_misses), - cache_misses=int(payload.get("cache_misses", 0)) + retry_native_misses, + cache_hits=int(payload.get("cache_hits", 0)) + custom.cache_hits, + cache_misses=int(payload.get("cache_misses", 0)) + custom.cache_misses, built_in_ms=int(payload.get("built_in_ms", 0)), - custom_ms=int(payload.get("custom_ms", 0)), + custom_ms=custom.custom_ms, ) -def rules_cache_exists(project_dir: Path) -> bool: - """Return whether this project already has persisted rules-cache state.""" - - return (project_dir / "target" / "rules-cache").exists() +def _selected_custom_rules( + *, + config: RulesConfig, + catalogue: tuple[Rule, ...], + custom_payloads: list[dict[str, object]], +) -> tuple[Rule, ...]: + if not custom_payloads: + return () + selected: frozenset[str] = frozenset( + _selected_codes(config=config, custom_payloads=custom_payloads) + ) + return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) def _evaluate_request(request: dict[str, object]) -> str: @@ -264,33 +244,19 @@ def native_selected_codes( ) -> tuple[str, ...]: """Resolve the active ruleset through the native Fensu adapter.""" - closures: dict[str, tuple[Path, ...]] = {} + return _selected_codes( + config=config, + custom_payloads=_custom_rule_payloads(catalogue=catalogue, project_dir=project_dir), + ) + + +def _selected_codes( + *, config: RulesConfig, custom_payloads: list[dict[str, object]] +) -> tuple[str, ...]: request: dict[str, object] = { "version": RULES_NATIVE_API_VERSION, "config": _config_payload(config), - "custom_rules": [ - { - "code": rule.code, - "family": rule.family, - "slug": rule.slug, - "message": rule.message, - "remediation": rule.remediation, - "enabled_by_default": rule.enabled_by_default, - "implementation_fingerprint": custom_rule_implementation_fingerprint( - rule=rule, - project_dir=project_dir, - import_closure=_shared_import_closure( - rule=rule, project_dir=project_dir, cache=closures - ), - ), - **_custom_rule_source_payload(rule=rule, project_dir=project_dir), - "project_wide": rule.project_wide, - "check_name": getattr(rule.check, "__name__", ""), - "test_case_count": 0, - } - for rule in catalogue - if rule.custom - ], + "custom_rules": custom_payloads, } try: payload: object = json.loads( @@ -586,50 +552,25 @@ def _typed_value_payload(value: SqlValue) -> object: def _custom_rule_payloads( - *, - catalogue: tuple[Rule, ...], - project: CompiledProject, - project_dir: Path, - cache_enabled: bool, + *, catalogue: tuple[Rule, ...], project_dir: Path ) -> list[dict[str, object]]: - fingerprints: dict[frozenset[str], str] = {} closures: dict[str, tuple[Path, ...]] = {} - payloads: list[dict[str, object]] = [] - for rule in catalogue: - if not rule.custom: - continue - closure: tuple[Path, ...] = _shared_import_closure( - rule=rule, project_dir=project_dir, cache=closures - ) - attributes: frozenset[str] = custom_rule_project_fact_attributes( - rule=rule, project_dir=project_dir, import_closure=closure - ) - if cache_enabled and attributes not in fingerprints: - fingerprints[attributes] = _custom_fact_fingerprint( - project=project, attributes=attributes - ) - fact_fingerprint: str = fingerprints[attributes] if cache_enabled else "" - payloads.append( - _custom_rule_payload( - rule=rule, - project_dir=project_dir, - project_attributes=attributes, - fact_fingerprint=fact_fingerprint, - import_closure=closure, - ) + return [ + _custom_rule_payload( + rule=rule, + project_dir=project_dir, + import_closure=_shared_import_closure( + rule=rule, project_dir=project_dir, cache=closures + ), ) - return payloads + for rule in catalogue + if rule.custom + ] def _custom_rule_payload( - *, - rule: Rule, - project_dir: Path, - project_attributes: frozenset[str], - fact_fingerprint: str, - import_closure: tuple[Path, ...], + *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] ) -> dict[str, object]: - check_name: str = getattr(rule.check, "__name__", "") return { "code": rule.code, "family": rule.family, @@ -642,9 +583,7 @@ def _custom_rule_payload( ), **_custom_rule_source_payload(rule=rule, project_dir=project_dir), "project_wide": rule.project_wide, - "project_dependent": RULE_CONTEXT_PROJECT_FACT in project_attributes, - "fact_fingerprint": fact_fingerprint, - "check_name": check_name, + "check_name": getattr(rule.check, "__name__", ""), "test_case_count": len(custom_rule_test_evidence(rule=rule, project_dir=project_dir)), } @@ -660,82 +599,6 @@ def _shared_import_closure( return cache[source] -def _custom_fact_fingerprint(*, project: CompiledProject, attributes: frozenset[str]) -> str: - facts: dict[str, object] = {} - if RULE_CONTEXT_GRAPH_FACT in attributes: - facts[RULE_CONTEXT_GRAPH_FACT] = [ - (model.name, model.relative_path, model.deps) for model in project.models - ] - if RULE_CONTEXT_TESTS_FACT in attributes: - facts[RULE_CONTEXT_TESTS_FACT] = project.sql_tests - if RULE_CONTEXT_AUDITS_FACT in attributes: - facts[RULE_CONTEXT_AUDITS_FACT] = project.audits - if RULE_CONTEXT_DECLARATIONS_FACT in attributes: - facts[RULE_CONTEXT_DECLARATIONS_FACT] = ( - project.public_enums, - project.public_constants, - tuple(model.enum_declarations for model in project.models), - tuple(model.constant_declarations for model in project.models), - ) - if RULE_CONTEXT_PROJECT_FACT in attributes: - facts[RULE_CONTEXT_PROJECT_FACT] = _project_fact_models(project=project) - encoded: bytes = orjson.dumps(facts, option=orjson.OPT_SORT_KEYS, default=str) - return hashlib.sha256(encoded).hexdigest() - - -def _project_fact_models(*, project: CompiledProject) -> tuple[dict[str, object], ...]: - """Project models with lazy lineage represented by stable semantic data only.""" - - projected: list[dict[str, object]] = [] - for model in project.models: - payload: dict[str, object] = { - field.name: getattr(model, field.name) for field in fields(model) - } - payload["fast_lineage_columns"] = _lineage_fingerprint_payload( - lineage=model.fast_lineage_columns - ) - projected.append(payload) - return tuple(projected) - - -def _lineage_fingerprint_payload( - *, lineage: Sequence[CompiledLineageColumnFact] | None -) -> tuple[tuple[object, ...], ...] | None: - """Hash semantic lineage without rehydrating compact facts into Python objects.""" - - if lineage is None: - return None - payloads: list[tuple[object, ...]] = [] - if isinstance(lineage, CompactLineageFacts): - for name_index, transform_code, confidence_code, sources in lineage.rows: - upstream: tuple[tuple[str, str, str], ...] = tuple( - ( - lineage.string_pool[source[0]], - lineage.resource_name(source[1]), - lineage.string_pool[source[2]], - ) - for source in sources - ) - payloads.append( - ( - lineage.string_pool[name_index], - upstream, - lineage.transform_kind(transform_code), - lineage.confidence(confidence_code), - ) - ) - else: - for column in lineage: - upstream = tuple( - (source.resource_type, source.resource_name, source.column_name) - for source in column.upstream_columns - ) - payloads.append( - (column.output_column, upstream, column.transform_kind, column.confidence) - ) - return tuple(payloads) - - def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, object]: source: str | None = rule.source if source is not None: @@ -756,62 +619,6 @@ def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, o } -def _custom_host_payload( - *, - project: CompiledProject, - config: RulesConfig, - project_dir: Path, - catalogue: tuple[Rule, ...], - dialect: str, - verify_determinism: bool, -) -> tuple[dict[str, object] | None, Path | None]: - if not any(rule.custom for rule in catalogue): - return None, None - serializable_project: CompiledProject = replace( - project, - binding_catalog=None, - sql_expansions={}, - loaded_macros={}, - loader_functions=(), - hook_functions=(), - materialization_files=(), - external_sql_reference_resolver=None, - ) - input_dir: Path = project_dir / "target" / "rules-cache" / "host-inputs" - input_dir.mkdir(parents=True, exist_ok=True) - input_path: Path | None = None - try: - with tempfile.NamedTemporaryFile( - dir=input_dir, - prefix="project-", - suffix=".pickle", - delete=False, - ) as handle: - input_path = Path(handle.name) - pickle.dump((serializable_project, config), handle) - except Exception: - if input_path is not None: - input_path.unlink(missing_ok=True) - raise - if input_path is None: - raise RulesError("custom host project payload was not created") - return ( - { - "program": sys.executable, - "arguments": ["-m", CUSTOM_HOST_LAUNCH_MODULE], - "timeout_millis": 120_000, - "runtime_version": CUSTOM_HOST_RUNTIME_VERSION, - "payload": { - "project_pickle_path": str(input_path.resolve()), - "project_dir": str(project_dir.resolve()), - "dialect": dialect, - "verify_determinism": verify_determinism, - }, - }, - input_path, - ) - - def decode_rule_finding(value: object) -> Finding: if not isinstance(value, dict): raise RulesError("native rules engine returned an invalid finding") diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index b77e900b88..b3db580108 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -4,9 +4,17 @@ from collections import Counter from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass +from itertools import chain from pathlib import Path from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.rule_engine._helpers.host.module_state import ( + module_state_token, + rule_namespaces, +) +from sqlbuild.rule_engine._helpers.host.tracked_facts import tracked_fact_views +from sqlbuild.rule_engine.classes.fact_reads import FactReads from sqlbuild.rule_engine.classes.project_tree import public_model from sqlbuild.rule_engine.classes.rule_context import ( EvaluationRuleContext, @@ -14,13 +22,37 @@ build_rule_fact_views, ) from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard -from sqlbuild.rule_engine.exceptions import NonHermeticRuleError, RulesError -from sqlbuild.rule_engine.models import Finding, Model, Project, Rule, RulesConfig -from sqlbuild.rule_engine.types import RuleSubject +from sqlbuild.rule_engine.exceptions import ( + NonHermeticRuleError, + OpaqueModuleStateError, + RulesError, +) +from sqlbuild.rule_engine.models import ( + CustomRuleEvaluation, + CustomRuleRun, + Finding, + Model, + Project, + Rule, + RulesConfig, +) +from sqlbuild.rule_engine.types import CustomRulePlan, RuleSubject type _FindingKey = tuple[str, str, int, int, str, str] +@dataclass(frozen=True) +class _PassInputs: + project: CompiledProject + config: RulesConfig + project_dir: Path + selected_rules: tuple[Rule, ...] + dialect: str + facts: RuleFactViews + plan: CustomRulePlan | None + guard: RuntimeGuard | None + + def evaluate_custom_rules( *, project: CompiledProject, @@ -28,118 +60,198 @@ def evaluate_custom_rules( project_dir: Path, selected_rules: tuple[Rule, ...], dialect: str = "generic", - selected_model_paths: frozenset[str] | None = None, + plan: CustomRulePlan | None = None, + track_reads: bool = False, verify_determinism: bool = False, guard: RuntimeGuard | None = None, -) -> list[Finding]: - """Run only selected custom rules through the Python authoring API.""" +) -> CustomRuleRun: + """Run selected custom rules for every planned subject through the Python authoring API.""" - custom_rules: tuple[Rule, ...] = tuple(rule for rule in selected_rules if rule.custom) + custom_rules: tuple[Rule, ...] = tuple( + rule for rule in selected_rules if rule.custom and (plan is None or rule.code in plan) + ) if not custom_rules: - return [] + return CustomRuleRun(evaluations=(), untracked_codes=frozenset()) models: tuple[CompiledModel, ...] = tuple( - model - for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) - if selected_model_paths is None or model.relative_path.as_posix() in selected_model_paths + sorted(project.models, key=lambda item: item.relative_path.as_posix()) ) facts: RuleFactViews = build_rule_fact_views( project=project, project_dir=project_dir, dialect=dialect ) + reads: FactReads | None = FactReads() if track_reads else None + tracker: _ReadTracker = _ReadTracker( + reads=reads, rules=custom_rules, rules_root=project_dir / "rules" + ) - def run_pass(*, rules: tuple[Rule, ...], subjects: tuple[CompiledModel, ...]) -> list[Finding]: - return _evaluate_pass( - project=project, - config=config, - project_dir=project_dir, - selected_rules=selected_rules, - dialect=dialect, - facts=facts, - rules=rules, - models=subjects, - guard=guard, - ) + inputs: _PassInputs = _PassInputs( + project=project, + config=config, + project_dir=project_dir, + selected_rules=selected_rules, + dialect=dialect, + facts=facts, + plan=plan, + guard=guard, + ) + + def run_pass( + *, rules: tuple[Rule, ...], subjects: tuple[CompiledModel, ...], active: _ReadTracker + ) -> list[CustomRuleEvaluation]: + return _evaluate_pass(inputs=inputs, rules=rules, models=subjects, tracker=active) - findings: list[Finding] = run_pass(rules=custom_rules, subjects=models) + evaluations: list[CustomRuleEvaluation] = run_pass( + rules=custom_rules, subjects=models, active=tracker + ) if verify_determinism: - repeated: list[Finding] = run_pass(rules=custom_rules[::-1], subjects=models[::-1]) - _verify_repeatable(rules=custom_rules, first=findings, repeated=repeated) - return findings + repeated: list[CustomRuleEvaluation] = run_pass( + rules=custom_rules[::-1], + subjects=models[::-1], + active=_ReadTracker(reads=None, rules=(), rules_root=project_dir / "rules"), + ) + _verify_repeatable( + rules=custom_rules, + first=list(chain.from_iterable(item.findings for item in evaluations)), + repeated=list(chain.from_iterable(item.findings for item in repeated)), + ) + return CustomRuleRun( + evaluations=tuple(evaluations), untracked_codes=frozenset(tracker.untracked_codes) + ) + + +class _ReadTracker: + """Attribute reads to one invocation; rules whose module state changes become untracked.""" + + def __init__( + self, *, reads: FactReads | None, rules: tuple[Rule, ...], rules_root: Path + ) -> None: + self.reads: FactReads | None = reads + self.untracked_codes: set[str] = set() + self._rules_root: Path = rules_root + self._namespaces: dict[str, tuple[dict[str, object], ...]] = ( + {rule.code: rule_namespaces((rule.check,)) for rule in rules} + if reads is not None + else {} + ) + self._states: dict[str, str | None] = { + code: self._state_token(namespaces) for code, namespaces in self._namespaces.items() + } + + def start(self) -> None: + if self.reads is not None: + self.reads.current = set() + self.reads.untracked = False + + def finish(self, *, code: str) -> frozenset[tuple[object, ...]] | None: + reads: FactReads | None = self.reads + if reads is None: + return None + observed: set[tuple[object, ...]] | None = reads.current + reads.current = None + if reads.untracked: + self.untracked_codes.add(code) + for rule_code, namespaces in self._namespaces.items(): + if rule_code in self.untracked_codes: + continue + state: str | None = self._state_token(namespaces) + if state is None or state != self._states[rule_code]: + self.untracked_codes.add(rule_code) + return None if observed is None else frozenset(observed) + + def _state_token(self, namespaces: tuple[dict[str, object], ...]) -> str | None: + try: + return module_state_token(namespaces=namespaces, rules_root=self._rules_root) + except OpaqueModuleStateError: + return None def _evaluate_pass( *, - project: CompiledProject, - config: RulesConfig, - project_dir: Path, - selected_rules: tuple[Rule, ...], - dialect: str, - facts: RuleFactViews, + inputs: _PassInputs, rules: tuple[Rule, ...], models: tuple[CompiledModel, ...], - guard: RuntimeGuard | None, -) -> list[Finding]: - findings: list[Finding] = [] + tracker: _ReadTracker, +) -> list[CustomRuleEvaluation]: + evaluations: list[CustomRuleEvaluation] = [] + project: CompiledProject = inputs.project + plan: CustomRulePlan | None = inputs.plan + guard: RuntimeGuard | None = inputs.guard + context_facts: RuleFactViews = ( + inputs.facts + if tracker.reads is None + else tracked_fact_views(views=inputs.facts, reads=tracker.reads) + ) + + def context(rule: Rule) -> EvaluationRuleContext: + return EvaluationRuleContext( + model=None, + rule=rule, + config=inputs.config, + project=project, + project_dir=inputs.project_dir, + selected_rules=inputs.selected_rules, + dialect=inputs.dialect, + facts=context_facts, + reads=tracker.reads, + ) + project_rules: tuple[Rule, ...] = tuple( rule for rule in rules if rule.subject is RuleSubject.PROJECT ) model_rules: tuple[Rule, ...] = tuple( rule for rule in rules if rule.subject is RuleSubject.MODEL ) - model_contexts: tuple[tuple[Rule, EvaluationRuleContext], ...] = tuple( - ( - rule, - EvaluationRuleContext( - model=None, - rule=rule, - config=config, - project=project, - project_dir=project_dir, - selected_rules=selected_rules, - dialect=dialect, - facts=facts, - ), - ) - for rule in model_rules + model_contexts: tuple[tuple[Rule, EvaluationRuleContext, frozenset[str] | None], ...] = tuple( + (rule, context(rule), None if plan is None else plan.get(rule.code)) for rule in model_rules ) for rule in project_rules: - ctx: EvaluationRuleContext = EvaluationRuleContext( - model=None, - rule=rule, - config=config, - project=project, - project_dir=project_dir, - selected_rules=selected_rules, - dialect=dialect, - facts=facts, - ) + ctx: EvaluationRuleContext = context(rule) + tracker.start() try: - findings.extend( - _invoke( - rule=rule, - subject=Project( - model_count=len(project.models), - target_name=project.effective_target_name, - ), - ctx=ctx, - guard=guard, - ) + findings: list[Finding] = _invoke( + rule=rule, + subject=Project( + model_count=len(project.models), + target_name=project.effective_target_name, + ), + ctx=ctx, + guard=guard, ) except NonHermeticRuleError: raise except Exception as error: raise RulesError(f"rule {rule.code} failed for the project: {error}") from error + evaluations.append( + CustomRuleEvaluation( + code=rule.code, + subject=None, + findings=tuple(findings), + reads=tracker.finish(code=rule.code), + ) + ) for model in models: subject: Model = public_model(model) - for rule, ctx in model_contexts: + path: str = model.relative_path.as_posix() + for rule, ctx, planned in model_contexts: + if planned is not None and path not in planned: + continue + tracker.start() try: - findings.extend(_invoke(rule=rule, subject=subject, ctx=ctx, guard=guard)) + findings = _invoke(rule=rule, subject=subject, ctx=ctx, guard=guard) except NonHermeticRuleError: raise except Exception as error: raise RulesError( f"rule {rule.code} failed for {model.relative_path}: {error}" ) from error - return findings + evaluations.append( + CustomRuleEvaluation( + code=rule.code, + subject=path, + findings=tuple(findings), + reads=tracker.finish(code=rule.code), + ) + ) + return evaluations def _invoke( diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 3bacf498ef..9234488a90 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -14,17 +14,20 @@ from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue +from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard from sqlbuild.rule_engine.constants import ( CUSTOM_HOST_HASH_SEED, CUSTOM_HOST_INPUT_TUPLE_SIZE, CUSTOM_HOST_LAUNCH_MODULE, + CUSTOM_HOST_MAX_TRACKED_READS, CUSTOM_HOST_PROTOCOL_VERSION, CUSTOM_HOST_RUNTIME_VERSION, ) from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig +from sqlbuild.rule_engine.models import CustomRuleRun, Finding, Rule, RulesConfig +from sqlbuild.rule_engine.types import CustomRulePlan def main() -> int: @@ -51,13 +54,8 @@ def main() -> int: project_dir: Path = Path(str(payload["project_dir"])).resolve() dialect: str = str(payload.get("dialect", "generic")) verify_determinism: bool = payload.get("verify_determinism") is True - selected_codes: tuple[str, ...] = tuple(str(code) for code in payload["selected_codes"]) - raw_model_paths: object = payload.get("selected_model_paths") - selected_model_paths: frozenset[str] | None = ( - frozenset(str(path) for path in raw_model_paths) - if isinstance(raw_model_paths, list) - else None - ) + track_reads: bool = payload.get("track_reads") is True + plan: CustomRulePlan = _decode_plan(payload["plan"]) os.environ.clear() os.chdir(project_dir) guard = RuntimeGuard(project_dir=project_dir) @@ -68,14 +66,15 @@ def main() -> int: catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) guard.raise_violation() by_code: dict[str, Rule] = {rule.code: rule for rule in catalogue} - selected: tuple[Rule, ...] = tuple(by_code[code] for code in selected_codes) - findings: list[Finding] = evaluate_custom_rules( + selected: tuple[Rule, ...] = tuple(by_code[code] for code in sorted(plan)) + run: CustomRuleRun = evaluate_custom_rules( project=project, config=config, project_dir=project_dir, selected_rules=selected, dialect=dialect, - selected_model_paths=selected_model_paths, + plan=plan, + track_reads=track_reads, verify_determinism=verify_determinism, guard=guard, ) @@ -86,7 +85,7 @@ def main() -> int: "protocol": CUSTOM_HOST_PROTOCOL_VERSION, "runtime_version": CUSTOM_HOST_RUNTIME_VERSION, "error": None, - "payload": [_finding_payload(finding) for finding in findings], + "payload": _run_payload(run), "messages": messages.getvalue().splitlines(), } sys.stdout.write(json.dumps(response, sort_keys=True)) @@ -115,6 +114,49 @@ def _decode_inputs(payload: dict[str, Any]) -> tuple[CompiledProject, RulesConfi return decoded +def _decode_plan(value: object) -> CustomRulePlan: + if not isinstance(value, dict): + raise RulesError("custom host plan must be an object") + plan: CustomRulePlan = {} + for code, subjects in value.items(): + if subjects is not None and not isinstance(subjects, list): + raise RulesError("custom host plan subjects must be a list or null") + plan[str(code)] = ( + None if subjects is None else frozenset(str(subject) for subject in subjects) + ) + return plan + + +def _run_payload(run: CustomRuleRun) -> dict[str, object]: + """Encode invocations with read sets deduplicated across subjects and bounded in size.""" + + readsets: dict[frozenset[tuple[object, ...]], int] = {} + evaluations: list[dict[str, object]] = [] + for evaluation in run.evaluations: + index: int | None = None + if evaluation.reads is not None and evaluation.code not in run.untracked_codes: + index = readsets.setdefault(evaluation.reads, len(readsets)) + evaluations.append( + { + "code": evaluation.code, + "subject": evaluation.subject, + "findings": [_finding_payload(finding) for finding in evaluation.findings], + "reads": index, + } + ) + untracked: set[str] = set(run.untracked_codes) + if sum(map(len, readsets)) > CUSTOM_HOST_MAX_TRACKED_READS: + untracked.update(str(item["code"]) for item in evaluations if item["reads"] is not None) + readsets = {} + for item in evaluations: + item["reads"] = None + return { + "evaluations": evaluations, + "readsets": [sorted(map(fact_key_payload, readset)) for readset in readsets], + "untracked": sorted(untracked), + } + + def _finding_payload(finding: Finding) -> dict[str, object]: payload: dict[str, object] = asdict(finding) payload.pop("affected_rules", None) diff --git a/src/sqlbuild/rule_engine/_helpers/host/module_state.py b/src/sqlbuild/rule_engine/_helpers/host/module_state.py new file mode 100644 index 0000000000..e0d0a26a39 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/host/module_state.py @@ -0,0 +1,218 @@ +"""Detect custom-rule module state that one evaluation could leave for another.""" + +from __future__ import annotations + +import enum +import hashlib +import os +import re +from collections import deque +from collections.abc import Iterable +from decimal import Decimal +from fractions import Fraction +from pathlib import Path, PurePath +from types import BuiltinFunctionType, FunctionType, MethodType, ModuleType +from typing import Any, cast + +from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError + +_MAX_STATE_NODES: int = 50_000 +_ATOMIC_TYPES: tuple[type, ...] = ( + type(None), + bool, + int, + float, + complex, + str, + bytes, + bytearray, + Decimal, + Fraction, + PurePath, + range, + re.Pattern, + enum.Enum, +) +_IMMUTABLE_LIBRARY_MODULES: frozenset[str] = frozenset({"typing", "types", "builtins"}) +_SKIPPED_FUNCTION_ATTRIBUTES: frozenset[str] = frozenset({"__sqlbuild_rule__", "__wrapped__"}) + + +def rule_namespaces(rules: Iterable[object]) -> tuple[dict[str, object], ...]: + """Return the module globals of every loaded custom-rule check.""" + + namespaces: dict[int, dict[str, object]] = {} + for check in rules: + module_globals: object = getattr(check, "__globals__", None) + if isinstance(module_globals, dict): + namespaces.setdefault(id(module_globals), module_globals) + return tuple(namespaces.values()) + + +def module_state_token(*, namespaces: tuple[dict[str, object], ...], rules_root: Path) -> str: + """Fingerprint every mutable value reachable from rule-module globals.""" + + walker: _StateWalker = _StateWalker(rules_root=f"{rules_root.resolve()}{os.sep}") + for namespace in namespaces: + walker.visit_namespace(namespace) + return walker.digest.hexdigest() + + +class _StateWalker: + def __init__(self, *, rules_root: str) -> None: + self.digest: Any = hashlib.sha256() + self._rules_root: str = rules_root + self._namespaces: set[int] = set() + self._module_names: set[str] = set() + self._visited: dict[int, int] = {} + self._nodes: int = 0 + + def visit_namespace(self, namespace: dict[str, object]) -> None: + if id(namespace) in self._namespaces: + return + self._namespaces.add(id(namespace)) + name: object = namespace.get("__name__") + self._module_names.add(str(name)) + self.emit(f"namespace:{name}") + for key, value in sorted(namespace.items(), key=lambda item: item[0]): + if key.startswith("__") and key.endswith("__"): + continue + self.emit(f"name:{key}") + self.visit(value) + + def _owned_code(self, function: FunctionType) -> bool: + return function.__code__.co_filename.startswith(self._rules_root) + + def emit(self, text: str) -> None: + self.digest.update(text.encode()) + self.digest.update(b"\0") + + def visit(self, value: object) -> None: + self._nodes += 1 + if self._nodes > _MAX_STATE_NODES: + raise OpaqueModuleStateError("custom rule module state is too large to observe") + if isinstance(value, _ATOMIC_TYPES) and not isinstance(value, (list, dict, set)): + self.emit(f"atom:{type(value).__qualname__}:{value!r}") + return + identity: int = id(value) + seen: int | None = self._visited.get(identity) + if seen is not None: + self.emit(f"ref:{seen}") + return + self._visited[identity] = len(self._visited) + self.emit(f"object:{identity}:{type(value).__qualname__}") + self._visit_structure(value) + + def _visit_structure(self, value: object) -> None: + if isinstance(value, ModuleType): + location: object = getattr(value, "__file__", None) + if isinstance(location, str) and location.startswith(self._rules_root): + self.visit_namespace(vars(value)) + else: + self.emit(f"module:{value.__name__}") + elif isinstance(value, dict): + self._visit_mapping(cast(dict[object, object], value)) + elif isinstance(value, (list, tuple, deque)): + self.emit(f"len:{len(value)}") + for item in value: + self.visit(item) + elif isinstance(value, (set, frozenset)): + self.emit(f"len:{len(value)}") + tokens: list[str] = sorted(_atomic_text(item) for item in value) + for token in tokens: + self.emit(token) + elif isinstance(value, FunctionType): + self._visit_function(value) + elif isinstance(value, MethodType): + self.visit(value.__func__) + self.visit(value.__self__) + elif isinstance(value, type): + self._visit_type(value) + elif hasattr(value, "cache_info") and hasattr(value, "__wrapped__"): + cached: Any = cast(Any, value) + self.emit(f"cache:{cached.cache_info().currsize}") + self.visit(cached.__wrapped__) + elif isinstance(value, BuiltinFunctionType): + self.emit(f"builtin:{getattr(value, '__module__', '')}:{value.__qualname__}") + else: + self._visit_instance(value) + + def _visit_mapping(self, value: dict[object, object]) -> None: + self.emit(f"len:{len(value)}") + factory: object = getattr(value, "default_factory", None) + if factory is not None: + self.visit(factory) + for key, item in value.items(): + self.visit(key) + self.visit(item) + + def _visit_function(self, value: FunctionType) -> None: + if not self._owned_code(value): + self.emit(f"function:{value.__module__}:{value.__qualname__}") + return + self.visit_namespace(value.__globals__) + self.visit(value.__defaults__) + self.visit(value.__kwdefaults__) + for name, item in sorted(vars(value).items(), key=lambda entry: entry[0]): + if name in _SKIPPED_FUNCTION_ATTRIBUTES: + continue + self.emit(f"attribute:{name}") + self.visit(item) + for cell in value.__closure__ or (): + try: + contents: object = cell.cell_contents + except ValueError: + self.emit("empty-cell") + continue + self.visit(contents) + + def _visit_type(self, value: type) -> None: + owned: bool = value.__module__ in self._module_names or any( + isinstance(item, FunctionType) and self._owned_code(item) + for item in vars(value).values() + ) + if not owned: + self.emit(f"type:{value.__module__}:{value.__qualname__}") + return + for name, item in sorted(vars(value).items(), key=lambda entry: entry[0]): + if name.startswith("__") and name.endswith("__"): + continue + self.emit(f"attribute:{name}") + self.visit(item) + + def _visit_instance(self, value: object) -> None: + state: dict[str, object] = {} + attributes: object = getattr(value, "__dict__", None) + if isinstance(attributes, dict): + state.update(attributes) + for owner in type(value).__mro__: + for slot in getattr(owner, "__slots__", ()): + if isinstance(slot, str) and hasattr(value, slot): + state[slot] = getattr(value, slot) + if not state and type(value).__module__ not in self._module_names: + if _is_immutable_library_value(value): + self.emit(f"value:{type(value).__module__}:{type(value).__qualname__}") + return + raise OpaqueModuleStateError( + f"custom rule module state holds an opaque {type(value).__qualname__}" + ) + for name, item in sorted(state.items(), key=lambda entry: entry[0]): + self.emit(f"attribute:{name}") + self.visit(item) + + +def _atomic_text(value: object) -> str: + if isinstance(value, _ATOMIC_TYPES): + return f"{type(value).__qualname__}:{value!r}" + if isinstance(value, tuple): + return "(" + ",".join(_atomic_text(item) for item in value) + ")" + raise OpaqueModuleStateError( + f"custom rule module state holds an unhashable-order {type(value).__qualname__}" + ) + + +def _is_immutable_library_value(value: object) -> bool: + return ( + isinstance(value, (staticmethod, classmethod, property)) + or callable(value) + or type(value).__module__ in _IMMUTABLE_LIBRARY_MODULES + ) diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py new file mode 100644 index 0000000000..5462bd801b --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -0,0 +1,258 @@ +"""Custom-rule fact views that record which compiler facts each evaluation reads.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from sqlbuild.compiler.compile.models import CompiledAudit, CompiledSqlTest, InferredColumn +from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration +from sqlbuild.rule_engine.classes.fact_reads import FactReads +from sqlbuild.rule_engine.constants import ( + FACT_AUDITS_ALL, + FACT_AUDITS_FOR_MODEL, + FACT_COLUMNS_DECLARED, + FACT_COLUMNS_INFERRED, + FACT_COLUMNS_NAMES, + FACT_CONTRACTS_ENFORCED, + FACT_CONTRACTS_GRAIN, + FACT_DECLARATIONS_CONSTANTS, + FACT_DECLARATIONS_ENUMS, + FACT_DECLARATIONS_PUBLIC_CONSTANTS, + FACT_DECLARATIONS_PUBLIC_ENUMS, + FACT_GRAPH_DEPENDENCIES, + FACT_GRAPH_DEPENDENTS, + FACT_PROJECT_AUDITS, + FACT_PROJECT_FUNCTIONS, + FACT_PROJECT_MODELS, + FACT_PROJECT_SEEDS, + FACT_PROJECT_SOURCES, + FACT_PROJECT_TESTS, + FACT_SQL_FOR_MODEL, + FACT_TESTS_ALL, + FACT_TESTS_FOR_MODEL, + FACT_TREE_CHILDREN, + FACT_TREE_DESCENDANTS, + FACT_TREE_GLOB, + FACT_TREE_PATHS, + FACT_TREE_READ_TEXT, + FACT_TREE_RELATIVE_PARTS, + FACT_TREE_RESOURCES_UNDER, +) +from sqlbuild.rule_engine.models import Model, ModelSql, ProjectPath, RuleFactViews + + +class _TrackedView: + """Delegate unknown attribute access to the wrapped view and mark the evaluation untracked.""" + + __slots__ = ("_tracked_reads", "_tracked_view") + + _tracked_reads: FactReads + _tracked_view: Any + + def __init__(self, *, view: object, reads: FactReads) -> None: + object.__setattr__(self, "_tracked_view", view) + object.__setattr__(self, "_tracked_reads", reads) + + def __getattr__(self, name: str) -> object: + self._tracked_reads.untracked = True + return getattr(self._tracked_view, name) + + def __setattr__(self, name: str, value: object) -> None: + self._tracked_reads.untracked = True + setattr(self._tracked_view, name, value) + + +class _TrackedProjectTree(_TrackedView): + __slots__ = () + + def paths(self) -> tuple[ProjectPath, ...]: + self._tracked_reads.record(key=(FACT_TREE_PATHS,)) + return self._tracked_view.paths() + + def children(self, path: str = "") -> tuple[ProjectPath, ...]: + self._tracked_reads.record_text(fact=FACT_TREE_CHILDREN, values=(path,)) + return self._tracked_view.children(path) + + def descendants(self, path: str = "") -> tuple[ProjectPath, ...]: + self._tracked_reads.record_text(fact=FACT_TREE_DESCENDANTS, values=(path,)) + return self._tracked_view.descendants(path) + + def glob(self, pattern: str) -> tuple[ProjectPath, ...]: + self._tracked_reads.record_text(fact=FACT_TREE_GLOB, values=(pattern,)) + return self._tracked_view.glob(pattern) + + def relative_parts(self, *, path: Path | str, under: str = "") -> tuple[str, ...]: + self._tracked_reads.record_text( + fact=FACT_TREE_RELATIVE_PARTS, + values=( + path, + under, + ), + ) + return self._tracked_view.relative_parts(path=path, under=under) + + def resources_under(self, path: str) -> tuple[Model, ...]: + self._tracked_reads.record_text(fact=FACT_TREE_RESOURCES_UNDER, values=(path,)) + return self._tracked_view.resources_under(path) + + def read_text(self, path: str) -> str: + self._tracked_reads.record_text(fact=FACT_TREE_READ_TEXT, values=(path,)) + return self._tracked_view.read_text(path) + + +class _TrackedProjectFacts(_TrackedView): + __slots__ = ("_tracked_tree",) + + _tracked_tree: _TrackedProjectTree + + def __init__(self, *, view: object, reads: FactReads) -> None: + super().__init__(view=view, reads=reads) + object.__setattr__( + self, "_tracked_tree", _TrackedProjectTree(view=self._tracked_view.tree, reads=reads) + ) + + @property + def models(self) -> tuple[Model, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_MODELS,)) + return self._tracked_view.models + + @property + def sources(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_SOURCES,)) + return self._tracked_view.sources + + @property + def seeds(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_SEEDS,)) + return self._tracked_view.seeds + + @property + def functions(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_FUNCTIONS,)) + return self._tracked_view.functions + + @property + def tests(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_TESTS,)) + return self._tracked_view.tests + + @property + def audits(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_AUDITS,)) + return self._tracked_view.audits + + @property + def tree(self) -> _TrackedProjectTree: + return self._tracked_tree + + +class _TrackedSqlFacts(_TrackedView): + __slots__ = () + + def for_model(self, model: Model) -> ModelSql: + self._tracked_reads.record_model_path(fact=FACT_SQL_FOR_MODEL, model=model) + return self._tracked_view.for_model(model) + + +class _TrackedGraphFacts(_TrackedView): + __slots__ = () + + def dependencies(self, model: Model) -> tuple[Model, ...]: + self._tracked_reads.record_model_path(fact=FACT_GRAPH_DEPENDENCIES, model=model) + return self._tracked_view.dependencies(model) + + def dependents(self, model: Model) -> tuple[Model, ...]: + self._tracked_reads.record_model_path(fact=FACT_GRAPH_DEPENDENTS, model=model) + return self._tracked_view.dependents(model) + + +class _TrackedColumnFacts(_TrackedView): + __slots__ = () + + def declared(self, model: Model) -> tuple[object, ...]: + self._tracked_reads.record_model_path(fact=FACT_COLUMNS_DECLARED, model=model) + return self._tracked_view.declared(model) + + def inferred(self, model: Model) -> tuple[InferredColumn, ...] | None: + self._tracked_reads.record_model_path(fact=FACT_COLUMNS_INFERRED, model=model) + return self._tracked_view.inferred(model) + + def names(self, model: Model) -> tuple[str, ...] | None: + self._tracked_reads.record_model_path(fact=FACT_COLUMNS_NAMES, model=model) + return self._tracked_view.names(model) + + +class _TrackedContractFacts(_TrackedView): + __slots__ = () + + def enforced(self, model: Model) -> bool: + self._tracked_reads.record_model_path(fact=FACT_CONTRACTS_ENFORCED, model=model) + return self._tracked_view.enforced(model) + + def grain(self, model: Model) -> tuple[str, ...]: + self._tracked_reads.record_model_path(fact=FACT_CONTRACTS_GRAIN, model=model) + return self._tracked_view.grain(model) + + +class _TrackedTestFacts(_TrackedView): + __slots__ = () + + def all(self) -> tuple[CompiledSqlTest, ...]: + self._tracked_reads.record(key=(FACT_TESTS_ALL,)) + return self._tracked_view.all() + + def for_model(self, model: Model) -> tuple[CompiledSqlTest, ...]: + self._tracked_reads.record_model_name(fact=FACT_TESTS_FOR_MODEL, model=model) + return self._tracked_view.for_model(model) + + +class _TrackedAuditFacts(_TrackedView): + __slots__ = () + + def all(self) -> tuple[CompiledAudit, ...]: + self._tracked_reads.record(key=(FACT_AUDITS_ALL,)) + return self._tracked_view.all() + + def for_model(self, model: Model) -> tuple[CompiledAudit, ...]: + self._tracked_reads.record_model_name(fact=FACT_AUDITS_FOR_MODEL, model=model) + return self._tracked_view.for_model(model) + + +class _TrackedDeclarationFacts(_TrackedView): + __slots__ = () + + @property + def public_enums(self) -> tuple[EnumDeclaration, ...]: + self._tracked_reads.record(key=(FACT_DECLARATIONS_PUBLIC_ENUMS,)) + return self._tracked_view.public_enums + + @property + def public_constants(self) -> tuple[ConstantDeclaration, ...]: + self._tracked_reads.record(key=(FACT_DECLARATIONS_PUBLIC_CONSTANTS,)) + return self._tracked_view.public_constants + + @property + def enums(self) -> tuple[EnumDeclaration, ...]: + self._tracked_reads.record(key=(FACT_DECLARATIONS_ENUMS,)) + return self._tracked_view.enums + + @property + def constants(self) -> tuple[ConstantDeclaration, ...]: + self._tracked_reads.record(key=(FACT_DECLARATIONS_CONSTANTS,)) + return self._tracked_view.constants + + +def tracked_fact_views(*, views: RuleFactViews, reads: FactReads) -> RuleFactViews: + """Wrap every public fact view so reads are attributed to the active evaluation.""" + + return RuleFactViews( + project=_TrackedProjectFacts(view=views.project, reads=reads), + sql=_TrackedSqlFacts(view=views.sql, reads=reads), + graph=_TrackedGraphFacts(view=views.graph, reads=reads), + columns=_TrackedColumnFacts(view=views.columns, reads=reads), + contracts=_TrackedContractFacts(view=views.contracts, reads=reads), + tests=_TrackedTestFacts(view=views.tests, reads=reads), + audits=_TrackedAuditFacts(view=views.audits, reads=reads), + declarations=_TrackedDeclarationFacts(view=views.declarations, reads=reads), + ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 4022e48b90..929e79dcca 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -52,7 +52,6 @@ evaluate_native, finalize_native_findings, native_catalogue, - rules_cache_exists, ) from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine._helpers.run.literal_duplicates import with_duplicate_literal_hints @@ -160,7 +159,6 @@ def evaluate_rules( catalogue=catalogue, dialect=dialect, defer_suppressions=True, - rules_cache_present_at_start=rules_cache_exists(resolved_project_dir), ) sql_started: float = time.monotonic() sql_result: _SqlRulesEvaluation = _run_sql_rules( diff --git a/src/sqlbuild/rule_engine/classes/fact_digests.py b/src/sqlbuild/rule_engine/classes/fact_digests.py new file mode 100644 index 0000000000..05c663cc44 --- /dev/null +++ b/src/sqlbuild/rule_engine/classes/fact_digests.py @@ -0,0 +1,47 @@ +"""Memoized digests of recorded custom-rule facts.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable, Iterable +from typing import Any + +import orjson + +from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_outcome_digest +from sqlbuild.rule_engine.models import RuleFactViews +from sqlbuild.rule_engine.types import FactKey + + +class FactDigests: + """Memoized digests of recorded fact keys against one compiled project snapshot.""" + + def __init__(self, *, views: Callable[[], RuleFactViews]) -> None: + self._views_factory: Callable[[], RuleFactViews] = views + self._views: RuleFactViews | None = None + self._digests: dict[FactKey, str] = {} + + @property + def views(self) -> RuleFactViews: + if self._views is None: + self._views = self._views_factory() + return self._views + + def digest(self, key: FactKey) -> str: + """Return one fact digest, raising FactDigestError when it cannot be reproduced.""" + + cached: str | None = self._digests.get(key) + if cached is not None: + return cached + digest: str = fact_outcome_digest(views=self.views, key=key) + self._digests[key] = digest + return digest + + def combined(self, keys: Iterable[FactKey]) -> str: + """Digest one read set in canonical key order.""" + + combined: Any = hashlib.sha256() + for key in sorted(keys): + combined.update(orjson.dumps(key)) + combined.update(self.digest(key).encode()) + return combined.hexdigest() diff --git a/src/sqlbuild/rule_engine/classes/fact_reads.py b/src/sqlbuild/rule_engine/classes/fact_reads.py new file mode 100644 index 0000000000..f099e393a1 --- /dev/null +++ b/src/sqlbuild/rule_engine/classes/fact_reads.py @@ -0,0 +1,50 @@ +"""Recorder for the compiler facts one custom-rule evaluation observes.""" + +from __future__ import annotations + +import os + +from sqlbuild.rule_engine.models import Model + + +class FactReads: + """Collect the fact keys one custom-rule evaluation observes.""" + + __slots__ = ("current", "untracked") + + def __init__(self) -> None: + self.current: set[tuple[object, ...]] | None = None + self.untracked: bool = False + + def record(self, *, key: tuple[object, ...]) -> None: + current: set[tuple[object, ...]] | None = self.current + if current is not None: + current.add(key) + + def record_model_path(self, *, fact: str, model: object) -> None: + self._record_model(fact=fact, model=model, by_name=False) + + def record_model_name(self, *, fact: str, model: object) -> None: + self._record_model(fact=fact, model=model, by_name=True) + + def _record_model(self, *, fact: str, model: object, by_name: bool) -> None: + current: set[tuple[object, ...]] | None = self.current + if current is None: + return + if type(model) is Model: + current.add((fact, model.name if by_name else model.path)) + else: + self.untracked = True + + def record_text(self, *, fact: str, values: tuple[object, ...]) -> None: + current: set[tuple[object, ...]] | None = self.current + if current is None: + return + texts: list[str] = [] + for value in values: + if isinstance(value, (str, os.PathLike)): + texts.append(os.fspath(value)) + else: + self.untracked = True + return + current.add((fact, *texts)) diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index 741caf6caf..5bec6e4632 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -20,11 +20,17 @@ class ProjectTree: def __init__(self, *, project_dir: Path, project: CompiledProject) -> None: self._root: Path = project_dir.resolve() - self._nodes: tuple[ProjectPath, ...] = self._discover() + self._discovered: tuple[ProjectPath, ...] | None = None self._resource_paths: dict[str, Model] = { item.relative_path.as_posix(): public_model(item) for item in project.models } + @property + def _nodes(self) -> tuple[ProjectPath, ...]: + if self._discovered is None: + self._discovered = self._discover() + return self._discovered + def paths(self) -> tuple[ProjectPath, ...]: return self._nodes diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index db86706c1f..f6cbceb13a 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -10,6 +10,7 @@ from sqlbuild.rule_engine.classes.column_facts import ColumnFacts from sqlbuild.rule_engine.classes.contract_facts import ContractFacts from sqlbuild.rule_engine.classes.declaration_facts import DeclarationFacts +from sqlbuild.rule_engine.classes.fact_reads import FactReads from sqlbuild.rule_engine.classes.graph_facts import GraphFacts from sqlbuild.rule_engine.classes.model_test_facts import TestFacts from sqlbuild.rule_engine.classes.project_facts import ProjectFacts @@ -59,6 +60,7 @@ def __init__( selected_rules: tuple[Rule, ...], dialect: str, facts: RuleFactViews | None = None, + reads: FactReads | None = None, ) -> None: del model, selected_rules self._rule = rule @@ -74,6 +76,13 @@ def __init__( self.tests = views.tests self.audits = views.audits self.declarations = views.declarations + self._reads: FactReads | None = reads + + def __setattr__(self, name: str, value: object) -> None: + reads: FactReads | None = self.__dict__.get("_reads") + if reads is not None: + reads.untracked = True + object.__setattr__(self, name, value) def option[T](self, option: RuleOption[T]) -> T: """Return one declared option after native configuration validation.""" diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index f2685811ee..2a22a6e537 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -8,7 +8,10 @@ RULES_NATIVE_API_VERSION: int = 1 CUSTOM_HOST_PROTOCOL_VERSION: int = 1 -CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v1" +CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" +CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 +CUSTOM_RULES_CACHE_VERSION: str = "custom-rules-reads-v1" +CUSTOM_RULES_CACHE_FILE: str = "target/rules-cache/bulk/custom-rules.json" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 CUSTOM_HOST_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host" CUSTOM_HOST_LAUNCH_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host_launch" @@ -79,11 +82,6 @@ TARGET_DIRECTORY_NAME: str = "target" RULE_DECORATOR_TOKEN: str = "@rule" INIT_MODULE_NAME: str = "__init__" -RULE_CONTEXT_PROJECT_FACT: str = "project" -RULE_CONTEXT_GRAPH_FACT: str = "graph" -RULE_CONTEXT_TESTS_FACT: str = "tests" -RULE_CONTEXT_AUDITS_FACT: str = "audits" -RULE_CONTEXT_DECLARATIONS_FACT: str = "declarations" PARENT_DIRECTORY_TOKEN: str = ".." MIN_AUDITS_PER_MODEL: str = "min_audits_per_model" @@ -123,3 +121,32 @@ RULE_FIX_AVAILABLE_NOTE: str = ( "Fix available: `sqb format --fix` applies it after compiler verification" ) +FACT_PROJECT_MODELS: str = "project.models" +FACT_PROJECT_SOURCES: str = "project.sources" +FACT_PROJECT_SEEDS: str = "project.seeds" +FACT_PROJECT_FUNCTIONS: str = "project.functions" +FACT_PROJECT_TESTS: str = "project.tests" +FACT_PROJECT_AUDITS: str = "project.audits" +FACT_TREE_PATHS: str = "tree.paths" +FACT_TREE_CHILDREN: str = "tree.children" +FACT_TREE_DESCENDANTS: str = "tree.descendants" +FACT_TREE_GLOB: str = "tree.glob" +FACT_TREE_RELATIVE_PARTS: str = "tree.relative_parts" +FACT_TREE_RESOURCES_UNDER: str = "tree.resources_under" +FACT_TREE_READ_TEXT: str = "tree.read_text" +FACT_SQL_FOR_MODEL: str = "sql.for_model" +FACT_GRAPH_DEPENDENCIES: str = "graph.dependencies" +FACT_GRAPH_DEPENDENTS: str = "graph.dependents" +FACT_COLUMNS_DECLARED: str = "columns.declared" +FACT_COLUMNS_INFERRED: str = "columns.inferred" +FACT_COLUMNS_NAMES: str = "columns.names" +FACT_CONTRACTS_ENFORCED: str = "contracts.enforced" +FACT_CONTRACTS_GRAIN: str = "contracts.grain" +FACT_TESTS_ALL: str = "tests.all" +FACT_TESTS_FOR_MODEL: str = "tests.for_model" +FACT_AUDITS_ALL: str = "audits.all" +FACT_AUDITS_FOR_MODEL: str = "audits.for_model" +FACT_DECLARATIONS_PUBLIC_ENUMS: str = "declarations.public_enums" +FACT_DECLARATIONS_PUBLIC_CONSTANTS: str = "declarations.public_constants" +FACT_DECLARATIONS_ENUMS: str = "declarations.enums" +FACT_DECLARATIONS_CONSTANTS: str = "declarations.constants" diff --git a/src/sqlbuild/rule_engine/exceptions.py b/src/sqlbuild/rule_engine/exceptions.py index 2d4ace09b4..2d164a99a2 100644 --- a/src/sqlbuild/rule_engine/exceptions.py +++ b/src/sqlbuild/rule_engine/exceptions.py @@ -15,3 +15,11 @@ class NonHermeticRuleError(RulesError): class RuleAssertionError(RulesError): """Raised when a custom-rule harness case produces an unexpected result.""" + + +class FactDigestError(RulesError): + """Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.""" + + +class OpaqueModuleStateError(RulesError): + """Raised when custom-rule module state holds a value whose mutations cannot be observed.""" diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 9c0b57c0d0..d464c229e4 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -381,3 +381,31 @@ class RuleResult: @property def finding_count(self) -> int: return len(self.findings) + + +@dataclass(frozen=True) +class CustomRuleEvaluation: + """Findings and observed fact reads of one rule invocation for one subject.""" + + code: str + subject: str | None + findings: tuple[Finding, ...] + reads: frozenset[tuple[object, ...]] | None + + +@dataclass(frozen=True) +class CustomRuleRun: + """Every invocation of one host run plus rules whose reads could not be attributed.""" + + evaluations: tuple[CustomRuleEvaluation, ...] + untracked_codes: frozenset[str] + + +@dataclass(frozen=True) +class CustomRulesOutcome: + """Custom-rule finding payloads in canonical order and their cache accounting.""" + + findings: tuple[dict[str, object], ...] + cache_hits: int + cache_misses: int + custom_ms: int diff --git a/src/sqlbuild/rule_engine/types.py b/src/sqlbuild/rule_engine/types.py index af7766b5d5..ccaedcbba2 100644 --- a/src/sqlbuild/rule_engine/types.py +++ b/src/sqlbuild/rule_engine/types.py @@ -25,6 +25,8 @@ ) type RuleOptionValue = bool | int | str | tuple[str, ...] | tuple[int, ...] +type FactKey = tuple[str, ...] +type CustomRulePlan = dict[str, frozenset[str] | None] class RuleSubject(StrEnum): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 6a4bb69c45..2f84b7dea5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -1,6 +1,8 @@ """Test case types for Rules performance guards.""" +from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path @dataclass(frozen=True) @@ -189,3 +191,12 @@ class DuplicateLiteralCase: description: str literals: tuple[tuple[str, str], ...] expected_hints: tuple[tuple[str, str | None], ...] + + +@dataclass(frozen=True) +class RulesCacheEditCase: + """One authored input edit applied after a warm compile with every rule selected.""" + + description: str + edit: Callable[[Path], None] + expected_exit_code: int = 1 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 4f9e255e1e..831de8a32c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -1,5 +1,6 @@ """Compiler Rules end-to-end fixture helpers.""" +import hashlib import json import os import subprocess @@ -226,3 +227,147 @@ def long_literal_hints(result: subprocess.CompletedProcess[str]) -> dict[str, st str(item["path"]): str(item["remediation"]).partition("Define the value once")[0].strip() for item in json.loads(result.stdout)["findings"] } + + +INCREMENTAL_RULES_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "warehouse.duckdb"\n\n' + '[targets.dev]\nschema = "dev"\n\n[targets.prod]\nschema = "prod"\n\n' + '[vars]\ndiscount_rate = "0.1"\n\n' + '[rules]\nselect = ["SQBR", "XSQBR"]\n\n' + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" + ), + "models/staging/stg_orders.sql": ( + "MODEL (\n materialized view,\n contract enforced,\n columns (\n" + " order_id (type INTEGER, nullable false, audits [not_null]),\n" + " amount (type DOUBLE),\n ),\n);\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(10.5 AS DOUBLE) AS amount\n" + ), + "models/marts/order_totals.sql": ( + "MODEL (\n columns (\n" + " order_id (type INTEGER, nullable false, audits [not_null]),\n" + " net_amount (type DOUBLE),\n ),\n);\n\n" + "SELECT\n o.order_id,\n" + ' CAST(@discounted("o.amount") * (1 - CAST(@@discount_rate AS DOUBLE)) AS DOUBLE)' + " AS net_amount\n" + 'FROM __ref("stg_orders") AS o\n' + ), + "models/marts/_sqlbuild/_macros/amounts.py": ( + 'def discounted(amount: str) -> str:\n return f"({amount}) * 0.9"\n' + ), + "tests/unit/test_order_totals.sql": ( + "TEST ();\n\nWITH\n__ref__stg_orders AS (\n SELECT 1 AS order_id, 10.0 AS amount\n),\n" + "__expected__order_totals AS (\n SELECT 1 AS order_id, 9.0 AS net_amount\n)\n" + "SELECT 1\n" + ), + "config/policy.yml": "strict: false\n", + "rules/limits.py": "MAX_NAME_LENGTH: int = 30\n", + "rules/memo.py": """from sqlbuild.rules import Finding, Model, RuleContext, rule + +_SEEN_SQL: dict[str, str] = {} + + +@rule(code="XSQBRGOV005", message="Models must not repeat SQL", remediation="Deduplicate.") +def unique_sql(*, model: Model, ctx: RuleContext) -> list[Finding]: + if not _SEEN_SQL: + for item in ctx.project.models: + _SEEN_SQL[item.name] = ctx.sql.for_model(item).expanded.source + own: str = _SEEN_SQL[model.name] + repeated: bool = sum(source == own for source in _SEEN_SQL.values()) > 1 + return [ctx.finding(subject=model)] if repeated else [] +""", + "rules/governance.py": """from sqlbuild.rules import Finding, Model, Project, RuleContext, rule + +from rules import limits + + +@rule(code="XSQBRGOV001", message="Contracts must be enforced", remediation="Enforce it.") +def enforced_contracts(*, model: Model, ctx: RuleContext) -> list[Finding]: + if "SELECT" not in ctx.sql.for_model(model).authored.source: + return [] + return [] if ctx.contracts.enforced(model) else [ctx.finding(subject=model)] + + +@rule(code="XSQBRGOV002", message="Staging models need a consumer", remediation="Use it.") +def staging_consumers(*, model: Model, ctx: RuleContext) -> list[Finding]: + if not model.name.startswith("stg_"): + return [] + return [] if ctx.graph.dependents(model) else [ctx.finding(subject=model)] + + +@rule(code="XSQBRGOV003", message="Strict policy limits models", remediation="Relax it.") +def strict_policy(*, project: Project, ctx: RuleContext) -> list[Finding]: + del project + findings: list[Finding] = [] + if "strict: true" in ctx.project.tree.read_text("config/policy.yml"): + findings.append(ctx.finding(subject="config/policy.yml", message="strict policy")) + if len(ctx.project.tree.glob("config/*.yml")) > 1: + findings.append(ctx.finding(subject="config/policy.yml", message="several policies")) + if len(ctx.project.models) > 2: + findings.append(ctx.finding(subject="config/policy.yml", message="too many models")) + return findings + + +@rule(code="XSQBRGOV004", message="Model names are bounded", remediation="Shorten it.") +def bounded_names(*, model: Model, ctx: RuleContext) -> list[Finding]: + names: list[str] = [test.name for test in ctx.tests.for_model(model)] + too_long: bool = len(model.name) > limits.MAX_NAME_LENGTH + unnamed: bool = any("discount" not in name for name in names) + return [ctx.finding(subject=model)] if too_long or unnamed else [] + + +@rule(code="XSQBRGOV006", message="Expanded SQL uses a flagged value", remediation="Review.") +def flagged_values(*, model: Model, ctx: RuleContext) -> list[Finding]: + expanded: str = ctx.sql.for_model(model).expanded.source + flagged: bool = any(value in expanded for value in ("12.5", "0.8", "0.2")) + return [ctx.finding(subject=model)] if flagged else [] +""", +} + + +def write_project_files(*, project_dir: Path, files: dict[str, str]) -> None: + """Write authored project files beneath one project directory.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def rules_compile_outcome(project_dir: Path, *arguments: str) -> tuple[int, str, str]: + """Compile in a fresh process and return its exit code, diagnostics, and artifacts digest.""" + + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--no-color", + "--project-dir", + str(project_dir), + "compile", + "--json", + *arguments, + ], + capture_output=True, + text=True, + check=False, + ) + payload: dict[str, Any] = json.loads(result.stdout or "{}") + artifacts: Any = hashlib.sha256() + compiled: Path = project_dir / "target" / "compiled" + for path in sorted(filter(Path.is_file, compiled.rglob("*"))): + artifacts.update(path.relative_to(compiled).as_posix().encode()) + artifacts.update(path.read_bytes()) + return ( + result.returncode, + json.dumps(payload.get("diagnostics"), sort_keys=True), + artifacts.hexdigest(), + ) + + +def rule_cache_counts(project_dir: Path) -> tuple[int, int]: + """Return rule cache hits and misses of one fresh-process compile.""" + + result: subprocess.CompletedProcess[str] = run_compile_cli(project_dir) + timings: dict[str, int] = json.loads(result.stdout)["compile_timings"] + return timings["rule_cache_hits"], timings["rule_cache_misses"] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py new file mode 100644 index 0000000000..96611b12f6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py @@ -0,0 +1,140 @@ +"""A warm Rules compile after any single input edit must equal a compile without caches.""" + +import shutil +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + replace_project_text, + write_project_file, +) +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import RulesCacheEditCase +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + INCREMENTAL_RULES_PROJECT, + rule_cache_counts, + rules_compile_outcome, + write_project_files, +) + + +@pytest.mark.parametrize( + "test_case", + ( + RulesCacheEditCase( + "model_sql_edit", + lambda root: replace_project_text( + root, + "models/staging/stg_orders.sql", + "CAST(10.5 AS DOUBLE)", + "CAST(12.5 AS DOUBLE)", + ), + ), + RulesCacheEditCase( + "model_contract_edit", + lambda root: replace_project_text( + root, "models/staging/stg_orders.sql", " contract enforced,\n", "" + ), + ), + RulesCacheEditCase( + "model_added", + lambda root: write_project_file( + root, + "models/marts/order_counts.sql", + 'MODEL ();\n\nSELECT order_id FROM __ref("stg_orders")\n', + ), + ), + RulesCacheEditCase( + "consumer_removed", + lambda root: replace_project_text( + root, + "models/marts/order_totals.sql", + 'FROM __ref("stg_orders") AS o\n', + "FROM (SELECT 1 AS order_id, 1.0 AS amount) AS o\n", + ), + ), + RulesCacheEditCase( + "test_edit", + lambda root: replace_project_text( + root, + "tests/unit/test_order_totals.sql", + "TEST ();", + 'TEST (name "order_totals_applies_discount");', + ), + ), + RulesCacheEditCase( + "macro_edit", + lambda root: replace_project_text( + root, "models/marts/_sqlbuild/_macros/amounts.py", "* 0.9", "* 0.8" + ), + ), + RulesCacheEditCase( + "var_edit", + lambda root: replace_project_text( + root, "sqlbuild_project.toml", 'discount_rate = "0.1"', 'discount_rate = "0.2"' + ), + ), + RulesCacheEditCase( + "rule_source_edit", + lambda root: replace_project_text( + root, + "rules/governance.py", + '"Contracts must be enforced"', + '"Output contracts must be enforced"', + ), + ), + RulesCacheEditCase( + "rule_helper_edit", + lambda root: replace_project_text(root, "rules/limits.py", "= 30", "= 5"), + ), + RulesCacheEditCase( + "tree_file_edit", + lambda root: replace_project_text(root, "config/policy.yml", "false", "true"), + ), + RulesCacheEditCase( + "tree_file_added", + lambda root: write_project_file(root, "config/extra.yml", "enabled: true\n"), + ), + RulesCacheEditCase( + "rules_threshold_edit", + lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + "min_custom_rule_test_cases = 0\n", + "min_custom_rule_test_cases = 0\nmin_tests_per_model = 0\n", + ), + ), + RulesCacheEditCase( + "target_switch", + lambda root: write_project_file(root, "sqlbuild_local.toml", 'target = "prod"\n'), + ), + ), + ids=lambda case: case.description, +) +def test_given_warm_rules_cache_when_one_input_changes_then_compile_equals_uncached_compile( + tmp_path: Path, test_case: RulesCacheEditCase +) -> None: + project_dir: Path = tmp_path / "orders" + write_project_files(project_dir=project_dir, files=INCREMENTAL_RULES_PROJECT) + cold: tuple[int, str, str] = rules_compile_outcome(project_dir) + assert cold[0] == 1 + assert '"XSQBRGOV001"' in cold[1] + hits, misses = rule_cache_counts(project_dir) + assert hits > 0 + assert misses == 0 + + test_case.edit(project_dir) + edited: tuple[int, str, str] = rules_compile_outcome(project_dir) + oracle_dir: Path = tmp_path / "oracle" + shutil.copytree(project_dir, oracle_dir, ignore=shutil.ignore_patterns("target")) + oracle: tuple[int, str, str] = rules_compile_outcome(oracle_dir, "--no-cache") + rewarmed: tuple[int, str, str] = rules_compile_outcome(project_dir) + + assert edited[0] == test_case.expected_exit_code + assert edited == oracle + assert rewarmed == oracle + assert rule_cache_counts(project_dir)[1] == 0 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index b6c5142b08..33fb11fea3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -import threading from collections.abc import Callable from pathlib import Path from typing import cast @@ -14,13 +13,12 @@ import sqlbuild.compiler.compile._helpers.attachment.declaration_scope as declaration_scope_module import sqlbuild.compiler.compile.main._build_compile_inputs as compile_inputs_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module -import sqlbuild.rule_engine._helpers.engine.native as native_module +import sqlbuild.rule_engine._helpers.engine.custom_rules as custom_rules_module import sqlbuild.rule_engine._helpers.run.rules as rules_module from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile.models import CompiledProject, DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.rule_engine.models import Rule from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( DynamicPivotRulesIntegrationTestCase, ExplicitContractOutputRuleIntegrationTestCase, @@ -921,10 +919,10 @@ def counting_collect( @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("custom host starts with the first request", 1, "XSQBRARCH001")], + [RulesIntegrationTestCase("custom host runs once and is skipped when warm", 1, "XSQBRARCH001")], ids=lambda case: case.description, ) -def test_given_sql_rules_create_cache_first_when_compiling_then_custom_host_is_not_retried( +def test_given_sql_and_custom_rules_when_compiling_twice_then_custom_host_runs_only_cold( test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -962,43 +960,21 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: """, encoding="utf-8", ) - sql_cache_written: threading.Event = threading.Event() - requests: list[bool] = [] - original_write: Callable[..., None] = rules_module._write_sql_rule_cache - original_payloads: Callable[..., list[dict[str, object]]] = native_module._custom_rule_payloads - original_request: Callable[[dict[str, object]], str] = native_module._evaluate_request + host_runs: list[str] = [] + original_host: Callable[[str], str] = custom_rules_module._native.run_custom_host_json - def signalling_write(*, project_dir: Path, bucket: dict[str, dict[str, object]]) -> None: - original_write(project_dir=project_dir, bucket=bucket) - sql_cache_written.set() + def recording_host(spec_json: str) -> str: + host_runs.append(spec_json) + return original_host(spec_json) - def delayed_payloads( - *, - catalogue: tuple[Rule, ...], - project: CompiledProject, - project_dir: Path, - cache_enabled: bool, - ) -> list[dict[str, object]]: - _ = sql_cache_written.wait(timeout=30) - return original_payloads( - catalogue=catalogue, - project=project, - project_dir=project_dir, - cache_enabled=cache_enabled, - ) - - def recording_request(request: dict[str, object]) -> str: - requests.append(request["custom_host"] is not None) - return original_request(request) - - monkeypatch.setattr(rules_module, "_write_sql_rule_cache", signalling_write) - monkeypatch.setattr(native_module, "_custom_rule_payloads", delayed_payloads) - monkeypatch.setattr(native_module, "_evaluate_request", recording_request) + monkeypatch.setattr(custom_rules_module._native, "run_custom_host_json", recording_host) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) - assert sql_cache_written.is_set() - assert requests == [True] + warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + + assert len(host_runs) == 1 + assert warm == cold assert { f"models/orders.sql:{test_case.expected_code}", "tests/unit/test_orders.sql:SQBRSQL004", diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index ed35db4454..6197220f68 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -1,6 +1,6 @@ """Test case declarations for rules engine helpers.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from sqlbuild.rule_engine.models import RuleExemption, RuleIgnore @@ -111,12 +111,6 @@ class ScopePayloadTestCase: expected_result: bool -@dataclass(frozen=True) -class ProjectFingerprintRepresentationTestCase: - description: str - expected_equivalent: bool - - @dataclass(frozen=True) class NativeFactPayloadTestCase: description: str @@ -155,3 +149,15 @@ class CustomRuleImportTestCase: extra_files: tuple[tuple[str, str], ...] = () expected_rule_codes: tuple[str, ...] = ("XSQBRT101",) expected_error_pattern: str = "" + + +@dataclass(frozen=True) +class CustomRuleReadTrackingTestCase: + """One model-subject rule, an edit to the second model, and the expected reuse.""" + + description: str + body: str + expected_hits_after_edit: int + module_prelude: str = "" + edited_customers_sql: str = "SELECT 2 AS customer_id" + edited_customers_config: dict[str, object] = field(default_factory=dict) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index bb91a7a532..e569ca97dd 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -22,8 +22,9 @@ from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import native from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue +from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.models import Rule, RulesCacheConfig, RulesConfig +from sqlbuild.rule_engine.models import CustomRulesOutcome, Rule, RulesCacheConfig, RulesConfig from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project @@ -58,6 +59,7 @@ def evaluate_json(request_json: str) -> str: "evaluated_models": 0, "cache_hits": 0, "cache_misses": 0, + "selected_codes": [], } ) @@ -226,3 +228,39 @@ def custom_rules_with_imports( config=RulesConfig(select=("XSQBRT101",)), project_dir=project_dir ) return tuple(filter(lambda rule: rule.custom, catalogue)) + + +def two_model_project( + *, customers_sql: str, customers_config: dict[str, object] +) -> CompiledProject: + """Return an orders and customers project with configurable customers facts.""" + + orders: CompiledProject = build_project( + name="orders", + relative_path="models/orders.sql", + sql="SELECT 1 AS order_id", + config_values={}, + ) + customers: CompiledProject = build_project( + name="customers", + relative_path="models/customers.sql", + sql=customers_sql, + config_values=customers_config, + ) + return replace(orders, models=(*orders.models, *customers.models)) + + +def evaluate_cached_custom_rules( + *, project: CompiledProject, project_dir: Path, cache_enabled: bool +) -> CustomRulesOutcome: + """Evaluate the selected custom rule through the incremental read-tracking cache.""" + + config: RulesConfig = RulesConfig( + select=("XSQBRT101",), cache=RulesCacheConfig(enabled=cache_enabled) + ) + rules: tuple[Rule, ...] = tuple( + filter(attrgetter("custom"), build_catalogue(config=config, project_dir=project_dir)) + ) + return evaluate_custom_rules_cached( + project=project, config=config, project_dir=project_dir, rules=rules, dialect="duckdb" + ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py new file mode 100644 index 0000000000..36572d0ed3 --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py @@ -0,0 +1,122 @@ +"""Custom-rule results are reused only when every fact the invocation read is unchanged.""" + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.rule_engine.models import CustomRulesOutcome +from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( + CustomRuleReadTrackingTestCase, +) +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( + evaluate_cached_custom_rules, + two_model_project, + write_rule, +) + +_CUSTOMERS_SQL: str = "SELECT 1 AS customer_id" +_FLAG_ORDERS: str = ( + ' return [ctx.finding(subject=model)] if model.name == "orders" and {condition} else []' +) + + +@pytest.mark.parametrize( + "test_case", + ( + CustomRuleReadTrackingTestCase( + description="own SQL read reuses the unedited subject", + body="_ = ctx.sql.for_model(model).expanded.source\n" + + _FLAG_ORDERS.format(condition="True"), + expected_hits_after_edit=1, + ), + CustomRuleReadTrackingTestCase( + description="reading every model SQL invalidates every subject", + body="sources = [ctx.sql.for_model(item).expanded.source " + "for item in ctx.project.models]\n" + _FLAG_ORDERS.format(condition="len(sources) > 1"), + expected_hits_after_edit=0, + ), + CustomRuleReadTrackingTestCase( + description="unchanged contracts of every model keep both subjects", + body="enforced = [ctx.contracts.enforced(item) for item in ctx.project.models]\n" + + _FLAG_ORDERS.format(condition="not any(enforced)"), + expected_hits_after_edit=2, + ), + CustomRuleReadTrackingTestCase( + description="changed contract of another model invalidates readers", + body="enforced = [ctx.contracts.enforced(item) for item in ctx.project.models]\n" + + _FLAG_ORDERS.format(condition="not any(enforced)"), + expected_hits_after_edit=0, + edited_customers_sql=_CUSTOMERS_SQL, + edited_customers_config={"contract": "enforced"}, + ), + CustomRuleReadTrackingTestCase( + description="module-level memo makes the rule untracked", + module_prelude="_MEMO: dict[str, object] = {}", + body="if 'sql' not in _MEMO:\n" + " _MEMO['sql'] = [ctx.sql.for_model(item).expanded.source " + "for item in ctx.project.models]\n" + + _FLAG_ORDERS.format(condition="'SELECT 2' in ' '.join(_MEMO['sql'])"), + expected_hits_after_edit=0, + ), + CustomRuleReadTrackingTestCase( + description="cached helper makes the rule untracked", + module_prelude=( + "import functools\n\n" + "@functools.cache\n" + "def _sources(ctx: RuleContext) -> str:\n" + " return ' '.join(ctx.sql.for_model(item).expanded.source " + "for item in ctx.project.models)\n" + ), + body=_FLAG_ORDERS.format(condition="'SELECT 2' in _sources(ctx)").strip(), + expected_hits_after_edit=0, + ), + CustomRuleReadTrackingTestCase( + description="private view access makes the rule untracked", + body="_ = ctx.sql._models\n" + _FLAG_ORDERS.format(condition="True"), + expected_hits_after_edit=0, + ), + CustomRuleReadTrackingTestCase( + description="context attribute state makes the rule untracked", + body="ctx.seen = model.name\n" + _FLAG_ORDERS.format(condition="True"), + expected_hits_after_edit=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_recorded_reads_when_one_model_changes_then_only_dependent_results_rerun( + tmp_path: Path, + test_case: CustomRuleReadTrackingTestCase, +) -> None: + _ = write_rule(root=tmp_path, body=test_case.body, module_import=test_case.module_prelude) + original: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + edited: CompiledProject = two_model_project( + customers_sql=test_case.edited_customers_sql, + customers_config=test_case.edited_customers_config, + ) + + cold: CustomRulesOutcome = evaluate_cached_custom_rules( + project=original, project_dir=tmp_path, cache_enabled=True + ) + warm: CustomRulesOutcome = evaluate_cached_custom_rules( + project=original, project_dir=tmp_path, cache_enabled=True + ) + after_edit: CustomRulesOutcome = evaluate_cached_custom_rules( + project=edited, project_dir=tmp_path, cache_enabled=True + ) + oracle: CustomRulesOutcome = evaluate_cached_custom_rules( + project=edited, project_dir=tmp_path, cache_enabled=False + ) + + assert (cold.cache_hits, cold.cache_misses) == (0, 2) + assert (warm.cache_hits, warm.cache_misses) == (2, 0) + assert warm.findings == cold.findings + assert (after_edit.cache_hits, after_edit.cache_misses) == ( + test_case.expected_hits_after_edit, + 2 - test_case.expected_hits_after_edit, + ) + assert after_edit.findings == oracle.findings + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 584fe81a61..12f1d9a917 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -170,19 +170,25 @@ def test_given_hermetic_custom_rule_when_cache_required_then_second_run_hits_cac description="model-only rule ignores unrelated project file changes", body="del model\n return []", changed_path="config/new.yml", - expected_cache_hits=1, + expected_cache_hits=2, ), CustomRuleCacheDependencyTestCase( description="graph-observing rule ignores unrelated project file changes", body="_ = ctx.graph.dependencies(model)\n return []", changed_path="config/new.yml", + expected_cache_hits=2, + ), + CustomRuleCacheDependencyTestCase( + description="project glob observes a newly matching file", + body=('del model\n _ = ctx.project.tree.glob("config/*.yml")\n return []'), + changed_path="config/new.yml", expected_cache_hits=1, ), CustomRuleCacheDependencyTestCase( - description="negative project glob observes a newly matching file", + description="project glob ignores a file it does not match", body=('del model\n _ = ctx.project.tree.glob("config/**/*.yml")\n return []'), changed_path="config/new.yml", - expected_cache_hits=0, + expected_cache_hits=2, ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index a170d9e678..586c0a081f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -5,7 +5,6 @@ import pytest from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, CompiledObjectKey, CompiledProject, CompiledSqlScenario, @@ -15,11 +14,8 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredSqlScenarioFile, ) -from sqlbuild.rule_engine._helpers.engine.native import _custom_fact_fingerprint -from sqlbuild.rule_engine.constants import RULE_CONTEXT_PROJECT_FACT from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( NativeFactPayloadTestCase, - ProjectFingerprintRepresentationTestCase, ) from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( captured_native_request, @@ -143,54 +139,5 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar assert "secret fixture value" not in str(request["sql_tests"] + request["sql_scenarios"]) -@pytest.mark.parametrize( - "test_case", - ( - ProjectFingerprintRepresentationTestCase( - "lineage representation preserves cache identity", True - ), - ), - ids=lambda case: case.description, -) -def test_given_compact_or_materialized_lineage_when_fingerprinting_then_semantics_define_identity( - test_case: ProjectFingerprintRepresentationTestCase, -) -> None: - project: CompiledProject = build_project( - name="selected_orders", - relative_path="models/selected_orders.sql", - sql="SELECT order_id FROM orders", - config_values={}, - ) - lineage: CompactLineageFacts = CompactLineageFacts( - string_pool=("order_id", "source", "canonical_input", "orders", "other_orders"), - rows=((0, 0, 1, ((1, 2, 0),)),), - resource_name_indexes={2: 3}, - ) - compact: CompiledProject = replace( - project, models=(replace(project.models[0], fast_lineage_columns=lineage),) - ) - materialized: CompiledProject = replace( - project, models=(replace(project.models[0], fast_lineage_columns=tuple(lineage)),) - ) - changed: CompiledProject = replace( - project, - models=( - replace( - project.models[0], - fast_lineage_columns=replace(lineage, resource_name_indexes={2: 4}), - ), - ), - ) - attributes: frozenset[str] = frozenset({RULE_CONTEXT_PROJECT_FACT}) - compact_fingerprint: str = _custom_fact_fingerprint(project=compact, attributes=attributes) - materialized_fingerprint: str = _custom_fact_fingerprint( - project=materialized, attributes=attributes - ) - changed_fingerprint: str = _custom_fact_fingerprint(project=changed, attributes=attributes) - - assert (compact_fingerprint == materialized_fingerprint) is test_case.expected_equivalent - assert changed_fingerprint != compact_fingerprint - - if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) From 893242c9aa3bd50c01d938d03d3b2f09d3a5b18f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:44:12 +0100 Subject: [PATCH 2/5] perf(compile): reuse the built-in rules response for an identical native request Warm compiles re-sent an unchanged 60 MB request to the native engine, which spent about a second re-validating per-model cache entries. The complete response is now memoized under a digest of the serialized request and the installed code identity, so any fact, scope, config or code change re-evaluates through the native per-model cache. Refs CHI-445 --- .../rule_engine/_helpers/engine/native.py | 37 +++++++++--- .../rule_engine/_helpers/run/native_memo.py | 49 ++++++++++++++++ src/sqlbuild/rule_engine/constants.py | 2 + .../_helpers/engine/_test_types.py | 11 ++++ .../rule_engine/_helpers/engine/helpers.py | 28 ++++++++- .../_helpers/engine/test_native_memo.py | 58 +++++++++++++++++++ 6 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 src/sqlbuild/rule_engine/_helpers/run/native_memo.py create mode 100644 tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index fe56e76ec2..a609ba1cc8 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -33,6 +33,11 @@ custom_rule_test_evidence, ) from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached +from sqlbuild.rule_engine._helpers.run.native_memo import ( + native_request_identity, + read_native_response, + write_native_response, +) from sqlbuild.rule_engine.constants import ( RULES_NATIVE_API_VERSION, TYPE_PROOF_RULE_CODES, @@ -108,7 +113,12 @@ def evaluate_native( else None ) try: - response: object = orjson.loads(_evaluate_request(request)) + response_json: str + reused: bool + response_json, reused = _evaluate_request( + request=request, project_dir=project_dir, cache_enabled=config.cache.enabled + ) + response: object = orjson.loads(response_json) except (ValueError, TypeError) as error: raise RulesError(str(error)) from error custom: CustomRulesOutcome = ( @@ -135,12 +145,14 @@ def evaluate_native( evaluated_codes=tuple(str(code) for code in selected_codes), findings=findings, ) + native_hits: int = int(payload.get("cache_hits", 0)) + native_misses: int = int(payload.get("cache_misses", 0)) return RulesResult( findings=findings, evaluated_models=int(payload.get("evaluated_models", 0)), - cache_hits=int(payload.get("cache_hits", 0)) + custom.cache_hits, - cache_misses=int(payload.get("cache_misses", 0)) + custom.cache_misses, - built_in_ms=int(payload.get("built_in_ms", 0)), + cache_hits=(native_hits + native_misses if reused else native_hits) + custom.cache_hits, + cache_misses=(0 if reused else native_misses) + custom.cache_misses, + built_in_ms=0 if reused else int(payload.get("built_in_ms", 0)), custom_ms=custom.custom_ms, ) @@ -159,10 +171,19 @@ def _selected_custom_rules( return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) -def _evaluate_request(request: dict[str, object]) -> str: - return _native.evaluate_json( - orjson.dumps(request, option=orjson.OPT_SORT_KEYS, default=str).decode() - ) +def _evaluate_request( + *, request: dict[str, object], project_dir: Path, cache_enabled: bool +) -> tuple[str, bool]: + request_json: bytes = orjson.dumps(request, option=orjson.OPT_SORT_KEYS, default=str) + if not cache_enabled: + return _native.evaluate_json(request_json.decode()), False + identity: str = native_request_identity(request_json) + reused: str | None = read_native_response(project_dir=project_dir, identity=identity) + if reused is not None: + return reused, True + response: str = _native.evaluate_json(request_json.decode()) + write_native_response(project_dir=project_dir, identity=identity, response=response) + return response, False def finalize_native_findings( diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py new file mode 100644 index 0000000000..e11cf9dc2b --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py @@ -0,0 +1,49 @@ +"""Reuse the complete built-in rules response for a byte-identical native request.""" + +from __future__ import annotations + +import hashlib +import os +import threading +from pathlib import Path +from typing import Any + +import orjson + +from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.rule_engine.constants import NATIVE_RULES_MEMO_FILE, NATIVE_RULES_MEMO_VERSION + + +def native_request_identity(request_json: bytes) -> str: + """Digest one serialized native request together with the installed code identity.""" + + digest: Any = hashlib.sha256() + digest.update(NATIVE_RULES_MEMO_VERSION.encode()) + digest.update(b"\0") + digest.update(compiled_code_identity().encode()) + digest.update(b"\0") + digest.update(request_json) + return digest.hexdigest() + + +def read_native_response(*, project_dir: Path, identity: str) -> str | None: + """Return the stored response for this exact request identity, if any.""" + + try: + payload: object = orjson.loads((project_dir / NATIVE_RULES_MEMO_FILE).read_bytes()) + except (OSError, orjson.JSONDecodeError): + return None + if not isinstance(payload, dict) or payload.get("identity") != identity: + return None + response: object = payload.get("response") + return response if isinstance(response, str) else None + + +def write_native_response(*, project_dir: Path, identity: str, response: str) -> None: + """Atomically replace the stored response with the latest evaluated request.""" + + path: Path = project_dir / NATIVE_RULES_MEMO_FILE + path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") + temporary.write_bytes(orjson.dumps({"identity": identity, "response": response})) + temporary.replace(path) diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index 2a22a6e537..b9e266f999 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -12,6 +12,8 @@ CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 CUSTOM_RULES_CACHE_VERSION: str = "custom-rules-reads-v1" CUSTOM_RULES_CACHE_FILE: str = "target/rules-cache/bulk/custom-rules.json" +NATIVE_RULES_MEMO_VERSION: str = "native-rules-response-v1" +NATIVE_RULES_MEMO_FILE: str = "target/rules-cache/bulk/native-response.json" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 CUSTOM_HOST_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host" CUSTOM_HOST_LAUNCH_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host_launch" diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index 6197220f68..d33ed9681a 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -161,3 +161,14 @@ class CustomRuleReadTrackingTestCase: module_prelude: str = "" edited_customers_sql: str = "SELECT 2 AS customer_id" edited_customers_config: dict[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True) +class NativeMemoTestCase: + """One warm built-in evaluation followed by an edit that must invalidate the memo.""" + + description: str + original_config: dict[str, object] + edited_config: dict[str, object] + expected_original_codes: tuple[str, ...] + expected_edited_codes: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index e569ca97dd..3c277bebc1 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -24,7 +24,14 @@ from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.models import CustomRulesOutcome, Rule, RulesCacheConfig, RulesConfig +from sqlbuild.rule_engine.main._evaluate import evaluate +from sqlbuild.rule_engine.models import ( + CustomRulesOutcome, + Rule, + RulesCacheConfig, + RulesConfig, + RulesResult, +) from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project @@ -66,7 +73,7 @@ def evaluate_json(request_json: str) -> str: monkeypatch.setattr(native._native, "evaluate_json", evaluate_json) native.evaluate_native( project=project, - config=RulesConfig(), + config=RulesConfig(cache=RulesCacheConfig(enabled=False)), project_dir=project_dir, catalogue=(), ) @@ -264,3 +271,20 @@ def evaluate_cached_custom_rules( return evaluate_custom_rules_cached( project=project, config=config, project_dir=project_dir, rules=rules, dialect="duckdb" ) + + +def evaluate_contract_rule( + *, config_values: dict[str, object], project_dir: Path, cache_enabled: bool +) -> RulesResult: + """Evaluate one built-in contract rule over a single mart model.""" + + config: RulesConfig = RulesConfig( + select=("SQBRCONTRACT101",), cache=RulesCacheConfig(enabled=cache_enabled) + ) + project: CompiledProject = build_project( + name="commerce__mart__orders", + relative_path="models/mart/commerce__mart__orders.sql", + sql="WITH orders AS (SELECT id FROM source_orders) SELECT id FROM orders", + config_values=config_values, + ) + return evaluate(project=project, config=config, project_dir=project_dir) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py new file mode 100644 index 0000000000..e66b18c738 --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py @@ -0,0 +1,58 @@ +from pathlib import Path + +import pytest + +from sqlbuild.rule_engine.models import RulesResult +from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import NativeMemoTestCase +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule + + +@pytest.mark.parametrize( + "test_case", + [ + NativeMemoTestCase( + description="contract_enforced_after_warm_run", + original_config={"materialized": "table"}, + edited_config={"materialized": "table", "contract": "enforced"}, + expected_original_codes=("SQBRCONTRACT101",), + expected_edited_codes=(), + ), + NativeMemoTestCase( + description="contract_removed_after_warm_run", + original_config={"materialized": "table", "contract": "enforced"}, + edited_config={"materialized": "table"}, + expected_original_codes=(), + expected_edited_codes=("SQBRCONTRACT101",), + ), + ], + ids=lambda case: case.description, +) +def test_given_warm_built_in_memo_when_request_repeats_or_changes_then_matches_uncached( + tmp_path: Path, test_case: NativeMemoTestCase +) -> None: + cached_dir: Path = tmp_path / "cached" + cached_dir.mkdir() + + first: RulesResult = evaluate_contract_rule( + config_values=test_case.original_config, project_dir=cached_dir, cache_enabled=True + ) + repeated: RulesResult = evaluate_contract_rule( + config_values=test_case.original_config, project_dir=cached_dir, cache_enabled=True + ) + edited: RulesResult = evaluate_contract_rule( + config_values=test_case.edited_config, project_dir=cached_dir, cache_enabled=True + ) + uncached_edited: RulesResult = evaluate_contract_rule( + config_values=test_case.edited_config, project_dir=tmp_path, cache_enabled=False + ) + + assert tuple(finding.code for finding in first.findings) == test_case.expected_original_codes + assert repeated.findings == first.findings + assert (repeated.cache_misses, repeated.built_in_ms) == (0, 0) + assert tuple(finding.code for finding in edited.findings) == test_case.expected_edited_codes + assert edited.findings == uncached_edited.findings + assert edited.cache_misses > 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 70273ffe88bf9cac3815349ce61b88ca8637a3a2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:51:41 +0100 Subject: [PATCH 3/5] perf(compile): resolve macro context injection once per loaded macro Every macro call rebuilt the callable's inspect signature to decide whether to inject the macro context. The decision is now a cached property of the loaded macro, removing about 3,600 signature constructions from a dense 3,000-model compile. Refs CHI-442 --- .../compiler/compile/_helpers/render/macros.py | 9 +-------- src/sqlbuild/compiler/compile/models.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index c7e1674f1a..9228c39ac3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -1717,14 +1717,7 @@ def _call_loaded_macro( args: tuple[object, ...], kwargs: dict[str, object], ) -> object: - signature: inspect.Signature = inspect.signature(loaded_macro.function) - parameters: tuple[inspect.Parameter, ...] = tuple(signature.parameters.values()) - if ( - parameters - and parameters[0].kind - in {inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD} - and parameters[0].name == MACRO_CONTEXT_PARAMETER_NAME - ): + if loaded_macro.injects_context: if MACRO_CONTEXT_PARAMETER_NAME in kwargs: raise CompileInputError( f"Macro '@{loaded_macro.name}' must not be called with keyword argument 'ctx'; " diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index ca50211cf0..282597ae36 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -2,14 +2,16 @@ from __future__ import annotations +import inspect from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass, field +from functools import cached_property from pathlib import Path from typing import Any, overload from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity -from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE +from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, @@ -234,6 +236,20 @@ class LoadedMacro: function: Callable[..., object] dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) + @cached_property + def injects_context(self) -> bool: + """Whether the first positional parameter receives the injected macro context.""" + + parameters: tuple[inspect.Parameter, ...] = tuple( + inspect.signature(self.function).parameters.values() + ) + return bool( + parameters + and parameters[0].kind + in {inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD} + and parameters[0].name == MACRO_CONTEXT_PARAMETER_NAME + ) + @dataclass(frozen=True) class StaticMacroExport: From c6ae7ef48eefb2e60b2c6124d75756bcd71990aa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:06:03 +0100 Subject: [PATCH 4/5] perf(compile): guard dense warm and one-edit compile budgets and oracles Adds 3,000- and 5,000-model dense guards that compile cold, warm, and after a single model edit, then compare the edit with a from-scratch uncached oracle. They assert exact fingerprints, complete warm cache reuse, and that the edit re-evaluates only the edited model's built-in rules while the project-wide custom rule is reused. The dense job timeout grows to cover the extra compiles. Refs CHI-441 --- .github/workflows/verify.yml | 2 +- .../cli/commands/main/compile/_test_types.py | 13 +++ .../cli/commands/main/compile/helpers.py | 45 ++++++++++ .../compile/test_dense_compile_performance.py | 90 +++++++++++++++++++ 4 files changed, 149 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index bf38f76037..9c4a376e34 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -365,7 +365,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 20 strategy: fail-fast: false matrix: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 8d1d5a5a68..cf7997e7fb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -240,6 +240,19 @@ class DenseCompileGuardTestCase: expected_fingerprint: str +@dataclass(frozen=True) +class DenseWarmEditCompileGuardTestCase: + description: str + model_count: int + edited_model_index: int + expected_warm_max_seconds: float + expected_edit_max_seconds: float + expected_max_rss_bytes: int + expected_cold_fingerprint: str + expected_edit_fingerprint: str + expected_edit_rule_cache_misses: int + + @dataclass(frozen=True) class RuleGatedTestDiagnosticsTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index a178d3dfa3..fde1b109b2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -840,6 +840,51 @@ def prepare_small_inspection_project(*, project_dir: Path, model_count: int) -> ) +def run_dense_warm_edit_benchmark( + *, + project_dir: Path, + edited_model_path: Path, + expected_warm_max_seconds: float, + expected_edit_max_seconds: float, +) -> dict[str, FreshProcessCompileBenchmarkResult]: + """Measure cached cold, warm, and one-model edit plus a from-scratch oracle of the edit.""" + + measurements: dict[str, FreshProcessCompileBenchmarkResult] = { + "cold": _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label="dense-cold", + expected_max_wall_seconds=4 * expected_warm_max_seconds, + compile_args=(), + ), + "warm": _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label="dense-warm", + expected_max_wall_seconds=expected_warm_max_seconds, + compile_args=(), + ), + } + _replace_benchmark_text( + path=edited_model_path, + old="COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)", + new="COALESCE(b.amount, 1000) + CAST(b.id AS DOUBLE)", + ) + measurements["edit"] = _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label="dense-edit", + expected_max_wall_seconds=expected_edit_max_seconds, + compile_args=(), + ) + oracle_dir: Path = project_dir.with_name(f"{project_dir.name}_oracle") + _ = shutil.copytree(project_dir, oracle_dir, ignore=shutil.ignore_patterns("target")) + measurements["oracle"] = _run_fresh_process_compile_benchmark( + project_dir=oracle_dir, + label="dense-oracle", + expected_max_wall_seconds=4 * expected_edit_max_seconds, + compile_args=("--no-cache",), + ) + return measurements + + def _append_benchmark_edit(path: Path, label: str) -> None: path.write_text( path.read_text(encoding="utf-8") + f"\n-- one {label} edit\n", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 9a358f069e..5dc59b4235 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -8,6 +8,7 @@ import pytest from scripts.cold_compile_performance._helpers.dense_project import ( + dense_model_name, write_dense_compile_project, ) from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( @@ -18,11 +19,14 @@ from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, + DenseWarmEditCompileGuardTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, + fresh_process_compile_cache_metrics, measure_model_sql_bytes, + run_dense_warm_edit_benchmark, ) _GIB: int = 1024 * 1024 * 1024 @@ -125,5 +129,91 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ assert result.peak_rss_bytes < test_case.expected_max_rss_bytes +@pytest.mark.performance +@pytest.mark.cold_compile_performance +@pytest.mark.parametrize( + "test_case", + ( + DenseWarmEditCompileGuardTestCase( + "dense_models_3000_warm_and_one_edit", + 3000, + 1521, + 22.0, + 24.0, + 11 * _GIB // 4, + "c35ea7ec71e6e81059146bc1cd72f10e6493248dc6274aba7737d616d434585b", + "255c19bb188491b53d44c2e5d41472137f763806c699f5acb3899c6b47c5360b", + 3, + ), + DenseWarmEditCompileGuardTestCase( + "dense_models_5000_warm_and_one_edit", + 5000, + 2521, + 36.0, + 38.0, + 13 * _GIB // 4, + "b07422282a6800de7830b9c4950f76bdc6fd555ebd3f21152b8649f976be74dc", + "c239f53fb5fa787d250f5f7bf404b3731b480b77d2690dd31fb41ba727475fc0", + 3, + ), + ), + ids=lambda case: case.description, +) +def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches_oracle_in_budget( + test_case: DenseWarmEditCompileGuardTestCase, + tmp_path: Path, +) -> None: + assert_required_cgroup_memory_limit() + project_dir: Path = tmp_path / "dense_orders" + write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) + index: int = test_case.edited_model_index + edited_model_path: Path = ( + project_dir + / "models" + / f"sales{index // 1000:03d}" + / "intermediate" + / "clean" + / f"batch{index // 5:05d}" + / f"{dense_model_name(index)}.sql" + ) + measurements: dict[str, FreshProcessCompileBenchmarkResult] = run_dense_warm_edit_benchmark( + project_dir=project_dir, + edited_model_path=edited_model_path, + expected_warm_max_seconds=test_case.expected_warm_max_seconds, + expected_edit_max_seconds=test_case.expected_edit_max_seconds, + ) + for label, result in measurements.items(): + print( + f"dense {label} models={test_case.model_count} wall={result.elapsed_seconds:.2f}s " + f"cpu={result.cpu_seconds:.2f}s fingerprint={result.semantic_fingerprint}", + flush=True, + ) + assert result.payload["diagnostics"] == [] + assert result.payload["has_errors"] is False + assert result.peak_rss_bytes < test_case.expected_max_rss_bytes + cold: FreshProcessCompileBenchmarkResult = measurements["cold"] + warm: FreshProcessCompileBenchmarkResult = measurements["warm"] + edit: FreshProcessCompileBenchmarkResult = measurements["edit"] + assert cold.semantic_fingerprint == test_case.expected_cold_fingerprint + assert warm.semantic_fingerprint == test_case.expected_cold_fingerprint + assert edit.semantic_fingerprint == test_case.expected_edit_fingerprint + assert measurements["oracle"].semantic_fingerprint == test_case.expected_edit_fingerprint + warm_timings: dict[str, int] = cast(dict[str, int], warm.payload["compile_timings"]) + edit_timings: dict[str, int] = cast(dict[str, int], edit.payload["compile_timings"]) + batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(warm) + assert (batch_hits + entry_hits, misses, bypasses) == (test_case.model_count, 0, 0) + assert warm_timings["rule_cache_misses"] == 0 + batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(edit) + assert 0 < misses < test_case.model_count + assert batch_hits + entry_hits + misses == test_case.model_count + assert edit_timings["rule_cache_misses"] == test_case.expected_edit_rule_cache_misses + oracle_timings: dict[str, int] = cast( + dict[str, int], measurements["oracle"].payload["compile_timings"] + ) + assert oracle_timings["rule_cache_hits"] == 0 + assert warm.elapsed_seconds < test_case.expected_warm_max_seconds + assert edit.elapsed_seconds < test_case.expected_edit_max_seconds + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) From c0b3908779c2daf8f80cce36f7da8eadd1c9cbe3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:30:47 +0100 Subject: [PATCH 5/5] fix(rules): verify filesystem reads and conservatively key untracked custom-rule results - Record the digest of each project-tree value a rule actually read in the host and refuse to store a result whose filesystem facts changed before the parent recorded the fingerprint. - Key untracked rules on their recorded reads plus every observable fact, so files outside the discovered tree (hidden directories) still invalidate them. - Never cache rules that touch private or unknown fact-view attributes. - Include the thread id in the custom-rule cache temporary file name. --- .../_helpers/engine/custom_rules.py | 80 +++++-- .../_helpers/engine/fact_replay.py | 21 +- .../_helpers/host/custom_evaluation.py | 9 +- .../rule_engine/_helpers/host/custom_host.py | 14 +- .../_helpers/host/tracked_facts.py | 78 +++++-- .../rule_engine/classes/fact_reads.py | 22 +- src/sqlbuild/rule_engine/models.py | 2 + .../_helpers/engine/_test_types.py | 30 +++ .../rule_engine/_helpers/engine/helpers.py | 27 +++ .../_helpers/engine/test_custom_rule_reads.py | 196 +++++++++++++++++- .../_helpers/engine/test_native_memo.py | 40 +++- 11 files changed, 460 insertions(+), 59 deletions(-) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 80c170899d..4b500db707 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -7,6 +7,7 @@ import pickle import sys import tempfile +import threading import time from collections.abc import Callable from dataclasses import asdict, dataclass, replace @@ -22,7 +23,7 @@ custom_rule_implementation_fingerprint, custom_rule_import_closure, ) -from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys +from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys, is_tree_fact from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( @@ -51,6 +52,14 @@ class _Evaluated: reads: tuple[FactKey, ...] | None +@dataclass(frozen=True) +class _HostRun: + evaluated: dict[tuple[str, str], _Evaluated] + untracked_reads: dict[str, tuple[FactKey, ...]] + uncacheable: frozenset[str] + observed: dict[FactKey, str | None] + + @dataclass(frozen=True) class _CachedEntry: subject_identity: str @@ -82,7 +91,7 @@ def evaluate_custom_rules_cached( if not custom: return CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) if not config.cache.enabled: - evaluated: dict[tuple[str, str], _Evaluated] = _run_host( + uncached: _HostRun = _run_host( project=project, config=config, project_dir=project_dir, @@ -95,7 +104,7 @@ def evaluate_custom_rules_cached( findings=_ordered_findings( rules=custom, subjects=_subjects(project=project, rules=custom), - results={key: value.findings for key, value in evaluated.items()}, + results={key: value.findings for key, value in uncached.evaluated.items()}, ), cache_hits=0, cache_misses=len(custom), @@ -143,7 +152,7 @@ def evaluate_custom_rules_cached( len(subjects[code]) if planned is None else len(planned) for code, planned in plan.items() ) if plan: - evaluated = _run_host( + run: _HostRun = _run_host( project=project, config=config, project_dir=project_dir, @@ -152,15 +161,15 @@ def evaluate_custom_rules_cached( track_reads=True, verify_determinism=verify_determinism, ) - _validate_paths(project=project, rules=custom, evaluated=evaluated) + _validate_paths(project=project, rules=custom, evaluated=run.evaluated) stored = _store( stored=stored, identities=identities, subjects=subjects, - evaluated=evaluated, + run=run, digests=digests, ) - for key, value in evaluated.items(): + for key, value in run.evaluated.items(): results[key] = value.findings _write_cache(path=cache_path, stored=stored, subjects=subjects) return CustomRulesOutcome( @@ -260,7 +269,7 @@ def _run_host( plan: dict[str, list[str] | None], track_reads: bool, verify_determinism: bool, -) -> dict[tuple[str, str], _Evaluated]: +) -> _HostRun: input_dir: Path = project_dir / "target" / "rules-cache" / "host-inputs" input_dir.mkdir(parents=True, exist_ok=True) input_path: Path | None = None @@ -309,21 +318,28 @@ def _host_project(project: CompiledProject) -> CompiledProject: ) -def _decode_host_response(response: object) -> dict[tuple[str, str], _Evaluated]: +def _decode_host_response(response: object) -> _HostRun: if not isinstance(response, dict): raise RulesError("custom rule host returned an invalid payload") payload: dict[str, Any] = cast(dict[str, Any], response) raw_readsets: Any = payload.get("readsets") raw_evaluations: Any = payload.get("evaluations") raw_untracked: Any = payload.get("untracked") + raw_uncacheable: Any = payload.get("uncacheable") + raw_observed: Any = payload.get("observed") if ( not isinstance(raw_readsets, list) or not isinstance(raw_evaluations, list) - or not isinstance(raw_untracked, list) + or not isinstance(raw_untracked, dict) + or not isinstance(raw_uncacheable, list) + or not isinstance(raw_observed, list) ): raise RulesError("custom rule host returned an invalid payload") readsets: list[tuple[FactKey, ...]] = list(map(_decode_readset, raw_readsets)) - untracked: frozenset[str] = frozenset(str(code) for code in raw_untracked) + untracked: dict[str, tuple[FactKey, ...]] = { + str(code): _decode_readset(reads) for code, reads in raw_untracked.items() + } + uncacheable: frozenset[str] = frozenset(str(code) for code in raw_uncacheable) evaluated: dict[tuple[str, str], _Evaluated] = {} for raw_item in raw_evaluations: if not isinstance(raw_item, dict): @@ -339,7 +355,15 @@ def _decode_host_response(response: object) -> dict[tuple[str, str], _Evaluated] findings=tuple(map(_finding_payload, findings)), reads=(readsets[reads] if isinstance(reads, int) and code not in untracked else None), ) - return evaluated + return _HostRun( + evaluated=evaluated, + untracked_reads=untracked, + uncacheable=uncacheable, + observed={ + tuple(map(str, key)): None if digest is None else str(digest) + for key, digest in raw_observed + }, + ) def _validate_paths( @@ -373,7 +397,7 @@ def _store( stored: dict[str, _RuleCache], identities: dict[str, str], subjects: dict[str, tuple[_Subject, ...]], - evaluated: dict[tuple[str, str], _Evaluated], + run: _HostRun, digests: FactDigests, ) -> dict[str, _RuleCache]: updated: dict[str, _RuleCache] = {} @@ -395,20 +419,22 @@ def _store( readset: index for index, readset in enumerate(current.readsets) } combined_by_readset: dict[tuple[FactKey, ...], str | None] = {} - for (evaluated_code, subject_key), result in evaluated.items(): + for (evaluated_code, subject_key), result in run.evaluated.items(): if evaluated_code != code: continue + if code in run.uncacheable: + current.entries.pop(subject_key, None) + continue reads: tuple[FactKey, ...] | None = result.reads if reads is None: if full_keys is None: full_keys = full_fact_keys(digests.views) - reads = full_keys + reads = (*run.untracked_reads.get(code, ()), *full_keys) readset: tuple[FactKey, ...] = tuple(sorted(set(reads))) if readset not in combined_by_readset: - try: - combined_by_readset[readset] = digests.combined(readset) - except FactDigestError: - combined_by_readset[readset] = None + combined_by_readset[readset] = _observed_digest( + readset=readset, observed=run.observed, digests=digests + ) combined: str | None = combined_by_readset[readset] if combined is None: current.entries.pop(subject_key, None) @@ -427,6 +453,20 @@ def _store( return updated +def _observed_digest( + *, readset: tuple[FactKey, ...], observed: dict[FactKey, str | None], digests: FactDigests +) -> str | None: + """Digest a read set only when its filesystem facts still match what the rule observed.""" + + try: + for key in readset: + if is_tree_fact(key) and key in observed and observed[key] != digests.digest(key): + return None + return digests.combined(readset) + except FactDigestError: + return None + + def _ordered_findings( *, rules: tuple[Rule, ...], @@ -493,7 +533,7 @@ def _write_cache( }, } path.parent.mkdir(parents=True, exist_ok=True) - temporary: Path = path.with_suffix(f".tmp-{os.getpid()}") + temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") temporary.write_bytes(orjson.dumps({"version": CUSTOM_RULES_CACHE_VERSION, "rules": rules})) temporary.replace(path) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py index 99c3f2d3f0..7404be19f1 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py @@ -97,6 +97,7 @@ FACT_TREE_RESOURCES_UNDER: 1, FACT_TREE_READ_TEXT: 1, } +_TREE_FACTS: frozenset[str] = frozenset({FACT_TREE_PATHS, *_TEXT_FACTS}) def fact_key_payload(key: tuple[object, ...]) -> list[str]: @@ -130,10 +131,28 @@ def fact_outcome_digest(*, views: RuleFactViews, key: FactKey) -> str: except FactDigestError: raise except Exception as error: - return _encoded_digest(("error", type(error).__name__, str(error))) + return fact_error_digest(error) + return fact_value_digest(value) + + +def fact_value_digest(value: object) -> str: + """Digest one fact value exactly as a recomputed fact would digest it.""" + return _encoded_digest(("value", value)) +def fact_error_digest(error: Exception) -> str: + """Digest one raised fact error exactly as a recomputed fact would digest it.""" + + return _encoded_digest(("error", type(error).__name__, str(error))) + + +def is_tree_fact(key: FactKey) -> bool: + """Return whether a fact is read from the live project filesystem.""" + + return bool(key) and key[0] in _TREE_FACTS + + def _fact_value(*, views: RuleFactViews, key: FactKey) -> object: if not key: raise FactDigestError("empty fact key") diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index b3db580108..17fbe73fe7 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -114,7 +114,10 @@ def run_pass( repeated=list(chain.from_iterable(item.findings for item in repeated)), ) return CustomRuleRun( - evaluations=tuple(evaluations), untracked_codes=frozenset(tracker.untracked_codes) + evaluations=tuple(evaluations), + untracked_codes=frozenset(tracker.untracked_codes), + uncacheable_codes=frozenset(tracker.uncacheable_codes), + observed=() if reads is None else tuple(reads.observed.items()), ) @@ -126,6 +129,7 @@ def __init__( ) -> None: self.reads: FactReads | None = reads self.untracked_codes: set[str] = set() + self.uncacheable_codes: set[str] = set() self._rules_root: Path = rules_root self._namespaces: dict[str, tuple[dict[str, object], ...]] = ( {rule.code: rule_namespaces((rule.check,)) for rule in rules} @@ -140,6 +144,7 @@ def start(self) -> None: if self.reads is not None: self.reads.current = set() self.reads.untracked = False + self.reads.uncacheable = False def finish(self, *, code: str) -> frozenset[tuple[object, ...]] | None: reads: FactReads | None = self.reads @@ -149,6 +154,8 @@ def finish(self, *, code: str) -> frozenset[tuple[object, ...]] | None: reads.current = None if reads.untracked: self.untracked_codes.add(code) + if reads.uncacheable: + self.uncacheable_codes.add(code) for rule_code, namespaces in self._namespaces.items(): if rule_code in self.untracked_codes: continue diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 9234488a90..ce315b2a0b 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -132,9 +132,10 @@ def _run_payload(run: CustomRuleRun) -> dict[str, object]: readsets: dict[frozenset[tuple[object, ...]], int] = {} evaluations: list[dict[str, object]] = [] + untracked: set[str] = set(run.untracked_codes) for evaluation in run.evaluations: index: int | None = None - if evaluation.reads is not None and evaluation.code not in run.untracked_codes: + if evaluation.reads is not None and evaluation.code not in untracked: index = readsets.setdefault(evaluation.reads, len(readsets)) evaluations.append( { @@ -144,16 +145,23 @@ def _run_payload(run: CustomRuleRun) -> dict[str, object]: "reads": index, } ) - untracked: set[str] = set(run.untracked_codes) if sum(map(len, readsets)) > CUSTOM_HOST_MAX_TRACKED_READS: untracked.update(str(item["code"]) for item in evaluations if item["reads"] is not None) readsets = {} for item in evaluations: item["reads"] = None + untracked_reads: dict[str, set[tuple[object, ...]]] = {code: set() for code in untracked} + for evaluation in run.evaluations: + if evaluation.code in untracked_reads and evaluation.reads is not None: + untracked_reads[evaluation.code].update(evaluation.reads) return { "evaluations": evaluations, "readsets": [sorted(map(fact_key_payload, readset)) for readset in readsets], - "untracked": sorted(untracked), + "untracked": { + code: sorted(map(fact_key_payload, reads)) for code, reads in untracked_reads.items() + }, + "uncacheable": sorted(run.uncacheable_codes), + "observed": sorted([list(key), digest] for key, digest in run.observed), } diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py index 5462bd801b..6864c9ca68 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -2,11 +2,13 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path from typing import Any from sqlbuild.compiler.compile.models import CompiledAudit, CompiledSqlTest, InferredColumn from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration +from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_error_digest, fact_value_digest from sqlbuild.rule_engine.classes.fact_reads import FactReads from sqlbuild.rule_engine.constants import ( FACT_AUDITS_ALL, @@ -39,11 +41,12 @@ FACT_TREE_RELATIVE_PARTS, FACT_TREE_RESOURCES_UNDER, ) +from sqlbuild.rule_engine.exceptions import FactDigestError from sqlbuild.rule_engine.models import Model, ModelSql, ProjectPath, RuleFactViews class _TrackedView: - """Delegate unknown attribute access to the wrapped view and mark the evaluation untracked.""" + """Delegate unknown attribute access to the wrapped view and mark the evaluation uncacheable.""" __slots__ = ("_tracked_reads", "_tracked_view") @@ -55,50 +58,81 @@ def __init__(self, *, view: object, reads: FactReads) -> None: object.__setattr__(self, "_tracked_reads", reads) def __getattr__(self, name: str) -> object: - self._tracked_reads.untracked = True + self._tracked_reads.uncacheable = True return getattr(self._tracked_view, name) def __setattr__(self, name: str, value: object) -> None: - self._tracked_reads.untracked = True + self._tracked_reads.uncacheable = True setattr(self._tracked_view, name, value) class _TrackedProjectTree(_TrackedView): + """Record live filesystem reads with the digest of the value each read returned.""" + __slots__ = () def paths(self) -> tuple[ProjectPath, ...]: - self._tracked_reads.record(key=(FACT_TREE_PATHS,)) - return self._tracked_view.paths() + key: tuple[str, ...] = (FACT_TREE_PATHS,) + self._tracked_reads.record(key=key) + observed: tuple[str, ...] | None = key if self._tracked_reads.current is not None else None + return self._observed(key=observed, read=self._tracked_view.paths) def children(self, path: str = "") -> tuple[ProjectPath, ...]: - self._tracked_reads.record_text(fact=FACT_TREE_CHILDREN, values=(path,)) - return self._tracked_view.children(path) + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_CHILDREN, values=(path,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.children(path)) def descendants(self, path: str = "") -> tuple[ProjectPath, ...]: - self._tracked_reads.record_text(fact=FACT_TREE_DESCENDANTS, values=(path,)) - return self._tracked_view.descendants(path) + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_DESCENDANTS, values=(path,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.descendants(path)) def glob(self, pattern: str) -> tuple[ProjectPath, ...]: - self._tracked_reads.record_text(fact=FACT_TREE_GLOB, values=(pattern,)) - return self._tracked_view.glob(pattern) + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_GLOB, values=(pattern,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.glob(pattern)) def relative_parts(self, *, path: Path | str, under: str = "") -> tuple[str, ...]: - self._tracked_reads.record_text( - fact=FACT_TREE_RELATIVE_PARTS, - values=( - path, - under, - ), + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_RELATIVE_PARTS, values=(path, under) + ) + return self._observed( + key=key, read=lambda: self._tracked_view.relative_parts(path=path, under=under) ) - return self._tracked_view.relative_parts(path=path, under=under) def resources_under(self, path: str) -> tuple[Model, ...]: - self._tracked_reads.record_text(fact=FACT_TREE_RESOURCES_UNDER, values=(path,)) - return self._tracked_view.resources_under(path) + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_RESOURCES_UNDER, values=(path,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.resources_under(path)) def read_text(self, path: str) -> str: - self._tracked_reads.record_text(fact=FACT_TREE_READ_TEXT, values=(path,)) - return self._tracked_view.read_text(path) + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_READ_TEXT, values=(path,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.read_text(path)) + + def _observed[T](self, *, key: tuple[str, ...] | None, read: Callable[[], T]) -> T: + reads: FactReads = self._tracked_reads + if key is None or (key[0] != FACT_TREE_READ_TEXT and key in reads.observed): + return read() + try: + value: T = read() + except Exception as error: + reads.observe(key=key, digest=_digest_or_none(error=error)) + raise + reads.observe(key=key, digest=_digest_or_none(value=value)) + return value + + +def _digest_or_none(*, value: object = None, error: Exception | None = None) -> str | None: + try: + return fact_value_digest(value) if error is None else fact_error_digest(error) + except FactDigestError: + return None class _TrackedProjectFacts(_TrackedView): diff --git a/src/sqlbuild/rule_engine/classes/fact_reads.py b/src/sqlbuild/rule_engine/classes/fact_reads.py index f099e393a1..8680946276 100644 --- a/src/sqlbuild/rule_engine/classes/fact_reads.py +++ b/src/sqlbuild/rule_engine/classes/fact_reads.py @@ -10,11 +10,13 @@ class FactReads: """Collect the fact keys one custom-rule evaluation observes.""" - __slots__ = ("current", "untracked") + __slots__ = ("current", "observed", "uncacheable", "untracked") def __init__(self) -> None: self.current: set[tuple[object, ...]] | None = None self.untracked: bool = False + self.uncacheable: bool = False + self.observed: dict[tuple[str, ...], str | None] = {} def record(self, *, key: tuple[object, ...]) -> None: current: set[tuple[object, ...]] | None = self.current @@ -36,15 +38,25 @@ def _record_model(self, *, fact: str, model: object, by_name: bool) -> None: else: self.untracked = True - def record_text(self, *, fact: str, values: tuple[object, ...]) -> None: + def record_text(self, *, fact: str, values: tuple[object, ...]) -> tuple[str, ...] | None: current: set[tuple[object, ...]] | None = self.current if current is None: - return + return None texts: list[str] = [] for value in values: if isinstance(value, (str, os.PathLike)): texts.append(os.fspath(value)) else: self.untracked = True - return - current.add((fact, *texts)) + return None + key: tuple[str, ...] = (fact, *texts) + current.add(key) + return key + + def observe(self, *, key: tuple[str, ...], digest: str | None) -> None: + """Remember the digest a live filesystem fact had when the rule read it.""" + + if key in self.observed and self.observed[key] != digest: + self.observed[key] = None + else: + self.observed.setdefault(key, digest) diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index d464c229e4..cb745b884b 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -399,6 +399,8 @@ class CustomRuleRun: evaluations: tuple[CustomRuleEvaluation, ...] untracked_codes: frozenset[str] + uncacheable_codes: frozenset[str] = frozenset() + observed: tuple[tuple[tuple[str, ...], str | None], ...] = () @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index d33ed9681a..10d5f74f42 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -163,6 +163,27 @@ class CustomRuleReadTrackingTestCase: edited_customers_config: dict[str, object] = field(default_factory=dict) +@dataclass(frozen=True) +class CustomRuleUncacheableTestCase: + """One model-subject rule whose fact access cannot be attributed to recorded reads.""" + + description: str + body: str + expected_warm_hits: int = 0 + + +@dataclass(frozen=True) +class CustomRuleProjectFileTestCase: + """One rule reading a project file whose contents change from customers to orders.""" + + description: str + file_path: str + body: str + expected_rerun_hits: int + expected_finding_paths: tuple[str, ...] + module_prelude: str = "" + + @dataclass(frozen=True) class NativeMemoTestCase: """One warm built-in evaluation followed by an edit that must invalidate the memo.""" @@ -172,3 +193,12 @@ class NativeMemoTestCase: edited_config: dict[str, object] expected_original_codes: tuple[str, ...] expected_edited_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeBuildIdentityTestCase: + """One cached evaluation repeated after the native extension reports another build.""" + + description: str + expected_rebuilt_evaluations: int + expected_warm_evaluations: int diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 3c277bebc1..600de06dda 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -1,6 +1,7 @@ """Test helpers for native rules engine boundaries.""" import json +from collections.abc import Callable from dataclasses import replace from operator import attrgetter from pathlib import Path @@ -187,6 +188,18 @@ def write_rule( return path.relative_to(root) +def write_project_file_rule( + *, root: Path, file_path: str, contents: str, body: str, module_import: str +) -> Path: + """Write one project file plus a custom rule and return the absolute file path.""" + + path: Path = root / file_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + _ = write_rule(root=root, body=body, module_import=module_import) + return path + + def load_custom_rule(*, root: Path, configured_path: Path) -> Rule: del configured_path config: RulesConfig = RulesConfig() @@ -288,3 +301,17 @@ def evaluate_contract_rule( config_values=config_values, ) return evaluate(project=project, config=config, project_dir=project_dir) + + +def record_native_evaluations(*, monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Count every request that reaches the native built-in rules engine.""" + + requests: list[str] = [] + evaluate_json: Callable[[str], str] = native._native.evaluate_json + + def recording(request_json: str) -> str: + requests.append(request_json) + return evaluate_json(request_json) + + monkeypatch.setattr(native._native, "evaluate_json", recording) + return requests diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py index 36572d0ed3..a0fe767bcf 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py @@ -1,21 +1,31 @@ """Custom-rule results are reused only when every fact the invocation read is unchanged.""" +from collections.abc import Callable from pathlib import Path +from typing import Any import pytest +import sqlbuild._native as native_module from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.rule_engine._helpers.engine import custom_rules from sqlbuild.rule_engine.models import CustomRulesOutcome from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( + CustomRuleProjectFileTestCase, CustomRuleReadTrackingTestCase, + CustomRuleUncacheableTestCase, + NativeBuildIdentityTestCase, ) from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( evaluate_cached_custom_rules, two_model_project, + write_project_file_rule, write_rule, ) _CUSTOMERS_SQL: str = "SELECT 1 AS customer_id" +_OWNERS_BEFORE: str = "owners: [customers]\n" +_OWNERS_AFTER: str = "owners: [orders]\n" _FLAG_ORDERS: str = ( ' return [ctx.finding(subject=model)] if model.name == "orders" and {condition} else []' ) @@ -71,11 +81,6 @@ body=_FLAG_ORDERS.format(condition="'SELECT 2' in _sources(ctx)").strip(), expected_hits_after_edit=0, ), - CustomRuleReadTrackingTestCase( - description="private view access makes the rule untracked", - body="_ = ctx.sql._models\n" + _FLAG_ORDERS.format(condition="True"), - expected_hits_after_edit=0, - ), CustomRuleReadTrackingTestCase( description="context attribute state makes the rule untracked", body="ctx.seen = model.name\n" + _FLAG_ORDERS.format(condition="True"), @@ -118,5 +123,186 @@ def test_given_recorded_reads_when_one_model_changes_then_only_dependent_results assert after_edit.findings == oracle.findings +@pytest.mark.parametrize( + "test_case", + ( + CustomRuleUncacheableTestCase( + description="private SQL view state", + body="_ = ctx.sql._models\n" + _FLAG_ORDERS.format(condition="True"), + ), + CustomRuleUncacheableTestCase( + description="private column view state", + body="_ = ctx.columns._models\n" + _FLAG_ORDERS.format(condition="True"), + ), + CustomRuleUncacheableTestCase( + description="private graph view state", + body="_ = ctx.graph._compiled_by_path\n" + _FLAG_ORDERS.format(condition="True"), + ), + ), + ids=lambda case: case.description, +) +def test_given_private_view_access_when_evaluating_repeatedly_then_rule_is_never_reused( + tmp_path: Path, + test_case: CustomRuleUncacheableTestCase, +) -> None: + _ = write_rule(root=tmp_path, body=test_case.body) + project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + + cold: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + warm: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + + assert (cold.cache_hits, cold.cache_misses) == (0, 2) + assert (warm.cache_hits, warm.cache_misses) == ( + test_case.expected_warm_hits, + 2 - test_case.expected_warm_hits, + ) + assert warm.findings == cold.findings + + +@pytest.mark.parametrize( + "test_case", + ( + CustomRuleProjectFileTestCase( + description="hidden file read through a cached helper", + file_path=".config/owners.yaml", + module_prelude=( + "import functools\n\n" + "@functools.cache\n" + "def _owners(ctx: RuleContext) -> str:\n" + " return ctx.project.tree.read_text('.config/owners.yaml')\n" + ), + body=_FLAG_ORDERS.format(condition="'orders' in _owners(ctx)").strip(), + expected_rerun_hits=0, + expected_finding_paths=("models/orders.sql",), + ), + ), + ids=lambda case: case.description, +) +def test_given_untracked_rule_reading_hidden_file_when_file_changes_then_rule_reruns( + tmp_path: Path, + test_case: CustomRuleProjectFileTestCase, +) -> None: + owners: Path = write_project_file_rule( + root=tmp_path, + file_path=test_case.file_path, + contents=_OWNERS_BEFORE, + body=test_case.body, + module_import=test_case.module_prelude, + ) + project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + + cold: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + warm: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + owners.write_text(_OWNERS_AFTER, encoding="utf-8") + after_edit: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + + assert cold.findings == () + assert (warm.cache_hits, warm.cache_misses) == (2, 0) + assert (after_edit.cache_hits, after_edit.cache_misses) == ( + test_case.expected_rerun_hits, + 2 - test_case.expected_rerun_hits, + ) + assert tuple(finding["path"] for finding in after_edit.findings) == ( + test_case.expected_finding_paths + ) + + +@pytest.mark.parametrize( + "test_case", + ( + CustomRuleProjectFileTestCase( + description="file edited after the host read it", + file_path="config/owners.yaml", + body=_FLAG_ORDERS.format( + condition="'orders' in ctx.project.tree.read_text('config/owners.yaml')" + ).strip(), + expected_rerun_hits=1, + expected_finding_paths=("models/orders.sql",), + ), + ), + ids=lambda case: case.description, +) +def test_given_file_edited_while_rules_run_when_rerunning_then_new_contents_are_evaluated( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: CustomRuleProjectFileTestCase, +) -> None: + owners: Path = write_project_file_rule( + root=tmp_path, + file_path=test_case.file_path, + contents=_OWNERS_BEFORE, + body=test_case.body, + module_import=test_case.module_prelude, + ) + project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + run_host: Callable[..., object] = custom_rules._run_host + + def run_host_then_edit(**kwargs: Any) -> object: + result: object = run_host(**kwargs) + owners.write_text(_OWNERS_AFTER, encoding="utf-8") + return result + + monkeypatch.setattr(custom_rules, "_run_host", run_host_then_edit) + during_edit: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + monkeypatch.undo() + rerun: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + oracle: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=False + ) + + assert during_edit.findings == () + assert (rerun.cache_hits, rerun.cache_misses) == ( + test_case.expected_rerun_hits, + 2 - test_case.expected_rerun_hits, + ) + assert rerun.findings == oracle.findings + assert tuple(finding["path"] for finding in rerun.findings) == ( + test_case.expected_finding_paths + ) + + +@pytest.mark.parametrize( + "test_case", + [NativeBuildIdentityTestCase("custom results follow the native build", 2, 0)], + ids=lambda case: case.description, +) +def test_given_custom_rule_cache_from_another_native_build_when_evaluating_then_rules_rerun( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: NativeBuildIdentityTestCase, +) -> None: + _ = write_rule(root=tmp_path, body=_FLAG_ORDERS.format(condition="True").strip()) + project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + + cold: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + monkeypatch.setattr(native_module, "BUILD_IDENTITY", f"{native_module.BUILD_IDENTITY}-rebuilt") + rebuilt: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + warm: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + + assert rebuilt.cache_misses == test_case.expected_rebuilt_evaluations + assert warm.cache_misses == test_case.expected_warm_evaluations + assert rebuilt.findings == warm.findings == cold.findings + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py index e66b18c738..50b39076a0 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py @@ -2,9 +2,16 @@ import pytest +import sqlbuild._native as native_module from sqlbuild.rule_engine.models import RulesResult -from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import NativeMemoTestCase -from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule +from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( + NativeBuildIdentityTestCase, + NativeMemoTestCase, +) +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( + evaluate_contract_rule, + record_native_evaluations, +) @pytest.mark.parametrize( @@ -54,5 +61,34 @@ def test_given_warm_built_in_memo_when_request_repeats_or_changes_then_matches_u assert edited.cache_misses > 0 +@pytest.mark.parametrize( + "test_case", + [NativeBuildIdentityTestCase("memoized response follows the native build", 1, 0)], + ids=lambda case: case.description, +) +def test_given_memoized_response_from_another_native_build_when_evaluating_then_rules_rerun( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: NativeBuildIdentityTestCase, +) -> None: + config_values: dict[str, object] = {"materialized": "table"} + cold: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + native_calls: list[str] = record_native_evaluations(monkeypatch=monkeypatch) + monkeypatch.setattr(native_module, "BUILD_IDENTITY", f"{native_module.BUILD_IDENTITY}-rebuilt") + rebuilt: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + rebuilt_calls: int = len(native_calls) + warm: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + + assert rebuilt_calls == test_case.expected_rebuilt_evaluations + assert len(native_calls) - rebuilt_calls == test_case.expected_warm_evaluations + assert rebuilt.findings == warm.findings == cold.findings + + if __name__ == "__main__": pytest.main([__file__, "-vv"])