diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index bf38f76037..9c4a376e34 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -365,7 +365,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 20 strategy: fail-fast: false matrix: diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 696e9578db..fdc5c9b548 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -73,6 +73,12 @@ fn evaluate_json(py: Python<'_>, request_json: &str) -> PyResult { .map_err(value_error) } +#[pyfunction] +fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { + py.compiler_detach(|| crate::engine::main::custom_host::run_custom_host_json(spec_json)) + .map_err(value_error) +} + #[pyfunction] fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| crate::sql_lint::main::engine::lint_json(request_json)) @@ -422,6 +428,7 @@ fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(evaluate_json, module)?)?; module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?; + module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; module.add( diff --git a/crates/sqlbuild-rules-native/src/constants.rs b/crates/sqlbuild-rules-native/src/constants.rs index 1d2d0a3c31..6e4aec2a8e 100644 --- a/crates/sqlbuild-rules-native/src/constants.rs +++ b/crates/sqlbuild-rules-native/src/constants.rs @@ -13,7 +13,6 @@ pub(crate) const GENERIC_BASIC_NAME: &str = "basic"; pub(crate) const GENERIC_SCENARIO_NAME: &str = "scenario"; pub(crate) const GENERIC_CASE_NAME: &str = "case"; pub(crate) const CUSTOM_RULE_NAMESPACE: &str = "XSQBR"; -pub(crate) const TARGET_DIRECTORY: &str = "target"; pub(crate) const REFERENCE_KIND: &str = "ref"; pub(crate) const MODEL_SCHEMA_CONFIG_KEY: &str = "schema"; pub(crate) const STAGING_LAYER_DIRECTORY: &str = "staging"; @@ -33,11 +32,6 @@ pub(crate) const NEGATION_OPERATOR: &str = "-"; pub(crate) const DECLARATION_DOMAIN_COMPONENTS: usize = 3; pub(crate) const DIRECT_DEPENDENCY_PATH_LENGTH: usize = 2; pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3; -pub(crate) const CUSTOM_HOST_REQUIRED_ERROR: &str = "selected custom rules require a custom host"; -pub(crate) const LOGS_DIRECTORY: &str = "logs"; -pub(crate) const GIT_DIRECTORY: &str = ".git"; -pub(crate) const RULES_DIRECTORY: &str = "rules"; -pub(crate) const PYTHON_EXTENSION: &str = "py"; pub(crate) const SQL_WILDCARD: &str = "*"; pub(crate) const UNKNOWN_SQL_TYPE: &str = "UNKNOWN"; pub(crate) const VARCHAR_SQL_TYPE: &str = "VARCHAR"; diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs index 07585624e0..ed1749bf96 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs @@ -1,6 +1,4 @@ use crate::models::Fault; -use fensu_policy::lifecycle::models::CacheRead; -use fensu_policy::{read_cache, write_cache}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -28,36 +26,6 @@ impl Cache { }) } - pub(crate) fn get(&self, path: &str, fingerprint: &str) -> Result>, String> { - match read_cache::>(&self.root, path, fingerprint) - .map_err(|error| format!("could not read rules cache: {error}"))? - { - CacheRead::Hit(faults) => Ok(Some(faults)), - CacheRead::Miss | CacheRead::Invalidated => Ok(None), - } - } - - pub(crate) fn put( - &self, - path: &str, - fingerprint: &str, - faults: &[Fault], - ) -> Result<(), String> { - write_cache(&self.root, path, fingerprint, &faults) - .map_err(|error| format!("could not write rules cache: {error}")) - } - - pub(crate) fn custom_rule_bucket( - &self, - code: &str, - rule_identity: &str, - ) -> Result { - read_bucket( - &self.root.join("bulk").join(format!("custom-{code}.json")), - rule_identity, - ) - } - pub(crate) fn native_rules_bucket( &self, ruleset_identity: &str, @@ -76,19 +44,6 @@ impl Cache { bucket, ) } - - pub(crate) fn put_custom_rule_bucket( - &self, - code: &str, - rule_identity: &str, - bucket: &RuleCacheBucket, - ) -> Result<(), String> { - write_bucket( - &self.root.join("bulk").join(format!("custom-{code}.json")), - rule_identity, - bucket, - ) - } } fn read_bucket(path: &Path, fingerprint: &str) -> Result { diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs new file mode 100644 index 0000000000..f46a484e0e --- /dev/null +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs @@ -0,0 +1,29 @@ +use crate::models::CustomHostSpec; +use fensu_policy::lifecycle::models::{ + CustomHostInvocation, CustomHostOutputLimits, CustomHostRequest, +}; +use fensu_policy::run_custom_host; +use std::path::Path; +use std::time::Duration; + +pub(crate) fn run_custom_host_json(spec_json: &str) -> Result { + let spec: CustomHostSpec = serde_json::from_str(spec_json) + .map_err(|error| format!("invalid custom host request: {error}"))?; + let request = CustomHostRequest { + protocol: fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION, + runtime_version: spec.runtime_version.clone(), + payload: spec.payload, + }; + let response = run_custom_host::<_, serde_json::Value>(CustomHostInvocation { + program: Path::new(&spec.program), + arguments: &spec.arguments, + timeout: Duration::from_millis(spec.timeout_millis), + output_limits: CustomHostOutputLimits::default(), + request: &request, + }) + .map_err(|error| format!("invalid rules lifecycle: {error}"))?; + let payload = response.payload.ok_or_else(|| { + "custom rule host returned no payload after successful validation".to_owned() + })?; + serde_json::to_string(&payload).map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs index 4fad5b6fc6..746de6cda6 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs @@ -1,9 +1,6 @@ use crate::configuration::main::validate as config; -use crate::constants::{ - API_VERSION, CUSTOM_HOST_REQUIRED_ERROR, GIT_DIRECTORY, LOGS_DIRECTORY, NATIVE_BUILD_IDENTITY, - PYTHON_EXTENSION, RULES_DIRECTORY, TARGET_DIRECTORY, -}; -use crate::engine::_helpers::cache::{Cache, RuleCacheBucket, RuleCacheEntry}; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; +use crate::engine::_helpers::cache::{Cache, RuleCacheEntry}; use crate::models::{ EvaluateRequest, EvaluateResponse, Fault, Model, RuleMetadata, RulesCodeGrammar, }; @@ -17,21 +14,21 @@ use crate::rules::models::{ use fensu_policy::lifecycle::constants::ANALYSIS_BATCH_SCHEMA_VERSION; use fensu_policy::lifecycle::errors::LifecycleError; use fensu_policy::lifecycle::models::{ - AnalysisBatchRequest, AnalysisInput, ApplySuppressionsRequest, CustomHostInvocation, - CustomHostOutputLimits, CustomHostRequest, ExactSuppression, Finding, FindingSeverity, - RuntimeIdentity, ScopedIgnore, + AnalysisBatchRequest, AnalysisInput, ApplySuppressionsRequest, ExactSuppression, Finding, + FindingSeverity, RuntimeIdentity, ScopedIgnore, }; -use fensu_policy::{apply_suppressions, evaluate_batch, run_custom_host}; +use fensu_policy::{apply_suppressions, evaluate_batch}; use rayon::iter::{ IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, }; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant}; +use std::path::Path; +use std::time::Instant; const NATIVE_RULE_WORKERS: usize = 4; const NATIVE_RULE_STACK_BYTES: usize = 16 * 1024 * 1024; +const COMPILER_FACTS_FINGERPRINT_SEED: &str = "compiler-owned-rules-facts-v2"; struct PendingModelRule<'a> { index: usize, @@ -225,103 +222,51 @@ pub(crate) fn evaluate_json(request_json: &str) -> Result { cache_hits: 0, cache_misses: 0, ruleset_fingerprint, + selected_codes: Vec::new(), built_in_ms: 0, - custom_ms: 0, }) .map_err(|error| error.to_string()); } - let custom_selected = selected.iter().any(|rule| rule.custom); let cache_enabled = request.config.cache.enabled; let project_aware = selected.iter().any(|rule| rule.project_wide || rule.custom); - let project_fingerprint = if project_aware { - Some(if custom_selected { - match &request.project_fingerprint { - Some(value) => fingerprint_request_facts(value.as_bytes(), &request)?, - None => fingerprint_project(Path::new(&request.project_dir), &request)?, - } - } else { - fingerprint_request_facts(b"compiler-owned-rules-facts-v1", &request)? + let project_fingerprint = project_aware + .then(|| { + fingerprint_request_facts( + request + .project_fingerprint + .as_deref() + .unwrap_or(COMPILER_FACTS_FINGERPRINT_SEED) + .as_bytes(), + &request, + ) }) - } else { - None - }; + .transpose()?; let cache = cache_enabled .then(|| Cache::open(Path::new(&request.project_dir))) .transpose()?; - let (cached_custom, custom_probe_ms) = probe_custom_rules_cache(CustomRulesCacheRequest { - request: &request, - selected: &selected_by_code, + let built_in_started = Instant::now(); + let mut raw_faults = request.initial_findings.clone(); + raw_faults.extend(evaluate_project::evaluate_project( + ProjectEvaluationRequest { + selected: &selected_by_code, + request: &request, + }, + )?); + let native = evaluate_native_models(NativeModelRuleRequest { + context: ModelRuleBatchContext { + request: &request, + selected: &selected_by_code, + threshold_overrides: &threshold_overrides, + }, cache: cache.as_ref(), + ruleset_fingerprint: &ruleset_fingerprint, project_fingerprint: project_fingerprint.as_deref(), })?; - let (raw_faults, cache_hits, cache_misses, built_in_ms, custom_ms) = - std::thread::scope(|scope| { - let custom_parallel_started = Instant::now(); - let custom_task = cached_custom.is_none().then(|| { - scope.spawn(|| { - evaluate_custom_rules_cached(CustomRulesCacheRequest { - request: &request, - selected: &selected_by_code, - cache: cache.as_ref(), - project_fingerprint: project_fingerprint.as_deref(), - }) - }) - }); - let built_in_started = Instant::now(); - let mut raw_faults = request.initial_findings.clone(); - raw_faults.extend(evaluate_project::evaluate_project( - ProjectEvaluationRequest { - selected: &selected_by_code, - request: &request, - }, - )?); - let native = evaluate_native_models(NativeModelRuleRequest { - context: ModelRuleBatchContext { - request: &request, - selected: &selected_by_code, - threshold_overrides: &threshold_overrides, - }, - cache: cache.as_ref(), - ruleset_fingerprint: &ruleset_fingerprint, - project_fingerprint: project_fingerprint.as_deref(), - })?; - let mut cache_hits = native.hits; - let mut cache_misses = native.misses; - raw_faults.extend(native.faults); - let built_in_ms = built_in_started.elapsed().as_millis() as u64; - let custom_started = if custom_task.is_some() { - custom_parallel_started - } else { - Instant::now() - }; - let custom_result = if let Some(task) = custom_task { - task.join() - .map_err(|_| "custom rule evaluation worker panicked".to_string())? - } else { - match cached_custom { - Some(custom) => Ok(custom), - None => evaluate_custom_rules_cached(CustomRulesCacheRequest { - request: &request, - selected: &selected_by_code, - cache: cache.as_ref(), - project_fingerprint: project_fingerprint.as_deref(), - }), - } - }; - let custom = match custom_result { - Ok(value) => value, - Err(error) if error == CUSTOM_HOST_REQUIRED_ERROR => { - return Err(format!("{error} [native_cache_misses={cache_misses}]")); - } - Err(error) => return Err(error), - }; - cache_hits += custom.hits; - cache_misses += custom.misses; - raw_faults.extend(custom.faults); - let custom_ms = custom_probe_ms + custom_started.elapsed().as_millis() as u64; - Ok::<_, String>((raw_faults, cache_hits, cache_misses, built_in_ms, custom_ms)) - })?; + let cache_hits = native.hits; + let cache_misses = native.misses; + raw_faults.extend(native.faults); + let built_in_ms = built_in_started.elapsed().as_millis() as u64; validate_exception_paths(&request)?; let faults = if request.defer_suppressions { raw_faults @@ -335,263 +280,12 @@ pub(crate) fn evaluate_json(request_json: &str) -> Result { cache_hits, cache_misses, ruleset_fingerprint, + selected_codes: selected.iter().map(|rule| rule.code.clone()).collect(), built_in_ms, - custom_ms, }) .map_err(|error| error.to_string()) } -struct CustomRulesCacheRequest<'a> { - request: &'a EvaluateRequest, - selected: &'a BTreeMap, - cache: Option<&'a Cache>, - project_fingerprint: Option<&'a str>, -} - -struct CustomEvaluation { - faults: Vec, - hits: usize, - misses: usize, -} - -fn probe_custom_rules_cache( - request: CustomRulesCacheRequest<'_>, -) -> Result<(Option, u64), String> { - let started = Instant::now(); - let cached = if request.request.custom_host.is_none() { - Some(evaluate_custom_rules_cached(request)?) - } else { - None - }; - Ok((cached, started.elapsed().as_millis() as u64)) -} - -type LocalRuleMiss<'a> = ( - &'a RuleMetadata, - String, - RuleCacheBucket, - BTreeMap, -); - -fn count_custom_rules(selected: &BTreeMap) -> usize { - let mut count = 0; - for rule in selected.values() { - if rule.custom { - count += 1; - } - } - count -} - -fn evaluate_custom_rules_cached( - request: CustomRulesCacheRequest<'_>, -) -> Result { - let CustomRulesCacheRequest { - request, - selected, - cache, - project_fingerprint, - } = request; - let custom_selected = selected.values().any(|rule| rule.custom); - if !custom_selected { - return Ok(CustomEvaluation { - faults: Vec::new(), - hits: 0, - misses: 0, - }); - } - let Some(cache) = cache else { - return evaluate_custom_rules(request, selected, None).map(|faults| CustomEvaluation { - faults, - hits: 0, - misses: count_custom_rules(selected), - }); - }; - let mut faults: Vec = Vec::new(); - let mut hits = 0; - let mut misses_count = 0; - let mut project_misses: BTreeMap = BTreeMap::new(); - let mut local_misses: BTreeMap> = BTreeMap::new(); - let mut selected_misses: BTreeMap = BTreeMap::new(); - let mut selected_model_paths: BTreeSet = BTreeSet::new(); - let model_fact_identities = custom_model_fact_identities(&request.models)?; - for (code, rule) in selected - .iter() - .filter(|(_, rule)| rule.custom && rule.project_wide) - { - let rule_identity = custom_rule_identity(rule, request)?; - let identity = custom_project_identity(&rule_identity, project_fingerprint); - let cache_path = format!("__custom_rules__/{code}/project"); - if let Some(cached) = cache.get(&cache_path, &identity)? { - faults.extend(cached); - hits += 1; - } else { - project_misses.insert(code.clone(), (*rule, identity)); - selected_misses.insert(code.clone(), *rule); - misses_count += 1; - } - } - for (code, rule) in selected - .iter() - .filter(|(_, rule)| rule.custom && !rule.project_wide) - { - let rule_identity = custom_rule_identity(rule, request)?; - let project_dependent = request - .custom_rules - .iter() - .find(|custom| custom.code == *code) - .is_some_and(|custom| custom.project_dependent); - let bucket = cache.custom_rule_bucket(code, &rule_identity)?; - let mut misses: BTreeMap = BTreeMap::new(); - for model in &request.models { - let model_identity = model_fact_identities - .get(&model.relative_path) - .ok_or_else(|| format!("missing custom rule model identity for {}", model.name))?; - let identity = custom_local_identity( - model_identity, - &rule_identity, - project_dependent.then_some(project_fingerprint).flatten(), - ); - if let Some(entry) = bucket - .entries - .get(&model.relative_path) - .filter(|entry| entry.identity == identity) - { - faults.extend(entry.faults.clone()); - hits += 1; - } else { - misses.insert(model.relative_path.clone(), identity); - selected_model_paths.insert(model.relative_path.clone()); - misses_count += 1; - } - } - if !misses.is_empty() { - selected_misses.insert(code.clone(), *rule); - local_misses.insert(code.clone(), (*rule, rule_identity, bucket, misses)); - } - } - if selected_misses.is_empty() { - return Ok(CustomEvaluation { - faults, - hits, - misses: misses_count, - }); - } - let selected_paths = (!selected_model_paths.is_empty()).then_some(&selected_model_paths); - let evaluated = evaluate_custom_rules(request, &selected_misses, selected_paths)?; - for (code, (_, _, _, _)) in &local_misses { - if let Some(fault) = evaluated - .iter() - .find(|fault| fault.code == *code && !selected_model_paths.contains(&fault.path)) - { - return Err(format!( - "model-subject custom rule returned a finding for unrelated path {}", - fault.path - )); - } - } - for (code, (_, rule_identity, mut bucket, misses)) in local_misses { - for (path, identity) in misses { - let model_faults: Vec = evaluated - .iter() - .filter(|fault| fault.code == code && fault.path == path) - .cloned() - .collect(); - let _ = bucket.entries.insert( - path, - RuleCacheEntry { - identity, - faults: model_faults.clone(), - }, - ); - faults.extend(model_faults); - } - cache.put_custom_rule_bucket(&code, &rule_identity, &bucket)?; - } - for (code, (_, identity)) in project_misses { - let project_faults: Vec = evaluated - .iter() - .filter(|fault| fault.code == code) - .cloned() - .collect(); - cache.put( - &format!("__custom_rules__/{code}/project"), - &identity, - &project_faults, - )?; - faults.extend(project_faults); - } - Ok(CustomEvaluation { - faults, - hits, - misses: misses_count, - }) -} - -fn custom_project_identity(rule: &str, project: Option<&str>) -> String { - let mut digest = Sha256::new(); - digest.update(b"custom-rule-project-v3"); - digest.update(rule.as_bytes()); - digest.update(project.unwrap_or("no-project-fingerprint").as_bytes()); - format!("{:x}", digest.finalize()) -} - -fn custom_local_identity(model_identity: &str, rule: &str, project: Option<&str>) -> String { - let mut digest = Sha256::new(); - digest.update(b"custom-rule-model-v3"); - digest.update(model_identity.as_bytes()); - digest.update(rule.as_bytes()); - digest.update(project.unwrap_or("model-local").as_bytes()); - format!("{:x}", digest.finalize()) -} - -fn custom_model_fact_identities( - models: &[crate::models::Model], -) -> Result, String> { - let identity = |model: &crate::models::Model| -> Result<(String, String), String> { - let mut digest = Sha256::new(); - digest.update(b"custom-rule-model-facts-v1"); - digest.update(ANALYSIS_BATCH_SCHEMA_VERSION.to_le_bytes()); - digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); - digest.update(serde_json::to_vec(model).map_err(|error| error.to_string())?); - Ok(( - model.relative_path.clone(), - format!("{:x}", digest.finalize()), - )) - }; - let results: Vec> = if models.len() <= 1 { - models.iter().map(identity).collect() - } else { - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(NATIVE_RULE_WORKERS.min(models.len())) - .stack_size(NATIVE_RULE_STACK_BYTES) - .build() - .map_err(|error| error.to_string())?; - pool.install(|| models.par_iter().map(identity).collect()) - }; - results.into_iter().collect() -} - -fn custom_rule_identity(rule: &RuleMetadata, request: &EvaluateRequest) -> Result { - let mut digest = Sha256::new(); - digest.update(b"custom-rule-identity-v1"); - digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); - digest.update(serde_json::to_vec(rule).map_err(|error| error.to_string())?); - digest.update( - serde_json::to_vec(&request.config.rule_options.get(&rule.code)) - .map_err(|error| error.to_string())?, - ); - digest.update(request.dialect.as_bytes()); - if let Some(custom) = request - .custom_rules - .iter() - .find(|custom| custom.code == rule.code) - { - digest.update(custom.fact_fingerprint.as_bytes()); - } - Ok(format!("{:x}", digest.finalize())) -} - fn model_cache_identities( models: &[&crate::models::Model], ruleset: &str, @@ -639,23 +333,6 @@ fn model_cache_identity( .map_err(lifecycle_error) } -fn fingerprint_project(project_dir: &Path, request: &EvaluateRequest) -> Result { - let mut files = collect_rule_input_files(project_dir, project_dir)?; - files.sort(); - let mut digest = Sha256::new(); - for path in files { - let relative = path.strip_prefix(project_dir).unwrap_or(&path); - digest.update(relative.to_string_lossy().as_bytes()); - digest.update(std::fs::read(&path).map_err(|error| { - format!( - "could not fingerprint rules project file {}: {error}", - path.display() - ) - })?); - } - fingerprint_request_facts(&digest.finalize(), request) -} - fn fingerprint_request_facts(seed: &[u8], request: &EvaluateRequest) -> Result { let mut digest = Sha256::new(); digest.update(seed); @@ -666,48 +343,14 @@ fn fingerprint_request_facts(seed: &[u8], request: &EvaluateRequest) -> Result Result, String> { - let mut output: Vec = Vec::new(); - for entry in std::fs::read_dir(directory) - .map_err(|error| format!("could not read rules project directory: {error}"))? - { - let entry = - entry.map_err(|error| format!("could not read rules project entry: {error}"))?; - let path = entry.path(); - if path.is_dir() { - if path.file_name().is_some_and(|name| { - name == TARGET_DIRECTORY || name == LOGS_DIRECTORY || name == GIT_DIRECTORY - }) { - continue; - } - output.extend(collect_rule_input_files(root, &path)?); - continue; - } - let supported = path - .extension() - .and_then(|value| value.to_str()) - .is_some_and(|value| { - matches!(value, PYTHON_EXTENSION | "sql" | "toml" | "yaml" | "yml") - }); - let is_rule_python = path.strip_prefix(root).is_ok_and(|relative| { - relative.starts_with(RULES_DIRECTORY) - && path - .extension() - .is_some_and(|value| value == PYTHON_EXTENSION) - }); - if supported && !is_rule_python && path.starts_with(root) { - output.push(path); - } - } - Ok(output) -} - fn validate_suppression_codes( request: &EvaluateRequest, catalogue: &[RuleMetadata], @@ -736,83 +379,6 @@ fn validate_suppression_codes( Ok(()) } -fn validate_custom_faults( - faults: &[Fault], - selected: &BTreeMap, -) -> Result<(), String> { - for fault in faults { - if !selected.get(&fault.code).is_some_and(|rule| rule.custom) { - return Err(format!( - "custom rule host returned a finding for unselected rule {}", - fault.code - )); - } - } - Ok(()) -} - -fn evaluate_custom_rules( - request: &EvaluateRequest, - selected: &BTreeMap, - model_paths: Option<&BTreeSet>, -) -> Result, String> { - let custom_selected = selected.values().any(|rule| rule.custom); - let Some(spec) = &request.custom_host else { - return if custom_selected { - Err("selected custom rules require a custom host".to_owned()) - } else { - Ok(Vec::new()) - }; - }; - if !custom_selected { - return Ok(Vec::new()); - } - let mut payload = spec.payload.clone(); - let payload_object = payload - .as_object_mut() - .ok_or_else(|| "custom rule host payload must be an object".to_owned())?; - payload_object.insert( - "selected_codes".to_owned(), - serde_json::Value::Array( - selected - .values() - .filter(|rule| rule.custom) - .map(|rule| serde_json::Value::String(rule.code.clone())) - .collect(), - ), - ); - payload_object.insert( - "selected_model_paths".to_owned(), - model_paths.map_or(serde_json::Value::Null, |paths| { - serde_json::Value::Array( - paths - .iter() - .cloned() - .map(serde_json::Value::String) - .collect(), - ) - }), - ); - let host_request = CustomHostRequest { - protocol: fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION, - runtime_version: spec.runtime_version.clone(), - payload, - }; - let response = run_custom_host::<_, Vec>(CustomHostInvocation { - program: Path::new(&spec.program), - arguments: &spec.arguments, - timeout: Duration::from_millis(spec.timeout_millis), - output_limits: CustomHostOutputLimits::default(), - request: &host_request, - }) - .map_err(lifecycle_error)?; - let faults = response.payload.ok_or_else(|| { - "custom rule host returned no finding payload after successful validation".to_owned() - })?; - validate_custom_faults(&faults, selected)?; - Ok(faults) -} - fn validate_exception_paths(request: &EvaluateRequest) -> Result<(), String> { let root = Path::new(&request.project_dir); for entry in &request.config.rule_exceptions { diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs index e45aa12145..d6c96b34b0 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod cache; +pub(crate) mod custom_host; pub(crate) mod evaluation; diff --git a/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs b/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs new file mode 100644 index 0000000000..b59958c6dd --- /dev/null +++ b/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs @@ -0,0 +1,3 @@ +pub(crate) fn run_custom_host_json(spec_json: &str) -> Result { + crate::engine::_helpers::custom_host::run_custom_host_json(spec_json) +} diff --git a/crates/sqlbuild-rules-native/src/engine/main/mod.rs b/crates/sqlbuild-rules-native/src/engine/main/mod.rs index 0dea504272..ae4f2a1b3e 100644 --- a/crates/sqlbuild-rules-native/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/engine/main/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod custom_host; pub(crate) mod evaluate; pub(crate) mod finalize; diff --git a/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs b/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs index 62d143eb92..ee51aec85c 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs @@ -212,7 +212,6 @@ pub(crate) fn request_with_scope(scope: Value) -> Value { "public_enums": [], "public_constants": [], "custom_rules": [], - "custom_host": null, "project_fingerprint": null, "scope_index": scope }) diff --git a/crates/sqlbuild-rules-native/src/models.rs b/crates/sqlbuild-rules-native/src/models.rs index b537e9d86d..e20ba27d73 100644 --- a/crates/sqlbuild-rules-native/src/models.rs +++ b/crates/sqlbuild-rules-native/src/models.rs @@ -691,10 +691,6 @@ pub(crate) struct CustomRule { #[serde(default)] pub project_wide: bool, #[serde(default)] - pub project_dependent: bool, - #[serde(default)] - pub fact_fingerprint: String, - #[serde(default)] pub check_name: String, #[serde(default)] pub test_case_count: u32, @@ -729,7 +725,6 @@ pub(crate) struct EvaluateRequest { pub initial_findings: Vec, pub defer_suppressions: bool, pub custom_rules: Vec, - pub custom_host: Option, pub project_fingerprint: Option, } @@ -767,7 +762,6 @@ impl Default for EvaluateRequest { initial_findings: vec![], defer_suppressions: false, custom_rules: vec![], - custom_host: None, project_fingerprint: None, } } @@ -791,8 +785,8 @@ pub(crate) struct EvaluateResponse { pub cache_hits: usize, pub cache_misses: usize, pub ruleset_fingerprint: String, + pub selected_codes: Vec, pub built_in_ms: u64, - pub custom_ms: u64, } #[derive(Clone, Debug, Serialize)] diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 118f3d2f5d..5c54c88e00 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -14,6 +14,7 @@ def analyze_column_references_json(request_json: str) -> str: ... def catalogue_json() -> str: ... def evaluate_json(request_json: str) -> str: ... def finalize_rule_findings_json(request_json: str) -> str: ... +def run_custom_host_json(spec_json: str) -> str: ... def format_sql_json(request_json: str) -> str: ... def format_sql_batch_json(request_json: str) -> str: ... def query_fingerprint(sql: str, dialect: str) -> str: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index c7e1674f1a..9228c39ac3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -1717,14 +1717,7 @@ def _call_loaded_macro( args: tuple[object, ...], kwargs: dict[str, object], ) -> object: - signature: inspect.Signature = inspect.signature(loaded_macro.function) - parameters: tuple[inspect.Parameter, ...] = tuple(signature.parameters.values()) - if ( - parameters - and parameters[0].kind - in {inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD} - and parameters[0].name == MACRO_CONTEXT_PARAMETER_NAME - ): + if loaded_macro.injects_context: if MACRO_CONTEXT_PARAMETER_NAME in kwargs: raise CompileInputError( f"Macro '@{loaded_macro.name}' must not be called with keyword argument 'ctx'; " diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index ca50211cf0..282597ae36 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -2,14 +2,16 @@ from __future__ import annotations +import inspect from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass, field +from functools import cached_property from pathlib import Path from typing import Any, overload from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity -from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE +from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, @@ -234,6 +236,20 @@ class LoadedMacro: function: Callable[..., object] dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) + @cached_property + def injects_context(self) -> bool: + """Whether the first positional parameter receives the injected macro context.""" + + parameters: tuple[inspect.Parameter, ...] = tuple( + inspect.signature(self.function).parameters.values() + ) + return bool( + parameters + and parameters[0].kind + in {inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD} + and parameters[0].name == MACRO_CONTEXT_PARAMETER_NAME + ) + @dataclass(frozen=True) class StaticMacroExport: diff --git a/src/sqlbuild/compiler/fact_cache/main/code_identity.py b/src/sqlbuild/compiler/fact_cache/main/code_identity.py new file mode 100644 index 0000000000..afdc7f4ea1 --- /dev/null +++ b/src/sqlbuild/compiler/fact_cache/main/code_identity.py @@ -0,0 +1,14 @@ +"""Public identity of the installed compiler code for derived caches.""" + +import hashlib + +import sqlbuild._native as _native +from sqlbuild.compiler.fact_cache._helpers.code_identity import installed_code_identity + + +def compiled_code_identity() -> str: + """Return the digest that changes with every released, locally edited, or rebuilt version.""" + + return hashlib.sha256( + f"{installed_code_identity()}\0{_native.BUILD_IDENTITY}".encode() + ).hexdigest() diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py index d049e0ee9d..e91ac52abc 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rule_evidence.py @@ -19,9 +19,6 @@ _PARAMETRIZE_ATTRIBUTE: str = "parametrize" _PYTEST_MARK_ATTRIBUTE: str = "mark" _PYTEST_MODULE_NAME: str = "pytest" -_PROJECT_FACT_ATTRIBUTES: frozenset[str] = frozenset( - {"audits", "declarations", "graph", "project", "tests"} -) @dataclass(frozen=True, order=True) @@ -92,38 +89,6 @@ def _fingerprint_label(*, path: Path, root: Path) -> str: return path.relative_to(root).as_posix() if path.is_relative_to(root) else path.as_posix() -def custom_rule_project_fact_attributes( - *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] | None = None -) -> frozenset[str]: - """Return project-wide context attributes used by implementation or helper code.""" - - if rule.project_wide: - return frozenset({"project"}) - if rule.source is None: - sources: tuple[str, ...] = (inspect.getsource(rule.check),) - else: - helper_paths: tuple[Path, ...] = ( - _import_closure( - source_path=Path(rule.source).resolve(), project_dir=project_dir.resolve() - ) - if import_closure is None - else import_closure - ) - sources = ( - inspect.getsource(rule.check), - *(path.read_text(encoding="utf-8") for path in helper_paths), - ) - attributes: set[str] = set() - for source in sources: - tree: ast.Module = ast.parse(source) - attributes.update( - node.attr - for node in ast.walk(tree) - if isinstance(node, ast.Attribute) and node.attr in _PROJECT_FACT_ATTRIBUTES - ) - return frozenset(attributes) - - def _import_closure(*, source_path: Path, project_dir: Path) -> tuple[Path, ...]: rules_root: Path = (project_dir / _RULES_DIRECTORY_NAME).resolve() pending: list[Path] = [source_path] diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py new file mode 100644 index 0000000000..4b500db707 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -0,0 +1,563 @@ +"""Incremental custom-rule evaluation keyed on the compiler facts each invocation read.""" + +from __future__ import annotations + +import hashlib +import os +import pickle +import sys +import tempfile +import threading +import time +from collections.abc import Callable +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Any, cast + +import orjson + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( + custom_rule_implementation_fingerprint, + custom_rule_import_closure, +) +from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys, is_tree_fact +from sqlbuild.rule_engine.classes.fact_digests import FactDigests +from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views +from sqlbuild.rule_engine.constants import ( + CUSTOM_HOST_LAUNCH_MODULE, + CUSTOM_HOST_RUNTIME_VERSION, + CUSTOM_RULES_CACHE_FILE, + CUSTOM_RULES_CACHE_VERSION, +) +from sqlbuild.rule_engine.exceptions import FactDigestError, RulesError +from sqlbuild.rule_engine.models import CustomRulesOutcome, Rule, RulesConfig +from sqlbuild.rule_engine.types import FactKey, RuleSubject + +_HOST_TIMEOUT_MILLIS: int = 120_000 +_PROJECT_SUBJECT: str = "" + + +@dataclass(frozen=True) +class _Subject: + key: str + identity: str + + +@dataclass(frozen=True) +class _Evaluated: + findings: tuple[dict[str, object], ...] + reads: tuple[FactKey, ...] | None + + +@dataclass(frozen=True) +class _HostRun: + evaluated: dict[tuple[str, str], _Evaluated] + untracked_reads: dict[str, tuple[FactKey, ...]] + uncacheable: frozenset[str] + observed: dict[FactKey, str | None] + + +@dataclass(frozen=True) +class _CachedEntry: + subject_identity: str + readset: int + reads_digest: str + findings: tuple[dict[str, object], ...] + + +@dataclass +class _RuleCache: + identity: str + readsets: list[tuple[FactKey, ...]] + entries: dict[str, _CachedEntry] + + +def evaluate_custom_rules_cached( + *, + project: CompiledProject, + config: RulesConfig, + project_dir: Path, + rules: tuple[Rule, ...], + dialect: str, + verify_determinism: bool = False, +) -> CustomRulesOutcome: + """Reuse results whose recorded fact reads are unchanged and evaluate only the rest.""" + + started: float = time.monotonic() + custom: tuple[Rule, ...] = tuple(sorted(rules, key=lambda rule: rule.code)) + if not custom: + return CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) + if not config.cache.enabled: + uncached: _HostRun = _run_host( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + plan={rule.code: None for rule in custom}, + track_reads=False, + verify_determinism=verify_determinism, + ) + return CustomRulesOutcome( + findings=_ordered_findings( + rules=custom, + subjects=_subjects(project=project, rules=custom), + results={key: value.findings for key, value in uncached.evaluated.items()}, + ), + cache_hits=0, + cache_misses=len(custom), + custom_ms=_elapsed_ms(started), + ) + digests: FactDigests = FactDigests( + views=_views_factory(project=project, project_dir=project_dir, dialect=dialect) + ) + subjects: dict[str, tuple[_Subject, ...]] = _subjects(project=project, rules=custom) + identities: dict[str, str] = { + rule.code: _rule_identity( + rule=rule, config=config, project_dir=project_dir, dialect=dialect + ) + for rule in custom + } + cache_path: Path = project_dir / CUSTOM_RULES_CACHE_FILE + stored: dict[str, _RuleCache] = _read_cache(cache_path) + results: dict[tuple[str, str], tuple[dict[str, object], ...]] = {} + plan: dict[str, list[str] | None] = {} + hits: int = 0 + for rule in custom: + cached: _RuleCache | None = stored.get(rule.code) + if cached is not None and cached.identity != identities[rule.code]: + cached = None + current_digests: dict[int, str | None] = {} + missing: list[str] = [] + for subject in subjects[rule.code]: + entry: _CachedEntry | None = None if cached is None else cached.entries.get(subject.key) + if ( + entry is not None + and cached is not None + and entry.subject_identity == subject.identity + and _readset_digest( + cached=cached, index=entry.readset, digests=digests, memo=current_digests + ) + == entry.reads_digest + ): + results[(rule.code, subject.key)] = entry.findings + hits += 1 + else: + missing.append(subject.key) + if missing: + plan[rule.code] = None if len(missing) == len(subjects[rule.code]) else sorted(missing) + misses: int = sum( + len(subjects[code]) if planned is None else len(planned) for code, planned in plan.items() + ) + if plan: + run: _HostRun = _run_host( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + plan=plan, + track_reads=True, + verify_determinism=verify_determinism, + ) + _validate_paths(project=project, rules=custom, evaluated=run.evaluated) + stored = _store( + stored=stored, + identities=identities, + subjects=subjects, + run=run, + digests=digests, + ) + for key, value in run.evaluated.items(): + results[key] = value.findings + _write_cache(path=cache_path, stored=stored, subjects=subjects) + return CustomRulesOutcome( + findings=_ordered_findings( + rules=custom, + subjects=subjects, + results=results, + ), + cache_hits=hits, + cache_misses=misses, + custom_ms=_elapsed_ms(started), + ) + + +def _views_factory( + *, project: CompiledProject, project_dir: Path, dialect: str +) -> Callable[[], RuleFactViews]: + def build() -> RuleFactViews: + return build_rule_fact_views( + project=_host_project(project), project_dir=project_dir, dialect=dialect + ) + + return build + + +def _subjects( + *, project: CompiledProject, rules: tuple[Rule, ...] +) -> dict[str, tuple[_Subject, ...]]: + project_subject: tuple[_Subject, ...] = ( + _Subject( + key=_PROJECT_SUBJECT, + identity=_digest([len(project.models), project.effective_target_name]), + ), + ) + model_subjects: tuple[_Subject, ...] = tuple( + _Subject( + key=model.relative_path.as_posix(), + identity=_digest([model.name, model.relative_path.as_posix(), _materialization(model)]), + ) + for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) + ) + return { + rule.code: project_subject if rule.subject is RuleSubject.PROJECT else model_subjects + for rule in rules + } + + +def _materialization(model: Any) -> str | None: + value: object = model.config.values.get("materialized") + return value if isinstance(value, str) else None + + +def _rule_identity(*, rule: Rule, config: RulesConfig, project_dir: Path, dialect: str) -> str: + closure: tuple[Path, ...] = custom_rule_import_closure(rule=rule, project_dir=project_dir) + return _digest( + [ + CUSTOM_RULES_CACHE_VERSION, + CUSTOM_HOST_RUNTIME_VERSION, + compiled_code_identity(), + rule.code, + rule.family, + rule.slug, + rule.message, + rule.remediation, + None if rule.subject is None else rule.subject.value, + custom_rule_implementation_fingerprint( + rule=rule, project_dir=project_dir, import_closure=closure + ), + asdict(config), + dialect, + str(project_dir.resolve()), + ] + ) + + +def _readset_digest( + *, cached: _RuleCache, index: int, digests: FactDigests, memo: dict[int, str | None] +) -> str | None: + if index not in memo: + try: + memo[index] = ( + digests.combined(cached.readsets[index]) + if 0 <= index < len(cached.readsets) + else None + ) + except FactDigestError: + memo[index] = None + return memo[index] + + +def _run_host( + *, + project: CompiledProject, + config: RulesConfig, + project_dir: Path, + dialect: str, + plan: dict[str, list[str] | None], + track_reads: bool, + verify_determinism: bool, +) -> _HostRun: + input_dir: Path = project_dir / "target" / "rules-cache" / "host-inputs" + input_dir.mkdir(parents=True, exist_ok=True) + input_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + dir=input_dir, prefix="project-", suffix=".pickle", delete=False + ) as handle: + input_path = Path(handle.name) + pickle.dump((_host_project(project), config), handle) + response_json: str = _native.run_custom_host_json( + orjson.dumps( + { + "program": sys.executable, + "arguments": ["-m", CUSTOM_HOST_LAUNCH_MODULE], + "timeout_millis": _HOST_TIMEOUT_MILLIS, + "runtime_version": CUSTOM_HOST_RUNTIME_VERSION, + "payload": { + "project_pickle_path": str(input_path.resolve()), + "project_dir": str(project_dir.resolve()), + "dialect": dialect, + "verify_determinism": verify_determinism, + "track_reads": track_reads, + "plan": plan, + }, + } + ).decode() + ) + except (ValueError, TypeError) as error: + raise RulesError(str(error)) from error + finally: + if input_path is not None: + input_path.unlink(missing_ok=True) + return _decode_host_response(orjson.loads(response_json)) + + +def _host_project(project: CompiledProject) -> CompiledProject: + return replace( + project, + binding_catalog=None, + sql_expansions={}, + loaded_macros={}, + loader_functions=(), + hook_functions=(), + materialization_files=(), + external_sql_reference_resolver=None, + ) + + +def _decode_host_response(response: object) -> _HostRun: + if not isinstance(response, dict): + raise RulesError("custom rule host returned an invalid payload") + payload: dict[str, Any] = cast(dict[str, Any], response) + raw_readsets: Any = payload.get("readsets") + raw_evaluations: Any = payload.get("evaluations") + raw_untracked: Any = payload.get("untracked") + raw_uncacheable: Any = payload.get("uncacheable") + raw_observed: Any = payload.get("observed") + if ( + not isinstance(raw_readsets, list) + or not isinstance(raw_evaluations, list) + or not isinstance(raw_untracked, dict) + or not isinstance(raw_uncacheable, list) + or not isinstance(raw_observed, list) + ): + raise RulesError("custom rule host returned an invalid payload") + readsets: list[tuple[FactKey, ...]] = list(map(_decode_readset, raw_readsets)) + untracked: dict[str, tuple[FactKey, ...]] = { + str(code): _decode_readset(reads) for code, reads in raw_untracked.items() + } + uncacheable: frozenset[str] = frozenset(str(code) for code in raw_uncacheable) + evaluated: dict[tuple[str, str], _Evaluated] = {} + for raw_item in raw_evaluations: + if not isinstance(raw_item, dict): + raise RulesError("custom rule host returned an invalid evaluation") + item: dict[str, Any] = cast(dict[str, Any], raw_item) + code: str = str(item["code"]) + subject: object = item.get("subject") + findings: Any = item.get("findings") + reads: object = item.get("reads") + if not isinstance(findings, list): + raise RulesError("custom rule host returned invalid findings") + evaluated[(code, _PROJECT_SUBJECT if subject is None else str(subject))] = _Evaluated( + findings=tuple(map(_finding_payload, findings)), + reads=(readsets[reads] if isinstance(reads, int) and code not in untracked else None), + ) + return _HostRun( + evaluated=evaluated, + untracked_reads=untracked, + uncacheable=uncacheable, + observed={ + tuple(map(str, key)): None if digest is None else str(digest) + for key, digest in raw_observed + }, + ) + + +def _validate_paths( + *, + project: CompiledProject, + rules: tuple[Rule, ...], + evaluated: dict[tuple[str, str], _Evaluated], +) -> None: + codes: frozenset[str] = frozenset(rule.code for rule in rules) + model_rules: frozenset[str] = frozenset( + rule.code for rule in rules if rule.subject is RuleSubject.MODEL + ) + model_paths: frozenset[str] = frozenset( + model.relative_path.as_posix() for model in project.models + ) + for (code, _subject), result in evaluated.items(): + for finding in result.findings: + if finding.get("code") not in codes: + raise RulesError( + f"custom rule host returned a finding for unselected rule {finding.get('code')}" + ) + if code in model_rules and finding.get("path") not in model_paths: + raise RulesError( + "model-subject custom rule returned a finding for unrelated path " + f"{finding.get('path')}" + ) + + +def _store( + *, + stored: dict[str, _RuleCache], + identities: dict[str, str], + subjects: dict[str, tuple[_Subject, ...]], + run: _HostRun, + digests: FactDigests, +) -> dict[str, _RuleCache]: + updated: dict[str, _RuleCache] = {} + full_keys: tuple[FactKey, ...] | None = None + for code, identity in identities.items(): + previous: _RuleCache | None = stored.get(code) + current: _RuleCache = ( + _RuleCache(identity=identity, readsets=[], entries={}) + if previous is None or previous.identity != identity + else _RuleCache( + identity=identity, + readsets=list(previous.readsets), + entries=dict(previous.entries), + ) + ) + updated[code] = current + by_subject: dict[str, _Subject] = {subject.key: subject for subject in subjects[code]} + positions: dict[tuple[FactKey, ...], int] = { + readset: index for index, readset in enumerate(current.readsets) + } + combined_by_readset: dict[tuple[FactKey, ...], str | None] = {} + for (evaluated_code, subject_key), result in run.evaluated.items(): + if evaluated_code != code: + continue + if code in run.uncacheable: + current.entries.pop(subject_key, None) + continue + reads: tuple[FactKey, ...] | None = result.reads + if reads is None: + if full_keys is None: + full_keys = full_fact_keys(digests.views) + reads = (*run.untracked_reads.get(code, ()), *full_keys) + readset: tuple[FactKey, ...] = tuple(sorted(set(reads))) + if readset not in combined_by_readset: + combined_by_readset[readset] = _observed_digest( + readset=readset, observed=run.observed, digests=digests + ) + combined: str | None = combined_by_readset[readset] + if combined is None: + current.entries.pop(subject_key, None) + continue + index: int | None = positions.get(readset) + if index is None: + index = len(current.readsets) + current.readsets.append(readset) + positions[readset] = index + current.entries[subject_key] = _CachedEntry( + subject_identity=by_subject[subject_key].identity, + readset=index, + reads_digest=combined, + findings=result.findings, + ) + return updated + + +def _observed_digest( + *, readset: tuple[FactKey, ...], observed: dict[FactKey, str | None], digests: FactDigests +) -> str | None: + """Digest a read set only when its filesystem facts still match what the rule observed.""" + + try: + for key in readset: + if is_tree_fact(key) and key in observed and observed[key] != digests.digest(key): + return None + return digests.combined(readset) + except FactDigestError: + return None + + +def _ordered_findings( + *, + rules: tuple[Rule, ...], + subjects: dict[str, tuple[_Subject, ...]], + results: dict[tuple[str, str], tuple[dict[str, object], ...]], +) -> tuple[dict[str, object], ...]: + findings: list[dict[str, object]] = [] + for rule in rules: + for subject in subjects[rule.code]: + findings.extend(results.get((rule.code, subject.key), ())) + return tuple(findings) + + +def _read_cache(path: Path) -> dict[str, _RuleCache]: + try: + payload: object = orjson.loads(path.read_bytes()) + except (OSError, orjson.JSONDecodeError): + return {} + if not isinstance(payload, dict) or payload.get("version") != CUSTOM_RULES_CACHE_VERSION: + return {} + rules: object = payload.get("rules") + if not isinstance(rules, dict): + return {} + stored: dict[str, _RuleCache] = {} + for code, value in rules.items(): + try: + stored[str(code)] = _RuleCache( + identity=str(value["identity"]), + readsets=list(map(_decode_readset, value["readsets"])), + entries={ + str(subject): _decode_entry(entry) + for subject, entry in value["entries"].items() + }, + ) + except (KeyError, TypeError, ValueError, IndexError): + continue + return stored + + +def _write_cache( + *, path: Path, stored: dict[str, _RuleCache], subjects: dict[str, tuple[_Subject, ...]] +) -> None: + rules: dict[str, object] = {} + for code, cached in stored.items(): + if code not in subjects: + continue + live: frozenset[str] = frozenset(subject.key for subject in subjects[code]) + entries: dict[str, _CachedEntry] = { + key: entry for key, entry in cached.entries.items() if key in live + } + used: list[int] = sorted({entry.readset for entry in entries.values()}) + remap: dict[int, int] = {old: new for new, old in enumerate(used)} + rules[code] = { + "identity": cached.identity, + "readsets": [list(map(list, cached.readsets[index])) for index in used], + "entries": { + key: [ + entry.subject_identity, + remap[entry.readset], + entry.reads_digest, + list(entry.findings), + ] + for key, entry in entries.items() + }, + } + path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") + temporary.write_bytes(orjson.dumps({"version": CUSTOM_RULES_CACHE_VERSION, "rules": rules})) + temporary.replace(path) + + +def _decode_readset(value: Any) -> tuple[FactKey, ...]: + return tuple(tuple(map(str, key)) for key in value) + + +def _decode_entry(value: Any) -> _CachedEntry: + return _CachedEntry( + subject_identity=str(value[0]), + readset=int(value[1]), + reads_digest=str(value[2]), + findings=tuple(map(_finding_payload, value[3])), + ) + + +def _finding_payload(value: Any) -> dict[str, object]: + return {str(key): item for key, item in value.items()} + + +def _digest(value: object) -> str: + return hashlib.sha256(orjson.dumps(value, default=str)).hexdigest() + + +def _elapsed_ms(started: float) -> int: + return round((time.monotonic() - started) * 1000) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py new file mode 100644 index 0000000000..7404be19f1 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py @@ -0,0 +1,198 @@ +"""Recompute and digest the compiler facts a custom-rule evaluation recorded.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable +from decimal import Decimal +from pathlib import Path, PurePath + +import orjson + +from sqlbuild.rule_engine.classes.sql_document import SqlDocument +from sqlbuild.rule_engine.constants import ( + FACT_AUDITS_ALL, + FACT_AUDITS_FOR_MODEL, + FACT_COLUMNS_DECLARED, + FACT_COLUMNS_INFERRED, + FACT_COLUMNS_NAMES, + FACT_CONTRACTS_ENFORCED, + FACT_CONTRACTS_GRAIN, + FACT_DECLARATIONS_CONSTANTS, + FACT_DECLARATIONS_ENUMS, + FACT_DECLARATIONS_PUBLIC_CONSTANTS, + FACT_DECLARATIONS_PUBLIC_ENUMS, + FACT_GRAPH_DEPENDENCIES, + FACT_GRAPH_DEPENDENTS, + FACT_PROJECT_AUDITS, + FACT_PROJECT_FUNCTIONS, + FACT_PROJECT_MODELS, + FACT_PROJECT_SEEDS, + FACT_PROJECT_SOURCES, + FACT_PROJECT_TESTS, + FACT_SQL_FOR_MODEL, + FACT_TESTS_ALL, + FACT_TESTS_FOR_MODEL, + FACT_TREE_CHILDREN, + FACT_TREE_DESCENDANTS, + FACT_TREE_GLOB, + FACT_TREE_PATHS, + FACT_TREE_READ_TEXT, + FACT_TREE_RELATIVE_PARTS, + FACT_TREE_RESOURCES_UNDER, +) +from sqlbuild.rule_engine.exceptions import FactDigestError +from sqlbuild.rule_engine.models import Model, RuleFactViews +from sqlbuild.rule_engine.types import FactKey + +_TRACKED_TEXT_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) +_ORJSON_OPTIONS: int = orjson.OPT_SORT_KEYS | orjson.OPT_NON_STR_KEYS + +_GLOBAL_FACTS: dict[str, Callable[[RuleFactViews], object]] = { + FACT_PROJECT_MODELS: lambda views: views.project.models, + FACT_PROJECT_SOURCES: lambda views: views.project.sources, + FACT_PROJECT_SEEDS: lambda views: views.project.seeds, + FACT_PROJECT_FUNCTIONS: lambda views: views.project.functions, + FACT_PROJECT_TESTS: lambda views: views.project.tests, + FACT_PROJECT_AUDITS: lambda views: views.project.audits, + FACT_TREE_PATHS: lambda views: views.project.tree.paths(), + FACT_TESTS_ALL: lambda views: views.tests.all(), + FACT_AUDITS_ALL: lambda views: views.audits.all(), + FACT_DECLARATIONS_PUBLIC_ENUMS: lambda views: views.declarations.public_enums, + FACT_DECLARATIONS_PUBLIC_CONSTANTS: lambda views: views.declarations.public_constants, + FACT_DECLARATIONS_ENUMS: lambda views: views.declarations.enums, + FACT_DECLARATIONS_CONSTANTS: lambda views: views.declarations.constants, +} +_MODEL_PATH_FACTS: dict[str, Callable[[RuleFactViews, Model], object]] = { + FACT_SQL_FOR_MODEL: lambda views, model: views.sql.for_model(model), + FACT_GRAPH_DEPENDENCIES: lambda views, model: views.graph.dependencies(model), + FACT_GRAPH_DEPENDENTS: lambda views, model: views.graph.dependents(model), + FACT_COLUMNS_DECLARED: lambda views, model: views.columns.declared(model), + FACT_COLUMNS_INFERRED: lambda views, model: views.columns.inferred(model), + FACT_COLUMNS_NAMES: lambda views, model: views.columns.names(model), + FACT_CONTRACTS_ENFORCED: lambda views, model: views.contracts.enforced(model), + FACT_CONTRACTS_GRAIN: lambda views, model: views.contracts.grain(model), +} +_MODEL_NAME_FACTS: dict[str, Callable[[RuleFactViews, Model], object]] = { + FACT_TESTS_FOR_MODEL: lambda views, model: views.tests.for_model(model), + FACT_AUDITS_FOR_MODEL: lambda views, model: views.audits.for_model(model), +} +_TEXT_FACTS: dict[str, Callable[[RuleFactViews, tuple[str, ...]], object]] = { + FACT_TREE_CHILDREN: lambda views, arguments: views.project.tree.children(*arguments), + FACT_TREE_DESCENDANTS: lambda views, arguments: views.project.tree.descendants(*arguments), + FACT_TREE_GLOB: lambda views, arguments: views.project.tree.glob(*arguments), + FACT_TREE_RELATIVE_PARTS: lambda views, arguments: views.project.tree.relative_parts( + path=arguments[0], under=arguments[1] + ), + FACT_TREE_RESOURCES_UNDER: lambda views, arguments: views.project.tree.resources_under( + *arguments + ), + FACT_TREE_READ_TEXT: lambda views, arguments: views.project.tree.read_text(*arguments), +} +_TEXT_FACT_ARITY: dict[str, int] = { + FACT_TREE_CHILDREN: 1, + FACT_TREE_DESCENDANTS: 1, + FACT_TREE_GLOB: 1, + FACT_TREE_RELATIVE_PARTS: 2, + FACT_TREE_RESOURCES_UNDER: 1, + FACT_TREE_READ_TEXT: 1, +} +_TREE_FACTS: frozenset[str] = frozenset({FACT_TREE_PATHS, *_TEXT_FACTS}) + + +def fact_key_payload(key: tuple[object, ...]) -> list[str]: + """Return the JSON form of one recorded fact key.""" + + return [value.as_posix() if isinstance(value, PurePath) else str(value) for value in key] + + +def full_fact_keys(views: RuleFactViews) -> tuple[FactKey, ...]: + """Return keys covering every fact a custom rule can observe through its context.""" + + keys: list[FactKey] = [(fact,) for fact in _GLOBAL_FACTS] + models: tuple[Model, ...] = views.project.models + for model in models: + path: str = model.path.as_posix() + keys.extend((fact, path) for fact in _MODEL_PATH_FACTS) + keys.extend((fact, model.name) for fact in _MODEL_NAME_FACTS) + keys.extend( + (FACT_TREE_READ_TEXT, node.value) + for node in views.project.tree.paths() + if node.is_file and Path(node.value).suffix.lower() in _TRACKED_TEXT_SUFFIXES + ) + return tuple(keys) + + +def fact_outcome_digest(*, views: RuleFactViews, key: FactKey) -> str: + """Recompute one recorded fact and digest its value or raised error.""" + + try: + value: object = _fact_value(views=views, key=key) + except FactDigestError: + raise + except Exception as error: + return fact_error_digest(error) + return fact_value_digest(value) + + +def fact_value_digest(value: object) -> str: + """Digest one fact value exactly as a recomputed fact would digest it.""" + + return _encoded_digest(("value", value)) + + +def fact_error_digest(error: Exception) -> str: + """Digest one raised fact error exactly as a recomputed fact would digest it.""" + + return _encoded_digest(("error", type(error).__name__, str(error))) + + +def is_tree_fact(key: FactKey) -> bool: + """Return whether a fact is read from the live project filesystem.""" + + return bool(key) and key[0] in _TREE_FACTS + + +def _fact_value(*, views: RuleFactViews, key: FactKey) -> object: + if not key: + raise FactDigestError("empty fact key") + fact: str = key[0] + arguments: tuple[str, ...] = key[1:] + if fact in _GLOBAL_FACTS and not arguments: + return _GLOBAL_FACTS[fact](views) + if fact in _MODEL_PATH_FACTS and len(arguments) == 1: + return _MODEL_PATH_FACTS[fact]( + views, Model(name="", path=Path(arguments[0]), materialization=None) + ) + if fact in _MODEL_NAME_FACTS and len(arguments) == 1: + return _MODEL_NAME_FACTS[fact]( + views, Model(name=arguments[0], path=Path(), materialization=None) + ) + if fact in _TEXT_FACTS and len(arguments) == _TEXT_FACT_ARITY[fact]: + return _TEXT_FACTS[fact](views, arguments) + raise FactDigestError(f"unknown fact key: {fact}") + + +def _encoded_digest(value: object) -> str: + try: + encoded: bytes = orjson.dumps(value, default=_encode_default, option=_ORJSON_OPTIONS) + except (TypeError, FactDigestError) as error: + raise FactDigestError(str(error)) from error + return hashlib.sha256(encoded).hexdigest() + + +def _encode_default(value: object) -> object: + if isinstance(value, PurePath): + return value.as_posix() + if isinstance(value, SqlDocument): + return {"source": value.source} + if isinstance(value, Decimal): + return str(value) + if isinstance(value, (set, frozenset)): + return sorted( + orjson.dumps(item, default=_encode_default, option=_ORJSON_OPTIONS).decode() + for item in value + ) + if isinstance(value, tuple): + return list(value) + raise FactDigestError(f"fact value of type {type(value).__name__} is not reproducible") diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 57555ec24c..a609ba1cc8 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -2,15 +2,10 @@ from __future__ import annotations -import hashlib import inspect import json -import pickle -import re -import sys -import tempfile -from collections.abc import Sequence -from dataclasses import asdict, fields, replace +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import asdict, replace from decimal import Decimal from pathlib import Path from typing import Any, cast @@ -20,8 +15,6 @@ import sqlbuild._native as _native from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, CompiledModel, CompiledModelSqlTestPayload, CompiledProject, @@ -37,22 +30,21 @@ from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( custom_rule_implementation_fingerprint, custom_rule_import_closure, - custom_rule_project_fact_attributes, custom_rule_test_evidence, ) +from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached +from sqlbuild.rule_engine._helpers.run.native_memo import ( + native_request_identity, + read_native_response, + write_native_response, +) from sqlbuild.rule_engine.constants import ( - CUSTOM_HOST_LAUNCH_MODULE, - CUSTOM_HOST_RUNTIME_VERSION, - RULE_CONTEXT_AUDITS_FACT, - RULE_CONTEXT_DECLARATIONS_FACT, - RULE_CONTEXT_GRAPH_FACT, - RULE_CONTEXT_PROJECT_FACT, - RULE_CONTEXT_TESTS_FACT, RULES_NATIVE_API_VERSION, TYPE_PROOF_RULE_CODES, ) from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import ( + CustomRulesOutcome, Finding, Rule, RulesConfig, @@ -61,9 +53,6 @@ from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind -_CUSTOM_HOST_REQUIRED: str = "selected custom rules require a custom host" -_NATIVE_CACHE_MISSES_PATTERN: re.Pattern[str] = re.compile(r"native_cache_misses=(\d+)") - def evaluate_native( *, @@ -75,15 +64,15 @@ def evaluate_native( initial_findings: tuple[Finding, ...] = (), defer_suppressions: bool = False, verify_determinism: bool = False, - rules_cache_present_at_start: bool | None = None, ) -> RulesResult: - """Evaluate one compiled model batch through the native engine.""" + """Evaluate built-in rules natively while custom rules run incrementally beside them.""" - selected_catalogue: list[Rule] = [] - for rule in catalogue: - selected_by_config: bool = any(rule.code.startswith(selector) for selector in config.select) - if rule.custom and selected_by_config: - selected_catalogue.append(rule) + custom_payloads: list[dict[str, object]] = _custom_rule_payloads( + catalogue=catalogue, project_dir=project_dir + ) + custom_rules: tuple[Rule, ...] = _selected_custom_rules( + config=config, catalogue=catalogue, custom_payloads=custom_payloads + ) request: dict[str, object] = { "version": RULES_NATIVE_API_VERSION, "project_dir": str(project_dir.resolve()), @@ -106,83 +95,95 @@ def evaluate_native( ], "scope_index": scope_metadata_projection(index=project.scope_index), "initial_findings": [_finding_payload(finding) for finding in initial_findings], - "defer_suppressions": defer_suppressions, - "custom_rules": _custom_rule_payloads( - catalogue=catalogue, - project=project, - project_dir=project_dir, - cache_enabled=config.cache.enabled, - ), + "defer_suppressions": True, + "custom_rules": custom_payloads, } - custom_host_input: Path | None = None - retry_native_misses: int = 0 - request["custom_host"] = None - try: - if not config.cache.enabled or not ( - rules_cache_exists(project_dir) - if rules_cache_present_at_start is None - else rules_cache_present_at_start - ): - custom_host, custom_host_input = _custom_host_payload( + with ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as pool: + custom_future: Future[CustomRulesOutcome] | None = ( + pool.submit( + evaluate_custom_rules_cached, project=project, config=config, - project_dir=project_dir, - catalogue=tuple(selected_catalogue), + project_dir=project_dir.resolve(), + rules=custom_rules, dialect=dialect, verify_determinism=verify_determinism, ) - request["custom_host"] = custom_host + if custom_rules + else None + ) try: - response_json: str = _evaluate_request(request) - except ValueError as error: - if _CUSTOM_HOST_REQUIRED not in str(error): - raise - match: re.Match[str] | None = _NATIVE_CACHE_MISSES_PATTERN.search(str(error)) - retry_native_misses = 0 if match is None else int(match.group(1)) - custom_host, custom_host_input = _custom_host_payload( - project=project, - config=config, - project_dir=project_dir, - catalogue=tuple(selected_catalogue), - dialect=dialect, - verify_determinism=verify_determinism, + response_json: str + reused: bool + response_json, reused = _evaluate_request( + request=request, project_dir=project_dir, cache_enabled=config.cache.enabled ) - request["custom_host"] = custom_host - response_json = _evaluate_request(request) - response: object = orjson.loads(response_json) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - finally: - if custom_host_input is not None: - custom_host_input.unlink(missing_ok=True) + response: object = orjson.loads(response_json) + except (ValueError, TypeError) as error: + raise RulesError(str(error)) from error + custom: CustomRulesOutcome = ( + CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) + if custom_future is None + else custom_future.result() + ) if not isinstance(response, dict): raise RulesError("native rules engine returned an invalid response") payload: dict[str, Any] = response if payload.get("version") != RULES_NATIVE_API_VERSION: raise RulesError("native rules engine returned an unsupported response version") raw_findings: object = payload.get("faults") - if not isinstance(raw_findings, list): + selected_codes: object = payload.get("selected_codes") + if not isinstance(raw_findings, list) or not isinstance(selected_codes, list): raise RulesError("native rules engine returned invalid findings") + findings: tuple[Finding, ...] = tuple( + decode_rule_finding(value) for value in (*raw_findings, *custom.findings) + ) + if not defer_suppressions: + findings = finalize_native_findings( + config=config, + project_dir=project_dir, + evaluated_codes=tuple(str(code) for code in selected_codes), + findings=findings, + ) + native_hits: int = int(payload.get("cache_hits", 0)) + native_misses: int = int(payload.get("cache_misses", 0)) return RulesResult( - findings=tuple(decode_rule_finding(value) for value in raw_findings), + findings=findings, evaluated_models=int(payload.get("evaluated_models", 0)), - cache_hits=max(0, int(payload.get("cache_hits", 0)) - retry_native_misses), - cache_misses=int(payload.get("cache_misses", 0)) + retry_native_misses, - built_in_ms=int(payload.get("built_in_ms", 0)), - custom_ms=int(payload.get("custom_ms", 0)), + cache_hits=(native_hits + native_misses if reused else native_hits) + custom.cache_hits, + cache_misses=(0 if reused else native_misses) + custom.cache_misses, + built_in_ms=0 if reused else int(payload.get("built_in_ms", 0)), + custom_ms=custom.custom_ms, ) -def rules_cache_exists(project_dir: Path) -> bool: - """Return whether this project already has persisted rules-cache state.""" - - return (project_dir / "target" / "rules-cache").exists() +def _selected_custom_rules( + *, + config: RulesConfig, + catalogue: tuple[Rule, ...], + custom_payloads: list[dict[str, object]], +) -> tuple[Rule, ...]: + if not custom_payloads: + return () + selected: frozenset[str] = frozenset( + _selected_codes(config=config, custom_payloads=custom_payloads) + ) + return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) -def _evaluate_request(request: dict[str, object]) -> str: - return _native.evaluate_json( - orjson.dumps(request, option=orjson.OPT_SORT_KEYS, default=str).decode() - ) +def _evaluate_request( + *, request: dict[str, object], project_dir: Path, cache_enabled: bool +) -> tuple[str, bool]: + request_json: bytes = orjson.dumps(request, option=orjson.OPT_SORT_KEYS, default=str) + if not cache_enabled: + return _native.evaluate_json(request_json.decode()), False + identity: str = native_request_identity(request_json) + reused: str | None = read_native_response(project_dir=project_dir, identity=identity) + if reused is not None: + return reused, True + response: str = _native.evaluate_json(request_json.decode()) + write_native_response(project_dir=project_dir, identity=identity, response=response) + return response, False def finalize_native_findings( @@ -264,33 +265,19 @@ def native_selected_codes( ) -> tuple[str, ...]: """Resolve the active ruleset through the native Fensu adapter.""" - closures: dict[str, tuple[Path, ...]] = {} + return _selected_codes( + config=config, + custom_payloads=_custom_rule_payloads(catalogue=catalogue, project_dir=project_dir), + ) + + +def _selected_codes( + *, config: RulesConfig, custom_payloads: list[dict[str, object]] +) -> tuple[str, ...]: request: dict[str, object] = { "version": RULES_NATIVE_API_VERSION, "config": _config_payload(config), - "custom_rules": [ - { - "code": rule.code, - "family": rule.family, - "slug": rule.slug, - "message": rule.message, - "remediation": rule.remediation, - "enabled_by_default": rule.enabled_by_default, - "implementation_fingerprint": custom_rule_implementation_fingerprint( - rule=rule, - project_dir=project_dir, - import_closure=_shared_import_closure( - rule=rule, project_dir=project_dir, cache=closures - ), - ), - **_custom_rule_source_payload(rule=rule, project_dir=project_dir), - "project_wide": rule.project_wide, - "check_name": getattr(rule.check, "__name__", ""), - "test_case_count": 0, - } - for rule in catalogue - if rule.custom - ], + "custom_rules": custom_payloads, } try: payload: object = json.loads( @@ -586,50 +573,25 @@ def _typed_value_payload(value: SqlValue) -> object: def _custom_rule_payloads( - *, - catalogue: tuple[Rule, ...], - project: CompiledProject, - project_dir: Path, - cache_enabled: bool, + *, catalogue: tuple[Rule, ...], project_dir: Path ) -> list[dict[str, object]]: - fingerprints: dict[frozenset[str], str] = {} closures: dict[str, tuple[Path, ...]] = {} - payloads: list[dict[str, object]] = [] - for rule in catalogue: - if not rule.custom: - continue - closure: tuple[Path, ...] = _shared_import_closure( - rule=rule, project_dir=project_dir, cache=closures - ) - attributes: frozenset[str] = custom_rule_project_fact_attributes( - rule=rule, project_dir=project_dir, import_closure=closure - ) - if cache_enabled and attributes not in fingerprints: - fingerprints[attributes] = _custom_fact_fingerprint( - project=project, attributes=attributes - ) - fact_fingerprint: str = fingerprints[attributes] if cache_enabled else "" - payloads.append( - _custom_rule_payload( - rule=rule, - project_dir=project_dir, - project_attributes=attributes, - fact_fingerprint=fact_fingerprint, - import_closure=closure, - ) + return [ + _custom_rule_payload( + rule=rule, + project_dir=project_dir, + import_closure=_shared_import_closure( + rule=rule, project_dir=project_dir, cache=closures + ), ) - return payloads + for rule in catalogue + if rule.custom + ] def _custom_rule_payload( - *, - rule: Rule, - project_dir: Path, - project_attributes: frozenset[str], - fact_fingerprint: str, - import_closure: tuple[Path, ...], + *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] ) -> dict[str, object]: - check_name: str = getattr(rule.check, "__name__", "") return { "code": rule.code, "family": rule.family, @@ -642,9 +604,7 @@ def _custom_rule_payload( ), **_custom_rule_source_payload(rule=rule, project_dir=project_dir), "project_wide": rule.project_wide, - "project_dependent": RULE_CONTEXT_PROJECT_FACT in project_attributes, - "fact_fingerprint": fact_fingerprint, - "check_name": check_name, + "check_name": getattr(rule.check, "__name__", ""), "test_case_count": len(custom_rule_test_evidence(rule=rule, project_dir=project_dir)), } @@ -660,82 +620,6 @@ def _shared_import_closure( return cache[source] -def _custom_fact_fingerprint(*, project: CompiledProject, attributes: frozenset[str]) -> str: - facts: dict[str, object] = {} - if RULE_CONTEXT_GRAPH_FACT in attributes: - facts[RULE_CONTEXT_GRAPH_FACT] = [ - (model.name, model.relative_path, model.deps) for model in project.models - ] - if RULE_CONTEXT_TESTS_FACT in attributes: - facts[RULE_CONTEXT_TESTS_FACT] = project.sql_tests - if RULE_CONTEXT_AUDITS_FACT in attributes: - facts[RULE_CONTEXT_AUDITS_FACT] = project.audits - if RULE_CONTEXT_DECLARATIONS_FACT in attributes: - facts[RULE_CONTEXT_DECLARATIONS_FACT] = ( - project.public_enums, - project.public_constants, - tuple(model.enum_declarations for model in project.models), - tuple(model.constant_declarations for model in project.models), - ) - if RULE_CONTEXT_PROJECT_FACT in attributes: - facts[RULE_CONTEXT_PROJECT_FACT] = _project_fact_models(project=project) - encoded: bytes = orjson.dumps(facts, option=orjson.OPT_SORT_KEYS, default=str) - return hashlib.sha256(encoded).hexdigest() - - -def _project_fact_models(*, project: CompiledProject) -> tuple[dict[str, object], ...]: - """Project models with lazy lineage represented by stable semantic data only.""" - - projected: list[dict[str, object]] = [] - for model in project.models: - payload: dict[str, object] = { - field.name: getattr(model, field.name) for field in fields(model) - } - payload["fast_lineage_columns"] = _lineage_fingerprint_payload( - lineage=model.fast_lineage_columns - ) - projected.append(payload) - return tuple(projected) - - -def _lineage_fingerprint_payload( - *, lineage: Sequence[CompiledLineageColumnFact] | None -) -> tuple[tuple[object, ...], ...] | None: - """Hash semantic lineage without rehydrating compact facts into Python objects.""" - - if lineage is None: - return None - payloads: list[tuple[object, ...]] = [] - if isinstance(lineage, CompactLineageFacts): - for name_index, transform_code, confidence_code, sources in lineage.rows: - upstream: tuple[tuple[str, str, str], ...] = tuple( - ( - lineage.string_pool[source[0]], - lineage.resource_name(source[1]), - lineage.string_pool[source[2]], - ) - for source in sources - ) - payloads.append( - ( - lineage.string_pool[name_index], - upstream, - lineage.transform_kind(transform_code), - lineage.confidence(confidence_code), - ) - ) - else: - for column in lineage: - upstream = tuple( - (source.resource_type, source.resource_name, source.column_name) - for source in column.upstream_columns - ) - payloads.append( - (column.output_column, upstream, column.transform_kind, column.confidence) - ) - return tuple(payloads) - - def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, object]: source: str | None = rule.source if source is not None: @@ -756,62 +640,6 @@ def _custom_rule_source_payload(*, rule: Rule, project_dir: Path) -> dict[str, o } -def _custom_host_payload( - *, - project: CompiledProject, - config: RulesConfig, - project_dir: Path, - catalogue: tuple[Rule, ...], - dialect: str, - verify_determinism: bool, -) -> tuple[dict[str, object] | None, Path | None]: - if not any(rule.custom for rule in catalogue): - return None, None - serializable_project: CompiledProject = replace( - project, - binding_catalog=None, - sql_expansions={}, - loaded_macros={}, - loader_functions=(), - hook_functions=(), - materialization_files=(), - external_sql_reference_resolver=None, - ) - input_dir: Path = project_dir / "target" / "rules-cache" / "host-inputs" - input_dir.mkdir(parents=True, exist_ok=True) - input_path: Path | None = None - try: - with tempfile.NamedTemporaryFile( - dir=input_dir, - prefix="project-", - suffix=".pickle", - delete=False, - ) as handle: - input_path = Path(handle.name) - pickle.dump((serializable_project, config), handle) - except Exception: - if input_path is not None: - input_path.unlink(missing_ok=True) - raise - if input_path is None: - raise RulesError("custom host project payload was not created") - return ( - { - "program": sys.executable, - "arguments": ["-m", CUSTOM_HOST_LAUNCH_MODULE], - "timeout_millis": 120_000, - "runtime_version": CUSTOM_HOST_RUNTIME_VERSION, - "payload": { - "project_pickle_path": str(input_path.resolve()), - "project_dir": str(project_dir.resolve()), - "dialect": dialect, - "verify_determinism": verify_determinism, - }, - }, - input_path, - ) - - def decode_rule_finding(value: object) -> Finding: if not isinstance(value, dict): raise RulesError("native rules engine returned an invalid finding") diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index b77e900b88..17fbe73fe7 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -4,9 +4,17 @@ from collections import Counter from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass +from itertools import chain from pathlib import Path from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.rule_engine._helpers.host.module_state import ( + module_state_token, + rule_namespaces, +) +from sqlbuild.rule_engine._helpers.host.tracked_facts import tracked_fact_views +from sqlbuild.rule_engine.classes.fact_reads import FactReads from sqlbuild.rule_engine.classes.project_tree import public_model from sqlbuild.rule_engine.classes.rule_context import ( EvaluationRuleContext, @@ -14,13 +22,37 @@ build_rule_fact_views, ) from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard -from sqlbuild.rule_engine.exceptions import NonHermeticRuleError, RulesError -from sqlbuild.rule_engine.models import Finding, Model, Project, Rule, RulesConfig -from sqlbuild.rule_engine.types import RuleSubject +from sqlbuild.rule_engine.exceptions import ( + NonHermeticRuleError, + OpaqueModuleStateError, + RulesError, +) +from sqlbuild.rule_engine.models import ( + CustomRuleEvaluation, + CustomRuleRun, + Finding, + Model, + Project, + Rule, + RulesConfig, +) +from sqlbuild.rule_engine.types import CustomRulePlan, RuleSubject type _FindingKey = tuple[str, str, int, int, str, str] +@dataclass(frozen=True) +class _PassInputs: + project: CompiledProject + config: RulesConfig + project_dir: Path + selected_rules: tuple[Rule, ...] + dialect: str + facts: RuleFactViews + plan: CustomRulePlan | None + guard: RuntimeGuard | None + + def evaluate_custom_rules( *, project: CompiledProject, @@ -28,118 +60,205 @@ def evaluate_custom_rules( project_dir: Path, selected_rules: tuple[Rule, ...], dialect: str = "generic", - selected_model_paths: frozenset[str] | None = None, + plan: CustomRulePlan | None = None, + track_reads: bool = False, verify_determinism: bool = False, guard: RuntimeGuard | None = None, -) -> list[Finding]: - """Run only selected custom rules through the Python authoring API.""" +) -> CustomRuleRun: + """Run selected custom rules for every planned subject through the Python authoring API.""" - custom_rules: tuple[Rule, ...] = tuple(rule for rule in selected_rules if rule.custom) + custom_rules: tuple[Rule, ...] = tuple( + rule for rule in selected_rules if rule.custom and (plan is None or rule.code in plan) + ) if not custom_rules: - return [] + return CustomRuleRun(evaluations=(), untracked_codes=frozenset()) models: tuple[CompiledModel, ...] = tuple( - model - for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) - if selected_model_paths is None or model.relative_path.as_posix() in selected_model_paths + sorted(project.models, key=lambda item: item.relative_path.as_posix()) ) facts: RuleFactViews = build_rule_fact_views( project=project, project_dir=project_dir, dialect=dialect ) + reads: FactReads | None = FactReads() if track_reads else None + tracker: _ReadTracker = _ReadTracker( + reads=reads, rules=custom_rules, rules_root=project_dir / "rules" + ) - def run_pass(*, rules: tuple[Rule, ...], subjects: tuple[CompiledModel, ...]) -> list[Finding]: - return _evaluate_pass( - project=project, - config=config, - project_dir=project_dir, - selected_rules=selected_rules, - dialect=dialect, - facts=facts, - rules=rules, - models=subjects, - guard=guard, - ) + inputs: _PassInputs = _PassInputs( + project=project, + config=config, + project_dir=project_dir, + selected_rules=selected_rules, + dialect=dialect, + facts=facts, + plan=plan, + guard=guard, + ) + + def run_pass( + *, rules: tuple[Rule, ...], subjects: tuple[CompiledModel, ...], active: _ReadTracker + ) -> list[CustomRuleEvaluation]: + return _evaluate_pass(inputs=inputs, rules=rules, models=subjects, tracker=active) - findings: list[Finding] = run_pass(rules=custom_rules, subjects=models) + evaluations: list[CustomRuleEvaluation] = run_pass( + rules=custom_rules, subjects=models, active=tracker + ) if verify_determinism: - repeated: list[Finding] = run_pass(rules=custom_rules[::-1], subjects=models[::-1]) - _verify_repeatable(rules=custom_rules, first=findings, repeated=repeated) - return findings + repeated: list[CustomRuleEvaluation] = run_pass( + rules=custom_rules[::-1], + subjects=models[::-1], + active=_ReadTracker(reads=None, rules=(), rules_root=project_dir / "rules"), + ) + _verify_repeatable( + rules=custom_rules, + first=list(chain.from_iterable(item.findings for item in evaluations)), + repeated=list(chain.from_iterable(item.findings for item in repeated)), + ) + return CustomRuleRun( + evaluations=tuple(evaluations), + untracked_codes=frozenset(tracker.untracked_codes), + uncacheable_codes=frozenset(tracker.uncacheable_codes), + observed=() if reads is None else tuple(reads.observed.items()), + ) + + +class _ReadTracker: + """Attribute reads to one invocation; rules whose module state changes become untracked.""" + + def __init__( + self, *, reads: FactReads | None, rules: tuple[Rule, ...], rules_root: Path + ) -> None: + self.reads: FactReads | None = reads + self.untracked_codes: set[str] = set() + self.uncacheable_codes: set[str] = set() + self._rules_root: Path = rules_root + self._namespaces: dict[str, tuple[dict[str, object], ...]] = ( + {rule.code: rule_namespaces((rule.check,)) for rule in rules} + if reads is not None + else {} + ) + self._states: dict[str, str | None] = { + code: self._state_token(namespaces) for code, namespaces in self._namespaces.items() + } + + def start(self) -> None: + if self.reads is not None: + self.reads.current = set() + self.reads.untracked = False + self.reads.uncacheable = False + + def finish(self, *, code: str) -> frozenset[tuple[object, ...]] | None: + reads: FactReads | None = self.reads + if reads is None: + return None + observed: set[tuple[object, ...]] | None = reads.current + reads.current = None + if reads.untracked: + self.untracked_codes.add(code) + if reads.uncacheable: + self.uncacheable_codes.add(code) + for rule_code, namespaces in self._namespaces.items(): + if rule_code in self.untracked_codes: + continue + state: str | None = self._state_token(namespaces) + if state is None or state != self._states[rule_code]: + self.untracked_codes.add(rule_code) + return None if observed is None else frozenset(observed) + + def _state_token(self, namespaces: tuple[dict[str, object], ...]) -> str | None: + try: + return module_state_token(namespaces=namespaces, rules_root=self._rules_root) + except OpaqueModuleStateError: + return None def _evaluate_pass( *, - project: CompiledProject, - config: RulesConfig, - project_dir: Path, - selected_rules: tuple[Rule, ...], - dialect: str, - facts: RuleFactViews, + inputs: _PassInputs, rules: tuple[Rule, ...], models: tuple[CompiledModel, ...], - guard: RuntimeGuard | None, -) -> list[Finding]: - findings: list[Finding] = [] + tracker: _ReadTracker, +) -> list[CustomRuleEvaluation]: + evaluations: list[CustomRuleEvaluation] = [] + project: CompiledProject = inputs.project + plan: CustomRulePlan | None = inputs.plan + guard: RuntimeGuard | None = inputs.guard + context_facts: RuleFactViews = ( + inputs.facts + if tracker.reads is None + else tracked_fact_views(views=inputs.facts, reads=tracker.reads) + ) + + def context(rule: Rule) -> EvaluationRuleContext: + return EvaluationRuleContext( + model=None, + rule=rule, + config=inputs.config, + project=project, + project_dir=inputs.project_dir, + selected_rules=inputs.selected_rules, + dialect=inputs.dialect, + facts=context_facts, + reads=tracker.reads, + ) + project_rules: tuple[Rule, ...] = tuple( rule for rule in rules if rule.subject is RuleSubject.PROJECT ) model_rules: tuple[Rule, ...] = tuple( rule for rule in rules if rule.subject is RuleSubject.MODEL ) - model_contexts: tuple[tuple[Rule, EvaluationRuleContext], ...] = tuple( - ( - rule, - EvaluationRuleContext( - model=None, - rule=rule, - config=config, - project=project, - project_dir=project_dir, - selected_rules=selected_rules, - dialect=dialect, - facts=facts, - ), - ) - for rule in model_rules + model_contexts: tuple[tuple[Rule, EvaluationRuleContext, frozenset[str] | None], ...] = tuple( + (rule, context(rule), None if plan is None else plan.get(rule.code)) for rule in model_rules ) for rule in project_rules: - ctx: EvaluationRuleContext = EvaluationRuleContext( - model=None, - rule=rule, - config=config, - project=project, - project_dir=project_dir, - selected_rules=selected_rules, - dialect=dialect, - facts=facts, - ) + ctx: EvaluationRuleContext = context(rule) + tracker.start() try: - findings.extend( - _invoke( - rule=rule, - subject=Project( - model_count=len(project.models), - target_name=project.effective_target_name, - ), - ctx=ctx, - guard=guard, - ) + findings: list[Finding] = _invoke( + rule=rule, + subject=Project( + model_count=len(project.models), + target_name=project.effective_target_name, + ), + ctx=ctx, + guard=guard, ) except NonHermeticRuleError: raise except Exception as error: raise RulesError(f"rule {rule.code} failed for the project: {error}") from error + evaluations.append( + CustomRuleEvaluation( + code=rule.code, + subject=None, + findings=tuple(findings), + reads=tracker.finish(code=rule.code), + ) + ) for model in models: subject: Model = public_model(model) - for rule, ctx in model_contexts: + path: str = model.relative_path.as_posix() + for rule, ctx, planned in model_contexts: + if planned is not None and path not in planned: + continue + tracker.start() try: - findings.extend(_invoke(rule=rule, subject=subject, ctx=ctx, guard=guard)) + findings = _invoke(rule=rule, subject=subject, ctx=ctx, guard=guard) except NonHermeticRuleError: raise except Exception as error: raise RulesError( f"rule {rule.code} failed for {model.relative_path}: {error}" ) from error - return findings + evaluations.append( + CustomRuleEvaluation( + code=rule.code, + subject=path, + findings=tuple(findings), + reads=tracker.finish(code=rule.code), + ) + ) + return evaluations def _invoke( diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 3bacf498ef..ce315b2a0b 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -14,17 +14,20 @@ from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue +from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard from sqlbuild.rule_engine.constants import ( CUSTOM_HOST_HASH_SEED, CUSTOM_HOST_INPUT_TUPLE_SIZE, CUSTOM_HOST_LAUNCH_MODULE, + CUSTOM_HOST_MAX_TRACKED_READS, CUSTOM_HOST_PROTOCOL_VERSION, CUSTOM_HOST_RUNTIME_VERSION, ) from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig +from sqlbuild.rule_engine.models import CustomRuleRun, Finding, Rule, RulesConfig +from sqlbuild.rule_engine.types import CustomRulePlan def main() -> int: @@ -51,13 +54,8 @@ def main() -> int: project_dir: Path = Path(str(payload["project_dir"])).resolve() dialect: str = str(payload.get("dialect", "generic")) verify_determinism: bool = payload.get("verify_determinism") is True - selected_codes: tuple[str, ...] = tuple(str(code) for code in payload["selected_codes"]) - raw_model_paths: object = payload.get("selected_model_paths") - selected_model_paths: frozenset[str] | None = ( - frozenset(str(path) for path in raw_model_paths) - if isinstance(raw_model_paths, list) - else None - ) + track_reads: bool = payload.get("track_reads") is True + plan: CustomRulePlan = _decode_plan(payload["plan"]) os.environ.clear() os.chdir(project_dir) guard = RuntimeGuard(project_dir=project_dir) @@ -68,14 +66,15 @@ def main() -> int: catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) guard.raise_violation() by_code: dict[str, Rule] = {rule.code: rule for rule in catalogue} - selected: tuple[Rule, ...] = tuple(by_code[code] for code in selected_codes) - findings: list[Finding] = evaluate_custom_rules( + selected: tuple[Rule, ...] = tuple(by_code[code] for code in sorted(plan)) + run: CustomRuleRun = evaluate_custom_rules( project=project, config=config, project_dir=project_dir, selected_rules=selected, dialect=dialect, - selected_model_paths=selected_model_paths, + plan=plan, + track_reads=track_reads, verify_determinism=verify_determinism, guard=guard, ) @@ -86,7 +85,7 @@ def main() -> int: "protocol": CUSTOM_HOST_PROTOCOL_VERSION, "runtime_version": CUSTOM_HOST_RUNTIME_VERSION, "error": None, - "payload": [_finding_payload(finding) for finding in findings], + "payload": _run_payload(run), "messages": messages.getvalue().splitlines(), } sys.stdout.write(json.dumps(response, sort_keys=True)) @@ -115,6 +114,57 @@ def _decode_inputs(payload: dict[str, Any]) -> tuple[CompiledProject, RulesConfi return decoded +def _decode_plan(value: object) -> CustomRulePlan: + if not isinstance(value, dict): + raise RulesError("custom host plan must be an object") + plan: CustomRulePlan = {} + for code, subjects in value.items(): + if subjects is not None and not isinstance(subjects, list): + raise RulesError("custom host plan subjects must be a list or null") + plan[str(code)] = ( + None if subjects is None else frozenset(str(subject) for subject in subjects) + ) + return plan + + +def _run_payload(run: CustomRuleRun) -> dict[str, object]: + """Encode invocations with read sets deduplicated across subjects and bounded in size.""" + + readsets: dict[frozenset[tuple[object, ...]], int] = {} + evaluations: list[dict[str, object]] = [] + untracked: set[str] = set(run.untracked_codes) + for evaluation in run.evaluations: + index: int | None = None + if evaluation.reads is not None and evaluation.code not in untracked: + index = readsets.setdefault(evaluation.reads, len(readsets)) + evaluations.append( + { + "code": evaluation.code, + "subject": evaluation.subject, + "findings": [_finding_payload(finding) for finding in evaluation.findings], + "reads": index, + } + ) + if sum(map(len, readsets)) > CUSTOM_HOST_MAX_TRACKED_READS: + untracked.update(str(item["code"]) for item in evaluations if item["reads"] is not None) + readsets = {} + for item in evaluations: + item["reads"] = None + untracked_reads: dict[str, set[tuple[object, ...]]] = {code: set() for code in untracked} + for evaluation in run.evaluations: + if evaluation.code in untracked_reads and evaluation.reads is not None: + untracked_reads[evaluation.code].update(evaluation.reads) + return { + "evaluations": evaluations, + "readsets": [sorted(map(fact_key_payload, readset)) for readset in readsets], + "untracked": { + code: sorted(map(fact_key_payload, reads)) for code, reads in untracked_reads.items() + }, + "uncacheable": sorted(run.uncacheable_codes), + "observed": sorted([list(key), digest] for key, digest in run.observed), + } + + def _finding_payload(finding: Finding) -> dict[str, object]: payload: dict[str, object] = asdict(finding) payload.pop("affected_rules", None) diff --git a/src/sqlbuild/rule_engine/_helpers/host/module_state.py b/src/sqlbuild/rule_engine/_helpers/host/module_state.py new file mode 100644 index 0000000000..e0d0a26a39 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/host/module_state.py @@ -0,0 +1,218 @@ +"""Detect custom-rule module state that one evaluation could leave for another.""" + +from __future__ import annotations + +import enum +import hashlib +import os +import re +from collections import deque +from collections.abc import Iterable +from decimal import Decimal +from fractions import Fraction +from pathlib import Path, PurePath +from types import BuiltinFunctionType, FunctionType, MethodType, ModuleType +from typing import Any, cast + +from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError + +_MAX_STATE_NODES: int = 50_000 +_ATOMIC_TYPES: tuple[type, ...] = ( + type(None), + bool, + int, + float, + complex, + str, + bytes, + bytearray, + Decimal, + Fraction, + PurePath, + range, + re.Pattern, + enum.Enum, +) +_IMMUTABLE_LIBRARY_MODULES: frozenset[str] = frozenset({"typing", "types", "builtins"}) +_SKIPPED_FUNCTION_ATTRIBUTES: frozenset[str] = frozenset({"__sqlbuild_rule__", "__wrapped__"}) + + +def rule_namespaces(rules: Iterable[object]) -> tuple[dict[str, object], ...]: + """Return the module globals of every loaded custom-rule check.""" + + namespaces: dict[int, dict[str, object]] = {} + for check in rules: + module_globals: object = getattr(check, "__globals__", None) + if isinstance(module_globals, dict): + namespaces.setdefault(id(module_globals), module_globals) + return tuple(namespaces.values()) + + +def module_state_token(*, namespaces: tuple[dict[str, object], ...], rules_root: Path) -> str: + """Fingerprint every mutable value reachable from rule-module globals.""" + + walker: _StateWalker = _StateWalker(rules_root=f"{rules_root.resolve()}{os.sep}") + for namespace in namespaces: + walker.visit_namespace(namespace) + return walker.digest.hexdigest() + + +class _StateWalker: + def __init__(self, *, rules_root: str) -> None: + self.digest: Any = hashlib.sha256() + self._rules_root: str = rules_root + self._namespaces: set[int] = set() + self._module_names: set[str] = set() + self._visited: dict[int, int] = {} + self._nodes: int = 0 + + def visit_namespace(self, namespace: dict[str, object]) -> None: + if id(namespace) in self._namespaces: + return + self._namespaces.add(id(namespace)) + name: object = namespace.get("__name__") + self._module_names.add(str(name)) + self.emit(f"namespace:{name}") + for key, value in sorted(namespace.items(), key=lambda item: item[0]): + if key.startswith("__") and key.endswith("__"): + continue + self.emit(f"name:{key}") + self.visit(value) + + def _owned_code(self, function: FunctionType) -> bool: + return function.__code__.co_filename.startswith(self._rules_root) + + def emit(self, text: str) -> None: + self.digest.update(text.encode()) + self.digest.update(b"\0") + + def visit(self, value: object) -> None: + self._nodes += 1 + if self._nodes > _MAX_STATE_NODES: + raise OpaqueModuleStateError("custom rule module state is too large to observe") + if isinstance(value, _ATOMIC_TYPES) and not isinstance(value, (list, dict, set)): + self.emit(f"atom:{type(value).__qualname__}:{value!r}") + return + identity: int = id(value) + seen: int | None = self._visited.get(identity) + if seen is not None: + self.emit(f"ref:{seen}") + return + self._visited[identity] = len(self._visited) + self.emit(f"object:{identity}:{type(value).__qualname__}") + self._visit_structure(value) + + def _visit_structure(self, value: object) -> None: + if isinstance(value, ModuleType): + location: object = getattr(value, "__file__", None) + if isinstance(location, str) and location.startswith(self._rules_root): + self.visit_namespace(vars(value)) + else: + self.emit(f"module:{value.__name__}") + elif isinstance(value, dict): + self._visit_mapping(cast(dict[object, object], value)) + elif isinstance(value, (list, tuple, deque)): + self.emit(f"len:{len(value)}") + for item in value: + self.visit(item) + elif isinstance(value, (set, frozenset)): + self.emit(f"len:{len(value)}") + tokens: list[str] = sorted(_atomic_text(item) for item in value) + for token in tokens: + self.emit(token) + elif isinstance(value, FunctionType): + self._visit_function(value) + elif isinstance(value, MethodType): + self.visit(value.__func__) + self.visit(value.__self__) + elif isinstance(value, type): + self._visit_type(value) + elif hasattr(value, "cache_info") and hasattr(value, "__wrapped__"): + cached: Any = cast(Any, value) + self.emit(f"cache:{cached.cache_info().currsize}") + self.visit(cached.__wrapped__) + elif isinstance(value, BuiltinFunctionType): + self.emit(f"builtin:{getattr(value, '__module__', '')}:{value.__qualname__}") + else: + self._visit_instance(value) + + def _visit_mapping(self, value: dict[object, object]) -> None: + self.emit(f"len:{len(value)}") + factory: object = getattr(value, "default_factory", None) + if factory is not None: + self.visit(factory) + for key, item in value.items(): + self.visit(key) + self.visit(item) + + def _visit_function(self, value: FunctionType) -> None: + if not self._owned_code(value): + self.emit(f"function:{value.__module__}:{value.__qualname__}") + return + self.visit_namespace(value.__globals__) + self.visit(value.__defaults__) + self.visit(value.__kwdefaults__) + for name, item in sorted(vars(value).items(), key=lambda entry: entry[0]): + if name in _SKIPPED_FUNCTION_ATTRIBUTES: + continue + self.emit(f"attribute:{name}") + self.visit(item) + for cell in value.__closure__ or (): + try: + contents: object = cell.cell_contents + except ValueError: + self.emit("empty-cell") + continue + self.visit(contents) + + def _visit_type(self, value: type) -> None: + owned: bool = value.__module__ in self._module_names or any( + isinstance(item, FunctionType) and self._owned_code(item) + for item in vars(value).values() + ) + if not owned: + self.emit(f"type:{value.__module__}:{value.__qualname__}") + return + for name, item in sorted(vars(value).items(), key=lambda entry: entry[0]): + if name.startswith("__") and name.endswith("__"): + continue + self.emit(f"attribute:{name}") + self.visit(item) + + def _visit_instance(self, value: object) -> None: + state: dict[str, object] = {} + attributes: object = getattr(value, "__dict__", None) + if isinstance(attributes, dict): + state.update(attributes) + for owner in type(value).__mro__: + for slot in getattr(owner, "__slots__", ()): + if isinstance(slot, str) and hasattr(value, slot): + state[slot] = getattr(value, slot) + if not state and type(value).__module__ not in self._module_names: + if _is_immutable_library_value(value): + self.emit(f"value:{type(value).__module__}:{type(value).__qualname__}") + return + raise OpaqueModuleStateError( + f"custom rule module state holds an opaque {type(value).__qualname__}" + ) + for name, item in sorted(state.items(), key=lambda entry: entry[0]): + self.emit(f"attribute:{name}") + self.visit(item) + + +def _atomic_text(value: object) -> str: + if isinstance(value, _ATOMIC_TYPES): + return f"{type(value).__qualname__}:{value!r}" + if isinstance(value, tuple): + return "(" + ",".join(_atomic_text(item) for item in value) + ")" + raise OpaqueModuleStateError( + f"custom rule module state holds an unhashable-order {type(value).__qualname__}" + ) + + +def _is_immutable_library_value(value: object) -> bool: + return ( + isinstance(value, (staticmethod, classmethod, property)) + or callable(value) + or type(value).__module__ in _IMMUTABLE_LIBRARY_MODULES + ) diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py new file mode 100644 index 0000000000..6864c9ca68 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -0,0 +1,292 @@ +"""Custom-rule fact views that record which compiler facts each evaluation reads.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from sqlbuild.compiler.compile.models import CompiledAudit, CompiledSqlTest, InferredColumn +from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration +from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_error_digest, fact_value_digest +from sqlbuild.rule_engine.classes.fact_reads import FactReads +from sqlbuild.rule_engine.constants import ( + FACT_AUDITS_ALL, + FACT_AUDITS_FOR_MODEL, + FACT_COLUMNS_DECLARED, + FACT_COLUMNS_INFERRED, + FACT_COLUMNS_NAMES, + FACT_CONTRACTS_ENFORCED, + FACT_CONTRACTS_GRAIN, + FACT_DECLARATIONS_CONSTANTS, + FACT_DECLARATIONS_ENUMS, + FACT_DECLARATIONS_PUBLIC_CONSTANTS, + FACT_DECLARATIONS_PUBLIC_ENUMS, + FACT_GRAPH_DEPENDENCIES, + FACT_GRAPH_DEPENDENTS, + FACT_PROJECT_AUDITS, + FACT_PROJECT_FUNCTIONS, + FACT_PROJECT_MODELS, + FACT_PROJECT_SEEDS, + FACT_PROJECT_SOURCES, + FACT_PROJECT_TESTS, + FACT_SQL_FOR_MODEL, + FACT_TESTS_ALL, + FACT_TESTS_FOR_MODEL, + FACT_TREE_CHILDREN, + FACT_TREE_DESCENDANTS, + FACT_TREE_GLOB, + FACT_TREE_PATHS, + FACT_TREE_READ_TEXT, + FACT_TREE_RELATIVE_PARTS, + FACT_TREE_RESOURCES_UNDER, +) +from sqlbuild.rule_engine.exceptions import FactDigestError +from sqlbuild.rule_engine.models import Model, ModelSql, ProjectPath, RuleFactViews + + +class _TrackedView: + """Delegate unknown attribute access to the wrapped view and mark the evaluation uncacheable.""" + + __slots__ = ("_tracked_reads", "_tracked_view") + + _tracked_reads: FactReads + _tracked_view: Any + + def __init__(self, *, view: object, reads: FactReads) -> None: + object.__setattr__(self, "_tracked_view", view) + object.__setattr__(self, "_tracked_reads", reads) + + def __getattr__(self, name: str) -> object: + self._tracked_reads.uncacheable = True + return getattr(self._tracked_view, name) + + def __setattr__(self, name: str, value: object) -> None: + self._tracked_reads.uncacheable = True + setattr(self._tracked_view, name, value) + + +class _TrackedProjectTree(_TrackedView): + """Record live filesystem reads with the digest of the value each read returned.""" + + __slots__ = () + + def paths(self) -> tuple[ProjectPath, ...]: + key: tuple[str, ...] = (FACT_TREE_PATHS,) + self._tracked_reads.record(key=key) + observed: tuple[str, ...] | None = key if self._tracked_reads.current is not None else None + return self._observed(key=observed, read=self._tracked_view.paths) + + def children(self, path: str = "") -> tuple[ProjectPath, ...]: + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_CHILDREN, values=(path,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.children(path)) + + def descendants(self, path: str = "") -> tuple[ProjectPath, ...]: + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_DESCENDANTS, values=(path,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.descendants(path)) + + def glob(self, pattern: str) -> tuple[ProjectPath, ...]: + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_GLOB, values=(pattern,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.glob(pattern)) + + def relative_parts(self, *, path: Path | str, under: str = "") -> tuple[str, ...]: + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_RELATIVE_PARTS, values=(path, under) + ) + return self._observed( + key=key, read=lambda: self._tracked_view.relative_parts(path=path, under=under) + ) + + def resources_under(self, path: str) -> tuple[Model, ...]: + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_RESOURCES_UNDER, values=(path,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.resources_under(path)) + + def read_text(self, path: str) -> str: + key: tuple[str, ...] | None = self._tracked_reads.record_text( + fact=FACT_TREE_READ_TEXT, values=(path,) + ) + return self._observed(key=key, read=lambda: self._tracked_view.read_text(path)) + + def _observed[T](self, *, key: tuple[str, ...] | None, read: Callable[[], T]) -> T: + reads: FactReads = self._tracked_reads + if key is None or (key[0] != FACT_TREE_READ_TEXT and key in reads.observed): + return read() + try: + value: T = read() + except Exception as error: + reads.observe(key=key, digest=_digest_or_none(error=error)) + raise + reads.observe(key=key, digest=_digest_or_none(value=value)) + return value + + +def _digest_or_none(*, value: object = None, error: Exception | None = None) -> str | None: + try: + return fact_value_digest(value) if error is None else fact_error_digest(error) + except FactDigestError: + return None + + +class _TrackedProjectFacts(_TrackedView): + __slots__ = ("_tracked_tree",) + + _tracked_tree: _TrackedProjectTree + + def __init__(self, *, view: object, reads: FactReads) -> None: + super().__init__(view=view, reads=reads) + object.__setattr__( + self, "_tracked_tree", _TrackedProjectTree(view=self._tracked_view.tree, reads=reads) + ) + + @property + def models(self) -> tuple[Model, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_MODELS,)) + return self._tracked_view.models + + @property + def sources(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_SOURCES,)) + return self._tracked_view.sources + + @property + def seeds(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_SEEDS,)) + return self._tracked_view.seeds + + @property + def functions(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_FUNCTIONS,)) + return self._tracked_view.functions + + @property + def tests(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_TESTS,)) + return self._tracked_view.tests + + @property + def audits(self) -> tuple[object, ...]: + self._tracked_reads.record(key=(FACT_PROJECT_AUDITS,)) + return self._tracked_view.audits + + @property + def tree(self) -> _TrackedProjectTree: + return self._tracked_tree + + +class _TrackedSqlFacts(_TrackedView): + __slots__ = () + + def for_model(self, model: Model) -> ModelSql: + self._tracked_reads.record_model_path(fact=FACT_SQL_FOR_MODEL, model=model) + return self._tracked_view.for_model(model) + + +class _TrackedGraphFacts(_TrackedView): + __slots__ = () + + def dependencies(self, model: Model) -> tuple[Model, ...]: + self._tracked_reads.record_model_path(fact=FACT_GRAPH_DEPENDENCIES, model=model) + return self._tracked_view.dependencies(model) + + def dependents(self, model: Model) -> tuple[Model, ...]: + self._tracked_reads.record_model_path(fact=FACT_GRAPH_DEPENDENTS, model=model) + return self._tracked_view.dependents(model) + + +class _TrackedColumnFacts(_TrackedView): + __slots__ = () + + def declared(self, model: Model) -> tuple[object, ...]: + self._tracked_reads.record_model_path(fact=FACT_COLUMNS_DECLARED, model=model) + return self._tracked_view.declared(model) + + def inferred(self, model: Model) -> tuple[InferredColumn, ...] | None: + self._tracked_reads.record_model_path(fact=FACT_COLUMNS_INFERRED, model=model) + return self._tracked_view.inferred(model) + + def names(self, model: Model) -> tuple[str, ...] | None: + self._tracked_reads.record_model_path(fact=FACT_COLUMNS_NAMES, model=model) + return self._tracked_view.names(model) + + +class _TrackedContractFacts(_TrackedView): + __slots__ = () + + def enforced(self, model: Model) -> bool: + self._tracked_reads.record_model_path(fact=FACT_CONTRACTS_ENFORCED, model=model) + return self._tracked_view.enforced(model) + + def grain(self, model: Model) -> tuple[str, ...]: + self._tracked_reads.record_model_path(fact=FACT_CONTRACTS_GRAIN, model=model) + return self._tracked_view.grain(model) + + +class _TrackedTestFacts(_TrackedView): + __slots__ = () + + def all(self) -> tuple[CompiledSqlTest, ...]: + self._tracked_reads.record(key=(FACT_TESTS_ALL,)) + return self._tracked_view.all() + + def for_model(self, model: Model) -> tuple[CompiledSqlTest, ...]: + self._tracked_reads.record_model_name(fact=FACT_TESTS_FOR_MODEL, model=model) + return self._tracked_view.for_model(model) + + +class _TrackedAuditFacts(_TrackedView): + __slots__ = () + + def all(self) -> tuple[CompiledAudit, ...]: + self._tracked_reads.record(key=(FACT_AUDITS_ALL,)) + return self._tracked_view.all() + + def for_model(self, model: Model) -> tuple[CompiledAudit, ...]: + self._tracked_reads.record_model_name(fact=FACT_AUDITS_FOR_MODEL, model=model) + return self._tracked_view.for_model(model) + + +class _TrackedDeclarationFacts(_TrackedView): + __slots__ = () + + @property + def public_enums(self) -> tuple[EnumDeclaration, ...]: + self._tracked_reads.record(key=(FACT_DECLARATIONS_PUBLIC_ENUMS,)) + return self._tracked_view.public_enums + + @property + def public_constants(self) -> tuple[ConstantDeclaration, ...]: + self._tracked_reads.record(key=(FACT_DECLARATIONS_PUBLIC_CONSTANTS,)) + return self._tracked_view.public_constants + + @property + def enums(self) -> tuple[EnumDeclaration, ...]: + self._tracked_reads.record(key=(FACT_DECLARATIONS_ENUMS,)) + return self._tracked_view.enums + + @property + def constants(self) -> tuple[ConstantDeclaration, ...]: + self._tracked_reads.record(key=(FACT_DECLARATIONS_CONSTANTS,)) + return self._tracked_view.constants + + +def tracked_fact_views(*, views: RuleFactViews, reads: FactReads) -> RuleFactViews: + """Wrap every public fact view so reads are attributed to the active evaluation.""" + + return RuleFactViews( + project=_TrackedProjectFacts(view=views.project, reads=reads), + sql=_TrackedSqlFacts(view=views.sql, reads=reads), + graph=_TrackedGraphFacts(view=views.graph, reads=reads), + columns=_TrackedColumnFacts(view=views.columns, reads=reads), + contracts=_TrackedContractFacts(view=views.contracts, reads=reads), + tests=_TrackedTestFacts(view=views.tests, reads=reads), + audits=_TrackedAuditFacts(view=views.audits, reads=reads), + declarations=_TrackedDeclarationFacts(view=views.declarations, reads=reads), + ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py new file mode 100644 index 0000000000..e11cf9dc2b --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py @@ -0,0 +1,49 @@ +"""Reuse the complete built-in rules response for a byte-identical native request.""" + +from __future__ import annotations + +import hashlib +import os +import threading +from pathlib import Path +from typing import Any + +import orjson + +from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.rule_engine.constants import NATIVE_RULES_MEMO_FILE, NATIVE_RULES_MEMO_VERSION + + +def native_request_identity(request_json: bytes) -> str: + """Digest one serialized native request together with the installed code identity.""" + + digest: Any = hashlib.sha256() + digest.update(NATIVE_RULES_MEMO_VERSION.encode()) + digest.update(b"\0") + digest.update(compiled_code_identity().encode()) + digest.update(b"\0") + digest.update(request_json) + return digest.hexdigest() + + +def read_native_response(*, project_dir: Path, identity: str) -> str | None: + """Return the stored response for this exact request identity, if any.""" + + try: + payload: object = orjson.loads((project_dir / NATIVE_RULES_MEMO_FILE).read_bytes()) + except (OSError, orjson.JSONDecodeError): + return None + if not isinstance(payload, dict) or payload.get("identity") != identity: + return None + response: object = payload.get("response") + return response if isinstance(response, str) else None + + +def write_native_response(*, project_dir: Path, identity: str, response: str) -> None: + """Atomically replace the stored response with the latest evaluated request.""" + + path: Path = project_dir / NATIVE_RULES_MEMO_FILE + path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") + temporary.write_bytes(orjson.dumps({"identity": identity, "response": response})) + temporary.replace(path) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 4022e48b90..929e79dcca 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -52,7 +52,6 @@ evaluate_native, finalize_native_findings, native_catalogue, - rules_cache_exists, ) from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine._helpers.run.literal_duplicates import with_duplicate_literal_hints @@ -160,7 +159,6 @@ def evaluate_rules( catalogue=catalogue, dialect=dialect, defer_suppressions=True, - rules_cache_present_at_start=rules_cache_exists(resolved_project_dir), ) sql_started: float = time.monotonic() sql_result: _SqlRulesEvaluation = _run_sql_rules( diff --git a/src/sqlbuild/rule_engine/classes/fact_digests.py b/src/sqlbuild/rule_engine/classes/fact_digests.py new file mode 100644 index 0000000000..05c663cc44 --- /dev/null +++ b/src/sqlbuild/rule_engine/classes/fact_digests.py @@ -0,0 +1,47 @@ +"""Memoized digests of recorded custom-rule facts.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable, Iterable +from typing import Any + +import orjson + +from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_outcome_digest +from sqlbuild.rule_engine.models import RuleFactViews +from sqlbuild.rule_engine.types import FactKey + + +class FactDigests: + """Memoized digests of recorded fact keys against one compiled project snapshot.""" + + def __init__(self, *, views: Callable[[], RuleFactViews]) -> None: + self._views_factory: Callable[[], RuleFactViews] = views + self._views: RuleFactViews | None = None + self._digests: dict[FactKey, str] = {} + + @property + def views(self) -> RuleFactViews: + if self._views is None: + self._views = self._views_factory() + return self._views + + def digest(self, key: FactKey) -> str: + """Return one fact digest, raising FactDigestError when it cannot be reproduced.""" + + cached: str | None = self._digests.get(key) + if cached is not None: + return cached + digest: str = fact_outcome_digest(views=self.views, key=key) + self._digests[key] = digest + return digest + + def combined(self, keys: Iterable[FactKey]) -> str: + """Digest one read set in canonical key order.""" + + combined: Any = hashlib.sha256() + for key in sorted(keys): + combined.update(orjson.dumps(key)) + combined.update(self.digest(key).encode()) + return combined.hexdigest() diff --git a/src/sqlbuild/rule_engine/classes/fact_reads.py b/src/sqlbuild/rule_engine/classes/fact_reads.py new file mode 100644 index 0000000000..8680946276 --- /dev/null +++ b/src/sqlbuild/rule_engine/classes/fact_reads.py @@ -0,0 +1,62 @@ +"""Recorder for the compiler facts one custom-rule evaluation observes.""" + +from __future__ import annotations + +import os + +from sqlbuild.rule_engine.models import Model + + +class FactReads: + """Collect the fact keys one custom-rule evaluation observes.""" + + __slots__ = ("current", "observed", "uncacheable", "untracked") + + def __init__(self) -> None: + self.current: set[tuple[object, ...]] | None = None + self.untracked: bool = False + self.uncacheable: bool = False + self.observed: dict[tuple[str, ...], str | None] = {} + + def record(self, *, key: tuple[object, ...]) -> None: + current: set[tuple[object, ...]] | None = self.current + if current is not None: + current.add(key) + + def record_model_path(self, *, fact: str, model: object) -> None: + self._record_model(fact=fact, model=model, by_name=False) + + def record_model_name(self, *, fact: str, model: object) -> None: + self._record_model(fact=fact, model=model, by_name=True) + + def _record_model(self, *, fact: str, model: object, by_name: bool) -> None: + current: set[tuple[object, ...]] | None = self.current + if current is None: + return + if type(model) is Model: + current.add((fact, model.name if by_name else model.path)) + else: + self.untracked = True + + def record_text(self, *, fact: str, values: tuple[object, ...]) -> tuple[str, ...] | None: + current: set[tuple[object, ...]] | None = self.current + if current is None: + return None + texts: list[str] = [] + for value in values: + if isinstance(value, (str, os.PathLike)): + texts.append(os.fspath(value)) + else: + self.untracked = True + return None + key: tuple[str, ...] = (fact, *texts) + current.add(key) + return key + + def observe(self, *, key: tuple[str, ...], digest: str | None) -> None: + """Remember the digest a live filesystem fact had when the rule read it.""" + + if key in self.observed and self.observed[key] != digest: + self.observed[key] = None + else: + self.observed.setdefault(key, digest) diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index 741caf6caf..5bec6e4632 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -20,11 +20,17 @@ class ProjectTree: def __init__(self, *, project_dir: Path, project: CompiledProject) -> None: self._root: Path = project_dir.resolve() - self._nodes: tuple[ProjectPath, ...] = self._discover() + self._discovered: tuple[ProjectPath, ...] | None = None self._resource_paths: dict[str, Model] = { item.relative_path.as_posix(): public_model(item) for item in project.models } + @property + def _nodes(self) -> tuple[ProjectPath, ...]: + if self._discovered is None: + self._discovered = self._discover() + return self._discovered + def paths(self) -> tuple[ProjectPath, ...]: return self._nodes diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index db86706c1f..f6cbceb13a 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -10,6 +10,7 @@ from sqlbuild.rule_engine.classes.column_facts import ColumnFacts from sqlbuild.rule_engine.classes.contract_facts import ContractFacts from sqlbuild.rule_engine.classes.declaration_facts import DeclarationFacts +from sqlbuild.rule_engine.classes.fact_reads import FactReads from sqlbuild.rule_engine.classes.graph_facts import GraphFacts from sqlbuild.rule_engine.classes.model_test_facts import TestFacts from sqlbuild.rule_engine.classes.project_facts import ProjectFacts @@ -59,6 +60,7 @@ def __init__( selected_rules: tuple[Rule, ...], dialect: str, facts: RuleFactViews | None = None, + reads: FactReads | None = None, ) -> None: del model, selected_rules self._rule = rule @@ -74,6 +76,13 @@ def __init__( self.tests = views.tests self.audits = views.audits self.declarations = views.declarations + self._reads: FactReads | None = reads + + def __setattr__(self, name: str, value: object) -> None: + reads: FactReads | None = self.__dict__.get("_reads") + if reads is not None: + reads.untracked = True + object.__setattr__(self, name, value) def option[T](self, option: RuleOption[T]) -> T: """Return one declared option after native configuration validation.""" diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index f2685811ee..b9e266f999 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -8,7 +8,12 @@ RULES_NATIVE_API_VERSION: int = 1 CUSTOM_HOST_PROTOCOL_VERSION: int = 1 -CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v1" +CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" +CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 +CUSTOM_RULES_CACHE_VERSION: str = "custom-rules-reads-v1" +CUSTOM_RULES_CACHE_FILE: str = "target/rules-cache/bulk/custom-rules.json" +NATIVE_RULES_MEMO_VERSION: str = "native-rules-response-v1" +NATIVE_RULES_MEMO_FILE: str = "target/rules-cache/bulk/native-response.json" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 CUSTOM_HOST_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host" CUSTOM_HOST_LAUNCH_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host_launch" @@ -79,11 +84,6 @@ TARGET_DIRECTORY_NAME: str = "target" RULE_DECORATOR_TOKEN: str = "@rule" INIT_MODULE_NAME: str = "__init__" -RULE_CONTEXT_PROJECT_FACT: str = "project" -RULE_CONTEXT_GRAPH_FACT: str = "graph" -RULE_CONTEXT_TESTS_FACT: str = "tests" -RULE_CONTEXT_AUDITS_FACT: str = "audits" -RULE_CONTEXT_DECLARATIONS_FACT: str = "declarations" PARENT_DIRECTORY_TOKEN: str = ".." MIN_AUDITS_PER_MODEL: str = "min_audits_per_model" @@ -123,3 +123,32 @@ RULE_FIX_AVAILABLE_NOTE: str = ( "Fix available: `sqb format --fix` applies it after compiler verification" ) +FACT_PROJECT_MODELS: str = "project.models" +FACT_PROJECT_SOURCES: str = "project.sources" +FACT_PROJECT_SEEDS: str = "project.seeds" +FACT_PROJECT_FUNCTIONS: str = "project.functions" +FACT_PROJECT_TESTS: str = "project.tests" +FACT_PROJECT_AUDITS: str = "project.audits" +FACT_TREE_PATHS: str = "tree.paths" +FACT_TREE_CHILDREN: str = "tree.children" +FACT_TREE_DESCENDANTS: str = "tree.descendants" +FACT_TREE_GLOB: str = "tree.glob" +FACT_TREE_RELATIVE_PARTS: str = "tree.relative_parts" +FACT_TREE_RESOURCES_UNDER: str = "tree.resources_under" +FACT_TREE_READ_TEXT: str = "tree.read_text" +FACT_SQL_FOR_MODEL: str = "sql.for_model" +FACT_GRAPH_DEPENDENCIES: str = "graph.dependencies" +FACT_GRAPH_DEPENDENTS: str = "graph.dependents" +FACT_COLUMNS_DECLARED: str = "columns.declared" +FACT_COLUMNS_INFERRED: str = "columns.inferred" +FACT_COLUMNS_NAMES: str = "columns.names" +FACT_CONTRACTS_ENFORCED: str = "contracts.enforced" +FACT_CONTRACTS_GRAIN: str = "contracts.grain" +FACT_TESTS_ALL: str = "tests.all" +FACT_TESTS_FOR_MODEL: str = "tests.for_model" +FACT_AUDITS_ALL: str = "audits.all" +FACT_AUDITS_FOR_MODEL: str = "audits.for_model" +FACT_DECLARATIONS_PUBLIC_ENUMS: str = "declarations.public_enums" +FACT_DECLARATIONS_PUBLIC_CONSTANTS: str = "declarations.public_constants" +FACT_DECLARATIONS_ENUMS: str = "declarations.enums" +FACT_DECLARATIONS_CONSTANTS: str = "declarations.constants" diff --git a/src/sqlbuild/rule_engine/exceptions.py b/src/sqlbuild/rule_engine/exceptions.py index 2d4ace09b4..2d164a99a2 100644 --- a/src/sqlbuild/rule_engine/exceptions.py +++ b/src/sqlbuild/rule_engine/exceptions.py @@ -15,3 +15,11 @@ class NonHermeticRuleError(RulesError): class RuleAssertionError(RulesError): """Raised when a custom-rule harness case produces an unexpected result.""" + + +class FactDigestError(RulesError): + """Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.""" + + +class OpaqueModuleStateError(RulesError): + """Raised when custom-rule module state holds a value whose mutations cannot be observed.""" diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 9c0b57c0d0..cb745b884b 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -381,3 +381,33 @@ class RuleResult: @property def finding_count(self) -> int: return len(self.findings) + + +@dataclass(frozen=True) +class CustomRuleEvaluation: + """Findings and observed fact reads of one rule invocation for one subject.""" + + code: str + subject: str | None + findings: tuple[Finding, ...] + reads: frozenset[tuple[object, ...]] | None + + +@dataclass(frozen=True) +class CustomRuleRun: + """Every invocation of one host run plus rules whose reads could not be attributed.""" + + evaluations: tuple[CustomRuleEvaluation, ...] + untracked_codes: frozenset[str] + uncacheable_codes: frozenset[str] = frozenset() + observed: tuple[tuple[tuple[str, ...], str | None], ...] = () + + +@dataclass(frozen=True) +class CustomRulesOutcome: + """Custom-rule finding payloads in canonical order and their cache accounting.""" + + findings: tuple[dict[str, object], ...] + cache_hits: int + cache_misses: int + custom_ms: int diff --git a/src/sqlbuild/rule_engine/types.py b/src/sqlbuild/rule_engine/types.py index af7766b5d5..ccaedcbba2 100644 --- a/src/sqlbuild/rule_engine/types.py +++ b/src/sqlbuild/rule_engine/types.py @@ -25,6 +25,8 @@ ) type RuleOptionValue = bool | int | str | tuple[str, ...] | tuple[int, ...] +type FactKey = tuple[str, ...] +type CustomRulePlan = dict[str, frozenset[str] | None] class RuleSubject(StrEnum): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 8d1d5a5a68..cf7997e7fb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -240,6 +240,19 @@ class DenseCompileGuardTestCase: expected_fingerprint: str +@dataclass(frozen=True) +class DenseWarmEditCompileGuardTestCase: + description: str + model_count: int + edited_model_index: int + expected_warm_max_seconds: float + expected_edit_max_seconds: float + expected_max_rss_bytes: int + expected_cold_fingerprint: str + expected_edit_fingerprint: str + expected_edit_rule_cache_misses: int + + @dataclass(frozen=True) class RuleGatedTestDiagnosticsTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index a178d3dfa3..fde1b109b2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -840,6 +840,51 @@ def prepare_small_inspection_project(*, project_dir: Path, model_count: int) -> ) +def run_dense_warm_edit_benchmark( + *, + project_dir: Path, + edited_model_path: Path, + expected_warm_max_seconds: float, + expected_edit_max_seconds: float, +) -> dict[str, FreshProcessCompileBenchmarkResult]: + """Measure cached cold, warm, and one-model edit plus a from-scratch oracle of the edit.""" + + measurements: dict[str, FreshProcessCompileBenchmarkResult] = { + "cold": _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label="dense-cold", + expected_max_wall_seconds=4 * expected_warm_max_seconds, + compile_args=(), + ), + "warm": _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label="dense-warm", + expected_max_wall_seconds=expected_warm_max_seconds, + compile_args=(), + ), + } + _replace_benchmark_text( + path=edited_model_path, + old="COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)", + new="COALESCE(b.amount, 1000) + CAST(b.id AS DOUBLE)", + ) + measurements["edit"] = _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label="dense-edit", + expected_max_wall_seconds=expected_edit_max_seconds, + compile_args=(), + ) + oracle_dir: Path = project_dir.with_name(f"{project_dir.name}_oracle") + _ = shutil.copytree(project_dir, oracle_dir, ignore=shutil.ignore_patterns("target")) + measurements["oracle"] = _run_fresh_process_compile_benchmark( + project_dir=oracle_dir, + label="dense-oracle", + expected_max_wall_seconds=4 * expected_edit_max_seconds, + compile_args=("--no-cache",), + ) + return measurements + + def _append_benchmark_edit(path: Path, label: str) -> None: path.write_text( path.read_text(encoding="utf-8") + f"\n-- one {label} edit\n", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 9a358f069e..5dc59b4235 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -8,6 +8,7 @@ import pytest from scripts.cold_compile_performance._helpers.dense_project import ( + dense_model_name, write_dense_compile_project, ) from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( @@ -18,11 +19,14 @@ from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, + DenseWarmEditCompileGuardTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, + fresh_process_compile_cache_metrics, measure_model_sql_bytes, + run_dense_warm_edit_benchmark, ) _GIB: int = 1024 * 1024 * 1024 @@ -125,5 +129,91 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ assert result.peak_rss_bytes < test_case.expected_max_rss_bytes +@pytest.mark.performance +@pytest.mark.cold_compile_performance +@pytest.mark.parametrize( + "test_case", + ( + DenseWarmEditCompileGuardTestCase( + "dense_models_3000_warm_and_one_edit", + 3000, + 1521, + 22.0, + 24.0, + 11 * _GIB // 4, + "c35ea7ec71e6e81059146bc1cd72f10e6493248dc6274aba7737d616d434585b", + "255c19bb188491b53d44c2e5d41472137f763806c699f5acb3899c6b47c5360b", + 3, + ), + DenseWarmEditCompileGuardTestCase( + "dense_models_5000_warm_and_one_edit", + 5000, + 2521, + 36.0, + 38.0, + 13 * _GIB // 4, + "b07422282a6800de7830b9c4950f76bdc6fd555ebd3f21152b8649f976be74dc", + "c239f53fb5fa787d250f5f7bf404b3731b480b77d2690dd31fb41ba727475fc0", + 3, + ), + ), + ids=lambda case: case.description, +) +def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches_oracle_in_budget( + test_case: DenseWarmEditCompileGuardTestCase, + tmp_path: Path, +) -> None: + assert_required_cgroup_memory_limit() + project_dir: Path = tmp_path / "dense_orders" + write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) + index: int = test_case.edited_model_index + edited_model_path: Path = ( + project_dir + / "models" + / f"sales{index // 1000:03d}" + / "intermediate" + / "clean" + / f"batch{index // 5:05d}" + / f"{dense_model_name(index)}.sql" + ) + measurements: dict[str, FreshProcessCompileBenchmarkResult] = run_dense_warm_edit_benchmark( + project_dir=project_dir, + edited_model_path=edited_model_path, + expected_warm_max_seconds=test_case.expected_warm_max_seconds, + expected_edit_max_seconds=test_case.expected_edit_max_seconds, + ) + for label, result in measurements.items(): + print( + f"dense {label} models={test_case.model_count} wall={result.elapsed_seconds:.2f}s " + f"cpu={result.cpu_seconds:.2f}s fingerprint={result.semantic_fingerprint}", + flush=True, + ) + assert result.payload["diagnostics"] == [] + assert result.payload["has_errors"] is False + assert result.peak_rss_bytes < test_case.expected_max_rss_bytes + cold: FreshProcessCompileBenchmarkResult = measurements["cold"] + warm: FreshProcessCompileBenchmarkResult = measurements["warm"] + edit: FreshProcessCompileBenchmarkResult = measurements["edit"] + assert cold.semantic_fingerprint == test_case.expected_cold_fingerprint + assert warm.semantic_fingerprint == test_case.expected_cold_fingerprint + assert edit.semantic_fingerprint == test_case.expected_edit_fingerprint + assert measurements["oracle"].semantic_fingerprint == test_case.expected_edit_fingerprint + warm_timings: dict[str, int] = cast(dict[str, int], warm.payload["compile_timings"]) + edit_timings: dict[str, int] = cast(dict[str, int], edit.payload["compile_timings"]) + batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(warm) + assert (batch_hits + entry_hits, misses, bypasses) == (test_case.model_count, 0, 0) + assert warm_timings["rule_cache_misses"] == 0 + batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(edit) + assert 0 < misses < test_case.model_count + assert batch_hits + entry_hits + misses == test_case.model_count + assert edit_timings["rule_cache_misses"] == test_case.expected_edit_rule_cache_misses + oracle_timings: dict[str, int] = cast( + dict[str, int], measurements["oracle"].payload["compile_timings"] + ) + assert oracle_timings["rule_cache_hits"] == 0 + assert warm.elapsed_seconds < test_case.expected_warm_max_seconds + assert edit.elapsed_seconds < test_case.expected_edit_max_seconds + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 6a4bb69c45..2f84b7dea5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -1,6 +1,8 @@ """Test case types for Rules performance guards.""" +from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path @dataclass(frozen=True) @@ -189,3 +191,12 @@ class DuplicateLiteralCase: description: str literals: tuple[tuple[str, str], ...] expected_hints: tuple[tuple[str, str | None], ...] + + +@dataclass(frozen=True) +class RulesCacheEditCase: + """One authored input edit applied after a warm compile with every rule selected.""" + + description: str + edit: Callable[[Path], None] + expected_exit_code: int = 1 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 4f9e255e1e..831de8a32c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -1,5 +1,6 @@ """Compiler Rules end-to-end fixture helpers.""" +import hashlib import json import os import subprocess @@ -226,3 +227,147 @@ def long_literal_hints(result: subprocess.CompletedProcess[str]) -> dict[str, st str(item["path"]): str(item["remediation"]).partition("Define the value once")[0].strip() for item in json.loads(result.stdout)["findings"] } + + +INCREMENTAL_RULES_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "warehouse.duckdb"\n\n' + '[targets.dev]\nschema = "dev"\n\n[targets.prod]\nschema = "prod"\n\n' + '[vars]\ndiscount_rate = "0.1"\n\n' + '[rules]\nselect = ["SQBR", "XSQBR"]\n\n' + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" + ), + "models/staging/stg_orders.sql": ( + "MODEL (\n materialized view,\n contract enforced,\n columns (\n" + " order_id (type INTEGER, nullable false, audits [not_null]),\n" + " amount (type DOUBLE),\n ),\n);\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(10.5 AS DOUBLE) AS amount\n" + ), + "models/marts/order_totals.sql": ( + "MODEL (\n columns (\n" + " order_id (type INTEGER, nullable false, audits [not_null]),\n" + " net_amount (type DOUBLE),\n ),\n);\n\n" + "SELECT\n o.order_id,\n" + ' CAST(@discounted("o.amount") * (1 - CAST(@@discount_rate AS DOUBLE)) AS DOUBLE)' + " AS net_amount\n" + 'FROM __ref("stg_orders") AS o\n' + ), + "models/marts/_sqlbuild/_macros/amounts.py": ( + 'def discounted(amount: str) -> str:\n return f"({amount}) * 0.9"\n' + ), + "tests/unit/test_order_totals.sql": ( + "TEST ();\n\nWITH\n__ref__stg_orders AS (\n SELECT 1 AS order_id, 10.0 AS amount\n),\n" + "__expected__order_totals AS (\n SELECT 1 AS order_id, 9.0 AS net_amount\n)\n" + "SELECT 1\n" + ), + "config/policy.yml": "strict: false\n", + "rules/limits.py": "MAX_NAME_LENGTH: int = 30\n", + "rules/memo.py": """from sqlbuild.rules import Finding, Model, RuleContext, rule + +_SEEN_SQL: dict[str, str] = {} + + +@rule(code="XSQBRGOV005", message="Models must not repeat SQL", remediation="Deduplicate.") +def unique_sql(*, model: Model, ctx: RuleContext) -> list[Finding]: + if not _SEEN_SQL: + for item in ctx.project.models: + _SEEN_SQL[item.name] = ctx.sql.for_model(item).expanded.source + own: str = _SEEN_SQL[model.name] + repeated: bool = sum(source == own for source in _SEEN_SQL.values()) > 1 + return [ctx.finding(subject=model)] if repeated else [] +""", + "rules/governance.py": """from sqlbuild.rules import Finding, Model, Project, RuleContext, rule + +from rules import limits + + +@rule(code="XSQBRGOV001", message="Contracts must be enforced", remediation="Enforce it.") +def enforced_contracts(*, model: Model, ctx: RuleContext) -> list[Finding]: + if "SELECT" not in ctx.sql.for_model(model).authored.source: + return [] + return [] if ctx.contracts.enforced(model) else [ctx.finding(subject=model)] + + +@rule(code="XSQBRGOV002", message="Staging models need a consumer", remediation="Use it.") +def staging_consumers(*, model: Model, ctx: RuleContext) -> list[Finding]: + if not model.name.startswith("stg_"): + return [] + return [] if ctx.graph.dependents(model) else [ctx.finding(subject=model)] + + +@rule(code="XSQBRGOV003", message="Strict policy limits models", remediation="Relax it.") +def strict_policy(*, project: Project, ctx: RuleContext) -> list[Finding]: + del project + findings: list[Finding] = [] + if "strict: true" in ctx.project.tree.read_text("config/policy.yml"): + findings.append(ctx.finding(subject="config/policy.yml", message="strict policy")) + if len(ctx.project.tree.glob("config/*.yml")) > 1: + findings.append(ctx.finding(subject="config/policy.yml", message="several policies")) + if len(ctx.project.models) > 2: + findings.append(ctx.finding(subject="config/policy.yml", message="too many models")) + return findings + + +@rule(code="XSQBRGOV004", message="Model names are bounded", remediation="Shorten it.") +def bounded_names(*, model: Model, ctx: RuleContext) -> list[Finding]: + names: list[str] = [test.name for test in ctx.tests.for_model(model)] + too_long: bool = len(model.name) > limits.MAX_NAME_LENGTH + unnamed: bool = any("discount" not in name for name in names) + return [ctx.finding(subject=model)] if too_long or unnamed else [] + + +@rule(code="XSQBRGOV006", message="Expanded SQL uses a flagged value", remediation="Review.") +def flagged_values(*, model: Model, ctx: RuleContext) -> list[Finding]: + expanded: str = ctx.sql.for_model(model).expanded.source + flagged: bool = any(value in expanded for value in ("12.5", "0.8", "0.2")) + return [ctx.finding(subject=model)] if flagged else [] +""", +} + + +def write_project_files(*, project_dir: Path, files: dict[str, str]) -> None: + """Write authored project files beneath one project directory.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def rules_compile_outcome(project_dir: Path, *arguments: str) -> tuple[int, str, str]: + """Compile in a fresh process and return its exit code, diagnostics, and artifacts digest.""" + + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--no-color", + "--project-dir", + str(project_dir), + "compile", + "--json", + *arguments, + ], + capture_output=True, + text=True, + check=False, + ) + payload: dict[str, Any] = json.loads(result.stdout or "{}") + artifacts: Any = hashlib.sha256() + compiled: Path = project_dir / "target" / "compiled" + for path in sorted(filter(Path.is_file, compiled.rglob("*"))): + artifacts.update(path.relative_to(compiled).as_posix().encode()) + artifacts.update(path.read_bytes()) + return ( + result.returncode, + json.dumps(payload.get("diagnostics"), sort_keys=True), + artifacts.hexdigest(), + ) + + +def rule_cache_counts(project_dir: Path) -> tuple[int, int]: + """Return rule cache hits and misses of one fresh-process compile.""" + + result: subprocess.CompletedProcess[str] = run_compile_cli(project_dir) + timings: dict[str, int] = json.loads(result.stdout)["compile_timings"] + return timings["rule_cache_hits"], timings["rule_cache_misses"] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py new file mode 100644 index 0000000000..96611b12f6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py @@ -0,0 +1,140 @@ +"""A warm Rules compile after any single input edit must equal a compile without caches.""" + +import shutil +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + replace_project_text, + write_project_file, +) +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import RulesCacheEditCase +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + INCREMENTAL_RULES_PROJECT, + rule_cache_counts, + rules_compile_outcome, + write_project_files, +) + + +@pytest.mark.parametrize( + "test_case", + ( + RulesCacheEditCase( + "model_sql_edit", + lambda root: replace_project_text( + root, + "models/staging/stg_orders.sql", + "CAST(10.5 AS DOUBLE)", + "CAST(12.5 AS DOUBLE)", + ), + ), + RulesCacheEditCase( + "model_contract_edit", + lambda root: replace_project_text( + root, "models/staging/stg_orders.sql", " contract enforced,\n", "" + ), + ), + RulesCacheEditCase( + "model_added", + lambda root: write_project_file( + root, + "models/marts/order_counts.sql", + 'MODEL ();\n\nSELECT order_id FROM __ref("stg_orders")\n', + ), + ), + RulesCacheEditCase( + "consumer_removed", + lambda root: replace_project_text( + root, + "models/marts/order_totals.sql", + 'FROM __ref("stg_orders") AS o\n', + "FROM (SELECT 1 AS order_id, 1.0 AS amount) AS o\n", + ), + ), + RulesCacheEditCase( + "test_edit", + lambda root: replace_project_text( + root, + "tests/unit/test_order_totals.sql", + "TEST ();", + 'TEST (name "order_totals_applies_discount");', + ), + ), + RulesCacheEditCase( + "macro_edit", + lambda root: replace_project_text( + root, "models/marts/_sqlbuild/_macros/amounts.py", "* 0.9", "* 0.8" + ), + ), + RulesCacheEditCase( + "var_edit", + lambda root: replace_project_text( + root, "sqlbuild_project.toml", 'discount_rate = "0.1"', 'discount_rate = "0.2"' + ), + ), + RulesCacheEditCase( + "rule_source_edit", + lambda root: replace_project_text( + root, + "rules/governance.py", + '"Contracts must be enforced"', + '"Output contracts must be enforced"', + ), + ), + RulesCacheEditCase( + "rule_helper_edit", + lambda root: replace_project_text(root, "rules/limits.py", "= 30", "= 5"), + ), + RulesCacheEditCase( + "tree_file_edit", + lambda root: replace_project_text(root, "config/policy.yml", "false", "true"), + ), + RulesCacheEditCase( + "tree_file_added", + lambda root: write_project_file(root, "config/extra.yml", "enabled: true\n"), + ), + RulesCacheEditCase( + "rules_threshold_edit", + lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + "min_custom_rule_test_cases = 0\n", + "min_custom_rule_test_cases = 0\nmin_tests_per_model = 0\n", + ), + ), + RulesCacheEditCase( + "target_switch", + lambda root: write_project_file(root, "sqlbuild_local.toml", 'target = "prod"\n'), + ), + ), + ids=lambda case: case.description, +) +def test_given_warm_rules_cache_when_one_input_changes_then_compile_equals_uncached_compile( + tmp_path: Path, test_case: RulesCacheEditCase +) -> None: + project_dir: Path = tmp_path / "orders" + write_project_files(project_dir=project_dir, files=INCREMENTAL_RULES_PROJECT) + cold: tuple[int, str, str] = rules_compile_outcome(project_dir) + assert cold[0] == 1 + assert '"XSQBRGOV001"' in cold[1] + hits, misses = rule_cache_counts(project_dir) + assert hits > 0 + assert misses == 0 + + test_case.edit(project_dir) + edited: tuple[int, str, str] = rules_compile_outcome(project_dir) + oracle_dir: Path = tmp_path / "oracle" + shutil.copytree(project_dir, oracle_dir, ignore=shutil.ignore_patterns("target")) + oracle: tuple[int, str, str] = rules_compile_outcome(oracle_dir, "--no-cache") + rewarmed: tuple[int, str, str] = rules_compile_outcome(project_dir) + + assert edited[0] == test_case.expected_exit_code + assert edited == oracle + assert rewarmed == oracle + assert rule_cache_counts(project_dir)[1] == 0 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index b6c5142b08..33fb11fea3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -import threading from collections.abc import Callable from pathlib import Path from typing import cast @@ -14,13 +13,12 @@ import sqlbuild.compiler.compile._helpers.attachment.declaration_scope as declaration_scope_module import sqlbuild.compiler.compile.main._build_compile_inputs as compile_inputs_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module -import sqlbuild.rule_engine._helpers.engine.native as native_module +import sqlbuild.rule_engine._helpers.engine.custom_rules as custom_rules_module import sqlbuild.rule_engine._helpers.run.rules as rules_module from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile.models import CompiledProject, DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.rule_engine.models import Rule from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( DynamicPivotRulesIntegrationTestCase, ExplicitContractOutputRuleIntegrationTestCase, @@ -921,10 +919,10 @@ def counting_collect( @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("custom host starts with the first request", 1, "XSQBRARCH001")], + [RulesIntegrationTestCase("custom host runs once and is skipped when warm", 1, "XSQBRARCH001")], ids=lambda case: case.description, ) -def test_given_sql_rules_create_cache_first_when_compiling_then_custom_host_is_not_retried( +def test_given_sql_and_custom_rules_when_compiling_twice_then_custom_host_runs_only_cold( test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -962,43 +960,21 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: """, encoding="utf-8", ) - sql_cache_written: threading.Event = threading.Event() - requests: list[bool] = [] - original_write: Callable[..., None] = rules_module._write_sql_rule_cache - original_payloads: Callable[..., list[dict[str, object]]] = native_module._custom_rule_payloads - original_request: Callable[[dict[str, object]], str] = native_module._evaluate_request + host_runs: list[str] = [] + original_host: Callable[[str], str] = custom_rules_module._native.run_custom_host_json - def signalling_write(*, project_dir: Path, bucket: dict[str, dict[str, object]]) -> None: - original_write(project_dir=project_dir, bucket=bucket) - sql_cache_written.set() + def recording_host(spec_json: str) -> str: + host_runs.append(spec_json) + return original_host(spec_json) - def delayed_payloads( - *, - catalogue: tuple[Rule, ...], - project: CompiledProject, - project_dir: Path, - cache_enabled: bool, - ) -> list[dict[str, object]]: - _ = sql_cache_written.wait(timeout=30) - return original_payloads( - catalogue=catalogue, - project=project, - project_dir=project_dir, - cache_enabled=cache_enabled, - ) - - def recording_request(request: dict[str, object]) -> str: - requests.append(request["custom_host"] is not None) - return original_request(request) - - monkeypatch.setattr(rules_module, "_write_sql_rule_cache", signalling_write) - monkeypatch.setattr(native_module, "_custom_rule_payloads", delayed_payloads) - monkeypatch.setattr(native_module, "_evaluate_request", recording_request) + monkeypatch.setattr(custom_rules_module._native, "run_custom_host_json", recording_host) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) - assert sql_cache_written.is_set() - assert requests == [True] + warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + + assert len(host_runs) == 1 + assert warm == cold assert { f"models/orders.sql:{test_case.expected_code}", "tests/unit/test_orders.sql:SQBRSQL004", diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index ed35db4454..10d5f74f42 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -1,6 +1,6 @@ """Test case declarations for rules engine helpers.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from sqlbuild.rule_engine.models import RuleExemption, RuleIgnore @@ -111,12 +111,6 @@ class ScopePayloadTestCase: expected_result: bool -@dataclass(frozen=True) -class ProjectFingerprintRepresentationTestCase: - description: str - expected_equivalent: bool - - @dataclass(frozen=True) class NativeFactPayloadTestCase: description: str @@ -155,3 +149,56 @@ class CustomRuleImportTestCase: extra_files: tuple[tuple[str, str], ...] = () expected_rule_codes: tuple[str, ...] = ("XSQBRT101",) expected_error_pattern: str = "" + + +@dataclass(frozen=True) +class CustomRuleReadTrackingTestCase: + """One model-subject rule, an edit to the second model, and the expected reuse.""" + + description: str + body: str + expected_hits_after_edit: int + module_prelude: str = "" + edited_customers_sql: str = "SELECT 2 AS customer_id" + edited_customers_config: dict[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CustomRuleUncacheableTestCase: + """One model-subject rule whose fact access cannot be attributed to recorded reads.""" + + description: str + body: str + expected_warm_hits: int = 0 + + +@dataclass(frozen=True) +class CustomRuleProjectFileTestCase: + """One rule reading a project file whose contents change from customers to orders.""" + + description: str + file_path: str + body: str + expected_rerun_hits: int + expected_finding_paths: tuple[str, ...] + module_prelude: str = "" + + +@dataclass(frozen=True) +class NativeMemoTestCase: + """One warm built-in evaluation followed by an edit that must invalidate the memo.""" + + description: str + original_config: dict[str, object] + edited_config: dict[str, object] + expected_original_codes: tuple[str, ...] + expected_edited_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeBuildIdentityTestCase: + """One cached evaluation repeated after the native extension reports another build.""" + + description: str + expected_rebuilt_evaluations: int + expected_warm_evaluations: int diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index bb91a7a532..600de06dda 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -1,6 +1,7 @@ """Test helpers for native rules engine boundaries.""" import json +from collections.abc import Callable from dataclasses import replace from operator import attrgetter from pathlib import Path @@ -22,8 +23,16 @@ from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import native from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue +from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.models import Rule, RulesCacheConfig, RulesConfig +from sqlbuild.rule_engine.main._evaluate import evaluate +from sqlbuild.rule_engine.models import ( + CustomRulesOutcome, + Rule, + RulesCacheConfig, + RulesConfig, + RulesResult, +) from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project @@ -58,13 +67,14 @@ def evaluate_json(request_json: str) -> str: "evaluated_models": 0, "cache_hits": 0, "cache_misses": 0, + "selected_codes": [], } ) monkeypatch.setattr(native._native, "evaluate_json", evaluate_json) native.evaluate_native( project=project, - config=RulesConfig(), + config=RulesConfig(cache=RulesCacheConfig(enabled=False)), project_dir=project_dir, catalogue=(), ) @@ -178,6 +188,18 @@ def write_rule( return path.relative_to(root) +def write_project_file_rule( + *, root: Path, file_path: str, contents: str, body: str, module_import: str +) -> Path: + """Write one project file plus a custom rule and return the absolute file path.""" + + path: Path = root / file_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + _ = write_rule(root=root, body=body, module_import=module_import) + return path + + def load_custom_rule(*, root: Path, configured_path: Path) -> Rule: del configured_path config: RulesConfig = RulesConfig() @@ -226,3 +248,70 @@ def custom_rules_with_imports( config=RulesConfig(select=("XSQBRT101",)), project_dir=project_dir ) return tuple(filter(lambda rule: rule.custom, catalogue)) + + +def two_model_project( + *, customers_sql: str, customers_config: dict[str, object] +) -> CompiledProject: + """Return an orders and customers project with configurable customers facts.""" + + orders: CompiledProject = build_project( + name="orders", + relative_path="models/orders.sql", + sql="SELECT 1 AS order_id", + config_values={}, + ) + customers: CompiledProject = build_project( + name="customers", + relative_path="models/customers.sql", + sql=customers_sql, + config_values=customers_config, + ) + return replace(orders, models=(*orders.models, *customers.models)) + + +def evaluate_cached_custom_rules( + *, project: CompiledProject, project_dir: Path, cache_enabled: bool +) -> CustomRulesOutcome: + """Evaluate the selected custom rule through the incremental read-tracking cache.""" + + config: RulesConfig = RulesConfig( + select=("XSQBRT101",), cache=RulesCacheConfig(enabled=cache_enabled) + ) + rules: tuple[Rule, ...] = tuple( + filter(attrgetter("custom"), build_catalogue(config=config, project_dir=project_dir)) + ) + return evaluate_custom_rules_cached( + project=project, config=config, project_dir=project_dir, rules=rules, dialect="duckdb" + ) + + +def evaluate_contract_rule( + *, config_values: dict[str, object], project_dir: Path, cache_enabled: bool +) -> RulesResult: + """Evaluate one built-in contract rule over a single mart model.""" + + config: RulesConfig = RulesConfig( + select=("SQBRCONTRACT101",), cache=RulesCacheConfig(enabled=cache_enabled) + ) + project: CompiledProject = build_project( + name="commerce__mart__orders", + relative_path="models/mart/commerce__mart__orders.sql", + sql="WITH orders AS (SELECT id FROM source_orders) SELECT id FROM orders", + config_values=config_values, + ) + return evaluate(project=project, config=config, project_dir=project_dir) + + +def record_native_evaluations(*, monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Count every request that reaches the native built-in rules engine.""" + + requests: list[str] = [] + evaluate_json: Callable[[str], str] = native._native.evaluate_json + + def recording(request_json: str) -> str: + requests.append(request_json) + return evaluate_json(request_json) + + monkeypatch.setattr(native._native, "evaluate_json", recording) + return requests diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py new file mode 100644 index 0000000000..a0fe767bcf --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py @@ -0,0 +1,308 @@ +"""Custom-rule results are reused only when every fact the invocation read is unchanged.""" + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +import sqlbuild._native as native_module +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.rule_engine._helpers.engine import custom_rules +from sqlbuild.rule_engine.models import CustomRulesOutcome +from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( + CustomRuleProjectFileTestCase, + CustomRuleReadTrackingTestCase, + CustomRuleUncacheableTestCase, + NativeBuildIdentityTestCase, +) +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( + evaluate_cached_custom_rules, + two_model_project, + write_project_file_rule, + write_rule, +) + +_CUSTOMERS_SQL: str = "SELECT 1 AS customer_id" +_OWNERS_BEFORE: str = "owners: [customers]\n" +_OWNERS_AFTER: str = "owners: [orders]\n" +_FLAG_ORDERS: str = ( + ' return [ctx.finding(subject=model)] if model.name == "orders" and {condition} else []' +) + + +@pytest.mark.parametrize( + "test_case", + ( + CustomRuleReadTrackingTestCase( + description="own SQL read reuses the unedited subject", + body="_ = ctx.sql.for_model(model).expanded.source\n" + + _FLAG_ORDERS.format(condition="True"), + expected_hits_after_edit=1, + ), + CustomRuleReadTrackingTestCase( + description="reading every model SQL invalidates every subject", + body="sources = [ctx.sql.for_model(item).expanded.source " + "for item in ctx.project.models]\n" + _FLAG_ORDERS.format(condition="len(sources) > 1"), + expected_hits_after_edit=0, + ), + CustomRuleReadTrackingTestCase( + description="unchanged contracts of every model keep both subjects", + body="enforced = [ctx.contracts.enforced(item) for item in ctx.project.models]\n" + + _FLAG_ORDERS.format(condition="not any(enforced)"), + expected_hits_after_edit=2, + ), + CustomRuleReadTrackingTestCase( + description="changed contract of another model invalidates readers", + body="enforced = [ctx.contracts.enforced(item) for item in ctx.project.models]\n" + + _FLAG_ORDERS.format(condition="not any(enforced)"), + expected_hits_after_edit=0, + edited_customers_sql=_CUSTOMERS_SQL, + edited_customers_config={"contract": "enforced"}, + ), + CustomRuleReadTrackingTestCase( + description="module-level memo makes the rule untracked", + module_prelude="_MEMO: dict[str, object] = {}", + body="if 'sql' not in _MEMO:\n" + " _MEMO['sql'] = [ctx.sql.for_model(item).expanded.source " + "for item in ctx.project.models]\n" + + _FLAG_ORDERS.format(condition="'SELECT 2' in ' '.join(_MEMO['sql'])"), + expected_hits_after_edit=0, + ), + CustomRuleReadTrackingTestCase( + description="cached helper makes the rule untracked", + module_prelude=( + "import functools\n\n" + "@functools.cache\n" + "def _sources(ctx: RuleContext) -> str:\n" + " return ' '.join(ctx.sql.for_model(item).expanded.source " + "for item in ctx.project.models)\n" + ), + body=_FLAG_ORDERS.format(condition="'SELECT 2' in _sources(ctx)").strip(), + expected_hits_after_edit=0, + ), + CustomRuleReadTrackingTestCase( + description="context attribute state makes the rule untracked", + body="ctx.seen = model.name\n" + _FLAG_ORDERS.format(condition="True"), + expected_hits_after_edit=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_recorded_reads_when_one_model_changes_then_only_dependent_results_rerun( + tmp_path: Path, + test_case: CustomRuleReadTrackingTestCase, +) -> None: + _ = write_rule(root=tmp_path, body=test_case.body, module_import=test_case.module_prelude) + original: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + edited: CompiledProject = two_model_project( + customers_sql=test_case.edited_customers_sql, + customers_config=test_case.edited_customers_config, + ) + + cold: CustomRulesOutcome = evaluate_cached_custom_rules( + project=original, project_dir=tmp_path, cache_enabled=True + ) + warm: CustomRulesOutcome = evaluate_cached_custom_rules( + project=original, project_dir=tmp_path, cache_enabled=True + ) + after_edit: CustomRulesOutcome = evaluate_cached_custom_rules( + project=edited, project_dir=tmp_path, cache_enabled=True + ) + oracle: CustomRulesOutcome = evaluate_cached_custom_rules( + project=edited, project_dir=tmp_path, cache_enabled=False + ) + + assert (cold.cache_hits, cold.cache_misses) == (0, 2) + assert (warm.cache_hits, warm.cache_misses) == (2, 0) + assert warm.findings == cold.findings + assert (after_edit.cache_hits, after_edit.cache_misses) == ( + test_case.expected_hits_after_edit, + 2 - test_case.expected_hits_after_edit, + ) + assert after_edit.findings == oracle.findings + + +@pytest.mark.parametrize( + "test_case", + ( + CustomRuleUncacheableTestCase( + description="private SQL view state", + body="_ = ctx.sql._models\n" + _FLAG_ORDERS.format(condition="True"), + ), + CustomRuleUncacheableTestCase( + description="private column view state", + body="_ = ctx.columns._models\n" + _FLAG_ORDERS.format(condition="True"), + ), + CustomRuleUncacheableTestCase( + description="private graph view state", + body="_ = ctx.graph._compiled_by_path\n" + _FLAG_ORDERS.format(condition="True"), + ), + ), + ids=lambda case: case.description, +) +def test_given_private_view_access_when_evaluating_repeatedly_then_rule_is_never_reused( + tmp_path: Path, + test_case: CustomRuleUncacheableTestCase, +) -> None: + _ = write_rule(root=tmp_path, body=test_case.body) + project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + + cold: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + warm: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + + assert (cold.cache_hits, cold.cache_misses) == (0, 2) + assert (warm.cache_hits, warm.cache_misses) == ( + test_case.expected_warm_hits, + 2 - test_case.expected_warm_hits, + ) + assert warm.findings == cold.findings + + +@pytest.mark.parametrize( + "test_case", + ( + CustomRuleProjectFileTestCase( + description="hidden file read through a cached helper", + file_path=".config/owners.yaml", + module_prelude=( + "import functools\n\n" + "@functools.cache\n" + "def _owners(ctx: RuleContext) -> str:\n" + " return ctx.project.tree.read_text('.config/owners.yaml')\n" + ), + body=_FLAG_ORDERS.format(condition="'orders' in _owners(ctx)").strip(), + expected_rerun_hits=0, + expected_finding_paths=("models/orders.sql",), + ), + ), + ids=lambda case: case.description, +) +def test_given_untracked_rule_reading_hidden_file_when_file_changes_then_rule_reruns( + tmp_path: Path, + test_case: CustomRuleProjectFileTestCase, +) -> None: + owners: Path = write_project_file_rule( + root=tmp_path, + file_path=test_case.file_path, + contents=_OWNERS_BEFORE, + body=test_case.body, + module_import=test_case.module_prelude, + ) + project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + + cold: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + warm: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + owners.write_text(_OWNERS_AFTER, encoding="utf-8") + after_edit: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + + assert cold.findings == () + assert (warm.cache_hits, warm.cache_misses) == (2, 0) + assert (after_edit.cache_hits, after_edit.cache_misses) == ( + test_case.expected_rerun_hits, + 2 - test_case.expected_rerun_hits, + ) + assert tuple(finding["path"] for finding in after_edit.findings) == ( + test_case.expected_finding_paths + ) + + +@pytest.mark.parametrize( + "test_case", + ( + CustomRuleProjectFileTestCase( + description="file edited after the host read it", + file_path="config/owners.yaml", + body=_FLAG_ORDERS.format( + condition="'orders' in ctx.project.tree.read_text('config/owners.yaml')" + ).strip(), + expected_rerun_hits=1, + expected_finding_paths=("models/orders.sql",), + ), + ), + ids=lambda case: case.description, +) +def test_given_file_edited_while_rules_run_when_rerunning_then_new_contents_are_evaluated( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: CustomRuleProjectFileTestCase, +) -> None: + owners: Path = write_project_file_rule( + root=tmp_path, + file_path=test_case.file_path, + contents=_OWNERS_BEFORE, + body=test_case.body, + module_import=test_case.module_prelude, + ) + project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + run_host: Callable[..., object] = custom_rules._run_host + + def run_host_then_edit(**kwargs: Any) -> object: + result: object = run_host(**kwargs) + owners.write_text(_OWNERS_AFTER, encoding="utf-8") + return result + + monkeypatch.setattr(custom_rules, "_run_host", run_host_then_edit) + during_edit: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + monkeypatch.undo() + rerun: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + oracle: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=False + ) + + assert during_edit.findings == () + assert (rerun.cache_hits, rerun.cache_misses) == ( + test_case.expected_rerun_hits, + 2 - test_case.expected_rerun_hits, + ) + assert rerun.findings == oracle.findings + assert tuple(finding["path"] for finding in rerun.findings) == ( + test_case.expected_finding_paths + ) + + +@pytest.mark.parametrize( + "test_case", + [NativeBuildIdentityTestCase("custom results follow the native build", 2, 0)], + ids=lambda case: case.description, +) +def test_given_custom_rule_cache_from_another_native_build_when_evaluating_then_rules_rerun( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: NativeBuildIdentityTestCase, +) -> None: + _ = write_rule(root=tmp_path, body=_FLAG_ORDERS.format(condition="True").strip()) + project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + + cold: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + monkeypatch.setattr(native_module, "BUILD_IDENTITY", f"{native_module.BUILD_IDENTITY}-rebuilt") + rebuilt: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + warm: CustomRulesOutcome = evaluate_cached_custom_rules( + project=project, project_dir=tmp_path, cache_enabled=True + ) + + assert rebuilt.cache_misses == test_case.expected_rebuilt_evaluations + assert warm.cache_misses == test_case.expected_warm_evaluations + assert rebuilt.findings == warm.findings == cold.findings + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 584fe81a61..12f1d9a917 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -170,19 +170,25 @@ def test_given_hermetic_custom_rule_when_cache_required_then_second_run_hits_cac description="model-only rule ignores unrelated project file changes", body="del model\n return []", changed_path="config/new.yml", - expected_cache_hits=1, + expected_cache_hits=2, ), CustomRuleCacheDependencyTestCase( description="graph-observing rule ignores unrelated project file changes", body="_ = ctx.graph.dependencies(model)\n return []", changed_path="config/new.yml", + expected_cache_hits=2, + ), + CustomRuleCacheDependencyTestCase( + description="project glob observes a newly matching file", + body=('del model\n _ = ctx.project.tree.glob("config/*.yml")\n return []'), + changed_path="config/new.yml", expected_cache_hits=1, ), CustomRuleCacheDependencyTestCase( - description="negative project glob observes a newly matching file", + description="project glob ignores a file it does not match", body=('del model\n _ = ctx.project.tree.glob("config/**/*.yml")\n return []'), changed_path="config/new.yml", - expected_cache_hits=0, + expected_cache_hits=2, ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index a170d9e678..586c0a081f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -5,7 +5,6 @@ import pytest from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, CompiledObjectKey, CompiledProject, CompiledSqlScenario, @@ -15,11 +14,8 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredSqlScenarioFile, ) -from sqlbuild.rule_engine._helpers.engine.native import _custom_fact_fingerprint -from sqlbuild.rule_engine.constants import RULE_CONTEXT_PROJECT_FACT from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( NativeFactPayloadTestCase, - ProjectFingerprintRepresentationTestCase, ) from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( captured_native_request, @@ -143,54 +139,5 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar assert "secret fixture value" not in str(request["sql_tests"] + request["sql_scenarios"]) -@pytest.mark.parametrize( - "test_case", - ( - ProjectFingerprintRepresentationTestCase( - "lineage representation preserves cache identity", True - ), - ), - ids=lambda case: case.description, -) -def test_given_compact_or_materialized_lineage_when_fingerprinting_then_semantics_define_identity( - test_case: ProjectFingerprintRepresentationTestCase, -) -> None: - project: CompiledProject = build_project( - name="selected_orders", - relative_path="models/selected_orders.sql", - sql="SELECT order_id FROM orders", - config_values={}, - ) - lineage: CompactLineageFacts = CompactLineageFacts( - string_pool=("order_id", "source", "canonical_input", "orders", "other_orders"), - rows=((0, 0, 1, ((1, 2, 0),)),), - resource_name_indexes={2: 3}, - ) - compact: CompiledProject = replace( - project, models=(replace(project.models[0], fast_lineage_columns=lineage),) - ) - materialized: CompiledProject = replace( - project, models=(replace(project.models[0], fast_lineage_columns=tuple(lineage)),) - ) - changed: CompiledProject = replace( - project, - models=( - replace( - project.models[0], - fast_lineage_columns=replace(lineage, resource_name_indexes={2: 4}), - ), - ), - ) - attributes: frozenset[str] = frozenset({RULE_CONTEXT_PROJECT_FACT}) - compact_fingerprint: str = _custom_fact_fingerprint(project=compact, attributes=attributes) - materialized_fingerprint: str = _custom_fact_fingerprint( - project=materialized, attributes=attributes - ) - changed_fingerprint: str = _custom_fact_fingerprint(project=changed, attributes=attributes) - - assert (compact_fingerprint == materialized_fingerprint) is test_case.expected_equivalent - assert changed_fingerprint != compact_fingerprint - - if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py new file mode 100644 index 0000000000..50b39076a0 --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py @@ -0,0 +1,94 @@ +from pathlib import Path + +import pytest + +import sqlbuild._native as native_module +from sqlbuild.rule_engine.models import RulesResult +from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( + NativeBuildIdentityTestCase, + NativeMemoTestCase, +) +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( + evaluate_contract_rule, + record_native_evaluations, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeMemoTestCase( + description="contract_enforced_after_warm_run", + original_config={"materialized": "table"}, + edited_config={"materialized": "table", "contract": "enforced"}, + expected_original_codes=("SQBRCONTRACT101",), + expected_edited_codes=(), + ), + NativeMemoTestCase( + description="contract_removed_after_warm_run", + original_config={"materialized": "table", "contract": "enforced"}, + edited_config={"materialized": "table"}, + expected_original_codes=(), + expected_edited_codes=("SQBRCONTRACT101",), + ), + ], + ids=lambda case: case.description, +) +def test_given_warm_built_in_memo_when_request_repeats_or_changes_then_matches_uncached( + tmp_path: Path, test_case: NativeMemoTestCase +) -> None: + cached_dir: Path = tmp_path / "cached" + cached_dir.mkdir() + + first: RulesResult = evaluate_contract_rule( + config_values=test_case.original_config, project_dir=cached_dir, cache_enabled=True + ) + repeated: RulesResult = evaluate_contract_rule( + config_values=test_case.original_config, project_dir=cached_dir, cache_enabled=True + ) + edited: RulesResult = evaluate_contract_rule( + config_values=test_case.edited_config, project_dir=cached_dir, cache_enabled=True + ) + uncached_edited: RulesResult = evaluate_contract_rule( + config_values=test_case.edited_config, project_dir=tmp_path, cache_enabled=False + ) + + assert tuple(finding.code for finding in first.findings) == test_case.expected_original_codes + assert repeated.findings == first.findings + assert (repeated.cache_misses, repeated.built_in_ms) == (0, 0) + assert tuple(finding.code for finding in edited.findings) == test_case.expected_edited_codes + assert edited.findings == uncached_edited.findings + assert edited.cache_misses > 0 + + +@pytest.mark.parametrize( + "test_case", + [NativeBuildIdentityTestCase("memoized response follows the native build", 1, 0)], + ids=lambda case: case.description, +) +def test_given_memoized_response_from_another_native_build_when_evaluating_then_rules_rerun( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: NativeBuildIdentityTestCase, +) -> None: + config_values: dict[str, object] = {"materialized": "table"} + cold: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + native_calls: list[str] = record_native_evaluations(monkeypatch=monkeypatch) + monkeypatch.setattr(native_module, "BUILD_IDENTITY", f"{native_module.BUILD_IDENTITY}-rebuilt") + rebuilt: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + rebuilt_calls: int = len(native_calls) + warm: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + + assert rebuilt_calls == test_case.expected_rebuilt_evaluations + assert len(native_calls) - rebuilt_calls == test_case.expected_warm_evaluations + assert rebuilt.findings == warm.findings == cold.findings + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])