From 85aa8a3a6ae420f2704abbec3b3c1ea1af8a317a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:53:08 +0100 Subject: [PATCH 01/14] perf: report per-rule custom rule timings and bypass the Rules cache with --no-cache --- .../_helpers/compile_driver/analysis.rs | 15 +- .../_helpers/compile_driver/command.rs | 14 ++ .../_helpers/compile_driver/plan_pipeline.rs | 18 +- .../bindings/_helpers/records/constants.rs | 1 + .../_helpers/rule_records/constants.rs | 5 + .../_helpers/rules/custom_evaluation.rs | 15 +- .../_helpers/rules/custom_rule_cache.rs | 13 ++ .../_helpers/rules/early_builtin_rules.rs | 7 +- .../bindings/_helpers/rules/rule_dispatch.rs | 8 +- .../src/bindings/_helpers/rules/rule_run.rs | 4 + .../bindings/_helpers/rules/rules_config.rs | 14 +- .../src/configuration/_helpers/loading.rs | 60 +++++-- crates/sqlbuild-rules/src/constants.rs | 2 + .../src/custom_host_pool/_helpers/decoding.rs | 29 ++- .../src/custom_host_pool/constants.rs | 10 ++ .../main/custom_rule_timings.rs | 49 +++++ .../main/merge_host_slices.rs | 7 +- .../src/custom_host_pool/main/mod.rs | 1 + .../custom_host_pool/main/without_codes.rs | 2 + .../src/custom_host_pool/models.rs | 31 ++++ .../src/custom_host_pool/tests/helpers.rs | 22 ++- .../tests/test_custom_host_pool.rs | 61 ++++++- .../src/custom_host_pool/tests/test_types.rs | 3 +- .../main/lookup_custom_rule_cache.rs | 7 +- .../src/custom_rule_cache/models.rs | 2 + .../tests/test_custom_rule_cache.rs | 6 + .../src/engine/tests/test_published_layers.rs | 59 ++++++ .../src/rules/_helpers/model_layers.rs | 68 ++++--- .../src/rules/main/builtin_options.rs | 7 +- src/sqlbuild/_native.pyi | 10 ++ .../_helpers/host/custom_evaluation.py | 72 ++++++-- .../rule_engine/_helpers/host/custom_host.py | 9 +- .../rule_engine/classes/runtime_guard.py | 28 ++- .../main/build/test_compile_json_behavior.py | 3 + .../cli/commands/main/rules/_test_types.py | 22 +++ .../cli/commands/main/rules/helpers.py | 33 ++++ .../main/rules/test_custom_rule_timings.py | 168 ++++++++++++++++++ .../main/rules/test_published_layer_rules.py | 55 ++++++ .../_helpers/engine/test_config.py | 35 ++++ website/src/content/docs/docs/cli/compile.mdx | 28 ++- .../rules/configuration-and-selection.mdx | 3 +- .../custom-rules/testing-and-determinism.mdx | 9 + .../concepts/rules/execution-and-caching.mdx | 4 + .../concepts/rules/published-layer-rules.mdx | 33 +++- 44 files changed, 960 insertions(+), 92 deletions(-) create mode 100644 crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 9840e1a8f1..ef7b081ad8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -31,7 +31,7 @@ use crate::bindings::_helpers::rules::early_builtin_rules::{EarlyRulesScope, Ear use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::rules_progress; use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; -use crate::bindings::_helpers::rules::rules_config::load_rules_config; +use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; const RICH_LINEAGE: &str = "rich"; const FAST_LINEAGE: &str = "fast"; @@ -114,6 +114,11 @@ impl<'py> AnalysisOptions<'py> { .unwrap_or_else(|| py.None().into_bound(py)), }) } + + /// Whether `--no-cache` asked this compile to bypass its caches, the Rules cache included. + fn no_cache(&self) -> PyResult { + native_field(&self.analysis_selection, "no_cache")?.is_truthy() + } } /// `{seconds:.2f}`, as Python formats elapsed milliseconds over a thousand. @@ -204,6 +209,7 @@ fn analyzed<'py>( project_dir: options.project_dir.extract()?, dialect: lint_dialect.extract()?, no_sql_analysis: options.no_sql_validation.is_truthy()?, + no_cache: options.no_cache()?, }); } let (graph, graph_ms): (Bound<'py, PyAny>, u64) = @@ -289,6 +295,10 @@ fn analyzed<'py>( "skipped_type_proof_rules", native_field(&rules_result, "skipped_type_proof_rules")?, ), + ( + "custom_rule_timings", + native_field(&rules_result, "custom_rule_timings")?, + ), ], ) } @@ -481,7 +491,8 @@ impl<'py> Driver<'_, 'py> { let options: &AnalysisOptions<'py> = self.options; let project_dir: &Bound<'py, PyAny> = &options.project_dir; let rules_config: Bound<'py, PyAny> = - load_rules_config(py, project_dir.extract()?)?.into_bound(py); + load_rules_config_with_cache(py, project_dir.extract()?, !options.no_cache()?)? + .into_bound(py); if !options.prepared_artifacts.is_none() && !any_error(core_diagnostics.as_any())? { let target_dir: Bound<'py, PyAny> = project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 624495ceca..902d4fa14c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -9,6 +9,9 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_host::compile_context::main::collect_compile_timings::collect_compile_timings; use sqlbuild_host::compile_context::main::timing_milliseconds::timing_milliseconds; use sqlbuild_host::compile_context::types::CompileTimingsHandle; +use sqlbuild_rules::custom_host_pool::constants::{ + CUSTOM_RULES_GUARD_KEY, CUSTOM_RULES_INVOCATIONS_KEY, CUSTOM_RULES_MONITOR_KEY, +}; use std::rc::Rc; use crate::bindings::_helpers::boundary::python_calls::{ @@ -487,6 +490,17 @@ fn timings_ms<'py>( ] { timings.set_item(key, native_field(analysis, attribute)?)?; } + for key in [ + CUSTOM_RULES_INVOCATIONS_KEY, + CUSTOM_RULES_MONITOR_KEY, + CUSTOM_RULES_GUARD_KEY, + ] { + timings.set_item(key, 0)?; + } + for entry in native_field(analysis, "custom_rule_timings")?.try_iter()? { + let (key, value): (String, u64) = entry?.extract()?; + timings.set_item(key, value)?; + } timings.set_item("write_ms", write_ms)?; for (key, value) in timing_milliseconds(detailed_timings) { timings.set_item(key, value)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index b10e4dd7e7..07e8d1173f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; -use crate::bindings::_helpers::rules::rules_config::load_rules_config; +use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; const GENERIC_DIALECT: &str = "generic"; @@ -80,16 +80,12 @@ fn run_rules_gate<'py>( rules_error.call1(("compiler rules require a resolved project directory",))?, )); } - let mut config: Bound<'py, PyAny> = - load_rules_config(py, project_dir.extract()?)?.into_bound(py); - if native_field(options, "no_cache")?.is_truthy()? { - let cache: Bound<'py, PyAny> = new_record_with( - py, - RecordId::RulesCacheConfig, - &[("enabled", false.into_pyobject(py)?.to_owned().into_any())], - )?; - config = replaced(&config, &[("cache", cache)])?; - } + let config: Bound<'py, PyAny> = load_rules_config_with_cache( + py, + project_dir.extract()?, + !native_field(options, "no_cache")?.is_truthy()?, + )? + .into_bound(py); progress(on_progress, "Evaluating configured rules...")?; let started: f64 = monotonic(py)?; let dialect: Bound<'py, PyAny> = adapter.call_method0("sql_analysis_dialect")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index dad59be8f8..0df2c24a75 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1024,6 +1024,7 @@ const COMPILE_ANALYSIS: RecordSpec = RecordSpec { FieldSpec::new("rule_cache_hits", FieldDefault::Int(0)), FieldSpec::new("rule_cache_misses", FieldDefault::Int(0)), FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), + FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs index 212db5027c..75fcfbcd84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -107,6 +107,7 @@ const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { FieldSpec::new("cache_hits", FieldDefault::Int(0)), FieldSpec::new("cache_misses", FieldDefault::Int(0)), FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), + FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), ], }; const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { @@ -131,6 +132,7 @@ const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { FieldSpec::required("cache_misses"), FieldSpec::required("custom_ms"), FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), ], }; const R_MODEL: RecordSpec = RecordSpec { @@ -295,6 +297,7 @@ const R_RULES_RESULT: RecordSpec = RecordSpec { FieldSpec::new("built_in_ms", FieldDefault::Int(0)), FieldSpec::new("custom_ms", FieldDefault::Int(0)), FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), ], }; const R_RULE_FILE: RecordSpec = RecordSpec { @@ -355,6 +358,8 @@ const R_CUSTOM_RULE_RUN: RecordSpec = RecordSpec { FieldSpec::new("uncacheable_codes", FieldDefault::EmptyFrozenset), FieldSpec::new("observed", FieldDefault::EmptyTuple), FieldSpec::new("stateful_codes", FieldDefault::EmptyFrozenset), + FieldSpec::new("rule_timings", FieldDefault::EmptyDict), + FieldSpec::new("monitor_ns", FieldDefault::Int(0)), ], }; const R_CUSTOM_HOST_PARTITION: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 98eac488eb..df0e087a38 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -10,7 +10,8 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, JsonValue, OrjsonOptions}; -use sqlbuild_rules::custom_host_pool::models::{HostRuleSubject, HostRun}; +use sqlbuild_rules::custom_host_pool::main::custom_rule_timings::custom_rule_timings; +use sqlbuild_rules::custom_host_pool::models::{HostRuleSubject, HostRun, HostTimings}; use sqlbuild_rules::custom_host_pool::types::{FactKey, HostPlan}; use sqlbuild_rules::custom_rule_cache::models::CustomEvaluation; use sqlbuild_rules::custom_rule_cache::types::ReadSet; @@ -84,7 +85,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( .collect::>()?; custom.sort_by(|left, right| left.0.cmp(&right.0)); if custom.is_empty() { - return outcome(py, (Vec::new(), 0, 0), (0, 0)); + return outcome(py, (Vec::new(), 0, 0), (0, 0), &[]); } let enabled: bool = native_field(&native_field(run.config, "cache")?, "enabled")?.is_truthy()?; @@ -152,6 +153,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( readset_digests.push(reproducible(digests.combined(readset))?); } let plan: HostPlan = cache.plan(readset_digests); + let mut host_timings: HostTimings = HostTimings::default(); let delta: Option> = injection(); if let Some(injection) = &delta && let Some(reason) = delta_plan_refusal(&plan, &cache, injection) @@ -163,7 +165,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( .iter() .map(|(code, rule)| Ok((code.clone(), host_subject(rule)?))) .collect::>()?; - let host_run: HostRun = run_custom_hosts( + let mut host_run: HostRun = run_custom_hosts( py, &HostRequest { project: run.project, @@ -178,6 +180,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( }, &plan, )?; + host_timings = std::mem::take(&mut host_run.timings); let evaluations: Vec = evaluations(host_run, &mut digests, enabled)?; if let Some(injection) = &delta && !evaluations @@ -203,6 +206,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( py, (cache.findings(py)?, cache.hits(), cache.misses()), (elapsed_ms, cpu_ms), + &custom_rule_timings(&host_timings, cache.counts_by_code()), ) } @@ -285,6 +289,7 @@ pub(crate) fn outcome<'py>( py: Python<'py>, (findings, hits, misses): (Vec>, usize, usize), (custom_ms, custom_cpu_ms): (i64, i64), + timings: &[(String, u64)], ) -> PyResult> { new_record_with( py, @@ -295,6 +300,10 @@ pub(crate) fn outcome<'py>( ("cache_misses", misses.into_bound_py_any(py)?), ("custom_ms", custom_ms.into_bound_py_any(py)?), ("custom_cpu_ms", custom_cpu_ms.into_bound_py_any(py)?), + ( + "custom_rule_timings", + PyTuple::new(py, timings.iter().cloned())?.into_any(), + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index b83586f846..a387f618df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -70,6 +70,7 @@ pub(crate) struct CustomRuleCache { results: SubjectResults, hits: usize, misses: usize, + counts_by_code: BTreeMap, } impl CustomRuleCache { @@ -112,6 +113,7 @@ impl CustomRuleCache { results: SubjectResults::new(), hits: 0, misses: 0, + counts_by_code: BTreeMap::new(), } } @@ -137,6 +139,11 @@ impl CustomRuleCache { ) -> BTreeMap>> { if self.path.is_none() { self.misses = self.rules.len(); + self.counts_by_code = self + .rules + .iter() + .map(|rule| (rule.code.clone(), (0, rule.subjects.len()))) + .collect(); return self .rules .iter() @@ -154,6 +161,7 @@ impl CustomRuleCache { self.results.extend(lookup.hits); self.hits = lookup.hit_count; self.misses = lookup.miss_count; + self.counts_by_code = lookup.counts_by_code; lookup.plan } @@ -206,6 +214,11 @@ impl CustomRuleCache { self.misses } + /// Each rule's reused and evaluated subject counts; every subject misses with caching off. + pub(crate) fn counts_by_code(&self) -> &BTreeMap { + &self.counts_by_code + } + fn extend_results(&mut self, evaluations: Vec) { for evaluation in evaluations { self.results diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs index 7baeba25dd..0445d7e3a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::rules::catalogue::references_custom_rules; use crate::bindings::_helpers::rules::rule_dispatch::speculative_rules_request; use crate::bindings::_helpers::rules::rule_plan::rule_run_plan; -use crate::bindings::_helpers::rules::rules_config::load_rules_config; +use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; use crate::bindings::types::CompilerDetach; const CANCELLED: &str = "native rules evaluation was cancelled"; @@ -40,6 +40,8 @@ pub(crate) struct EarlyRulesSettings { pub(crate) project_dir: PathBuf, pub(crate) dialect: String, pub(crate) no_sql_analysis: bool, + /// Whether `--no-cache` turned the Rules cache off, as the rules phase does. + pub(crate) no_cache: bool, } /// One compile thread's early built-in rules, from arming to the rules phase taking them. @@ -186,7 +188,8 @@ fn early_request( ) -> PyResult> { let py: Python<'_> = project.py(); let config: Bound<'_, PyAny> = - load_rules_config(py, settings.project_dir.clone())?.into_bound(py); + load_rules_config_with_cache(py, settings.project_dir.clone(), !settings.no_cache)? + .into_bound(py); if references_custom_rules(&config)? { return Ok(None); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index ff0b48d956..ef2dec1e10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -449,7 +449,7 @@ fn evaluate_native<'py>( cancel_on_interrupt(py, &evaluated, &run.cancel); let custom: PyResult> = match custom { Some(handle) => joined(py, handle, &run.cancel), - None => outcome(py, (Vec::new(), 0, 0), (0, 0)), + None => outcome(py, (Vec::new(), 0, 0), (0, 0), &[]), }; rules_result(run, defer_suppressions, evaluated, custom) } @@ -469,7 +469,7 @@ fn selectionless_result<'py>( run, true, evaluated, - outcome(py, (Vec::new(), 0, 0), (0, 0)), + outcome(py, (Vec::new(), 0, 0), (0, 0), &[]), ) } @@ -538,6 +538,10 @@ fn rules_result<'py>( ), ("custom_ms", field(&custom, "custom_ms")?), ("custom_cpu_ms", field(&custom, "custom_cpu_ms")?), + ( + "custom_rule_timings", + field(&custom, "custom_rule_timings")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index 0c59a06ca2..7bb4c3f5f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -127,6 +127,10 @@ pub(crate) fn finish_rule_run<'py>( ), ("custom_ms", record_field(result, "custom_ms")?), ("custom_cpu_ms", record_field(result, "custom_cpu_ms")?), + ( + "custom_rule_timings", + record_field(result, "custom_rule_timings")?, + ), ("cache_hits", total("cache_hits")?), ("cache_misses", total("cache_misses")?), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index c15e583ae7..24843d00ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -240,10 +240,22 @@ fn rules_error(py: Python<'_>, class: ErrorId, message: String, help: Option, project_dir: PathBuf) -> PyResult> { - let config: RulesConfig = match rules_config_of(&project_dir)? { + load_rules_config_with_cache(py, project_dir, true) +} + +/// The project's rules configuration, with the Rules cache off when `cache` is `false`. +/// +/// `--no-cache` passes `false`: the run neither reads nor writes built-in or custom rule results. +pub(crate) fn load_rules_config_with_cache( + py: Python<'_>, + project_dir: PathBuf, + cache: bool, +) -> PyResult> { + let mut config: RulesConfig = match rules_config_of(&project_dir)? { Ok(config) => config, Err(invalid) => return Err(invalid.into_pyerr(py)), }; + config.cache.enabled &= cache; Ok(config_object(py, &config)?.unbind()) } diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index f580959741..f1038d00e2 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -1,7 +1,7 @@ use crate::configuration::_helpers::removed_settings::removed_settings; use crate::configuration::errors::RulesConfigLoadError; use crate::constants::{ - ALLOWED_QUALIFIERS_OPTION, MIN_CUSTOM_RULE_TEST_CASES_KEY, MODEL_NAME_LAYERS, + ALLOWED_PATHS_KEY, ALLOWED_QUALIFIERS_OPTION, MIN_CUSTOM_RULE_TEST_CASES_KEY, MODEL_NAME_LAYERS, PUBLISHED_LAYERS_OPTION, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, RELATION_EXCEPTION_RULES, }; use crate::models::RulesConfig; @@ -307,27 +307,25 @@ fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> let label: String = format!("rules.rule_options.{PUBLISHED_SHAPE_CODE}.{ALLOWED_QUALIFIERS_OPTION}"); let shape: String = format!( - "{label} entries must be {{ qualifier = \"...\", reason = \"...\" }} tables with an \ - optional domain" + "{label} entries must be {{ qualifier = \"...\", reason = \"...\" }} or \ + {{ paths = [\"...\"], reason = \"...\" }} tables with an optional domain" ); let Some(entries) = entries.as_array() else { return Err(shape); }; - let mut seen: std::collections::BTreeSet<(Option<&str>, &str)> = + let mut seen: std::collections::BTreeSet<(Option<&str>, Option<&str>, Vec<&str>)> = std::collections::BTreeSet::new(); for entry in entries { let Some(table) = entry.as_object() else { return Err(shape); }; - if table - .keys() - .any(|key| !["domain", "qualifier", "reason"].contains(&key.as_str())) - { + if table.keys().any(|key| { + !["domain", "qualifier", ALLOWED_PATHS_KEY, "reason"].contains(&key.as_str()) + }) { return Err(shape); } let text = |key: &str| table.get(key).map(|value| value.as_str().map(str::trim)); - let (Some(Some(qualifier)), Some(Some(reason))) = (text("qualifier"), text("reason")) - else { + let Some(Some(reason)) = text("reason") else { return Err(shape); }; let domain: Option<&str> = match text("domain") { @@ -335,13 +333,49 @@ fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> Some(Some(domain)) if !domain.is_empty() => Some(domain), Some(_) => return Err(shape), }; - if qualifier.is_empty() || reason.is_empty() { + let qualifier: Option<&str> = match text("qualifier") { + None => None, + Some(Some(qualifier)) => Some(qualifier), + Some(None) => return Err(shape), + }; + let paths: Option> = match table.get(ALLOWED_PATHS_KEY) { + None => None, + Some(paths) => { + let Some(paths) = paths.as_array() else { + return Err(shape); + }; + let paths: Option> = + paths.iter().map(|path| path.as_str().map(str::trim)).collect(); + let Some(mut paths) = paths else { + return Err(shape); + }; + paths.sort_unstable(); + Some(paths) + } + }; + if qualifier.is_none() && paths.is_none() { + return Err(shape); + } + if qualifier.is_some_and(str::is_empty) || reason.is_empty() { return Err(format!( "{label} entries need a non-empty qualifier and reason" )); } - if !seen.insert((domain, qualifier)) { - return Err(format!("duplicate {label} entry: {qualifier}")); + if paths + .as_ref() + .is_some_and(|paths| paths.is_empty() || paths.iter().any(|path| path.is_empty())) + { + return Err(format!( + "{label} entries need a non-empty list of non-empty paths" + )); + } + let paths: Vec<&str> = paths.unwrap_or_default(); + let described: String = match qualifier { + Some(qualifier) => qualifier.to_owned(), + None => paths.join(", "), + }; + if !seen.insert((domain, qualifier, paths)) { + return Err(format!("duplicate {label} entry: {described}")); } } Ok(()) diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index ad865fb4e1..6952d90a7a 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -55,5 +55,7 @@ pub(crate) const PUBLISHED_SHAPE_CODE: &str = "SQBRPROJECT107"; pub(crate) const PUBLISHED_LAYERS_OPTION: &str = "published_layers"; /// The option listing qualifiers that do not count towards a domain's name shape. pub(crate) const ALLOWED_QUALIFIERS_OPTION: &str = "allowed_qualifiers"; +/// The `allowed_qualifiers` entry key whose globs match published models by file path. +pub(crate) const ALLOWED_PATHS_KEY: &str = "paths"; /// The published layers when a project does not configure them. pub(crate) const DEFAULT_PUBLISHED_LAYERS: [&str; 2] = ["mart", "mart_v"]; diff --git a/crates/sqlbuild-rules/src/custom_host_pool/_helpers/decoding.rs b/crates/sqlbuild-rules/src/custom_host_pool/_helpers/decoding.rs index a6f3b24df4..ce3024fa23 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/_helpers/decoding.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/_helpers/decoding.rs @@ -8,7 +8,7 @@ use crate::custom_host_pool::_helpers::payload_values::{fact_key, python_text, r use crate::custom_host_pool::constants::{ CUSTOM_RULE_PROJECT_SUBJECT, INVALID_EVALUATION, INVALID_FINDINGS, INVALID_PAYLOAD, }; -use crate::custom_host_pool::models::{HostEvaluation, HostSlice}; +use crate::custom_host_pool::models::{HostEvaluation, HostSlice, HostTimings, RuleTiming}; use crate::custom_host_pool::types::FactKey; /// The slice a host response describes, or the `RulesError` message of a malformed one. @@ -51,9 +51,36 @@ pub(crate) fn decoded_slice(response: &Value) -> Result { Some(_) | None => BTreeSet::new(), }, untracked_reads, + timings: payload.get("timings").map(timings).unwrap_or_default(), }) } +/// The host's reported timings; anything malformed counts as zero. +fn timings(value: &Value) -> HostTimings { + let count = |key: &str| value.get(key).and_then(Value::as_u64).unwrap_or(0); + let rules = match value.get("rules") { + Some(Value::Object(rules)) => rules + .iter() + .map(|(code, timing)| { + let part = |index: usize| timing.get(index).and_then(Value::as_u64).unwrap_or(0); + ( + code.clone(), + RuleTiming { + invocations: part(0), + cpu_ns: part(1), + }, + ) + }) + .collect(), + Some(_) | None => BTreeMap::new(), + }; + HostTimings { + rules, + monitor_ns: count("monitor_ns"), + guard_ns: count("guard_ns"), + } +} + fn evaluation( item: &Value, readsets: &[Vec], diff --git a/crates/sqlbuild-rules/src/custom_host_pool/constants.rs b/crates/sqlbuild-rules/src/custom_host_pool/constants.rs index aedde400c8..867aaa40d8 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/constants.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/constants.rs @@ -25,3 +25,13 @@ pub(crate) const INVALID_PAYLOAD: &str = "custom rule host returned an invalid p pub(crate) const INVALID_EVALUATION: &str = "custom rule host returned an invalid evaluation"; /// The `RulesError` message of findings that are not a list. pub(crate) const INVALID_FINDINGS: &str = "custom rule host returned invalid findings"; + +/// The `compile_timings` key of every custom-rule invocation the hosts ran. +pub const CUSTOM_RULES_INVOCATIONS_KEY: &str = "custom_rules_invocations"; +/// The `compile_timings` key of the hosts' module-state monitor CPU. +pub const CUSTOM_RULES_MONITOR_KEY: &str = "custom_rules_monitor_cpu_ms"; +/// The `compile_timings` key of the hosts' runtime-guard check CPU. +pub const CUSTOM_RULES_GUARD_KEY: &str = "custom_rules_guard_cpu_ms"; +/// The prefix of one rule code's `compile_timings` keys: `custom_rules..`. +pub(crate) const CUSTOM_RULE_KEY_PREFIX: &str = "custom_rules."; +pub(crate) const NANOSECONDS_PER_MILLISECOND: u64 = 1_000_000; diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs new file mode 100644 index 0000000000..a086b4ebac --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs @@ -0,0 +1,49 @@ +//! The flat `compile_timings` entries of one custom-rule evaluation. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::custom_host_pool::constants::{ + CUSTOM_RULE_KEY_PREFIX, CUSTOM_RULES_GUARD_KEY, CUSTOM_RULES_INVOCATIONS_KEY, + CUSTOM_RULES_MONITOR_KEY, NANOSECONDS_PER_MILLISECOND, +}; +use crate::custom_host_pool::models::{HostTimings, RuleTiming}; + +/// Run-wide totals, then each rule's invocations, CPU and cache counts in code order. +/// +/// `cache_counts` holds each rule's reused and missing subjects; rules the cache answered in +/// full report zero invocations. +#[must_use] +pub fn custom_rule_timings( + timings: &HostTimings, + cache_counts: &BTreeMap, +) -> Vec<(String, u64)> { + let invocations: u64 = timings + .rules + .values() + .map(|timing| timing.invocations) + .sum(); + let mut entries: Vec<(String, u64)> = vec![ + (CUSTOM_RULES_INVOCATIONS_KEY.to_owned(), invocations), + ( + CUSTOM_RULES_MONITOR_KEY.to_owned(), + timings.monitor_ns / NANOSECONDS_PER_MILLISECOND, + ), + ( + CUSTOM_RULES_GUARD_KEY.to_owned(), + timings.guard_ns / NANOSECONDS_PER_MILLISECOND, + ), + ]; + let codes: BTreeSet<&String> = timings.rules.keys().chain(cache_counts.keys()).collect(); + for code in codes { + let timing: RuleTiming = timings.rules.get(code).copied().unwrap_or_default(); + let (hits, misses): (usize, usize) = cache_counts.get(code).copied().unwrap_or_default(); + let key = |measure: &str| format!("{CUSTOM_RULE_KEY_PREFIX}{code}.{measure}"); + entries.extend([ + (key("invocations"), timing.invocations), + (key("cpu_ms"), timing.cpu_ns / NANOSECONDS_PER_MILLISECOND), + (key("cache_hits"), u64::try_from(hits).unwrap_or(u64::MAX)), + (key("cache_misses"), u64::try_from(misses).unwrap_or(u64::MAX)), + ]); + } + entries +} diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/merge_host_slices.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/merge_host_slices.rs index 282652d78d..f6ea3644c4 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/main/merge_host_slices.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/merge_host_slices.rs @@ -5,7 +5,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::custom_host_pool::_helpers::merging::{ collected_untracked_reads, demoted, merged_observations, ordered_evaluations, }; -use crate::custom_host_pool::models::{HostEvaluation, HostRun, HostSlice}; +use crate::custom_host_pool::models::{HostEvaluation, HostRun, HostSlice, HostTimings}; use crate::custom_host_pool::types::{FactKey, HostInvocation}; /// Merge slices in single-host invocation `order`, demoting tracking that grew too large. @@ -17,6 +17,10 @@ pub fn merge_host_slices(slices: Vec, order: &[HostInvocation]) -> Ho collected_untracked_reads(&slices, &untracked); let (evaluated, untracked_reads): (Vec, BTreeMap>) = demoted(evaluations, &untracked, untracked_reads); + let mut timings: HostTimings = HostTimings::default(); + for slice in &slices { + timings.add(&slice.timings); + } HostRun { evaluated, untracked_reads: untracked_reads @@ -28,5 +32,6 @@ pub fn merge_host_slices(slices: Vec, order: &[HostInvocation]) -> Ho .flat_map(|slice| slice.uncacheable.iter().cloned()) .collect(), observed: merged_observations(&slices), + timings, } } diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/mod.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/mod.rs index 14dbd95162..c32c0ed92e 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/main/mod.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/mod.rs @@ -1,4 +1,5 @@ pub mod available_cores; +pub mod custom_rule_timings; pub mod decode_host_response; pub mod host_count; pub mod merge_host_slices; diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/without_codes.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/without_codes.rs index c75fb8da37..e612e8cadd 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/main/without_codes.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/without_codes.rs @@ -5,6 +5,8 @@ use std::collections::BTreeSet; use crate::custom_host_pool::models::HostSlice; /// The slice without evaluations, untracked reads, uncacheable or stateful marks of `codes`. +/// +/// Timings stay: the slice's hosts spent that CPU even when a rerun replaces its results. #[must_use] pub fn without_codes(slice: HostSlice, codes: &BTreeSet) -> HostSlice { HostSlice { diff --git a/crates/sqlbuild-rules/src/custom_host_pool/models.rs b/crates/sqlbuild-rules/src/custom_host_pool/models.rs index a9cd86f651..5d944b3d90 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/models.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/models.rs @@ -30,6 +30,35 @@ pub struct HostSlice { pub observed: BTreeMap>, pub bounded: bool, pub stateful: BTreeSet, + pub timings: HostTimings, +} + +/// One rule's invocations and the CPU nanoseconds they took, excluding runtime-guard checks. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RuleTiming { + pub invocations: u64, + pub cpu_ns: u64, +} + +/// Where host CPU went: each rule, the module-state monitor and the runtime guard's checks. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HostTimings { + pub rules: BTreeMap, + pub monitor_ns: u64, + pub guard_ns: u64, +} + +impl HostTimings { + /// Add `other`'s invocations and CPU to these timings. + pub fn add(&mut self, other: &Self) { + for (code, timing) in &other.rules { + let total: &mut RuleTiming = self.rules.entry(code.clone()).or_default(); + total.invocations += timing.invocations; + total.cpu_ns += timing.cpu_ns; + } + self.monitor_ns += other.monitor_ns; + self.guard_ns += other.guard_ns; + } } /// Every host's results combined exactly as one host would have reported them. @@ -39,4 +68,6 @@ pub struct HostRun { pub untracked_reads: BTreeMap>, pub uncacheable: BTreeSet, pub observed: BTreeMap>, + /// Every host's timings summed, including hosts whose results a rerun replaced. + pub timings: HostTimings, } diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs index 1340930d24..adc33e2f55 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use crate::custom_host_pool::models::HostEvaluation; +use crate::custom_host_pool::models::{HostEvaluation, HostTimings, RuleTiming}; use crate::custom_host_pool::types::{HostInvocation, HostPlan}; /// Write a fake `/proc/self/cgroup` and a cgroup hierarchy below `root`. @@ -54,3 +54,23 @@ pub(super) fn evaluation(code: &str, subject: &str, reads: Option<&[&[&str]]>) - }), } } + +/// Host timings with the given `(code, invocations, cpu_ns)` rules. +pub(super) fn rule_timings(rules: &[(&str, u64, u64)], monitor_ns: u64, guard_ns: u64) -> HostTimings { + HostTimings { + rules: rules + .iter() + .map(|(code, invocations, cpu_ns)| { + ( + (*code).to_owned(), + RuleTiming { + invocations: *invocations, + cpu_ns: *cpu_ns, + }, + ) + }) + .collect(), + monitor_ns, + guard_ns, + } +} diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs index f35430e473..186dc27b79 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs @@ -5,12 +5,15 @@ use serde_json::json; use crate::custom_host_pool::constants::MIN_INVOCATIONS_PER_HOST; use crate::custom_host_pool::main::available_cores::available_cores; +use crate::custom_host_pool::main::custom_rule_timings::custom_rule_timings; use crate::custom_host_pool::main::decode_host_response::decode_host_response; use crate::custom_host_pool::main::host_count::host_count; use crate::custom_host_pool::main::merge_host_slices::merge_host_slices; use crate::custom_host_pool::main::partition_host_plan::partition_host_plan; -use crate::custom_host_pool::models::HostSlice; -use crate::custom_host_pool::tests::helpers::{evaluation, invocations, plan, write_cgroup_files}; +use crate::custom_host_pool::models::{HostSlice, HostTimings, RuleTiming}; +use crate::custom_host_pool::tests::helpers::{ + evaluation, invocations, plan, rule_timings, write_cgroup_files, +}; use crate::custom_host_pool::tests::test_types::{ DecodeTestCase, HostCapacityTestCase, HostCountTestCase, HostPlanTestCase, MergeTestCase, }; @@ -209,6 +212,7 @@ fn given_host_response_when_decoding_then_slice_or_rules_error_is_returned() { "observed": [[["tree.paths"], "abc"], [["tree.glob", "*.sql"], null]], "bounded": true, "stateful": ["R1"], + "timings": {"rules": {"R1": [2, 500], "R2": "bad"}, "monitor_ns": 7, "guard_ns": 3}, }), expected_slice: Ok(HostSlice { evaluations: vec![ @@ -229,6 +233,20 @@ fn given_host_response_when_decoding_then_slice_or_rules_error_is_returned() { ]), bounded: true, stateful: BTreeSet::from(["R1".to_owned()]), + timings: HostTimings { + rules: BTreeMap::from([ + ( + "R1".to_owned(), + RuleTiming { + invocations: 2, + cpu_ns: 500, + }, + ), + ("R2".to_owned(), RuleTiming::default()), + ]), + monitor_ns: 7, + guard_ns: 3, + }, }), }, DecodeTestCase { @@ -264,11 +282,13 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { HostSlice { evaluations: vec![evaluation("R1", "b", Some(&[&["k"]]))], observed: BTreeMap::from([(vec!["t".to_owned()], Some("1".to_owned()))]), + timings: rule_timings(&[("R1", 1, 40)], 5, 1), ..HostSlice::default() }, HostSlice { evaluations: vec![evaluation("R1", "a", Some(&[&["k"]]))], observed: BTreeMap::from([(vec!["t".to_owned()], Some("2".to_owned()))]), + timings: rule_timings(&[("R1", 1, 60), ("R2", 3, 9)], 7, 2), ..HostSlice::default() }, ], @@ -276,6 +296,7 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { expected_order: invocations(&[("R1", "a"), ("R1", "b")]), expected_untracked: vec![], expected_observed: vec![(vec!["t".to_owned()], None)], + expected_timings: rule_timings(&[("R1", 2, 100), ("R2", 3, 9)], 12, 3), }, MergeTestCase { description: "a bounded host untracks every tracked rule", @@ -288,6 +309,7 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { expected_order: invocations(&[("R1", "a")]), expected_untracked: vec!["R1".to_owned()], expected_observed: vec![], + expected_timings: HostTimings::default(), }, ]; for test_case in test_cases { @@ -311,5 +333,40 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { "{}", test_case.description ); + assert_eq!( + run.timings, test_case.expected_timings, + "{}", + test_case.description + ); } } + +#[test] +fn given_host_timings_and_cache_counts_when_reporting_then_flat_entries_cover_every_rule() { + let timings: HostTimings = rule_timings(&[("R2", 3, 4_500_000)], 2_900_000, 1_000_000); + let cache_counts: BTreeMap = + BTreeMap::from([("R1".to_owned(), (5, 0)), ("R2".to_owned(), (2, 3))]); + + let entries: Vec<(String, u64)> = custom_rule_timings(&timings, &cache_counts); + + let expected: Vec<(&str, u64)> = vec![ + ("custom_rules_invocations", 3), + ("custom_rules_monitor_cpu_ms", 2), + ("custom_rules_guard_cpu_ms", 1), + ("custom_rules.R1.invocations", 0), + ("custom_rules.R1.cpu_ms", 0), + ("custom_rules.R1.cache_hits", 5), + ("custom_rules.R1.cache_misses", 0), + ("custom_rules.R2.invocations", 3), + ("custom_rules.R2.cpu_ms", 4), + ("custom_rules.R2.cache_hits", 2), + ("custom_rules.R2.cache_misses", 3), + ]; + assert_eq!( + entries, + expected + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect::>() + ); +} diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs index 16ea800bd3..7bcde96b8c 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::custom_host_pool::models::HostSlice; +use crate::custom_host_pool::models::{HostSlice, HostTimings}; use crate::custom_host_pool::types::{HostInvocation, HostPlan}; pub(super) struct HostCapacityTestCase { @@ -37,4 +37,5 @@ pub(super) struct MergeTestCase { pub(super) expected_order: Vec, pub(super) expected_untracked: Vec, pub(super) expected_observed: Vec<(Vec, Option)>, + pub(super) expected_timings: HostTimings, } diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs index 7e05b58abf..1ca0843694 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs @@ -17,7 +17,7 @@ fn reusable<'a>( (digests.get(readset) == Some(&entry.reads_digest)).then_some(entry) } -/// Hits per rule and subject, and each rule's missing subjects (`None` when all miss). +/// Hits per rule and subject, each rule's missing subjects (`None` when all miss) and counts. #[must_use] pub fn lookup_custom_rule_cache( stored: &StoredRules, @@ -28,16 +28,21 @@ pub fn lookup_custom_rule_cache( for rule in rules { let cached: Option<&StoredRule> = current_rule(stored, rule); let mut missing: Vec = Vec::new(); + let mut hits: usize = 0; for subject in &rule.subjects { match reusable(cached, (&subject.key, &subject.identity), digests) { Some(entry) => { let key = (rule.code.clone(), subject.key.clone()); lookup.hits.push((key, entry.findings.clone())); lookup.hit_count += 1; + hits += 1; } None => missing.push(subject.key.clone()), } } + lookup + .counts_by_code + .insert(rule.code.clone(), (hits, missing.len())); if missing.is_empty() { continue; } diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/models.rs b/crates/sqlbuild-rules/src/custom_rule_cache/models.rs index d464c88238..d4fef8775f 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/models.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/models.rs @@ -76,6 +76,8 @@ pub struct CustomCacheLookup { pub plan: BTreeMap>>, pub hit_count: usize, pub miss_count: usize, + /// Each rule's reused and missing subject counts. + pub counts_by_code: BTreeMap, } /// One evaluated invocation, with its read set and digest when its result may be cached. diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs index 807431d92f..d253b278c4 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs @@ -122,5 +122,11 @@ fn given_stored_results_when_looking_up_then_unchanged_subjects_and_reads_hit() "{}", test_case.description ); + assert_eq!( + lookup.counts_by_code.values().copied().collect::>(), + vec![(lookup.hit_count, lookup.miss_count)], + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs index 80da53a289..920dca63b0 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs @@ -149,6 +149,65 @@ fn given_published_models_when_evaluating_layer_rules_then_reads_and_shapes_matc "allowed_qualifiers = [{ domain = \"sales\", qualifier = \"fra\"", )], }, + PublishedLayerTestCase { + description: "one qualifier glob replaces a literal entry per country", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__customers", &[]), + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__deu", &[]), + ("sales__mart__orders__ita", &[]), + ("sales__mart__returns__fra_snapshots", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"domain": "sales", "qualifier": "[a-z][a-z][a-z]", "reason": "Published once per country."}, + {"domain": "sales", "qualifier": "[a-z][a-z][a-z]_snapshots", "reason": "Daily snapshots per country."} + ]}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a qualifier glob is case-sensitive and leaves other qualifiers flagged", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__customers", &[]), + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__finance", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"qualifier": "[A-Z][A-Z][A-Z]", "reason": "Published once per country."}, + {"domain": "finance", "qualifier": "[a-z][a-z][a-z]", "reason": "Per country."} + ]}), + expected_faults: &[ + ( + "models/sales/mart/sales__mart__orders__finance.sql", + "domain sales publishes 2 models as ____ and 2 as", + ), + ( + "models/sales/mart/sales__mart__orders__fra.sql", + "sales__mart__orders__fra does not fit the majority shape", + ), + ], + }, + PublishedLayerTestCase { + description: "a paths entry leaves out the published models in its folder", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__deu", &[]), + ("finance__mart__revenue", &[]), + ("finance__mart__revenue__fra", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"paths": ["models/sales/mart/*"], "reason": "Published once per country."} + ]}), + expected_faults: &[( + "models/finance/mart/finance__mart__revenue__fra.sql", + "domain finance publishes 1 model as ____ and 1 as", + )], + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs index 8264cd1c62..1699004a5f 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::Value; use crate::constants::{ - ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, INTERMEDIATE_LAYER_DIRECTORY, + ALLOWED_PATHS_KEY, ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, INTERMEDIATE_LAYER_DIRECTORY, MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, STAGING_LAYER_DIRECTORY, }; @@ -11,6 +11,7 @@ use crate::models::{EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; use crate::rules::_helpers::evaluation::{parse_name, path_fault}; use crate::rules::models::{FaultCollector, ProjectEvaluationRequest}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::text::main::glob_match::glob_match; const NO_CANONICAL_LAYER_FOLDER: &str = "no canonical layer folder"; /// Where an intermediate model that replaces a published one lives. @@ -350,10 +351,33 @@ fn intermediate_move(model: &Published<'_>) -> Option { )) } -/// One literal qualifier that does not count towards a domain's name shape. +/// One `allowed_qualifiers` entry: the published models it leaves out of the name-shape check. +/// +/// `qualifier` and `paths` are case-sensitive globs; a model must match every part it sets. struct AllowedQualifier { domain: Option, - qualifier: String, + qualifier: Option, + paths: Vec, +} + +impl AllowedQualifier { + fn matches(&self, entry: &Published<'_>) -> bool { + self.domain + .as_ref() + .is_none_or(|domain| *domain == entry.name.domain) + && self.qualifier.as_ref().is_none_or(|pattern| { + entry + .name + .source + .as_ref() + .is_some_and(|qualifier| glob_match(qualifier, pattern)) + }) + && (self.paths.is_empty() + || self + .paths + .iter() + .any(|pattern| glob_match(&entry.model.relative_path, pattern))) + } } fn allowed_qualifiers(config: &RulesConfig, code: &str) -> Vec { @@ -369,27 +393,29 @@ fn allowed_qualifiers(config: &RulesConfig, code: &str) -> Vec .iter() .filter_map(Value::as_object) .filter_map(|entry| { - Some(AllowedQualifier { - domain: entry - .get("domain") - .and_then(Value::as_str) - .map(str::to_owned), - qualifier: entry.get("qualifier")?.as_str()?.to_owned(), - }) + let text = |key: &str| entry.get(key).and_then(Value::as_str).map(str::to_owned); + let allowed = AllowedQualifier { + domain: text("domain"), + qualifier: text("qualifier"), + paths: entry + .get(ALLOWED_PATHS_KEY) + .and_then(Value::as_array) + .map(|paths| { + paths + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default(), + }; + (allowed.qualifier.is_some() || !allowed.paths.is_empty()).then_some(allowed) }) .collect() } -fn allowlisted(allowed: &[AllowedQualifier], name: &PublishedName) -> bool { - name.source.as_ref().is_some_and(|qualifier| { - allowed.iter().any(|entry| { - entry.qualifier == *qualifier - && entry - .domain - .as_ref() - .is_none_or(|domain| *domain == name.domain) - }) - }) +fn allowlisted(allowed: &[AllowedQualifier], entry: &Published<'_>) -> bool { + allowed.iter().any(|allowed| allowed.matches(entry)) } /// Flag published models whose name shape differs from their domain's majority shape. @@ -405,7 +431,7 @@ pub(crate) fn published_name_shapes( let Some(entry) = published(model, &layers) else { continue; }; - if allowlisted(&allowed, &entry.name) { + if allowlisted(&allowed, &entry) { continue; } domains diff --git a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs index aa1069b965..41c222fea5 100644 --- a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs +++ b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs @@ -59,8 +59,11 @@ pub fn builtin_options(code: &str) -> Vec { BuiltinOption { kind: BuiltinOptionKind::TableList, name: ALLOWED_QUALIFIERS_OPTION, - description: "Literal qualifiers that do not count towards a domain's name \ - shape, as { qualifier, reason } tables with an optional domain.", + description: "Qualifiers that do not count towards a domain's name shape, as \ + { qualifier, reason } tables with an optional domain. qualifier \ + is a case-sensitive glob such as \"[a-z][a-z][a-z]\"; \ + { paths = [...], reason } instead leaves out published models \ + whose file path matches a glob.", }, ], "SQBRTEST203" => vec![BuiltinOption { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f22d759fa1..6a5efcb27f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -10704,6 +10704,7 @@ class RulesRunResult(NativeRecord): cache_hits: int cache_misses: int skipped_type_proof_rules: tuple[str, ...] + custom_rule_timings: tuple[tuple[str, int], ...] def __init__( self, findings: tuple[Finding, ...], @@ -10715,6 +10716,7 @@ class RulesRunResult(NativeRecord): cache_hits: int = ..., cache_misses: int = ..., skipped_type_proof_rules: tuple[str, ...] = ..., + custom_rule_timings: tuple[tuple[str, int], ...] = ..., ) -> None: ... class CustomRulesOutcome(NativeRecord): @@ -10725,6 +10727,7 @@ class CustomRulesOutcome(NativeRecord): cache_misses: int custom_ms: int custom_cpu_ms: int + custom_rule_timings: tuple[tuple[str, int], ...] def __init__( self, findings: tuple[Finding, ...], @@ -10732,6 +10735,7 @@ class CustomRulesOutcome(NativeRecord): cache_misses: int, custom_ms: int, custom_cpu_ms: int = ..., + custom_rule_timings: tuple[tuple[str, int], ...] = ..., ) -> None: ... class ResourceKind(StrEnum): @@ -15835,6 +15839,7 @@ class RulesResult(NativeRecord): built_in_ms: int custom_ms: int custom_cpu_ms: int + custom_rule_timings: tuple[tuple[str, int], ...] def __init__( self, findings: tuple[Finding, ...], @@ -15845,6 +15850,7 @@ class RulesResult(NativeRecord): built_in_ms: int = ..., custom_ms: int = ..., custom_cpu_ms: int = ..., + custom_rule_timings: tuple[tuple[str, int], ...] = ..., ) -> None: ... class RuleFile(NativeRecord): @@ -15929,6 +15935,8 @@ class CustomRuleRun(NativeRecord): uncacheable_codes: frozenset[str] observed: tuple[tuple[tuple[str, ...], str | None], ...] stateful_codes: frozenset[str] + rule_timings: dict[str, tuple[int, int]] + monitor_ns: int def __init__( self, evaluations: tuple[CustomRuleEvaluation, ...], @@ -15936,6 +15944,8 @@ class CustomRuleRun(NativeRecord): uncacheable_codes: frozenset[str] = ..., observed: tuple[tuple[tuple[str, ...], str | None], ...] = ..., stateful_codes: frozenset[str] = ..., + rule_timings: dict[str, tuple[int, int]] = ..., + monitor_ns: int = ..., ) -> None: ... class CustomHostPartition(NativeRecord): diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 355cf43f44..f4de4d1ff3 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -5,9 +5,10 @@ from collections import Counter from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext -from dataclasses import dataclass +from dataclasses import dataclass, field from itertools import chain from pathlib import Path +from time import thread_time_ns from sqlbuild._native import ( CUSTOM_RULE_PROJECT_SUBJECT, @@ -41,6 +42,18 @@ type _FindingKey = tuple[str, str, int, int, str, str] +@dataclass +class _RuleTimings: + """Invocations and CPU nanoseconds per rule code, excluding the runtime guard's checks.""" + + by_code: dict[str, list[int]] = field(default_factory=dict) + + def record(self, *, code: str, cpu_ns: int) -> None: + totals: list[int] = self.by_code.setdefault(code, [0, 0]) + totals[0] += 1 + totals[1] += max(cpu_ns, 0) + + @dataclass(frozen=True) class _PassInputs: project: RuleFactProject @@ -52,6 +65,7 @@ class _PassInputs: plan: CustomRulePlan | None guard: RuntimeGuard | None cancelled: Callable[[], bool] | None + timings: _RuleTimings def evaluate_custom_rules( @@ -98,6 +112,7 @@ def evaluate_custom_rules( plan=plan, guard=guard, cancelled=None if partition is None else partition.cancelled, + timings=_RuleTimings(), ) def run_pass( @@ -108,12 +123,15 @@ def run_pass( evaluations: list[CustomRuleEvaluation] = run_pass( rules=custom_rules, subjects=models, active=tracker ) + monitor_ns: int = tracker.monitor_ns if verify_determinism: + repeat_tracker: _ReadTracker = _ReadTracker( + reads=None, rules=(), rules_root=project_dir / "rules" + ) repeated: list[CustomRuleEvaluation] = run_pass( - rules=custom_rules[::-1], - subjects=models[::-1], - active=_ReadTracker(reads=None, rules=(), rules_root=project_dir / "rules"), + rules=custom_rules[::-1], subjects=models[::-1], active=repeat_tracker ) + monitor_ns += repeat_tracker.monitor_ns _verify_repeatable( rules=custom_rules, first=list(chain.from_iterable(item.findings for item in evaluations)), @@ -125,6 +143,11 @@ def run_pass( uncacheable_codes=frozenset(tracker.uncacheable_codes), observed=() if reads is None else tuple(reads.observed.items()), stateful_codes=frozenset(tracker.stateful_codes), + rule_timings={ + code: (invocations, cpu_ns) + for code, (invocations, cpu_ns) in inputs.timings.by_code.items() + }, + monitor_ns=monitor_ns, ) @@ -143,6 +166,8 @@ def __init__( self.untracked_codes: set[str] = set() self.uncacheable_codes: set[str] = set() self.stateful_codes: set[str] = set() + self.monitor_ns: int = 0 + started: int = thread_time_ns() self._state: ModuleStateMonitor | None = ( ModuleStateMonitor( namespaces={rule.code: rule_namespaces((rule.check,)) for rule in rules}, @@ -151,6 +176,7 @@ def __init__( if reads is not None or detect_state else None ) + self.monitor_ns += thread_time_ns() - started def start(self) -> None: if self.reads is not None: @@ -168,7 +194,11 @@ def finish(self, *, code: str) -> frozenset[tuple[object, ...]] | None: self.untracked_codes.add(code) if reads.uncacheable: self.uncacheable_codes.add(code) - changed: tuple[str, ...] = () if self._state is None else self._state.changed_codes() + changed: tuple[str, ...] = () + if self._state is not None: + started: int = thread_time_ns() + changed = self._state.changed_codes() + self.monitor_ns += thread_time_ns() - started self.stateful_codes.update(changed) if reads is not None: self.untracked_codes.update(changed) @@ -230,6 +260,7 @@ def context(rule: Rule) -> EvaluationRuleContext: ), ctx=ctx, guard=guard, + timings=inputs.timings, ) except NonHermeticRuleError: raise @@ -252,7 +283,9 @@ def context(rule: Rule) -> EvaluationRuleContext: _stop_if_cancelled(inputs) tracker.start() try: - findings = _invoke(rule=rule, subject=subject, ctx=ctx, guard=guard) + findings = _invoke( + rule=rule, subject=subject, ctx=ctx, guard=guard, timings=inputs.timings + ) except NonHermeticRuleError: raise except Exception as error: @@ -278,18 +311,29 @@ def _stop_if_cancelled(inputs: _PassInputs) -> None: def _invoke( - *, rule: Rule, subject: object, ctx: EvaluationRuleContext, guard: RuntimeGuard | None + *, + rule: Rule, + subject: object, + ctx: EvaluationRuleContext, + guard: RuntimeGuard | None, + timings: _RuleTimings, ) -> list[Finding]: if rule.subject_parameter is None or rule.context_parameter is None: raise RulesError(f"custom rule {rule.code} has no resolved typed signature") scope: AbstractContextManager[None] = nullcontext() if guard is None else guard.rule(rule.code) - with scope: - result: object = rule.check( - **{ - rule.subject_parameter: subject, - rule.context_parameter: ctx, - } - ) + guard_started: int = 0 if guard is None else guard.check_ns + started: int = thread_time_ns() + try: + with scope: + result: object = rule.check( + **{ + rule.subject_parameter: subject, + rule.context_parameter: ctx, + } + ) + finally: + guard_ns: int = 0 if guard is None else guard.check_ns - guard_started + timings.record(code=rule.code, cpu_ns=thread_time_ns() - started - guard_ns) if not isinstance(result, list) or any(not isinstance(finding, Finding) for finding in result): raise RulesError(f"custom rule {rule.code} must return list[Finding]") return result diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index fbc81eb9c7..4b1dad7dd5 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -108,7 +108,7 @@ def main() -> int: "protocol": CUSTOM_HOST_PROTOCOL_VERSION, "runtime_version": CUSTOM_HOST_RUNTIME_VERSION, "error": None, - "payload": _run_payload(run), + "payload": _run_payload(run=run, guard_ns=0 if guard is None else guard.check_ns), "messages": messages.getvalue().splitlines(), } sys.stdout.write(json.dumps(response, sort_keys=True)) @@ -193,7 +193,7 @@ def _decode_plan(value: object) -> CustomRulePlan: return plan -def _run_payload(run: CustomRuleRun) -> dict[str, object]: +def _run_payload(*, run: CustomRuleRun, guard_ns: int) -> dict[str, object]: """Encode invocations with read sets deduplicated across subjects and bounded in size.""" readsets: dict[frozenset[tuple[object, ...]], int] = {} @@ -231,6 +231,11 @@ def _run_payload(run: CustomRuleRun) -> dict[str, object]: "observed": sorted([list(key), digest] for key, digest in run.observed), "bounded": bounded, "stateful": sorted(run.stateful_codes), + "timings": { + "rules": {code: list(timing) for code, timing in sorted(run.rule_timings.items())}, + "monitor_ns": run.monitor_ns, + "guard_ns": guard_ns, + }, } diff --git a/src/sqlbuild/rule_engine/classes/runtime_guard.py b/src/sqlbuild/rule_engine/classes/runtime_guard.py index 03fc03ba1e..21c0374069 100644 --- a/src/sqlbuild/rule_engine/classes/runtime_guard.py +++ b/src/sqlbuild/rule_engine/classes/runtime_guard.py @@ -10,6 +10,7 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager from pathlib import Path +from time import thread_time_ns from types import CodeType, FrameType, FunctionType, ModuleType from sqlbuild._native import NonHermeticRuleError, allowed_custom_rule_import @@ -82,11 +83,18 @@ def __init__(self, *, project_dir: Path) -> None: self._metadata_scope: threading.local = threading.local() self.active_rule: str | None = None self.violation: NonHermeticRuleError | None = None + self.check_ns: int = 0 def __call__(self, event: str, arguments: tuple[object, ...]) -> None: if event not in _GUARDED_EVENTS and not event.startswith(_GUARDED_PREFIXES): return - frame: FrameType = sys._getframe(1) + started: int = thread_time_ns() + try: + self._check(event=event, arguments=arguments, frame=sys._getframe(1)) + finally: + self.check_ns += thread_time_ns() - started + + def _check(self, *, event: str, arguments: tuple[object, ...], frame: FrameType) -> None: rule_frame: FrameType | None = ( self._importing_rule_frame(frame) if event == _IMPORT_EVENT @@ -127,12 +135,18 @@ def _call_metadata( function: Callable[..., object] = guarded_function if getattr(self._metadata_scope, "trusted", False): return function(*arguments, **keywords) - paths: tuple[object, ...] = tuple(map(_plain_path, arguments)) - named: dict[str, object] = {key: _plain_path(value) for key, value in keywords.items()} - rule_frame: FrameType | None = self._responsible_rule_frame(sys._getframe(1)) - if rule_frame is not None: - target: str = f" for {paths[0]!r}" if paths else "" - self._reject(rule_frame=rule_frame, action=f"{_METADATA_ACTION}{target}") + started: int = thread_time_ns() + try: + paths: tuple[object, ...] = tuple(map(_plain_path, arguments)) + named: dict[str, object] = { + key: _plain_path(value) for key, value in keywords.items() + } + rule_frame: FrameType | None = self._responsible_rule_frame(sys._getframe(1)) + if rule_frame is not None: + target: str = f" for {paths[0]!r}" if paths else "" + self._reject(rule_frame=rule_frame, action=f"{_METADATA_ACTION}{target}") + finally: + self.check_ns += thread_time_ns() - started self._metadata_scope.trusted = True try: return function(*paths, **named) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 0107e1a80e..0947504e0d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -72,6 +72,9 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "built_in_rules_ms", "custom_rules_ms", "custom_rules_cpu_ms", + "custom_rules_invocations", + "custom_rules_monitor_cpu_ms", + "custom_rules_guard_cpu_ms", "early_lint_wait_ms", "model_analysis_ms", "model_analysis_cpu_ms", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index a8a6b4ddf9..0a8455576a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -315,3 +315,25 @@ class SelectedGraphRuleCase: expected_compile_findings: tuple[tuple[str, str], ...] expected_plan_findings: tuple[tuple[str, str], ...] expected_plan_exit: int + + +@dataclass(frozen=True) +class CustomRuleTimingCase: + """Custom Rules compiled cold, warm, with `--no-cache`, and warm again.""" + + description: str + model_count: int + selected_rules: tuple[str, ...] + files: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class ModuleStateAttributionCase: + """Rules whose module state changes persist or are undone within one model.""" + + description: str + model_count: int + selected_rules: tuple[str, ...] + files: tuple[tuple[str, str], ...] + edited_model: str + expected_misses_after_edit: dict[str, int] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index f8aaf7541f..d496b3808f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -114,6 +114,7 @@ def run_compile_cli( environment: tuple[tuple[str, str], ...] = (), working_directory: Path | None = None, preexec: Callable[[], None] | None = None, + arguments: tuple[str, ...] = (), ) -> subprocess.CompletedProcess[str]: """Run the real `sqb compile --json` command from an optional invocation directory.""" @@ -125,6 +126,7 @@ def run_compile_cli( str(project_dir), "compile", "--json", + *arguments, ], capture_output=True, text=True, @@ -206,6 +208,37 @@ def rule_cache_hits(result: subprocess.CompletedProcess[str]) -> int: return int(json.loads(result.stdout)["compile_timings"]["rule_cache_hits"]) +def custom_rule_timings( + result: subprocess.CompletedProcess[str], +) -> dict[str, tuple[int, int, int, int]]: + """Map each custom Rule code to its invocations, CPU ms, cache hits and cache misses.""" + + timings: dict[str, int] = json.loads(result.stdout)["compile_timings"] + codes: set[str] = { + key.split(".")[1] for key in timings if key.startswith("custom_rules.") + } + return { + code: ( + timings[f"custom_rules.{code}.invocations"], + timings[f"custom_rules.{code}.cpu_ms"], + timings[f"custom_rules.{code}.cache_hits"], + timings[f"custom_rules.{code}.cache_misses"], + ) + for code in codes + } + + +def custom_rule_cache_counts( + result: subprocess.CompletedProcess[str], +) -> dict[str, tuple[int, int, int]]: + """Map each custom Rule code to its invocations, cache hits and cache misses.""" + + return { + code: (invocations, hits, misses) + for code, (invocations, _, hits, misses) in custom_rule_timings(result).items() + } + + def string_set_order(*, names: tuple[str, ...], hash_seed: str) -> str: """Return the comma-joined iteration order of a string set under one hash seed.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py new file mode 100644 index 0000000000..5d426deaa8 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py @@ -0,0 +1,168 @@ +"""Real CLI coverage of per-Rule custom-rule timings, `--no-cache` and module-state attribution.""" + +import json +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( + CustomRuleTimingCase, + ModuleStateAttributionCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + custom_rule_cache_counts, + custom_rule_source, + custom_rule_timings, + rule_cache_hits, + run_compile_cli, + split_model_files, + write_custom_rule_project, +) + +_FULL_COMPILE: tuple[tuple[str, str], ...] = ( + ("SQLBUILD_DISABLE_COMPILE_REUSE", "1"), + ("SQLBUILD_DISABLE_DELTA_COMPILE", "1"), +) +_MODEL_COUNT: int = 4 +_RULE_HEADER: str = "from sqlbuild.rules import Finding, Model, RuleContext, rule\n" + + +def _rule(*, code: str, body: str, header: str = _RULE_HEADER, function: str = "check") -> str: + return custom_rule_source(code=code, header=header, body=body).replace( + "def check(", f"def {function}(", 1 + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CustomRuleTimingCase( + description="two model rules in separate files", + model_count=_MODEL_COUNT, + selected_rules=("XSQBRTIME001", "XSQBRTIME002"), + files=( + ( + "rules/names.py", + _rule(code="XSQBRTIME001", body=" return [] if model.name else []\n"), + ), + ( + "rules/paths.py", + _rule( + code="XSQBRTIME002", + body=" return [] if model.path.suffix == '.sql' else []\n", + ), + ), + *split_model_files(_MODEL_COUNT), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_custom_rules_when_compiling_then_timings_and_no_cache_report_each_rule( + tmp_path: Path, test_case: CustomRuleTimingCase +) -> None: + write_custom_rule_project( + project_dir=tmp_path, selected_rules=test_case.selected_rules, files=test_case.files + ) + subjects: int = test_case.model_count + 1 + + cold: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) + warm: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) + uncached: subprocess.CompletedProcess[str] = run_compile_cli( + tmp_path, environment=_FULL_COMPILE, arguments=("--no-cache",) + ) + rewarmed: subprocess.CompletedProcess[str] = run_compile_cli( + tmp_path, environment=_FULL_COMPILE + ) + + for result in (cold, warm, uncached, rewarmed): + assert result.returncode == 0, result.stdout + result.stderr + evaluated: dict[str, tuple[int, int, int]] = { + code: (subjects, 0, subjects) for code in test_case.selected_rules + } + reused: dict[str, tuple[int, int, int]] = { + code: (0, subjects, 0) for code in test_case.selected_rules + } + assert custom_rule_cache_counts(cold) == evaluated + assert custom_rule_cache_counts(warm) == reused + assert custom_rule_cache_counts(uncached) == evaluated + assert rule_cache_hits(uncached) == 0 + assert custom_rule_cache_counts(rewarmed) == reused + cold_timings: dict[str, int] = json.loads(cold.stdout)["compile_timings"] + assert cold_timings["custom_rules_invocations"] == subjects * len(test_case.selected_rules) + assert cold_timings["custom_rules_monitor_cpu_ms"] >= 0 + assert cold_timings["custom_rules_guard_cpu_ms"] >= 0 + assert all(cpu_ms >= 0 for _, cpu_ms, _, _ in custom_rule_timings(cold).values()) + assert json.loads(warm.stdout)["compile_timings"]["custom_rules_invocations"] == 0 + + +@pytest.mark.parametrize( + "test_case", + [ + ModuleStateAttributionCase( + description="a persistent mutation is attributed, one undone within a model is not", + model_count=_MODEL_COUNT, + selected_rules=("XSQBRSTATE001", "XSQBRSTATE002", "XSQBRSTATE003", "XSQBRSTATE004"), + files=( + ( + "rules/persistent.py", + _rule( + code="XSQBRSTATE001", + header=_RULE_HEADER + "SEEN: list[str] = []\n", + body=" SEEN.append(model.name)\n return []\n", + ), + ), + ( + "rules/transient.py", + _rule( + code="XSQBRSTATE002", + header=_RULE_HEADER + "SCRATCH: list[str] = []\n", + body=" SCRATCH.append(model.name)\n return []\n", + function="stage", + ) + + "\n\n" + + _rule( + code="XSQBRSTATE003", + header="", + body=" SCRATCH.clear()\n return []\n", + function="restore", + ), + ), + ( + "rules/clean.py", + _rule(code="XSQBRSTATE004", body=" return [] if model.name else []\n"), + ), + *split_model_files(_MODEL_COUNT), + ), + edited_model="models/orders_001.sql", + expected_misses_after_edit={ + "XSQBRSTATE001": _MODEL_COUNT + 1, + "XSQBRSTATE002": 0, + "XSQBRSTATE003": 0, + "XSQBRSTATE004": 0, + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_rule_module_state_when_recompiling_an_edit_then_only_persistent_state_is_untracked( + tmp_path: Path, test_case: ModuleStateAttributionCase +) -> None: + write_custom_rule_project( + project_dir=tmp_path, selected_rules=test_case.selected_rules, files=test_case.files + ) + + cold: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) + (tmp_path / test_case.edited_model).write_text( + 'MODEL (description "Orders");\nSELECT 2 AS order_id\n', encoding="utf-8" + ) + edited: subprocess.CompletedProcess[str] = run_compile_cli( + tmp_path, environment=_FULL_COMPILE + ) + + assert cold.returncode == 0, cold.stdout + cold.stderr + assert edited.returncode == 0, edited.stdout + edited.stderr + assert { + code: misses for code, (_, _, misses) in custom_rule_cache_counts(edited).items() + } == test_case.expected_misses_after_edit diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py index 72112b7c1e..6e71507791 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py @@ -57,6 +57,30 @@ "MODEL (description 'Finance view of orders.');\n" 'SELECT o.order_id FROM __ref("sales__mart__orders") AS o\n', ) +_COUNTRY_GLOB: str = ( + "\n[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ domain = "sales", qualifier = "[a-z][a-z][a-z]", ' + 'reason = "Published once per country." }]\n' +) +_REGIONAL_PATHS: str = ( + "\n[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = ["models/sales/mart/regional/*"], ' + 'reason = "Published once per country." }]\n' +) +_COUNTRIES: tuple[tuple[str, str], ...] = tuple( + ( + f"models/sales/mart/regional/sales__mart__orders__{country}.sql", + f"MODEL (description 'Orders published for {country}.');\n" + 'SELECT o.order_id FROM __ref("sales__int_clean__orders") AS o\n', + ) + for country in ("deu", "fra", "ita") +) +_FINANCE_FROM_INTERMEDIATE: tuple[str, str] = ( + _FINANCE_PATH, + "MODEL (description 'Orders reshaped for finance.');\n" + 'SELECT o.order_id FROM __ref("sales__int_clean__orders") AS o\n', +) +_PER_COUNTRY: tuple[tuple[str, str], ...] = (_INTERMEDIATE, _ORDERS, _CUSTOMERS, *_COUNTRIES) _ORDERS_FINANCE: tuple[tuple[str, str], ...] = ( _INTERMEDIATE, _ORDERS, @@ -83,6 +107,37 @@ expected_findings=(("SQBRGRAPH103", _FINANCE_PATH),), expected_exit=1, ), + PublishedLayerCase( + description="a per-country family without an entry outnumbers the unqualified models", + files=_PER_COUNTRY, + rules_toml="", + expected_findings=( + ("SQBRPROJECT107", "models/sales/mart/sales__mart__customers.sql"), + ("SQBRPROJECT107", "models/sales/mart/sales__mart__orders.sql"), + ), + expected_exit=1, + ), + PublishedLayerCase( + description="one qualifier glob replaces a literal entry per country", + files=_PER_COUNTRY, + rules_toml=_COUNTRY_GLOB, + expected_findings=(), + expected_exit=0, + ), + PublishedLayerCase( + description="a paths entry leaves out the per-country folder", + files=_PER_COUNTRY, + rules_toml=_REGIONAL_PATHS, + expected_findings=(), + expected_exit=0, + ), + PublishedLayerCase( + description="a qualifier the glob does not match is still flagged", + files=(*_PER_COUNTRY, _FINANCE_FROM_INTERMEDIATE), + rules_toml=_COUNTRY_GLOB, + expected_findings=(("SQBRPROJECT107", _FINANCE_PATH),), + expected_exit=1, + ), PublishedLayerCase( description="shared logic in intermediate and a view of its own entity", files=(_INTERMEDIATE, _ORDERS, _CUSTOMERS, _ORDERS_VIEW, _FINANCE_FROM_PUBLISHED), diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index ad73e6c27f..5a05c3244b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -194,6 +194,41 @@ ), expected_error_pattern="allowed_qualifiers entries must be", ), + RulesConfigErrorTestCase( + description="allowed qualifier entry with neither qualifier nor paths", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ domain = "sales", reason = "Regional exports." }]\n' + ), + expected_error_pattern="allowed_qualifiers entries must be", + ), + RulesConfigErrorTestCase( + description="allowed qualifier paths that are not a list", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = "models/sales/*", reason = "Exports." }]\n' + ), + expected_error_pattern="allowed_qualifiers entries must be", + ), + RulesConfigErrorTestCase( + description="allowed qualifier with an empty paths list", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = [], reason = "Exports." }]\n' + ), + expected_error_pattern="need a non-empty list of non-empty paths", + ), + RulesConfigErrorTestCase( + description="duplicate allowed qualifier glob", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + "allowed_qualifiers = [\n" + ' { qualifier = "[a-z][a-z][a-z]", reason = "Per country." },\n' + ' { qualifier = "[a-z][a-z][a-z]", reason = "Per region." },\n' + "]\n" + ), + expected_error_pattern=r"duplicate .*allowed_qualifiers entry: \[a-z\]\[a-z\]\[a-z\]", + ), RulesConfigErrorTestCase( description="unknown SQL test rules key", source='[rules.sql_tests]\npipeline_directory = "pipelines"\nroot = "tests"\n', diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 0973b91ad2..a0ac52bf6b 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -22,7 +22,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | Flag | Description | |------|-------------| | `--no-sql-analysis` | Disable SQL syntax, binding, type inference, and semantic validation (`--no-sql-validation` remains an alias) | -| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | +| `--no-cache` | Bypass the reusable compile-analysis cache and the Rules cache for this invocation | | `--defer-to` | Resolve unselected model references against another target | | `--json` | Output the full compile report as JSON | | `--manifest` | Generate `target/manifest.json` with project metadata | @@ -202,6 +202,11 @@ Known limits: - In these cases, or whenever a result looks stale, run `sqb compile --no-cache` or set `SQLBUILD_DISABLE_COMPILE_REUSE=1`. +`--no-cache` also turns the Rules cache off for that compile: built-in and custom Rules evaluate +every subject again, and the compile neither reads nor writes stored Rule results, as with +`[rules.cache] enabled = false`. A `--no-cache` compile reports `rule_cache_hits` as `0`. The next +compile without the flag reuses whatever the cache held before. + A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, @@ -291,6 +296,27 @@ Returns a structured report including: - `lineage_mode` - which lineage mode was used - `artifacts` - paths to written files +Every `compile_timings` value is a non-negative integer. Besides the phase timings, it attributes +custom Rule cost: + +| Key | Meaning | +|-----|---------| +| `custom_rules_ms`, `custom_rules_cpu_ms` | Wall and CPU time of the custom Rules phase, including its host processes | +| `custom_rules_invocations` | Custom Rule invocations the hosts ran, one per Rule and subject | +| `custom_rules_monitor_cpu_ms` | Host CPU spent checking Rule modules for state one subject could leave for another | +| `custom_rules_guard_cpu_ms` | Host CPU spent by the runtime guard deciding whether an audited operation, such as opening a file or importing a module, came from Rule code | +| `custom_rules..invocations` | Invocations of one custom Rule | +| `custom_rules..cpu_ms` | Host CPU of those invocations, excluding the guard's checks | +| `custom_rules..cache_hits` | Subjects whose stored result was reused | +| `custom_rules..cache_misses` | Subjects evaluated because no stored result applied; every subject when the Rules cache is off | + +Each selected custom Rule has the four `custom_rules..*` keys; a Rule the cache answered in +full reports `0` invocations. The keys are flat so that `compile_timings` stays one level deep, for +example `jq '.compile_timings["custom_rules.XORDERS001.cpu_ms"]'`. Invocations and CPU include a +Rule SQLBuild evaluates again in one process because its module state changed, so they can exceed +the Rule's subject count. Monitor and guard time are separate from each Rule's CPU; the hosts' +remaining CPU goes to starting Python, loading Rule modules, and reading the project. + If compile stops on an error before it can build the report, such as a model header that does not parse, `--json` still prints one JSON document. It has `command`, `has_errors: true`, `stopped: true`, and a `diagnostics` list with that error's `code`, `message`, and `help`, the same diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 070b4e38f4..fdd98992e3 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -62,7 +62,8 @@ reviewed exceptions. Listed tests are not flagged and still count toward `SQBRTE [Empty-input tests](/docs/concepts/testing#empty-input-tests). `SQBRGRAPH103` and `SQBRPROJECT107` accept `published_layers`, and `SQBRPROJECT107` accepts -`allowed_qualifiers`, a list of `{ qualifier, reason }` tables with an optional `domain`. See +`allowed_qualifiers`, a list of `{ qualifier, reason }` or `{ paths, reason }` tables with an +optional `domain`; `qualifier` and `paths` accept globs. See [Published-layer Rules](/docs/concepts/rules/published-layer-rules). Unknown Rule codes, option names, and invalid option values fail configuration. diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx index e0c011bbdc..13086f1b33 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx @@ -50,6 +50,15 @@ findings and cache entries match a single-process run. The result stays correct, longer runs in parallel and costs an extra host process. Keep each subject's findings independent of the others so Rules can run in parallel. +SQLBuild compares module state once per subject, after every selected Rule has evaluated it. A +change still present then is reported against each Rule whose module, or helper module it imports, +holds the changed value, the same Rules a check after every invocation would name. A change undone +before the subject's last Rule finishes, such as a scratch list one Rule fills and another empties +for the same model, is not detected. The same check decides which Rules the cache can track by the +facts they read: a Rule whose module state changed is cached against every project fact instead, so +any edit evaluates it again. `compile_timings` reports the time this check takes as +`custom_rules_monitor_cpu_ms`; see [`sqb compile`](/docs/cli/compile#json-output). + ## Helpers Custom Rules can import repository-owned helpers under `rules/` and these deterministic standard diff --git a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx index 7d9170ac54..3c9f1d8983 100644 --- a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx +++ b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx @@ -43,6 +43,10 @@ genuinely project-wide invariants, with correspondingly broader invalidation. Positive and negative project-tree observations are dependencies. If a Rule observes that a glob has no matches, adding a matching file invalidates that result. +`--no-cache` on `sqb compile`, `sqb plan`, and `sqb build` turns the Rules cache off for that +invocation: every built-in and custom Rule evaluates again, and stored results are neither read nor +written. Use it when a Rule finding looks stale. + ## Built-in-only execution When no custom Rule is selected, SQLBuild does not initialize the custom Python host. When every diff --git a/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx index a7fcb6472b..11f8757a65 100644 --- a/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx @@ -46,9 +46,9 @@ not fit the domain's majority shape are reported, with the remediation: example `sqb rename sales__mart__customers sales__mart__customers__core`. 3. If the qualifier is intentional and structural, allowlist it with a reason. -Allowlisted qualifiers are literal and do not count towards the shape: models that use them are left -out of the check. Each entry needs a reason, and an entry with a `domain` applies to that domain -only: +Allowlisted qualifiers do not count towards the shape: models that use them are left out of the +check and out of their domain's counts. Each entry needs a reason, and an entry with a `domain` +applies to that domain only: ```toml [rules.rule_options.SQBRPROJECT107] @@ -58,6 +58,33 @@ allowed_qualifiers = [ ] ``` +`qualifier` is a case-sensitive glob with the same grammar as selectors: `*` matches any run of +characters, `?` one character, and `[...]` one character from a set, with `[!...]` negating it. A +qualifier without wildcards matches only itself. One entry can cover a whole family of +qualifiers, such as a domain published once per country: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { domain = "sales", qualifier = "[a-z][a-z][a-z]", reason = "Published once per country." }, + { domain = "sales", qualifier = "[a-z][a-z][a-z]_snapshots", reason = "Daily snapshots per country." }, +] +``` + +An entry with `paths` instead leaves out the published models whose file path, relative to the +project, matches any of its globs. A family published once per qualifier usually lives in one +folder: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { paths = ["models/sales/mart/regional/*"], reason = "Published once per region." }, +] +``` + +An entry with both `qualifier` and `paths` leaves out only the models that match both. Published +models that match no entry are checked as before. + ## Published layers Both Rules treat `mart` and `mart_v` as the published layers. Set `published_layers` on either Rule From c9eac1f3d6c3f283df6855b920fafc5e8fa245bd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:03:01 +0100 Subject: [PATCH 02/14] perf: check custom rule module state once per subject --- .../_helpers/host/custom_evaluation.py | 32 ++++++++++++++----- .../src/sqlbuild/cli/commands/main/helpers.py | 7 ++-- .../sqlbuild/cli/commands/main/test_rules.py | 15 ++++++--- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index f4de4d1ff3..36a292e0f2 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -152,7 +152,13 @@ def run_pass( class _ReadTracker: - """Attribute reads to one invocation; rules whose module state changes become untracked.""" + """Attribute reads to one invocation; rules whose module state changes become untracked. + + Module state is compared once per subject, after every Rule evaluated it, rather than after + each invocation. A change that persists to the end of a subject is caught and attributed to + the codes whose module state it changed, exactly as a per-invocation check attributes it; a + change undone before the subject's last Rule finishes is not. + """ def __init__( self, @@ -194,15 +200,19 @@ def finish(self, *, code: str) -> frozenset[tuple[object, ...]] | None: self.untracked_codes.add(code) if reads.uncacheable: self.uncacheable_codes.add(code) - changed: tuple[str, ...] = () - if self._state is not None: - started: int = thread_time_ns() - changed = self._state.changed_codes() - self.monitor_ns += thread_time_ns() - started + return None if observed is None else frozenset(observed) + + def finish_subject(self) -> None: + """Mark codes whose module state changed since the previous subject finished.""" + + if self._state is None: + return + started: int = thread_time_ns() + changed: tuple[str, ...] = self._state.changed_codes() + self.monitor_ns += thread_time_ns() - started self.stateful_codes.update(changed) - if reads is not None: + if self.reads is not None: self.untracked_codes.update(changed) - return None if observed is None else frozenset(observed) def _evaluate_pass( @@ -274,12 +284,16 @@ def context(rule: Rule) -> EvaluationRuleContext: reads=tracker.finish(code=rule.code), ) ) + if any(item.subject is None for item in evaluations): + tracker.finish_subject() for model in models: subject: Model = public_model(model) path: str = model.relative_path.as_posix() + evaluated: bool = False for rule, ctx, planned in model_contexts: if planned is not None and path not in planned: continue + evaluated = True _stop_if_cancelled(inputs) tracker.start() try: @@ -300,6 +314,8 @@ def context(rule: Rule) -> EvaluationRuleContext: reads=tracker.finish(code=rule.code), ) ) + if evaluated: + tracker.finish_subject() return evaluations diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 3d401d105a..224035ad45 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -845,11 +845,12 @@ def write_project_files(*, project_dir: Path, files: dict[str, str]) -> None: def compile_findings( - *, project_dir: Path, capsys: pytest.CaptureFixture[str] + *, project_dir: Path, capsys: pytest.CaptureFixture[str], no_cache: bool = True ) -> tuple[tuple[str, str, int | None, int | None], ...]: - """Compile through the CLI with no cache and return each finding's code and location.""" + """Compile through the CLI, by default with no cache, and return each finding's location.""" - _ = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + flags: list[str] = ["--no-cache"] if no_cache else [] + _ = main(["--project-dir", str(project_dir), "compile", "--json", *flags]) payload: dict[str, object] = json.loads(capsys.readouterr().out) diagnostics: list[dict[str, Any]] = cast(list[dict[str, Any]], payload["diagnostics"]) return tuple( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 20ed28a8e4..3348a4c779 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -1054,21 +1054,26 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ test_case: EarlyLintInputsTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: + # `--no-cache` would skip the Rules cache this test upgrades; disable only the compile cache. + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_CACHE", "1") write_project_files(project_dir=tmp_path, files=EARLY_LINT_PROJECT_FILES) previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( - project_dir=tmp_path, capsys=capsys + project_dir=tmp_path, capsys=capsys, no_cache=False ) cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="sql.json") stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) stored["build_identity"] = "previous-build" cache_path.write_text(json.dumps(stored), encoding="utf-8") upgraded_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( - project_dir=tmp_path, capsys=capsys + project_dir=tmp_path, capsys=capsys, no_cache=False ) assert previous_findings == upgraded_findings == test_case.expected_findings - assert compile_findings(project_dir=tmp_path, capsys=capsys) == upgraded_findings + assert compile_findings(project_dir=tmp_path, capsys=capsys, no_cache=False) == ( + upgraded_findings + ) @pytest.mark.parametrize( @@ -1231,10 +1236,12 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: ) def compile_payload() -> dict[str, Any]: - exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) assert exit_code == 1 return json.loads(capsys.readouterr().out) + # `--no-cache` would skip the Rules cache this test compares; disable only the compile cache. + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_CACHE", "1") single_hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=1) single: dict[str, Any] = compile_payload() rules_bulk_cache_path(project_dir=tmp_path, file_name="custom-rules.json").parent.parent.rename( From c6fbaafbf55a3c6507823c17e0809508d9760c2a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:03:12 +0100 Subject: [PATCH 03/14] docs: regenerate the bundled docs skill --- .../sqlbuild/references/docs/cli/compile.md | 28 +++++++++++++++- .../rules/configuration-and-selection.md | 3 +- .../custom-rules/testing-and-determinism.md | 9 +++++ .../concepts/rules/execution-and-caching.md | 4 +++ .../concepts/rules/published-layer-rules.md | 33 +++++++++++++++++-- 5 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 09030d8ff4..e567a22d60 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -34,7 +34,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | Flag | Description | |------|-------------| | `--no-sql-analysis` | Disable SQL syntax, binding, type inference, and semantic validation (`--no-sql-validation` remains an alias) | -| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | +| `--no-cache` | Bypass the reusable compile-analysis cache and the Rules cache for this invocation | | `--defer-to` | Resolve unselected model references against another target | | `--json` | Output the full compile report as JSON | | `--manifest` | Generate `target/manifest.json` with project metadata | @@ -212,6 +212,11 @@ Known limits: - In these cases, or whenever a result looks stale, run `sqb compile --no-cache` or set `SQLBUILD_DISABLE_COMPILE_REUSE=1`. +`--no-cache` also turns the Rules cache off for that compile: built-in and custom Rules evaluate +every subject again, and the compile neither reads nor writes stored Rule results, as with +`[rules.cache] enabled = false`. A `--no-cache` compile reports `rule_cache_hits` as `0`. The next +compile without the flag reuses whatever the cache held before. + A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside @@ -297,6 +302,27 @@ Returns a structured report including: - `lineage_mode` - which lineage mode was used - `artifacts` - paths to written files +Every `compile_timings` value is a non-negative integer. Besides the phase timings, it attributes +custom Rule cost: + +| Key | Meaning | +|-----|---------| +| `custom_rules_ms`, `custom_rules_cpu_ms` | Wall and CPU time of the custom Rules phase, including its host processes | +| `custom_rules_invocations` | Custom Rule invocations the hosts ran, one per Rule and subject | +| `custom_rules_monitor_cpu_ms` | Host CPU spent checking Rule modules for state one subject could leave for another | +| `custom_rules_guard_cpu_ms` | Host CPU spent by the runtime guard deciding whether an audited operation, such as opening a file or importing a module, came from Rule code | +| `custom_rules..invocations` | Invocations of one custom Rule | +| `custom_rules..cpu_ms` | Host CPU of those invocations, excluding the guard's checks | +| `custom_rules..cache_hits` | Subjects whose stored result was reused | +| `custom_rules..cache_misses` | Subjects evaluated because no stored result applied; every subject when the Rules cache is off | + +Each selected custom Rule has the four `custom_rules..*` keys; a Rule the cache answered in +full reports `0` invocations. The keys are flat so that `compile_timings` stays one level deep, for +example `jq '.compile_timings["custom_rules.XORDERS001.cpu_ms"]'`. Invocations and CPU include a +Rule SQLBuild evaluates again in one process because its module state changed, so they can exceed +the Rule's subject count. Monitor and guard time are separate from each Rule's CPU; the hosts' +remaining CPU goes to starting Python, loading Rule modules, and reading the project. + If compile stops on an error before it can build the report, such as a model header that does not parse, `--json` still prints one JSON document. It has `command`, `has_errors: true`, `stopped: true`, and a `diagnostics` list with that error's `code`, `message`, and `help`, the same diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 6a835eb92f..6a8d53c6da 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -72,7 +72,8 @@ reviewed exceptions. Listed tests are not flagged and still count toward `SQBRTE [Empty-input tests](../testing.md#empty-input-tests). `SQBRGRAPH103` and `SQBRPROJECT107` accept `published_layers`, and `SQBRPROJECT107` accepts -`allowed_qualifiers`, a list of `{ qualifier, reason }` tables with an optional `domain`. See +`allowed_qualifiers`, a list of `{ qualifier, reason }` or `{ paths, reason }` tables with an +optional `domain`; `qualifier` and `paths` accept globs. See [Published-layer Rules](published-layer-rules.md). Unknown Rule codes, option names, and invalid option values fail configuration. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md index ef11cb0bf5..10ecd224a8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md @@ -60,6 +60,15 @@ findings and cache entries match a single-process run. The result stays correct, longer runs in parallel and costs an extra host process. Keep each subject's findings independent of the others so Rules can run in parallel. +SQLBuild compares module state once per subject, after every selected Rule has evaluated it. A +change still present then is reported against each Rule whose module, or helper module it imports, +holds the changed value, the same Rules a check after every invocation would name. A change undone +before the subject's last Rule finishes, such as a scratch list one Rule fills and another empties +for the same model, is not detected. The same check decides which Rules the cache can track by the +facts they read: a Rule whose module state changed is cached against every project fact instead, so +any edit evaluates it again. `compile_timings` reports the time this check takes as +`custom_rules_monitor_cpu_ms`; see [`sqb compile`](../../../cli/compile.md#json-output). + ## Helpers Custom Rules can import repository-owned helpers under `rules/` and these deterministic standard diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md index 3088640fdf..3fcbc90a80 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md @@ -46,6 +46,10 @@ genuinely project-wide invariants, with correspondingly broader invalidation. Positive and negative project-tree observations are dependencies. If a Rule observes that a glob has no matches, adding a matching file invalidates that result. +`--no-cache` on `sqb compile`, `sqb plan`, and `sqb build` turns the Rules cache off for that +invocation: every built-in and custom Rule evaluates again, and stored results are neither read nor +written. Use it when a Rule finding looks stale. + ## Built-in-only execution When no custom Rule is selected, SQLBuild does not initialize the custom Python host. When every diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md index 248639b4a9..862f9346e1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md @@ -49,9 +49,9 @@ not fit the domain's majority shape are reported, with the remediation: example `sqb rename sales__mart__customers sales__mart__customers__core`. 3. If the qualifier is intentional and structural, allowlist it with a reason. -Allowlisted qualifiers are literal and do not count towards the shape: models that use them are left -out of the check. Each entry needs a reason, and an entry with a `domain` applies to that domain -only: +Allowlisted qualifiers do not count towards the shape: models that use them are left out of the +check and out of their domain's counts. Each entry needs a reason, and an entry with a `domain` +applies to that domain only: ```toml [rules.rule_options.SQBRPROJECT107] @@ -61,6 +61,33 @@ allowed_qualifiers = [ ] ``` +`qualifier` is a case-sensitive glob with the same grammar as selectors: `*` matches any run of +characters, `?` one character, and `[...]` one character from a set, with `[!...]` negating it. A +qualifier without wildcards matches only itself. One entry can cover a whole family of +qualifiers, such as a domain published once per country: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { domain = "sales", qualifier = "[a-z][a-z][a-z]", reason = "Published once per country." }, + { domain = "sales", qualifier = "[a-z][a-z][a-z]_snapshots", reason = "Daily snapshots per country." }, +] +``` + +An entry with `paths` instead leaves out the published models whose file path, relative to the +project, matches any of its globs. A family published once per qualifier usually lives in one +folder: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { paths = ["models/sales/mart/regional/*"], reason = "Published once per region." }, +] +``` + +An entry with both `qualifier` and `paths` leaves out only the models that match both. Published +models that match no entry are checked as before. + ## Published layers Both Rules treat `mart` and `mart_v` as the published layers. Set `published_layers` on either Rule From 2b78f2aab8d7f2bafdf317cdbf5f16e8bc2132c9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:33:34 +0100 Subject: [PATCH 04/14] style: satisfy the layout and test policies for custom rule timings --- .../bindings/_helpers/rules/rules_config.rs | 2 - .../src/configuration/_helpers/loading.rs | 11 ++- .../main/custom_rule_timings.rs | 8 +- .../custom_host_pool/main/without_codes.rs | 2 - .../src/custom_host_pool/tests/helpers.rs | 6 +- .../tests/test_custom_host_pool.rs | 73 +++++++++++------ .../src/custom_host_pool/tests/test_types.rs | 7 ++ .../src/rules/_helpers/model_layers.rs | 32 ++++---- .../_helpers/host/custom_evaluation.py | 8 +- .../rule_engine/classes/runtime_guard.py | 4 +- .../cli/commands/main/rules/_test_types.py | 3 +- .../cli/commands/main/rules/helpers.py | 19 +++-- .../main/rules/test_custom_rule_timings.py | 80 ++++++++++--------- .../src/sqlbuild/cli/commands/main/helpers.py | 6 +- .../sqlbuild/cli/commands/main/test_rules.py | 10 +-- 15 files changed, 155 insertions(+), 116 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index 24843d00ef..a59d0e0da2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -244,8 +244,6 @@ pub(crate) fn load_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResul } /// The project's rules configuration, with the Rules cache off when `cache` is `false`. -/// -/// `--no-cache` passes `false`: the run neither reads nor writes built-in or custom rule results. pub(crate) fn load_rules_config_with_cache( py: Python<'_>, project_dir: PathBuf, diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index f1038d00e2..2556c10589 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -1,8 +1,9 @@ use crate::configuration::_helpers::removed_settings::removed_settings; use crate::configuration::errors::RulesConfigLoadError; use crate::constants::{ - ALLOWED_PATHS_KEY, ALLOWED_QUALIFIERS_OPTION, MIN_CUSTOM_RULE_TEST_CASES_KEY, MODEL_NAME_LAYERS, - PUBLISHED_LAYERS_OPTION, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, RELATION_EXCEPTION_RULES, + ALLOWED_PATHS_KEY, ALLOWED_QUALIFIERS_OPTION, MIN_CUSTOM_RULE_TEST_CASES_KEY, + MODEL_NAME_LAYERS, PUBLISHED_LAYERS_OPTION, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, + RELATION_EXCEPTION_RULES, }; use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; @@ -344,8 +345,10 @@ fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> let Some(paths) = paths.as_array() else { return Err(shape); }; - let paths: Option> = - paths.iter().map(|path| path.as_str().map(str::trim)).collect(); + let paths: Option> = paths + .iter() + .map(|path| path.as_str().map(str::trim)) + .collect(); let Some(mut paths) = paths else { return Err(shape); }; diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs index a086b4ebac..48055e2ed0 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs @@ -9,9 +9,6 @@ use crate::custom_host_pool::constants::{ use crate::custom_host_pool::models::{HostTimings, RuleTiming}; /// Run-wide totals, then each rule's invocations, CPU and cache counts in code order. -/// -/// `cache_counts` holds each rule's reused and missing subjects; rules the cache answered in -/// full report zero invocations. #[must_use] pub fn custom_rule_timings( timings: &HostTimings, @@ -42,7 +39,10 @@ pub fn custom_rule_timings( (key("invocations"), timing.invocations), (key("cpu_ms"), timing.cpu_ns / NANOSECONDS_PER_MILLISECOND), (key("cache_hits"), u64::try_from(hits).unwrap_or(u64::MAX)), - (key("cache_misses"), u64::try_from(misses).unwrap_or(u64::MAX)), + ( + key("cache_misses"), + u64::try_from(misses).unwrap_or(u64::MAX), + ), ]); } entries diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/without_codes.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/without_codes.rs index e612e8cadd..c75fb8da37 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/main/without_codes.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/without_codes.rs @@ -5,8 +5,6 @@ use std::collections::BTreeSet; use crate::custom_host_pool::models::HostSlice; /// The slice without evaluations, untracked reads, uncacheable or stateful marks of `codes`. -/// -/// Timings stay: the slice's hosts spent that CPU even when a rerun replaces its results. #[must_use] pub fn without_codes(slice: HostSlice, codes: &BTreeSet) -> HostSlice { HostSlice { diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs index adc33e2f55..382ac143c9 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs @@ -56,7 +56,11 @@ pub(super) fn evaluation(code: &str, subject: &str, reads: Option<&[&[&str]]>) - } /// Host timings with the given `(code, invocations, cpu_ns)` rules. -pub(super) fn rule_timings(rules: &[(&str, u64, u64)], monitor_ns: u64, guard_ns: u64) -> HostTimings { +pub(super) fn rule_timings( + rules: &[(&str, u64, u64)], + monitor_ns: u64, + guard_ns: u64, +) -> HostTimings { HostTimings { rules: rules .iter() diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs index 186dc27b79..d6a75dd132 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs @@ -16,6 +16,7 @@ use crate::custom_host_pool::tests::helpers::{ }; use crate::custom_host_pool::tests::test_types::{ DecodeTestCase, HostCapacityTestCase, HostCountTestCase, HostPlanTestCase, MergeTestCase, + TimingEntriesTestCase, }; const NODE_CPUS: usize = 64; @@ -343,30 +344,54 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { #[test] fn given_host_timings_and_cache_counts_when_reporting_then_flat_entries_cover_every_rule() { - let timings: HostTimings = rule_timings(&[("R2", 3, 4_500_000)], 2_900_000, 1_000_000); - let cache_counts: BTreeMap = - BTreeMap::from([("R1".to_owned(), (5, 0)), ("R2".to_owned(), (2, 3))]); + let test_cases = [ + TimingEntriesTestCase { + description: "a fully cached rule reports zero invocations beside an evaluated one", + timings: rule_timings(&[("R2", 3, 4_500_000)], 2_900_000, 1_000_000), + cache_counts: &[("R1", (5, 0)), ("R2", (2, 3))], + expected_entries: &[ + ("custom_rules_invocations", 3), + ("custom_rules_monitor_cpu_ms", 2), + ("custom_rules_guard_cpu_ms", 1), + ("custom_rules.R1.invocations", 0), + ("custom_rules.R1.cpu_ms", 0), + ("custom_rules.R1.cache_hits", 5), + ("custom_rules.R1.cache_misses", 0), + ("custom_rules.R2.invocations", 3), + ("custom_rules.R2.cpu_ms", 4), + ("custom_rules.R2.cache_hits", 2), + ("custom_rules.R2.cache_misses", 3), + ], + }, + TimingEntriesTestCase { + description: "no custom rules report only zero totals", + timings: HostTimings::default(), + cache_counts: &[], + expected_entries: &[ + ("custom_rules_invocations", 0), + ("custom_rules_monitor_cpu_ms", 0), + ("custom_rules_guard_cpu_ms", 0), + ], + }, + ]; + for test_case in test_cases { + let cache_counts: BTreeMap = test_case + .cache_counts + .iter() + .map(|(code, counts)| ((*code).to_owned(), *counts)) + .collect(); - let entries: Vec<(String, u64)> = custom_rule_timings(&timings, &cache_counts); + let entries: Vec<(String, u64)> = custom_rule_timings(&test_case.timings, &cache_counts); - let expected: Vec<(&str, u64)> = vec![ - ("custom_rules_invocations", 3), - ("custom_rules_monitor_cpu_ms", 2), - ("custom_rules_guard_cpu_ms", 1), - ("custom_rules.R1.invocations", 0), - ("custom_rules.R1.cpu_ms", 0), - ("custom_rules.R1.cache_hits", 5), - ("custom_rules.R1.cache_misses", 0), - ("custom_rules.R2.invocations", 3), - ("custom_rules.R2.cpu_ms", 4), - ("custom_rules.R2.cache_hits", 2), - ("custom_rules.R2.cache_misses", 3), - ]; - assert_eq!( - entries, - expected - .into_iter() - .map(|(key, value)| (key.to_owned(), value)) - .collect::>() - ); + assert_eq!( + entries, + test_case + .expected_entries + .iter() + .map(|(key, value)| ((*key).to_owned(), *value)) + .collect::>(), + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs index 7bcde96b8c..10ca81d91a 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs @@ -39,3 +39,10 @@ pub(super) struct MergeTestCase { pub(super) expected_observed: Vec<(Vec, Option)>, pub(super) expected_timings: HostTimings, } + +pub(super) struct TimingEntriesTestCase { + pub(super) description: &'static str, + pub(super) timings: HostTimings, + pub(super) cache_counts: &'static [(&'static str, (usize, usize))], + pub(super) expected_entries: &'static [(&'static str, u64)], +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs index 1699004a5f..1f1b11ff0c 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs @@ -3,9 +3,9 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::Value; use crate::constants::{ - ALLOWED_PATHS_KEY, ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, INTERMEDIATE_LAYER_DIRECTORY, - MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, - STAGING_LAYER_DIRECTORY, + ALLOWED_PATHS_KEY, ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, + INTERMEDIATE_LAYER_DIRECTORY, MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, + PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, STAGING_LAYER_DIRECTORY, }; use crate::models::{EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; use crate::rules::_helpers::evaluation::{parse_name, path_fault}; @@ -352,8 +352,6 @@ fn intermediate_move(model: &Published<'_>) -> Option { } /// One `allowed_qualifiers` entry: the published models it leaves out of the name-shape check. -/// -/// `qualifier` and `paths` are case-sensitive globs; a model must match every part it sets. struct AllowedQualifier { domain: Option, qualifier: Option, @@ -397,23 +395,25 @@ fn allowed_qualifiers(config: &RulesConfig, code: &str) -> Vec let allowed = AllowedQualifier { domain: text("domain"), qualifier: text("qualifier"), - paths: entry - .get(ALLOWED_PATHS_KEY) - .and_then(Value::as_array) - .map(|paths| { - paths - .iter() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect() - }) - .unwrap_or_default(), + paths: entry.get(ALLOWED_PATHS_KEY).map(texts).unwrap_or_default(), }; (allowed.qualifier.is_some() || !allowed.paths.is_empty()).then_some(allowed) }) .collect() } +/// The strings of a JSON list; anything else is empty. +fn texts(value: &Value) -> Vec { + let Some(items) = value.as_array() else { + return Vec::new(); + }; + items + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} + fn allowlisted(allowed: &[AllowedQualifier], entry: &Published<'_>) -> bool { allowed.iter().any(|allowed| allowed.matches(entry)) } diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 36a292e0f2..ba15b7f954 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -152,13 +152,7 @@ def run_pass( class _ReadTracker: - """Attribute reads to one invocation; rules whose module state changes become untracked. - - Module state is compared once per subject, after every Rule evaluated it, rather than after - each invocation. A change that persists to the end of a subject is caught and attributed to - the codes whose module state it changed, exactly as a per-invocation check attributes it; a - change undone before the subject's last Rule finishes is not. - """ + """Attribute reads to one invocation; rules whose module state changes become untracked.""" def __init__( self, diff --git a/src/sqlbuild/rule_engine/classes/runtime_guard.py b/src/sqlbuild/rule_engine/classes/runtime_guard.py index 21c0374069..fa59971da5 100644 --- a/src/sqlbuild/rule_engine/classes/runtime_guard.py +++ b/src/sqlbuild/rule_engine/classes/runtime_guard.py @@ -138,9 +138,7 @@ def _call_metadata( started: int = thread_time_ns() try: paths: tuple[object, ...] = tuple(map(_plain_path, arguments)) - named: dict[str, object] = { - key: _plain_path(value) for key, value in keywords.items() - } + named: dict[str, object] = {key: _plain_path(value) for key, value in keywords.items()} rule_frame: FrameType | None = self._responsible_rule_frame(sys._getframe(1)) if rule_frame is not None: target: str = f" for {paths[0]!r}" if paths else "" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 0a8455576a..654eaa77cc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -322,9 +322,9 @@ class CustomRuleTimingCase: """Custom Rules compiled cold, warm, with `--no-cache`, and warm again.""" description: str - model_count: int selected_rules: tuple[str, ...] files: tuple[tuple[str, str], ...] + expected_subjects: int @dataclass(frozen=True) @@ -332,7 +332,6 @@ class ModuleStateAttributionCase: """Rules whose module state changes persist or are undone within one model.""" description: str - model_count: int selected_rules: tuple[str, ...] files: tuple[tuple[str, str], ...] edited_model: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index d496b3808f..f9cf7dd094 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -209,14 +209,11 @@ def rule_cache_hits(result: subprocess.CompletedProcess[str]) -> int: def custom_rule_timings( - result: subprocess.CompletedProcess[str], + result: subprocess.CompletedProcess[str], codes: tuple[str, ...] ) -> dict[str, tuple[int, int, int, int]]: """Map each custom Rule code to its invocations, CPU ms, cache hits and cache misses.""" timings: dict[str, int] = json.loads(result.stdout)["compile_timings"] - codes: set[str] = { - key.split(".")[1] for key in timings if key.startswith("custom_rules.") - } return { code: ( timings[f"custom_rules.{code}.invocations"], @@ -229,16 +226,26 @@ def custom_rule_timings( def custom_rule_cache_counts( - result: subprocess.CompletedProcess[str], + result: subprocess.CompletedProcess[str], codes: tuple[str, ...] ) -> dict[str, tuple[int, int, int]]: """Map each custom Rule code to its invocations, cache hits and cache misses.""" return { code: (invocations, hits, misses) - for code, (invocations, _, hits, misses) in custom_rule_timings(result).items() + for code, (invocations, _, hits, misses) in custom_rule_timings(result, codes).items() } +def named_custom_rule_source( + *, code: str, body: str, function: str, header: str = CUSTOM_RULE_HEADER +) -> str: + """Return one model-subject custom Rule whose check function has the given name.""" + + return custom_rule_source(code=code, header=header, body=body).replace( + "def check(", f"def {function}(", 1 + ) + + def string_set_order(*, names: tuple[str, ...], hash_seed: str) -> str: """Return the comma-joined iteration order of a string set under one hash seed.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py index 5d426deaa8..b47ec59e4b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py @@ -12,8 +12,8 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( custom_rule_cache_counts, - custom_rule_source, custom_rule_timings, + named_custom_rule_source, rule_cache_hits, run_compile_cli, split_model_files, @@ -26,12 +26,7 @@ ) _MODEL_COUNT: int = 4 _RULE_HEADER: str = "from sqlbuild.rules import Finding, Model, RuleContext, rule\n" - - -def _rule(*, code: str, body: str, header: str = _RULE_HEADER, function: str = "check") -> str: - return custom_rule_source(code=code, header=header, body=body).replace( - "def check(", f"def {function}(", 1 - ) +_PURE_BODY: str = " return [] if model.name else []\n" @pytest.mark.parametrize( @@ -39,22 +34,25 @@ def _rule(*, code: str, body: str, header: str = _RULE_HEADER, function: str = " [ CustomRuleTimingCase( description="two model rules in separate files", - model_count=_MODEL_COUNT, selected_rules=("XSQBRTIME001", "XSQBRTIME002"), files=( ( "rules/names.py", - _rule(code="XSQBRTIME001", body=" return [] if model.name else []\n"), + named_custom_rule_source( + code="XSQBRTIME001", body=_PURE_BODY, function="names" + ), ), ( "rules/paths.py", - _rule( + named_custom_rule_source( code="XSQBRTIME002", body=" return [] if model.path.suffix == '.sql' else []\n", + function="paths", ), ), *split_model_files(_MODEL_COUNT), ), + expected_subjects=_MODEL_COUNT + 1, ) ], ids=lambda case: case.description, @@ -65,7 +63,13 @@ def test_given_custom_rules_when_compiling_then_timings_and_no_cache_report_each write_custom_rule_project( project_dir=tmp_path, selected_rules=test_case.selected_rules, files=test_case.files ) - subjects: int = test_case.model_count + 1 + codes: tuple[str, ...] = test_case.selected_rules + evaluated: dict[str, tuple[int, int, int]] = dict.fromkeys( + codes, (test_case.expected_subjects, 0, test_case.expected_subjects) + ) + reused: dict[str, tuple[int, int, int]] = dict.fromkeys( + codes, (0, test_case.expected_subjects, 0) + ) cold: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) warm: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) @@ -76,24 +80,21 @@ def test_given_custom_rules_when_compiling_then_timings_and_no_cache_report_each tmp_path, environment=_FULL_COMPILE ) - for result in (cold, warm, uncached, rewarmed): - assert result.returncode == 0, result.stdout + result.stderr - evaluated: dict[str, tuple[int, int, int]] = { - code: (subjects, 0, subjects) for code in test_case.selected_rules - } - reused: dict[str, tuple[int, int, int]] = { - code: (0, subjects, 0) for code in test_case.selected_rules - } - assert custom_rule_cache_counts(cold) == evaluated - assert custom_rule_cache_counts(warm) == reused - assert custom_rule_cache_counts(uncached) == evaluated + assert [result.returncode for result in (cold, warm, uncached, rewarmed)] == [0, 0, 0, 0], ( + cold.stderr + uncached.stderr + ) + assert custom_rule_cache_counts(cold, codes) == evaluated + assert custom_rule_cache_counts(warm, codes) == reused + assert custom_rule_cache_counts(uncached, codes) == evaluated assert rule_cache_hits(uncached) == 0 - assert custom_rule_cache_counts(rewarmed) == reused + assert custom_rule_cache_counts(rewarmed, codes) == reused cold_timings: dict[str, int] = json.loads(cold.stdout)["compile_timings"] - assert cold_timings["custom_rules_invocations"] == subjects * len(test_case.selected_rules) - assert cold_timings["custom_rules_monitor_cpu_ms"] >= 0 - assert cold_timings["custom_rules_guard_cpu_ms"] >= 0 - assert all(cpu_ms >= 0 for _, cpu_ms, _, _ in custom_rule_timings(cold).values()) + assert cold_timings["custom_rules_invocations"] == test_case.expected_subjects * len(codes) + assert ( + min(cold_timings["custom_rules_monitor_cpu_ms"], cold_timings["custom_rules_guard_cpu_ms"]) + >= 0 + ) + assert min(cpu_ms for _, cpu_ms, _, _ in custom_rule_timings(cold, codes).values()) >= 0 assert json.loads(warm.stdout)["compile_timings"]["custom_rules_invocations"] == 0 @@ -102,27 +103,27 @@ def test_given_custom_rules_when_compiling_then_timings_and_no_cache_report_each [ ModuleStateAttributionCase( description="a persistent mutation is attributed, one undone within a model is not", - model_count=_MODEL_COUNT, selected_rules=("XSQBRSTATE001", "XSQBRSTATE002", "XSQBRSTATE003", "XSQBRSTATE004"), files=( ( "rules/persistent.py", - _rule( + named_custom_rule_source( code="XSQBRSTATE001", header=_RULE_HEADER + "SEEN: list[str] = []\n", body=" SEEN.append(model.name)\n return []\n", + function="remember", ), ), ( "rules/transient.py", - _rule( + named_custom_rule_source( code="XSQBRSTATE002", header=_RULE_HEADER + "SCRATCH: list[str] = []\n", body=" SCRATCH.append(model.name)\n return []\n", function="stage", ) + "\n\n" - + _rule( + + named_custom_rule_source( code="XSQBRSTATE003", header="", body=" SCRATCH.clear()\n return []\n", @@ -131,7 +132,9 @@ def test_given_custom_rules_when_compiling_then_timings_and_no_cache_report_each ), ( "rules/clean.py", - _rule(code="XSQBRSTATE004", body=" return [] if model.name else []\n"), + named_custom_rule_source( + code="XSQBRSTATE004", body=_PURE_BODY, function="names" + ), ), *split_model_files(_MODEL_COUNT), ), @@ -157,12 +160,17 @@ def test_given_rule_module_state_when_recompiling_an_edit_then_only_persistent_s (tmp_path / test_case.edited_model).write_text( 'MODEL (description "Orders");\nSELECT 2 AS order_id\n', encoding="utf-8" ) - edited: subprocess.CompletedProcess[str] = run_compile_cli( - tmp_path, environment=_FULL_COMPILE - ) + edited: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) assert cold.returncode == 0, cold.stdout + cold.stderr assert edited.returncode == 0, edited.stdout + edited.stderr assert { - code: misses for code, (_, _, misses) in custom_rule_cache_counts(edited).items() + code: misses + for code, (_, _, misses) in custom_rule_cache_counts( + edited, test_case.selected_rules + ).items() } == test_case.expected_misses_after_edit + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 224035ad45..e47c69f1a0 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -845,11 +845,13 @@ def write_project_files(*, project_dir: Path, files: dict[str, str]) -> None: def compile_findings( - *, project_dir: Path, capsys: pytest.CaptureFixture[str], no_cache: bool = True + *, + project_dir: Path, + capsys: pytest.CaptureFixture[str], + flags: tuple[str, ...] = ("--no-cache",), ) -> tuple[tuple[str, str, int | None, int | None], ...]: """Compile through the CLI, by default with no cache, and return each finding's location.""" - flags: list[str] = ["--no-cache"] if no_cache else [] _ = main(["--project-dir", str(project_dir), "compile", "--json", *flags]) payload: dict[str, object] = json.loads(capsys.readouterr().out) diagnostics: list[dict[str, Any]] = cast(list[dict[str, Any]], payload["diagnostics"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 3348a4c779..91856b6fab 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -1056,24 +1056,21 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - # `--no-cache` would skip the Rules cache this test upgrades; disable only the compile cache. monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_CACHE", "1") write_project_files(project_dir=tmp_path, files=EARLY_LINT_PROJECT_FILES) previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( - project_dir=tmp_path, capsys=capsys, no_cache=False + project_dir=tmp_path, capsys=capsys, flags=() ) cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="sql.json") stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) stored["build_identity"] = "previous-build" cache_path.write_text(json.dumps(stored), encoding="utf-8") upgraded_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( - project_dir=tmp_path, capsys=capsys, no_cache=False + project_dir=tmp_path, capsys=capsys, flags=() ) assert previous_findings == upgraded_findings == test_case.expected_findings - assert compile_findings(project_dir=tmp_path, capsys=capsys, no_cache=False) == ( - upgraded_findings - ) + assert compile_findings(project_dir=tmp_path, capsys=capsys, flags=()) == (upgraded_findings) @pytest.mark.parametrize( @@ -1240,7 +1237,6 @@ def compile_payload() -> dict[str, Any]: assert exit_code == 1 return json.loads(capsys.readouterr().out) - # `--no-cache` would skip the Rules cache this test compares; disable only the compile cache. monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_CACHE", "1") single_hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=1) single: dict[str, Any] = compile_payload() From e703d46eb552a7d8744fa1307ec92f2380071338 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:35:01 +0100 Subject: [PATCH 05/14] fix: check project rules after each invocation and tighten SQBRPROJECT107 paths --- .../_helpers/compile_driver/analysis.rs | 3 +- .../bindings/_helpers/rules/early_sql_lint.rs | 28 +++++++----- .../src/configuration/_helpers/loading.rs | 36 +++++++++++----- .../src/engine/tests/test_published_layers.rs | 13 ++++++ .../custom-rules/testing-and-determinism.md | 12 +++--- .../concepts/rules/published-layer-rules.md | 12 +++++- src/sqlbuild/_native.pyi | 2 +- .../_helpers/host/custom_evaluation.py | 3 +- .../sqlbuild/cli/commands/main/_test_types.py | 10 +++++ .../src/sqlbuild/cli/commands/main/helpers.py | 43 +++++++++++++++++++ .../sqlbuild/cli/commands/main/test_rules.py | 37 ++++++++++++++++ .../_helpers/engine/test_config.py | 33 ++++++++++++++ .../custom-rules/testing-and-determinism.mdx | 12 +++--- .../concepts/rules/published-layer-rules.mdx | 5 ++- 14 files changed, 210 insertions(+), 39 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index ef7b081ad8..3ca9c5226b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -156,7 +156,8 @@ pub(crate) fn analyze<'py>( let enabled: bool = !(native_field(selection, "select")?.is_truthy()? || native_field(selection, "exclude")?.is_truthy()?) && crate::bindings::_helpers::compile_context::delta::injection().is_none(); - let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; + let early_lint: Bound<'py, EarlySqlLint> = + Bound::new(py, EarlySqlLint::new(enabled, options.no_cache()?))?; let early_rules: EarlyRulesScope = EarlyRulesScope::new(); let analysis: PyResult> = analyzed(py, &options, (&early_lint, enabled.then_some(&early_rules))); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 5781e5abd8..ce9c2b3e41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -84,7 +84,11 @@ struct EarlyLintPlan { } /// The plan early SQL lint runs with, when the project selects SQL rules without a warm cache. -fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult> { +fn early_lint_plan( + inputs: &Bound<'_, PyAny>, + dialect: &str, + no_cache: bool, +) -> PyResult> { let py: Python<'_> = inputs.py(); let project_dir: Bound<'_, PyAny> = record_field(&record_field(inputs, "discovered_inputs")?, "project_dir")?; @@ -97,9 +101,10 @@ fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult bool { /// Start early SQL lint on a native thread when the project selects SQL rules without a cache. fn start_early_lint( - inputs: &Bound<'_, PyAny>, - dialect: &str, + (inputs, dialect): (&Bound<'_, PyAny>, &str), + no_cache: bool, stop: &Arc, ) -> PyResult> { - let Some(plan) = early_lint_plan(inputs, dialect)? else { + let Some(plan) = early_lint_plan(inputs, dialect, no_cache)? else { return Ok(None); }; let mut compiled: HashMap = HashMap::new(); @@ -290,6 +295,8 @@ pub(crate) struct ExpansionReuse { pub(crate) struct EarlySqlLint { #[pyo3(get)] pub(crate) enabled: bool, + /// Whether `--no-cache` turned the Rules cache off, so a warm SQL rule cache is ignored. + no_cache: bool, preparation: Option>, expansion_reuse: Option, stop_requested: Arc, @@ -297,9 +304,10 @@ pub(crate) struct EarlySqlLint { impl EarlySqlLint { /// A manager that starts speculative checks only when `enabled`. - pub(crate) fn new(enabled: bool) -> Self { + pub(crate) fn new(enabled: bool, no_cache: bool) -> Self { Self { enabled, + no_cache, preparation: None, expansion_reuse: None, stop_requested: Arc::new(AtomicBool::new(false)), @@ -337,9 +345,9 @@ impl EarlySqlLint { #[pymethods] impl EarlySqlLint { #[new] - #[pyo3(signature = (*, enabled))] - fn py_new(enabled: bool) -> Self { - Self::new(enabled) + #[pyo3(signature = (*, enabled, no_cache=false))] + fn py_new(enabled: bool, no_cache: bool) -> Self { + Self::new(enabled, no_cache) } fn __enter__(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { @@ -363,7 +371,7 @@ impl EarlySqlLint { } let py: Python<'_> = inputs.py(); let stop: Arc = Arc::clone(&slf.stop_requested); - slf.preparation = start_early_lint(inputs, dialect, &stop)? + slf.preparation = start_early_lint((inputs, dialect), slf.no_cache, &stop)? .map(|preparation| Py::new(py, preparation)) .transpose()?; let declaration_scope: Bound<'_, PyAny> = record_field(inputs, "declaration_scope")?; diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index 2556c10589..6737b8cf28 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -359,18 +359,14 @@ fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> if qualifier.is_none() && paths.is_none() { return Err(shape); } - if qualifier.is_some_and(str::is_empty) || reason.is_empty() { - return Err(format!( - "{label} entries need a non-empty qualifier and reason" - )); + if reason.is_empty() { + return Err(format!("{label} entries need a non-empty reason")); } - if paths - .as_ref() - .is_some_and(|paths| paths.is_empty() || paths.iter().any(|path| path.is_empty())) - { - return Err(format!( - "{label} entries need a non-empty list of non-empty paths" - )); + if qualifier.is_some_and(str::is_empty) { + return Err(format!("{label} entries need a non-empty qualifier")); + } + if let Some(paths) = &paths { + validate_allowed_paths(paths, &label)?; } let paths: Vec<&str> = paths.unwrap_or_default(); let described: String = match qualifier { @@ -384,6 +380,24 @@ fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> Ok(()) } +/// One entry's sorted `paths` globs: non-empty, `/`-separated and each listed once. +fn validate_allowed_paths(paths: &[&str], label: &str) -> Result<(), String> { + if paths.is_empty() || paths.iter().any(|path| path.is_empty()) { + return Err(format!( + "{label} entries need a non-empty list of non-empty paths" + )); + } + if let Some(path) = paths.iter().find(|path| path.contains('\\')) { + return Err(format!( + "{label} paths must be project-relative globs with / separators: {path}" + )); + } + if let Some(pair) = paths.windows(2).find(|pair| pair[0] == pair[1]) { + return Err(format!("{label} entry lists path {} twice", pair[0])); + } + Ok(()) +} + /// A relation-level exception names a non-empty relation, for a Rule that reports one. fn validate_relation_scope(exception: &crate::models::RuleException) -> Result<(), String> { let blank = |value: &Option| value.as_ref().is_some_and(|text| text.trim().is_empty()); diff --git a/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs index 920dca63b0..510783eee1 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs @@ -190,6 +190,19 @@ fn given_published_models_when_evaluating_layer_rules_then_reads_and_shapes_matc ), ], }, + PublishedLayerTestCase { + description: "a paths star matches across folder separators", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__deu", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"paths": ["models/sales/*"], "reason": "Published once per country."} + ]}), + expected_faults: &[], + }, PublishedLayerTestCase { description: "a paths entry leaves out the published models in its folder", code: "SQBRPROJECT107", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md index 10ecd224a8..5fafa49622 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md @@ -60,11 +60,13 @@ findings and cache entries match a single-process run. The result stays correct, longer runs in parallel and costs an extra host process. Keep each subject's findings independent of the others so Rules can run in parallel. -SQLBuild compares module state once per subject, after every selected Rule has evaluated it. A -change still present then is reported against each Rule whose module, or helper module it imports, -holds the changed value, the same Rules a check after every invocation would name. A change undone -before the subject's last Rule finishes, such as a scratch list one Rule fills and another empties -for the same model, is not detected. The same check decides which Rules the cache can track by the +SQLBuild compares module state once per model, after every selected Rule has evaluated it, and +after each project-subject Rule invocation. A change still present then is reported against each +Rule whose module, or helper module it imports, holds the changed value, the same Rules a check after +every invocation would name. All of one model's Rules run in the same process, so a change undone +before the model's last Rule finishes, such as a scratch list one Rule fills and another empties for +the same model, is not detected and cannot change findings. Project Rules can run in different +processes, so they are still checked after each invocation. The same check decides which Rules the cache can track by the facts they read: a Rule whose module state changed is cached against every project fact instead, so any edit evaluates it again. `compile_timings` reports the time this check takes as `custom_rules_monitor_cpu_ms`; see [`sqb compile`](../../../cli/compile.md#json-output). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md index 862f9346e1..5be493a672 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md @@ -6,6 +6,13 @@ Online: https://sqlbuild.com/docs/concepts/rules/published-layer-rules/ +## Contents + +- SQBRGRAPH103: no published reads within a domain +- SQBRPROJECT107: one name shape per domain +- Published layers +- Deviating + A domain's published layer is its interface to the rest of the project. Under pressure to make something work, it is tempting to add a consumer-specific variant next to an existing published model: @@ -75,8 +82,9 @@ allowed_qualifiers = [ ``` An entry with `paths` instead leaves out the published models whose file path, relative to the -project, matches any of its globs. A family published once per qualifier usually lives in one -folder: +project and written with `/` separators, matches any of its globs. As in selectors and janitor +`exclude_patterns`, `*` also matches `/`, so `models/sales/mart/*` covers nested folders too. A +family published once per qualifier usually lives in one folder: ```toml [rules.rule_options.SQBRPROJECT107] diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6a5efcb27f..a92a67ff83 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5800,7 +5800,7 @@ def run_planned_sql_rules( class EarlySqlLint: """Own speculative SQL checks for one compilation invocation.""" - def __init__(self, *, enabled: bool) -> None: ... + def __init__(self, *, enabled: bool, no_cache: bool = False) -> None: ... @property def enabled(self) -> bool: ... def __enter__(self) -> EarlySqlLint: ... diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index ba15b7f954..132ae77ba9 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -197,7 +197,7 @@ def finish(self, *, code: str) -> frozenset[tuple[object, ...]] | None: return None if observed is None else frozenset(observed) def finish_subject(self) -> None: - """Mark codes whose module state changed since the previous subject finished.""" + """Mark codes whose module state changed since the last check; project rules check each.""" if self._state is None: return @@ -278,7 +278,6 @@ def context(rule: Rule) -> EvaluationRuleContext: reads=tracker.finish(code=rule.code), ) ) - if any(item.subject is None for item in evaluations): tracker.finish_subject() for model in models: subject: Model = public_model(model) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 8eaebda0bf..598e87a79b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -186,6 +186,16 @@ class CustomHostSplitIntegrationTestCase: expected_code: str +@dataclass(frozen=True) +class ProjectRuleScratchStateTestCase: + """Project rules sharing module state that one fills, one reads and one empties.""" + + description: str + hosts: int + expected_findings: tuple[tuple[str, str], ...] + expected_launches: int + + @dataclass(frozen=True) class CustomHostFailureIntegrationTestCase: """A compile whose custom rule fails for some models while its hosts are split.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index e47c69f1a0..d118f061f7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1146,3 +1146,46 @@ def rebuild_formatter(_project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> No """Pretend another SQLBuild build formats the project.""" rebuild_identity(monkeypatch=monkeypatch) + + +PROJECT_SCRATCH_RULES: str = """from sqlbuild.rules import Finding, Project, RuleContext, rule + +SCRATCH: list[str] = [] + + +@rule(code="XSQBRSCR001", message="Stage names", remediation="Keep rules pure.") +def stage(*, project: Project, ctx: RuleContext) -> list[Finding]: + del project + SCRATCH.extend(model.name for model in ctx.project.models) + return [] + + +@rule(code="XSQBRSCR002", message="Staged names are present", remediation="Keep rules pure.") +def read(*, project: Project, ctx: RuleContext) -> list[Finding]: + del project + return [ctx.finding(subject=ctx.project.models[0])] * int(bool(SCRATCH)) + + +@rule(code="XSQBRSCR003", message="Clear staged names", remediation="Keep rules pure.") +def clear(*, project: Project, ctx: RuleContext) -> list[Finding]: + del project + SCRATCH.clear() + return [] +""" + + +def write_project_scratch_rules(project_dir: Path) -> None: + """Write two order models and three project rules sharing one scratch list.""" + + write_project_files( + project_dir=project_dir, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["XSQBRSCR"]\n\n' + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" + ), + "models/orders.sql": 'MODEL (description "Orders");\nSELECT 1 AS order_id\n', + "models/customers.sql": ('MODEL (description "Customers");\nSELECT 1 AS customer_id\n'), + "rules/scratch.py": PROJECT_SCRATCH_RULES, + }, + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 91856b6fab..8c051b6461 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -27,6 +27,7 @@ CustomHostInterruptIntegrationTestCase, CustomHostOutcomeTestCase, CustomHostSplitIntegrationTestCase, + ProjectRuleScratchStateTestCase, CustomRuleCacheSequenceTestCase, CustomRuleSurfaceTestCase, DynamicPivotRulesIntegrationTestCase, @@ -50,6 +51,7 @@ observe_hosts, write_custom_host_project, write_project_files, + write_project_scratch_rules, ) _CACHE_RULES_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' @@ -1191,6 +1193,41 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: } <= cold +@pytest.mark.parametrize( + "test_case", + [ + ProjectRuleScratchStateTestCase( + description="one host runs the project rules in order", + hosts=1, + expected_findings=(("XSQBRSCR002", "models/customers.sql"),), + expected_launches=1, + ), + ProjectRuleScratchStateTestCase( + description="split project rules rerun in one host after the scratch list fills", + hosts=2, + expected_findings=(("XSQBRSCR002", "models/customers.sql"),), + expected_launches=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_rules_sharing_scratch_state_when_splitting_hosts_then_findings_match_one_host( + test_case: ProjectRuleScratchStateTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_project_scratch_rules(tmp_path) + hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=test_case.hosts) + + findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( + project_dir=tmp_path, capsys=capsys + ) + + assert tuple((code, path) for code, path, _, _ in findings) == test_case.expected_findings + assert len(hosts.launches()) == test_case.expected_launches + + @pytest.mark.parametrize( "test_case", [CustomHostSplitIntegrationTestCase("custom rules split across hosts", 9, 3, "XSQBRARCH001")], diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index 5a05c3244b..8a4696fe6b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -218,6 +218,39 @@ ), expected_error_pattern="need a non-empty list of non-empty paths", ), + RulesConfigErrorTestCase( + description="allowed paths entry with an empty reason", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = ["models/sales/*"], reason = " " }]\n' + ), + expected_error_pattern=r"allowed_qualifiers entries need a non-empty reason$", + ), + RulesConfigErrorTestCase( + description="allowed qualifier that is empty", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ qualifier = "", reason = "Exports." }]\n' + ), + expected_error_pattern=r"allowed_qualifiers entries need a non-empty qualifier$", + ), + RulesConfigErrorTestCase( + description="allowed path with backslash separators", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + "allowed_qualifiers = [{ paths = ['models\\sales\\*'], reason = \"Exports.\" }]\n" + ), + expected_error_pattern=r"paths must be project-relative globs with / separators", + ), + RulesConfigErrorTestCase( + description="allowed path listed twice in one entry", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = ["models/sales/*", "models/sales/*"], ' + 'reason = "Exports." }]\n' + ), + expected_error_pattern=r"allowed_qualifiers entry lists path models/sales/\* twice", + ), RulesConfigErrorTestCase( description="duplicate allowed qualifier glob", source=( diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx index 13086f1b33..972fb6bd07 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx @@ -50,11 +50,13 @@ findings and cache entries match a single-process run. The result stays correct, longer runs in parallel and costs an extra host process. Keep each subject's findings independent of the others so Rules can run in parallel. -SQLBuild compares module state once per subject, after every selected Rule has evaluated it. A -change still present then is reported against each Rule whose module, or helper module it imports, -holds the changed value, the same Rules a check after every invocation would name. A change undone -before the subject's last Rule finishes, such as a scratch list one Rule fills and another empties -for the same model, is not detected. The same check decides which Rules the cache can track by the +SQLBuild compares module state once per model, after every selected Rule has evaluated it, and +after each project-subject Rule invocation. A change still present then is reported against each +Rule whose module, or helper module it imports, holds the changed value, the same Rules a check after +every invocation would name. All of one model's Rules run in the same process, so a change undone +before the model's last Rule finishes, such as a scratch list one Rule fills and another empties for +the same model, is not detected and cannot change findings. Project Rules can run in different +processes, so they are still checked after each invocation. The same check decides which Rules the cache can track by the facts they read: a Rule whose module state changed is cached against every project fact instead, so any edit evaluates it again. `compile_timings` reports the time this check takes as `custom_rules_monitor_cpu_ms`; see [`sqb compile`](/docs/cli/compile#json-output). diff --git a/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx index 11f8757a65..4b5ec4d085 100644 --- a/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx @@ -72,8 +72,9 @@ allowed_qualifiers = [ ``` An entry with `paths` instead leaves out the published models whose file path, relative to the -project, matches any of its globs. A family published once per qualifier usually lives in one -folder: +project and written with `/` separators, matches any of its globs. As in selectors and janitor +`exclude_patterns`, `*` also matches `/`, so `models/sales/mart/*` covers nested folders too. A +family published once per qualifier usually lives in one folder: ```toml [rules.rule_options.SQBRPROJECT107] From 439a3af14a622b1779a27bc3a1371d0d779ca099 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:39:42 +0100 Subject: [PATCH 06/14] style: sort the rules integration test imports --- tests/integration/src/sqlbuild/cli/commands/main/test_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 8c051b6461..d2d435fb76 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -27,7 +27,6 @@ CustomHostInterruptIntegrationTestCase, CustomHostOutcomeTestCase, CustomHostSplitIntegrationTestCase, - ProjectRuleScratchStateTestCase, CustomRuleCacheSequenceTestCase, CustomRuleSurfaceTestCase, DynamicPivotRulesIntegrationTestCase, @@ -36,6 +35,7 @@ ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, NumericRangeDecisionIntegrationTestCase, + ProjectRuleScratchStateTestCase, RulePassIntegrationTestCase, RulesIntegrationTestCase, SetOperationStarRuleTestCase, From c4d4f81afda768a30e28bc6c4697640fbb7981a1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:05:12 +0100 Subject: [PATCH 07/14] feat: show which janitor candidates are in use and optionally keep them --- .../sqlbuild-cli/src/argv/command_table.json | 12 + crates/sqlbuild-cli/src/argv/constants.rs | 3 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 3 +- .../src/project/_helpers/sections.rs | 33 +- .../sqlbuild-config/src/project/constants.rs | 1 + .../src/project/tests/test_load_config.rs | 27 ++ .../src/janitor_text/_helpers/mod.rs | 1 + .../src/janitor_text/_helpers/plan_json.rs | 73 +++- .../janitor_text/_helpers/plan_sections.rs | 42 ++- .../src/janitor_text/_helpers/usage.rs | 116 +++++++ .../src/janitor_text/constants.rs | 12 + .../janitor_text/main/format_janitor_json.rs | 3 +- .../main/janitor_confirmation_prompt.rs | 14 + .../sqlbuild-core/src/janitor_text/models.rs | 32 ++ .../sqlbuild-core/src/janitor_text/tests.rs | 2 + .../src/janitor_text/tests/helpers.rs | 72 +++- .../tests/test_janitor_usage_text.rs | 198 +++++++++++ .../src/lifecycle_events/constants.rs | 3 +- .../src/entries/_helpers/executor.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + .../_helpers/contract_records/constants.rs | 2 + .../_helpers/janitor/janitor_command.rs | 6 + .../_helpers/janitor/janitor_phases.rs | 19 +- .../bindings/_helpers/janitor/janitor_plan.rs | 82 ++++- .../_helpers/janitor/janitor_usage.rs | 188 ++++++++++ .../src/bindings/_helpers/janitor/mod.rs | 1 + .../bindings/_helpers/records/constants.rs | 3 + fensu.toml | 2 +- .../sqlbuild/references/docs/cli/janitor.md | 44 ++- .../docs/concepts/project-configuration.md | 4 + src/sqlbuild/_native.pyi | 7 + src/sqlbuild/_native_host.py | 1 + .../adapter/contract/classes/base_adapter.py | 14 + .../contract/classes/duckdb_backed_adapter.py | 12 + .../contract/classes/strict_adapter.py | 18 + src/sqlbuild/adapter/contract/constants.py | 1 + .../main/unsupported_relation_usage.py | 12 + src/sqlbuild/adapter/contract/models.py | 21 ++ .../bigquery/classes/bigquery_adapter.py | 12 + .../clickhouse/classes/clickhouse_adapter.py | 12 + .../databricks/classes/databricks_adapter.py | 12 + .../postgres/classes/postgres_adapter.py | 12 + .../snowflake/_helpers/relation_usage.py | 199 +++++++++++ .../snowflake/classes/snowflake_adapter.py | 18 + .../adapters/sqlite/classes/sqlite_adapter.py | 12 + .../sqlserver/classes/sqlserver_adapter.py | 12 + src/sqlbuild/executor/janitor/models.py | 15 +- .../cli/commands/main/janitor/_test_types.py | 35 ++ .../cli/commands/main/janitor/helpers.py | 167 +++++++++ .../main/janitor/test_janitor_usage.py | 321 ++++++++++++++++++ .../adapters/snowflake/_test_types.py | 24 ++ .../adapters/snowflake/test_relation_usage.py | 277 +++++++++++++++ website/src/content/docs/docs/cli/janitor.mdx | 43 ++- .../docs/concepts/project-configuration.mdx | 4 + 54 files changed, 2222 insertions(+), 43 deletions(-) create mode 100644 crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs create mode 100644 src/sqlbuild/adapter/contract/main/unsupported_relation_usage.py create mode 100644 src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py create mode 100644 tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 66c16460bf..62e396c2bf 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -5273,6 +5273,18 @@ "help": "drop the compatibility view at this old model name before it expires; repeat for several views", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--keep-in-use" + ], + "dest": "keep_in_use", + "nargs": 0, + "const": true, + "default": false, + "help": "keep relations other queries read within janitor.in_use_window instead of archiving them (overrides janitor.keep_in_use)", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 912372680d..2cd040d890 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -64,7 +64,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 179] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 180] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -139,6 +139,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 179] = [ ("retention_days", NamespaceDefault::None), ("direct_state_history_versions", NamespaceDefault::None), ("drop_old_name_views", NamespaceDefault::EmptyList), + ("keep_in_use", NamespaceDefault::Bool(false)), ("bounded", NamespaceDefault::None), ("max_column_examples", NamespaceDefault::None), ("max_row_only_examples", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index e7a5de0990..7410b3e21d 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -486,7 +486,7 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); -const JANITOR_FIELDS: [Field; 10] = [ +const JANITOR_FIELDS: [Field; 11] = [ PROJECT_DIR, NO_COLOR, same("auto_approve"), @@ -496,6 +496,7 @@ const JANITOR_FIELDS: [Field; 10] = [ "drop_old_name_views", Source::TupleOf("drop_old_name_views"), ), + same("keep_in_use"), SELECTED_TARGET, same("as_target"), same("verbose"), diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 4dc70f4a5b..fea57e01e1 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -12,13 +12,13 @@ use crate::project::_helpers::targets::seconds_value; use crate::project::_helpers::values::{int_below, python_equal}; use crate::project::constants::{ AUDIT_SEVERITIES, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, - DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_QUERY_ARTIFACT_TTL, - FUTURE_CURSOR_ACTIONS, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, - MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, - REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, - SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, - SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, - USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, + DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_IN_USE_WINDOW, + DEFAULT_QUERY_ARTIFACT_TTL, FUTURE_CURSOR_ACTIONS, LEGACY_CONCURRENCY_KEY, + LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, + OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY, + RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, + SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, + UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, @@ -565,6 +565,20 @@ pub(crate) fn load_janitor(payload: Option<&ConfigValue>, file: ConfigFile) -> L get(mapping, "exclude_patterns"), place(file, "janitor.exclude_patterns"), )?; + let keep_in_use = optional_bool(mapping, "keep_in_use", false)?; + let in_use_window = match get(mapping, "in_use_window") { + None => DEFAULT_IN_USE_WINDOW.to_owned(), + Some(_) => optional_str(mapping, "in_use_window") + .ok() + .flatten() + .filter(|window| matches!(fixed_duration_seconds(window, false), Ok(Some(_)))) + .ok_or_else(|| { + ConfigFailure::at( + file, + "janitor.in_use_window must be a positive fixed duration such as '7d'", + ) + })?, + }; let checks = [ (retention_days.as_ref(), "retention_days"), (Some(&archive), "archive_retention_days"), @@ -591,6 +605,11 @@ pub(crate) fn load_janitor(payload: Option<&ConfigValue>, file: ConfigFile) -> L Built::Value(ConfigValue::Bool(delete_tracked_only)), ), ("exclude_patterns", Built::Tuple(exclude)), + ( + "in_use_window", + Built::Value(ConfigValue::String(in_use_window)), + ), + ("keep_in_use", Built::Value(ConfigValue::Bool(keep_in_use))), ], )) } diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 4c745d97c2..7cc378c799 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -27,6 +27,7 @@ pub(crate) const MODELS_DIRECTORY_NAME: &str = "models"; pub(crate) const UNLIMITED_DIFF_FULL_ROWS: &str = "unlimited"; pub(crate) const ZERO_FIXED_DURATION: &str = "0s"; pub(crate) const DEFAULT_QUERY_ARTIFACT_TTL: &str = "24h"; +pub(crate) const DEFAULT_IN_USE_WINDOW: &str = "7d"; pub(crate) const BOUNDED_REPLAY_PREFIX: &str = "bounded-"; pub(crate) const ZERO_FLOAT: f64 = 0.0; pub(crate) const SHUTDOWN_TIMEOUT_MAX_SECONDS: u64 = 600; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 1bcced4f42..969b1b5ade 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -40,6 +40,33 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { ), ), }, + ProjectFailureTestCase { + description: "a janitor usage window and keep_in_use load", + project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nenabled = true\n\ + in_use_window = '36h'\nkeep_in_use = true\n", + local: "", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "a calendar janitor usage window is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nin_use_window = '1mo'\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "janitor.in_use_window must be a positive fixed duration such as '7d'", + None, + ), + }, + ProjectFailureTestCase { + description: "a janitor usage window that is not a duration is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nin_use_window = 7\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "janitor.in_use_window must be a positive fixed duration such as '7d'", + None, + ), + }, ProjectFailureTestCase { description: "a git base ref loads", project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n", diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs index fa60de261a..f4f92c1ed8 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod counts; pub(crate) mod phases; pub(crate) mod plan_json; pub(crate) mod plan_sections; +pub(crate) mod usage; diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs index 3a52cb0afc..dc0a1ba8e2 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -1,9 +1,10 @@ //! The JSON payload of `sqb janitor --json`. +use crate::janitor_text::_helpers::usage::checked_usage; use crate::janitor_text::constants::{DROP_EARLY, DROP_EXPIRED}; use crate::janitor_text::models::{ - JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText, OldNameViewText, - SourceSchemaText, + ArchiveCandidateText, JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText, + OldNameViewText, SourceSchemaText, }; use crate::json::models::{JsonInteger, JsonValue}; @@ -61,6 +62,54 @@ fn old_name_view(view: &OldNameViewText, planned: &str) -> JsonValue { ]) } +/// `null` when usage was not checked, `false` when unread, else the reads. +fn in_use(candidate: &ArchiveCandidateText, checked: bool) -> JsonValue { + if !checked { + return JsonValue::Null; + } + candidate + .usage + .as_ref() + .map_or(JsonValue::Bool(false), |usage| { + object(vec![ + ("last_read", utc_timestamp(&usage.last_read_at)), + ("readers", texts(&usage.readers)), + ("query_count", integer(usage.query_count)), + ]) + }) +} + +fn archive_candidate(candidate: &ArchiveCandidateText, checked: bool) -> JsonValue { + object(vec![ + ("name", text(&candidate.name)), + ("archive_name", text(&candidate.archive_name)), + ("age_days", optional_integer(candidate.age_days)), + ("expires_at", utc_timestamp(&candidate.expires_at)), + ("deleted_in_run", JsonValue::Bool(candidate.expires_now)), + ("in_use", in_use(candidate, checked)), + ]) +} + +/// The top-level `usage_check`: its status and reason, or `null` when it did not run. +pub(crate) fn usage_check_payload(plan: Option<&JanitorPlanText>) -> JsonValue { + let Some(check) = plan.and_then(|plan| plan.usage_check.as_ref()) else { + return JsonValue::Null; + }; + object(vec![ + ( + "status", + text(if check.checked { + "checked" + } else { + "not_checked" + }), + ), + ("reason", optional_text(check.reason.as_deref())), + ("window", text(&check.window)), + ("keep_in_use", JsonValue::Bool(check.keep_in_use)), + ]) +} + fn source_schema(schema: &SourceSchemaText, blocked: bool) -> JsonValue { let mut fields: Vec<(&str, JsonValue)> = vec![ ("name", text(&schema.name)), @@ -96,6 +145,7 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { .map(|view| old_name_view(view, "missing")), ) .collect(); + let checked: bool = checked_usage(plan).is_some(); object(vec![ ("direct_mode", JsonValue::Bool(plan.direct_mode)), ("retention_days", integer(plan.retention_days)), @@ -114,15 +164,16 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { JsonValue::Array( plan.archive_candidates .iter() - .map(|candidate| { - object(vec![ - ("name", text(&candidate.name)), - ("archive_name", text(&candidate.archive_name)), - ("age_days", optional_integer(candidate.age_days)), - ("expires_at", utc_timestamp(&candidate.expires_at)), - ("deleted_in_run", JsonValue::Bool(candidate.expires_now)), - ]) - }) + .map(|candidate| archive_candidate(candidate, checked)) + .collect(), + ), + ), + ( + "kept_in_use", + JsonValue::Array( + plan.kept_in_use + .iter() + .map(|candidate| archive_candidate(candidate, checked)) .collect(), ), ), diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs index 776bf890c3..70283b3565 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs @@ -1,11 +1,14 @@ //! Sections of the janitor plan preview. use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::usage::{ + checked_usage, in_use_candidates, usage_detail, usage_notices, +}; use crate::janitor_text::constants::{ DROP_EARLY, DROP_EXPIRED, DROP_MISSING, LABEL_WIDTH, TREE_BLANK, TREE_BRANCH_GLYPH, TREE_CONTINUATION, TREE_LAST_GLYPH, VERBOSE_FLAG, }; -use crate::janitor_text::models::{JanitorPlanText, OldNameViewText}; +use crate::janitor_text::models::{ArchiveCandidateText, JanitorPlanText, OldNameViewText}; /// The first `limit` items, or all of them, and how many are hidden. fn visible(items: &[T], limit: Option) -> (&[T], usize) { @@ -91,6 +94,13 @@ pub(crate) fn summary(plan: &JanitorPlanText, style: CliStyle) -> String { "relations to archive", &plan.archive_candidates, )); + if let Some(check) = checked_usage(plan) { + if check.keep_in_use { + text.push_str(&count_row(style, "kept (in use)", &plan.kept_in_use)); + } else { + text.push_str(&count_row(style, "in use", &in_use_candidates(plan))); + } + } text.push_str(&count_row( style, "archives to delete", @@ -118,9 +128,38 @@ pub(crate) fn summary(plan: &JanitorPlanText, style: CliStyle) -> String { "objects skipped", &style.accent(&plan.skipped_relations.len().to_string()), )); + let notices: Vec = usage_notices(plan); + if !notices.is_empty() { + text.push('\n'); + for notice in notices { + text.push_str(&format!(" {}\n", style.warning(¬ice))); + } + } text } +/// In-use archive candidates with their readers, or the in-use relations kept instead. +fn in_use(plan: &JanitorPlanText, limit: Option, style: CliStyle) -> String { + let Some(check) = checked_usage(plan) else { + return String::new(); + }; + let (title, candidates): (&str, Vec<&ArchiveCandidateText>) = if check.keep_in_use { + ("Kept (in use)", plan.kept_in_use.iter().collect()) + } else { + ("In use", in_use_candidates(plan)) + }; + let lines: Vec<(String, Option)> = candidates + .into_iter() + .map(|candidate| { + ( + candidate.name.clone(), + usage_detail(candidate, &check.window), + ) + }) + .collect(); + listing((style, limit), title, &lines) +} + /// Schemas whose active sources block every janitor action. pub(crate) fn blocked(plan: &JanitorPlanText, limit: Option, style: CliStyle) -> String { if plan.blocked_schemas.is_empty() { @@ -190,6 +229,7 @@ pub(crate) fn archives(plan: &JanitorPlanText, limit: Option, style: CliS } text.push_str(&overflow(style, hidden, " ")); } + text.push_str(&in_use(plan, limit, style)); if !plan.archive_deletions.is_empty() { text.push_str(&format!("\n{}\n", style.success("Archives to delete"))); let (deletions, hidden) = visible(&plan.archive_deletions, limit); diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs new file mode 100644 index 0000000000..ca86885c3c --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs @@ -0,0 +1,116 @@ +//! Wording of the janitor usage check: who reads a candidate and what happens to it. + +use crate::cli_text::main::format_count_noun::format_count_noun; +use crate::janitor_text::_helpers::phases::noun; +use crate::janitor_text::constants::{ + KEEP_WITHOUT_USAGE_NOTICE, READER_LIMIT, USAGE_NOT_CHECKED_PREFIX, WINDOW_UNITS, +}; +use crate::janitor_text::models::{ArchiveCandidateText, JanitorPlanText, UsageCheckText}; + +/// The usage check when it ran and could read usage. +pub(crate) fn checked_usage(plan: &JanitorPlanText) -> Option<&UsageCheckText> { + plan.usage_check.as_ref().filter(|check| check.checked) +} + +/// Archive candidates with reads in the usage window. +pub(crate) fn in_use_candidates(plan: &JanitorPlanText) -> Vec<&ArchiveCandidateText> { + plan.archive_candidates + .iter() + .filter(|candidate| candidate.usage.is_some()) + .collect() +} + +/// `7 days` for a single-unit window such as `7d`, otherwise the window as written. +pub(crate) fn window_label(window: &str) -> String { + let digits: &str = window.trim_end_matches(|character: char| !character.is_ascii_digit()); + let unit: &str = &window[digits.len()..]; + match ( + digits.parse::(), + WINDOW_UNITS.iter().find(|(suffix, _)| *suffix == unit), + ) { + (Ok(count), Some((_, singular))) if !digits.is_empty() => noun_count(count, singular), + _ => window.to_owned(), + } +} + +fn noun_count(count: i64, singular: &str) -> String { + noun(usize::try_from(count).unwrap_or(usize::MAX), singular) +} + +/// `alice, bob, carol +2 more`: the first readers and how many are not named. +fn readers(names: &[String]) -> String { + let shown: usize = names.len().min(READER_LIMIT); + let mut text: String = names[..shown].join(", "); + if names.len() > shown { + text.push_str(&format!(" +{} more", names.len() - shown)); + } + text +} + +/// `queried by ( queries in )`, plus the frozen flag for tables. +pub(crate) fn usage_detail(candidate: &ArchiveCandidateText, window: &str) -> Option { + let usage = candidate.usage.as_ref()?; + let date: &str = usage.last_read_at.split(' ').next().unwrap_or_default(); + let mut text: String = format!( + "queried {date} by {} ({} in {})", + readers(&usage.readers), + format_count_noun(usage.query_count, "query", Some("queries")), + window_label(window) + ); + if candidate.is_table { + match &candidate.changed_on { + Some(changed_on) => { + text.push_str(&format!("; frozen since {changed_on}, still in use")); + } + None => text.push_str("; frozen, still in use"), + } + } + Some(text) +} + +/// The notice lines when usage could not be checked, empty otherwise. +pub(crate) fn usage_notices(plan: &JanitorPlanText) -> Vec { + let Some(check) = plan.usage_check.as_ref().filter(|check| !check.checked) else { + return Vec::new(); + }; + let mut notices: Vec = vec![format!( + "{USAGE_NOT_CHECKED_PREFIX}{}", + check.reason.as_deref().unwrap_or("unknown reason") + )]; + if check.keep_in_use { + notices.push(KEEP_WITHOUT_USAGE_NOTICE.to_owned()); + } + notices +} + +/// The prompt line stating how many archived relations are in use, or which are kept. +pub(crate) fn usage_prompt_line(plan: &JanitorPlanText) -> Option { + let check: &UsageCheckText = checked_usage(plan)?; + if check.keep_in_use && !plan.kept_in_use.is_empty() { + return Some(format!( + "{} in use {} kept.", + noun(plan.kept_in_use.len(), "relation"), + if plan.kept_in_use.len() == 1 { + "is" + } else { + "are" + } + )); + } + if plan.archive_candidates.is_empty() { + return None; + } + let in_use: usize = in_use_candidates(plan).len(); + let archived: usize = plan.archive_candidates.len(); + let in_use_text: String = match (archived, in_use) { + (1, 0) => "it is not in use".to_owned(), + (1, _) => "it is in use".to_owned(), + (_, 0) => "none of them are in use".to_owned(), + (_, 1) => "1 of them is in use".to_owned(), + _ => format!("{in_use} of them are in use"), + }; + Some(format!( + "{} will be archived; {in_use_text}.", + noun(archived, "relation") + )) +} diff --git a/crates/sqlbuild-core/src/janitor_text/constants.rs b/crates/sqlbuild-core/src/janitor_text/constants.rs index 13f8108f08..148a5f3ee0 100644 --- a/crates/sqlbuild-core/src/janitor_text/constants.rs +++ b/crates/sqlbuild-core/src/janitor_text/constants.rs @@ -22,3 +22,15 @@ pub(crate) const SECONDS_PER_MINUTE: u64 = 60; /// Durations below this many seconds show seconds with two decimals. pub(crate) const MINUTE_SECONDS: f64 = 60.0; pub(crate) const SECONDS_PER_HOUR: u64 = 3600; +/// How many readers each in-use line names before `+N more`. +pub(crate) const READER_LIMIT: usize = 3; +pub(crate) const USAGE_NOT_CHECKED_PREFIX: &str = "usage not checked: "; +pub(crate) const KEEP_WITHOUT_USAGE_NOTICE: &str = + "nothing kept in use: no usage could be checked, so relations in use are archived too"; +/// Single-unit usage windows and the noun of their unit. +pub(crate) const WINDOW_UNITS: [(&str, &str); 4] = [ + ("d", "day"), + ("h", "hour"), + ("m", "minute"), + ("s", "second"), +]; diff --git a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs index 8f644e58ab..8b626ab410 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs @@ -1,6 +1,6 @@ //! `sqb janitor --json`. -use crate::janitor_text::_helpers::plan_json::{plan_payload, run_payload}; +use crate::janitor_text::_helpers::plan_json::{plan_payload, run_payload, usage_check_payload}; use crate::janitor_text::models::{JanitorOutcome, JanitorPlanText, JanitorRunText}; use crate::json::main::dumps::dumps; use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; @@ -27,6 +27,7 @@ pub fn format_janitor_json( "plan".to_owned(), plan.map_or(JsonValue::Null, plan_payload), ), + ("usage_check".to_owned(), usage_check_payload(plan)), ( "result".to_owned(), run.map_or(JsonValue::Null, run_payload), diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs index 0db1e14284..e774f99687 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs @@ -3,6 +3,7 @@ use crate::janitor_text::_helpers::counts::{ archive_prefix, environment_label, physical_deletion_count, }; +use crate::janitor_text::_helpers::usage::{usage_notices, usage_prompt_line}; use crate::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation_phrase; use crate::janitor_text::models::JanitorPlanText; @@ -22,6 +23,12 @@ pub fn janitor_confirmation_prompt(plan: &JanitorPlanText) -> String { let items: usize = physical_deletion_count(plan) + plan.state_prunes.len(); format!("Janitor will {prefix}delete {items} items from {environment}.\n") }; + if let Some(line) = usage_prompt_line(plan) { + text.push_str(&format!("{line}\n")); + } + for notice in usage_notices(plan) { + text.push_str(&format!("{}\n", capitalized(¬ice))); + } if plan.retention_days == 0 { text.push_str("Retention: disabled (0 days)\nAge metadata will not be checked.\n"); } else { @@ -39,3 +46,10 @@ pub fn janitor_confirmation_prompt(plan: &JanitorPlanText) -> String { )); text } + +fn capitalized(text: &str) -> String { + let mut characters = text.chars(); + characters.next().map_or_else(String::new, |first| { + first.to_uppercase().chain(characters).collect() + }) +} diff --git a/crates/sqlbuild-core/src/janitor_text/models.rs b/crates/sqlbuild-core/src/janitor_text/models.rs index cbd49ccd0b..b20a159383 100644 --- a/crates/sqlbuild-core/src/janitor_text/models.rs +++ b/crates/sqlbuild-core/src/janitor_text/models.rs @@ -1,5 +1,15 @@ //! Display facts of a janitor plan and its execution, read from the executor's models. +/// Reads of one relation within the usage window. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RelationUsageText { + /// The latest read as `YYYY-MM-DD HH:MM:SS` UTC. + pub last_read_at: String, + /// Reader user names, most queries first. + pub readers: Vec, + pub query_count: i64, +} + /// A relation to archive: its name, archive name, age and expiry. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ArchiveCandidateText { @@ -11,6 +21,24 @@ pub struct ArchiveCandidateText { pub expires_now: bool, /// The expiry as `YYYY-MM-DD HH:MM:SS`. pub expires_at: String, + /// Whether the relation is a table, whose data stops changing once it is stale. + pub is_table: bool, + /// The date it last changed as `YYYY-MM-DD`, or none when unknown. + pub changed_on: Option, + /// Its reads within the usage window, or none when unread or not checked. + pub usage: Option, +} + +/// Whether the janitor checked reads of its archive candidates. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct UsageCheckText { + pub checked: bool, + /// Why usage was not checked. + pub reason: Option, + /// The usage window as configured, such as `7d`. + pub window: String, + /// Whether in-use relations are kept instead of archived. + pub keep_in_use: bool, } /// An archived relation to delete or retain. @@ -69,6 +97,10 @@ pub struct JanitorPlanText { pub skipped_relations: Vec<(String, String)>, pub skipped_schemas: Vec, pub blocked_schemas: Vec, + /// The usage check of the archive candidates, or none when it did not run. + pub usage_check: Option, + /// In-use relations kept instead of archived. + pub kept_in_use: Vec, } /// How many items one janitor run changed. diff --git a/crates/sqlbuild-core/src/janitor_text/tests.rs b/crates/sqlbuild-core/src/janitor_text/tests.rs index 3735ed1a78..53afbdab1c 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests.rs @@ -4,5 +4,7 @@ mod helpers; mod janitor_progress_text; #[path = "tests/test_janitor_text.rs"] mod janitor_text; +#[path = "tests/test_janitor_usage_text.rs"] +mod janitor_usage_text; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index 48c075193d..947d6b2693 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -1,6 +1,6 @@ use crate::janitor_text::models::{ ArchiveCandidateText, ArchivedRelationText, JanitorItemText, JanitorPhase, JanitorPlanText, - OldNameViewText, + OldNameViewText, RelationUsageText, UsageCheckText, }; /// A direct-mode plan for `dev` that archives one relation and deletes one archive. @@ -78,3 +78,73 @@ pub(super) fn old_name_plan(count: usize) -> JanitorPlanText { ..JanitorPlanText::default() } } + +/// Reads by `readers` of `count` queries, last on 2026-10-21. +fn usage(readers: &[&str], query_count: i64) -> RelationUsageText { + RelationUsageText { + last_read_at: "2026-10-21 09:30:00".to_owned(), + readers: readers.iter().map(|reader| (*reader).to_owned()).collect(), + query_count, + } +} + +/// Three candidates checked over `7d`: an in-use view, an in-use table read by five users, +/// and an unread table; `keep_in_use` moves the in-use two to `kept_in_use`. +pub(super) fn usage_plan(keep_in_use: bool) -> JanitorPlanText { + let view: ArchiveCandidateText = ArchiveCandidateText { + name: "dev.orders_daily".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__orders_daily".to_owned(), + usage: Some(usage(&["svc_reporting"], 14)), + ..ArchiveCandidateText::default() + }; + let table: ArchiveCandidateText = ArchiveCandidateText { + name: "dev.old_orders".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__old_orders".to_owned(), + is_table: true, + changed_on: Some("2026-09-01".to_owned()), + usage: Some(usage( + &["svc_reporting", "alice", "bob", "carol", "dave"], + 40, + )), + ..ArchiveCandidateText::default() + }; + let unread: ArchiveCandidateText = ArchiveCandidateText { + name: "dev.old_customers".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__old_customers".to_owned(), + is_table: true, + ..ArchiveCandidateText::default() + }; + let (archive_candidates, kept_in_use) = if keep_in_use { + (vec![unread], vec![view, table]) + } else { + (vec![view, table, unread], Vec::new()) + }; + JanitorPlanText { + target_name: Some("dev".to_owned()), + retention_days: 14, + direct_mode: true, + archive_retention_days: 14, + archive_candidates, + kept_in_use, + usage_check: Some(UsageCheckText { + checked: true, + reason: None, + window: "7d".to_owned(), + keep_in_use, + }), + ..JanitorPlanText::default() + } +} + +/// A plan whose usage could not be checked, for `reason`. +pub(super) fn unchecked_plan(keep_in_use: bool) -> JanitorPlanText { + JanitorPlanText { + usage_check: Some(UsageCheckText { + checked: false, + reason: Some("unsupported adapter".to_owned()), + window: "7d".to_owned(), + keep_in_use, + }), + ..direct_archive_plan() + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs new file mode 100644 index 0000000000..b951f809ef --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs @@ -0,0 +1,198 @@ +use crate::cli_text::models::CliStyle; +use crate::janitor_text::main::format_janitor_json::format_janitor_json; +use crate::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation_phrase; +use crate::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation_prompt; +use crate::janitor_text::main::render_janitor_plan::render_janitor_plan; +use crate::janitor_text::models::JanitorOutcome; +use crate::janitor_text::tests::helpers::{direct_archive_plan, unchecked_plan, usage_plan}; +use crate::janitor_text::tests::test_types::{JsonTestCase, PreviewTestCase}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_usage_check_when_rendering_preview_then_in_use_relations_show_readers() { + let test_cases = [ + PreviewTestCase { + description: "in-use candidates are listed with capped readers and archived anyway", + plan: usage_plan(false), + verbose: false, + expected_fragments: &[ + " relations to archive 3\n in use 2\n", + "\nIn use\n dev.orders_daily queried 2026-10-21 by svc_reporting \ + (14 queries in 7 days)\n", + " dev.old_orders queried 2026-10-21 by svc_reporting, alice, bob +2 more \ + (40 queries in 7 days); frozen since 2026-09-01, still in use\n", + " dev.old_customers -> dev._sqb_archive__", + ], + unexpected_fragments: &[ + "Kept (in use)", + "dev.old_customers queried", + "dev.orders_daily queried 2026-10-21 by svc_reporting (14 queries in 7 days); frozen", + "usage not checked", + ], + }, + PreviewTestCase { + description: "keep_in_use lists in-use relations as kept and leaves them unarchived", + plan: usage_plan(true), + verbose: false, + expected_fragments: &[ + " relations to archive 1\n kept (in use) 2\n", + "\nKept (in use)\n dev.orders_daily queried 2026-10-21", + " dev.old_orders queried 2026-10-21 by svc_reporting, alice, bob +2 more", + ], + unexpected_fragments: &["\nIn use\n", "dev.old_orders ->", "dev.orders_daily ->"], + }, + PreviewTestCase { + description: "an unchecked usage shows one notice and no in-use rows", + plan: unchecked_plan(false), + verbose: false, + expected_fragments: &["\n usage not checked: unsupported adapter\n"], + unexpected_fragments: &["in use ", "In use", "nothing kept in use"], + }, + PreviewTestCase { + description: "keep_in_use without a usage source warns that nothing is kept", + plan: unchecked_plan(true), + verbose: false, + expected_fragments: &[ + " usage not checked: unsupported adapter\n nothing kept in use: no usage \ + could be checked, so relations in use are archived too\n", + ], + unexpected_fragments: &["Kept (in use)"], + }, + PreviewTestCase { + description: "a plan without a usage check renders as before", + plan: direct_archive_plan(), + verbose: false, + expected_fragments: &[" relations to archive 1\n archives to delete"], + unexpected_fragments: &["usage", "in use"], + }, + ]; + + for test_case in test_cases { + let text: String = render_janitor_plan(&test_case.plan, test_case.verbose, PLAIN); + for fragment in test_case.expected_fragments { + assert!( + text.contains(fragment), + "{}: missing {fragment:?} in {text}", + test_case.description + ); + } + for fragment in test_case.unexpected_fragments { + assert!( + !text.contains(fragment), + "{}: unexpected {fragment:?} in {text}", + test_case.description + ); + } + } +} + +#[test] +fn given_usage_check_when_prompting_then_prompt_states_in_use_count() { + let test_cases: [(&str, crate::janitor_text::models::JanitorPlanText, &str); 4] = [ + ( + "the prompt counts in-use relations among those archived", + usage_plan(false), + "Janitor will archive 3 and delete 0 objects from dev.\n\ + 3 relations will be archived; 2 of them are in use.\nRetention: 14 days\n", + ), + ( + "keep_in_use counts the kept relations", + usage_plan(true), + "Janitor will archive 1 and delete 0 objects from dev.\n\ + 2 relations in use are kept.\nRetention: 14 days\n", + ), + ( + "an unchecked usage repeats the notice", + unchecked_plan(false), + "Janitor will archive 1 and delete 1 objects from dev.\n\ + Usage not checked: unsupported adapter\nRetention: 14 days\n", + ), + ( + "keep_in_use without a usage source repeats the warning", + unchecked_plan(true), + "Usage not checked: unsupported adapter\nNothing kept in use: no usage could be \ + checked, so relations in use are archived too\nRetention", + ), + ]; + + for (description, plan, expected) in test_cases { + let prompt: String = janitor_confirmation_prompt(&plan); + assert!( + prompt.contains(expected), + "{description}: missing {expected:?} in {prompt}" + ); + } + assert_eq!( + janitor_confirmation_phrase(&usage_plan(true)), + "archive 1 and delete 0 objects from dev" + ); +} + +#[test] +fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { + let test_cases = [ + JsonTestCase { + description: "checked candidates carry their reads or false, with the check status", + plan: Some(usage_plan(false)), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &[ + "\"in_use\": {\n \"last_read\": \"2026-10-21T09:30:00Z\",\n \ + \"readers\": [\n \"svc_reporting\"\n ],\n \ + \"query_count\": 14\n }", + "\"in_use\": false", + "\"kept_in_use\": []", + "\"usage_check\": {\n \"status\": \"checked\",\n \"reason\": null,\n \ + \"window\": \"7d\",\n \"keep_in_use\": false\n }", + ], + }, + JsonTestCase { + description: "kept relations are listed apart from the relations to archive", + plan: Some(usage_plan(true)), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &[ + "\"kept_in_use\": [\n {\n \"name\": \"dev.orders_daily\"", + "\"keep_in_use\": true", + ], + }, + JsonTestCase { + description: "an unchecked usage leaves in_use null and states the reason", + plan: Some(unchecked_plan(false)), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &[ + "\"in_use\": null", + "\"status\": \"not_checked\",\n \"reason\": \"unsupported adapter\"", + ], + }, + JsonTestCase { + description: "no usage check is null", + plan: Some(direct_archive_plan()), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &["\"usage_check\": null", "\"in_use\": null"], + }, + ]; + + for test_case in test_cases { + let text: String = format_janitor_json( + test_case.plan.as_ref(), + test_case.outcome, + test_case.run.as_ref(), + test_case.error, + ); + for fragment in test_case.expected_fragments { + assert!( + text.contains(fragment), + "{}: missing {fragment:?} in {text}", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index f6448e6691..cd6572eb5b 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -183,7 +183,7 @@ pub const OPERATION_KINDS: [&str; 10] = [ "subprocess", "warehouse", ]; -pub const OPERATION_NAMES: [&str; 56] = [ +pub const OPERATION_NAMES: [&str; 57] = [ "adbc_driver_install", "audit_evaluation", "clone_execution", @@ -208,6 +208,7 @@ pub const OPERATION_NAMES: [&str; 56] = [ "janitor_execution", "janitor_state_inspection", "janitor_target_connection", + "janitor_usage_check", "janitor_warehouse_inspection", "managed_source_load", "project_compile", diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 5e0d635827..06d7549249 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -284,6 +284,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Executor, }, + HostEntry { + name: "JanitorUsageCheck", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, HostEntry { name: "run_load_pipeline", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 85ba45b08f..531d06bd5c 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -257,6 +257,7 @@ pub enum ExecutorHost { JanitorRelationKey, JanitorSkippedRelation, JanitorSkippedSchema, + JanitorUsageCheck, RunLoadPipeline, LoadCallbacks, LoadExecutionResult, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 925d7e16ee..34186a05d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -360,6 +360,8 @@ const R_JANITOR_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("direct_state_history_versions", FieldDefault::Int(20)), FieldSpec::new("delete_tracked_only", FieldDefault::True), FieldSpec::new("exclude_patterns", FieldDefault::EmptyTuple), + FieldSpec::new("in_use_window", FieldDefault::Text("7d")), + FieldSpec::new("keep_in_use", FieldDefault::False), ], }; const R_MIGRATIONS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 5dba134a6c..ceea83e278 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -185,8 +185,14 @@ fn settings<'py>( "drop_old_name_views", &native_field(request, "drop_old_name_views")?, ), + ("in_use_window", &native_field(janitor, "in_use_window")?), ], )?; + options.set_item( + "keep_in_use", + native_field(request, "keep_in_use")?.is_truthy()? + || native_field(janitor, "keep_in_use")?.is_truthy()?, + )?; new_record(py, RecordId::JanitorSettings, (), Some(&options)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index ec56d16473..d06f9d89b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -22,6 +22,7 @@ use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; use crate::bindings::_helpers::janitor::janitor_planning::{ JanitorSettings, RelationScope, build_plan, }; +use crate::bindings::_helpers::janitor::janitor_usage::{UsageSettings, check_janitor_usage}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -284,6 +285,7 @@ pub(crate) fn build_janitor_execution_plan<'py>( Ok(plan) }, )?; + let has_candidates: bool = native_field(&plan, "archive_candidates")?.is_truthy()?; let options: Bound<'py, PyDict> = PyDict::new(py); options.set_item( "message", @@ -292,14 +294,27 @@ pub(crate) fn build_janitor_execution_plan<'py>( started.elapsed().as_secs_f64() ), )?; - options.set_item("blank_line_after", true)?; + options.set_item("blank_line_after", !has_candidates)?; status.call_method("complete", (), Some(&options))?; let unknown: Bound<'py, PyAny> = native_field(&plan, "old_name_views")?.getattr("unknown_requests")?; if unknown.is_truthy()? { return Err(unknown_view_error(py, &unknown)?); } - Ok(plan) + if !has_candidates { + return Ok(plan); + } + check_janitor_usage( + (&compiled.adapter, connection), + ( + &plan, + &status_reporter(&output_stream(invocation)?, use_color(invocation)?)?, + ), + &UsageSettings { + window: native_field(settings, "in_use_window")?.extract()?, + keep_in_use: native_field(settings, "keep_in_use")?.is_truthy()?, + }, + ) } /// Execute the cleanup plan on up to the command's concurrency, reporting to `progress`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs index 38a3a16c4c..08ca8190a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs @@ -8,11 +8,13 @@ use sqlbuild_core::janitor_text::main::render_janitor_completion::render_janitor use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::models::{ ArchiveCandidateText, ArchivedRelationText, JanitorPlanText, JanitorResultCounts, - OldNameViewText, SourceSchemaText, + OldNameViewText, RelationUsageText, SourceSchemaText, UsageCheckText, }; const TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S"; const DATE_FORMAT: &str = "%Y-%m-%d"; +/// Relation types containing this are views, whose reads stay live. +const VIEW_TYPE_FRAGMENT: &str = "view"; fn display_name(item: &Bound<'_, PyAny>) -> PyResult { item.call_method0("display_name")?.extract() @@ -121,21 +123,73 @@ fn source_schemas(schemas: &Bound<'_, PyAny>, blocked: bool) -> PyResult, pattern: &str) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(formatted(value, pattern)?)) +} + +fn relation_usage(usage: &Bound<'_, PyAny>) -> PyResult> { + if usage.is_none() { + return Ok(None); + } + Ok(Some(RelationUsageText { + last_read_at: formatted(&native_field(usage, "last_read_at")?, TIMESTAMP_FORMAT)?, + readers: native_field(usage, "readers")?.extract()?, + query_count: native_field(usage, "query_count")?.extract()?, + })) +} + +fn archive_candidate( + candidate: &Bound<'_, PyAny>, + planned_at: &Bound<'_, PyAny>, +) -> PyResult { + let expires_at: Bound<'_, PyAny> = native_field(candidate, "expires_at")?; + let age_timestamp: Bound<'_, PyAny> = native_field(candidate, "age_timestamp")?; + let relation_type: String = + native_field(&native_field(candidate, "relation")?, "relation_type")? + .str()? + .extract()?; + Ok(ArchiveCandidateText { + name: key_name(candidate)?, + archive_name: display_name(&native_field(candidate, "archive_key")?)?, + age_days: age_days(&age_timestamp, planned_at)?, + expires_now: expires_at.le(native_field(candidate, "archived_at")?)?, + expires_at: formatted(&expires_at, TIMESTAMP_FORMAT)?, + is_table: !relation_type.to_lowercase().contains(VIEW_TYPE_FRAGMENT), + changed_on: optional_formatted(&age_timestamp, DATE_FORMAT)?, + usage: relation_usage(&native_field(candidate, "usage")?)?, + }) +} + +fn candidate_texts( + items: &Bound<'_, PyAny>, + planned_at: &Bound<'_, PyAny>, +) -> PyResult> { + items + .try_iter()? + .map(|item| archive_candidate(&item?, planned_at)) + .collect() +} + +fn usage_check(check: &Bound<'_, PyAny>) -> PyResult> { + if check.is_none() { + return Ok(None); + } + Ok(Some(UsageCheckText { + checked: native_field(check, "checked")?.is_truthy()?, + reason: native_field(check, "reason")?.extract()?, + window: native_field(check, "window")?.extract()?, + keep_in_use: native_field(check, "keep_in_use")?.is_truthy()?, + })) +} + /// The display facts of one executor `JanitorPlan`. pub(crate) fn janitor_plan_text(plan: &Bound<'_, PyAny>) -> PyResult { let planned_at: Bound<'_, PyAny> = native_field(plan, "planned_at")?; - let mut archive_candidates: Vec = Vec::new(); - for candidate in native_field(plan, "archive_candidates")?.try_iter()? { - let candidate: Bound<'_, PyAny> = candidate?; - let expires_at: Bound<'_, PyAny> = native_field(&candidate, "expires_at")?; - archive_candidates.push(ArchiveCandidateText { - name: key_name(&candidate)?, - archive_name: display_name(&native_field(&candidate, "archive_key")?)?, - age_days: age_days(&native_field(&candidate, "age_timestamp")?, &planned_at)?, - expires_now: expires_at.le(native_field(&candidate, "archived_at")?)?, - expires_at: formatted(&expires_at, TIMESTAMP_FORMAT)?, - }); - } + let archive_candidates: Vec = + candidate_texts(&native_field(plan, "archive_candidates")?, &planned_at)?; let mut state_prunes: Vec<(String, i64)> = Vec::new(); for candidate in native_field(plan, "direct_state_prune_candidates")?.try_iter()? { let candidate: Bound<'_, PyAny> = candidate?; @@ -175,6 +229,8 @@ pub(crate) fn janitor_plan_text(plan: &Bound<'_, PyAny>) -> PyResult { + report: Bound<'py, PyAny>, + usages: HashMap>, +} + +fn lifecycle(py: Python<'_>) -> PyResult> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("operation_kind", "janitor")?; + options.set_item("operation_name", "janitor_usage_check")?; + host::entry(py, ExecutorHost::OperationLifecycle)?.call((), Some(&options)) +} + +fn replaced<'py>( + value: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = value.py(); + let options: Bound<'py, PyDict> = PyDict::new(py); + for (name, field) in fields { + options.set_item(*name, field)?; + } + imported(py, "dataclasses", "replace")?.call((value,), Some(&options)) +} + +fn read_usage<'py>( + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + candidates: &[Bound<'py, PyAny>], + window_seconds: i64, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let relations: Vec> = candidates + .iter() + .map(|candidate| native_field(candidate, "relation")) + .collect::>()?; + let report: Bound<'py, PyAny> = call_with( + &native_field(adapter, "relation_usage")?, + &[ + ("connection", connection.clone()), + ("relations", tuple(py, relations)?), + ( + "window_seconds", + window_seconds.into_pyobject(py)?.into_any(), + ), + ], + )?; + let mut usages: HashMap> = HashMap::new(); + if native_field(&report, "checked")?.is_truthy()? { + for usage in native_field(&report, "usages")?.try_iter()? { + let usage: Bound<'py, PyAny> = usage?; + usages.insert(Key::of(&usage)?.folded(), usage); + } + } + Ok(UsageOutcome { report, usages }) +} + +fn completion_message( + report: &Bound<'_, PyAny>, + (in_use, total): (usize, usize), + elapsed_seconds: f64, +) -> PyResult { + if !native_field(report, "checked")?.is_truthy()? { + let reason: Option = native_field(report, "reason")?.extract()?; + return Ok(format!( + "Relation usage not checked: {}. ({elapsed_seconds:.2}s)", + reason.unwrap_or_default() + )); + } + Ok(format!( + "Checked relation usage: {in_use} of {total} in use. ({elapsed_seconds:.2}s)" + )) +} + +/// The plan with each candidate's reads, the usage check, and in-use candidates kept when asked. +pub(crate) fn check_janitor_usage<'py>( + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (plan, status): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + settings: &UsageSettings, +) -> PyResult> { + let py: Python<'py> = plan.py(); + let candidates: Vec> = native_field(plan, "archive_candidates")? + .try_iter()? + .collect::>()?; + let window_seconds: i64 = fixed_duration_seconds(&settings.window) + .and_then(|seconds| i64::try_from(seconds).ok()) + .unwrap_or(i64::MAX); + let started: Instant = Instant::now(); + status.call_method1("start", (START_MESSAGE,))?; + let outcome: UsageOutcome<'py> = within(&lifecycle(py)?, |lifecycle| { + let outcome: UsageOutcome<'py> = + read_usage((adapter, connection), &candidates, window_seconds)?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", outcome.usages.len())?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("metadata", metadata)?; + lifecycle.call_method("completed", (), Some(&options))?; + Ok(outcome) + })?; + let checked: bool = native_field(&outcome.report, "checked")?.is_truthy()?; + let keep: bool = checked && settings.keep_in_use; + let mut archived: Vec> = Vec::new(); + let mut kept: Vec> = Vec::new(); + let mut kept_keys: HashSet = HashSet::new(); + let mut in_use: usize = 0; + for candidate in &candidates { + let key: Folded = Key::of(&native_field(candidate, "relation")?)?.folded(); + let Some(usage) = outcome.usages.get(&key) else { + archived.push(candidate.clone()); + continue; + }; + in_use += 1; + let read: Bound<'py, PyAny> = replaced(candidate, &[("usage", usage.clone())])?; + if keep { + kept_keys.insert(Key::of(&native_field(candidate, "key")?)?.folded()); + kept.push(read); + } else { + archived.push(read); + } + } + let mut deletions: Vec> = Vec::new(); + for deletion in native_field(plan, "archive_deletion_candidates")?.try_iter()? { + let deletion: Bound<'py, PyAny> = deletion?; + let original: Bound<'py, PyAny> = native_field(&deletion, "original_key")?; + if original.is_none() || !kept_keys.contains(&Key::of(&original)?.folded()) { + deletions.push(deletion); + } + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item( + "message", + completion_message( + &outcome.report, + (in_use, candidates.len()), + started.elapsed().as_secs_f64(), + )?, + )?; + options.set_item("blank_line_after", true)?; + status.call_method("complete", (), Some(&options))?; + let check: Bound<'py, PyAny> = record( + py, + ExecutorHost::JanitorUsageCheck, + &[ + ("checked", PyBool::new(py, checked).to_owned().into_any()), + ( + "window", + settings.window.as_str().into_pyobject(py)?.into_any(), + ), + ( + "keep_in_use", + PyBool::new(py, settings.keep_in_use).to_owned().into_any(), + ), + ("reason", native_field(&outcome.report, "reason")?), + ], + )?; + replaced( + plan, + &[ + ("archive_candidates", tuple(py, archived)?), + ("archive_deletion_candidates", tuple(py, deletions)?), + ("kept_in_use", tuple(py, kept)?), + ("usage_check", check), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs index d2231d7bc5..6c12b76ff8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs @@ -7,4 +7,5 @@ pub(crate) mod janitor_phases; pub(crate) mod janitor_plan; pub(crate) mod janitor_planning; pub(crate) mod janitor_progress; +pub(crate) mod janitor_usage; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 0df2c24a75..b56fb805f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -461,6 +461,7 @@ const JANITOR_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("retention_days", FieldDefault::None), FieldSpec::new("direct_state_history_versions", FieldDefault::None), FieldSpec::new("drop_old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new("keep_in_use", FieldDefault::False), FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("as_target", FieldDefault::None), FieldSpec::new("verbose", FieldDefault::False), @@ -488,6 +489,8 @@ const JANITOR_SETTINGS: RecordSpec = RecordSpec { FieldSpec::required("direct_state_history_versions"), FieldSpec::new("archive_retention_days", FieldDefault::Int(14)), FieldSpec::new("drop_old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new("in_use_window", FieldDefault::Text("7d")), + FieldSpec::new("keep_in_use", FieldDefault::False), ], }; const LOAD_COMMAND_REQUEST: RecordSpec = RecordSpec { diff --git a/fensu.toml b/fensu.toml index 66674b2c51..26113efe6e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -424,7 +424,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.executor.janitor.models"] -symbols = ["JanitorArchiveCandidate", "JanitorArchivedRelation", "JanitorBlockedSchema", "JanitorDeleteCandidate", "JanitorDirectStatePruneCandidate", "JanitorOldNameView", "JanitorOldNameViewPlanning", "JanitorPlan", "JanitorQueryDiffArtifactCandidate", "JanitorRelationKey", "JanitorSkippedRelation", "JanitorSkippedSchema"] +symbols = ["JanitorArchiveCandidate", "JanitorArchivedRelation", "JanitorBlockedSchema", "JanitorDeleteCandidate", "JanitorDirectStatePruneCandidate", "JanitorOldNameView", "JanitorOldNameViewPlanning", "JanitorPlan", "JanitorQueryDiffArtifactCandidate", "JanitorRelationKey", "JanitorSkippedRelation", "JanitorSkippedSchema", "JanitorUsageCheck"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 25074a0d76..6c73d40bdb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/cli/janitor/ - Usage - Flags - Output +- Relations in use - Speed - Targets - Lifecycle @@ -43,6 +44,7 @@ sqb --project-dir janitor [flags] | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | | `--verbose`, `-v` | List every entry of each preview section and every skipped object with its reason. Without it, each section shows its first 10 entries and skipped objects are counted by reason | | `--json` | Print the plan and the result as one JSON document on stdout; the preview, prompt and progress go to stderr | +| `--keep-in-use` | Keep relations that other queries read within `janitor.in_use_window` instead of archiving them, for this run. Same as `janitor.keep_in_use = true`. See [Relations in use](#relations-in-use) | | `--drop-old-name-view ` | Drop the [compatibility view](../concepts/models/migrations.md#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | @@ -73,7 +75,42 @@ Skipped objects (385) 175 relation is not tracked by SQLBuild ``` -With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. +With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `usage_check`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. + +## Relations in use + +Archiving renames a relation, so anything that still reads it fails from then on. Before archiving, the janitor looks up which candidates other queries read within `in_use_window` (default `7d`) and shows them, with their readers, in an `In use` section: + +``` +In use + analytics.orders_daily queried 2026-10-21 by svc_reporting, alice (14 queries in 7 days) + analytics.old_orders queried 2026-10-20 by svc_reporting, alice, bob +2 more (40 queries in 7 days); frozen since 2026-09-01, still in use +``` + +Each line names the latest read, up to three readers (most queries first) and how many queries read the relation. A stale table that is still read is also flagged `frozen since , still in use`: its data stopped changing on that date, so its readers are getting old data. Views are not flagged, because their reads stay live. The confirmation prompt states the count, for example `14 relations will be archived; 3 of them are in use.` + +By default in-use relations are still archived, which is also how their readers find out. To keep them instead, set `keep_in_use = true` under `[janitor]` or pass `--keep-in-use`. In-use relations are then listed under `Kept (in use)` with their readers and are left out of the archive list and the counts. A later run archives them once nothing reads them within the window. + +Only relations to archive are checked. Expired archives and state-table pruning are not. + +Usage comes from the warehouse's query history: + +| Adapter | Usage source | +|---------|--------------| +| Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`) joined with `QUERY_HISTORY` | +| Other adapters | Not available | + +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: + +```sql +GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; +``` + +Access history lags by up to about 3 hours, so reads in the last few hours may not show yet. + +When usage can't be checked, because the adapter has no usage source or the role can't read access history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. + +In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers` and `query_count`. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. ## Speed @@ -183,6 +220,8 @@ retention_days = 14 archive_retention_days = 14 delete_tracked_only = true exclude_patterns = ["audit_*", "tmp_*"] +in_use_window = "7d" +keep_in_use = false ``` When `retention_days` is not set, it defaults to `14`. See [Project Configuration](../concepts/project-configuration.md#janitor) for every janitor setting. @@ -204,6 +243,9 @@ sqb janitor --as prod # Clean up the prod target sqb janitor --target prod + +# Keep relations that are still read instead of archiving them +sqb janitor --keep-in-use ``` ## State history pruning diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index b37b0055d5..00ba7a33df 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -824,6 +824,8 @@ retention_days = 14 archive_retention_days = 14 delete_tracked_only = true exclude_patterns = ["audit_*", "tmp_*"] +in_use_window = "7d" +keep_in_use = false ``` | Field | Default | Description | @@ -834,6 +836,8 @@ exclude_patterns = ["audit_*", "tmp_*"] | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | | `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` | +| `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](../cli/janitor.md#relations-in-use) | +| `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | See [janitor](../cli/janitor.md) for the archive and delete lifecycle. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a92a67ff83..bf81edc288 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3130,6 +3130,7 @@ class CliNamespace: retention_days: int | None direct_state_history_versions: int | None drop_old_name_views: list[str] + keep_in_use: bool bounded: str | None max_column_examples: int | None max_row_only_examples: int | None @@ -6676,6 +6677,7 @@ class JanitorCommandRequest(NativeRecord): retention_days: int | None direct_state_history_versions: int | None drop_old_name_views: tuple[str, ...] + keep_in_use: bool selected_target: str | None as_target: str | None verbose: bool @@ -6688,6 +6690,7 @@ class JanitorCommandRequest(NativeRecord): retention_days: int | None = None, direct_state_history_versions: int | None = None, drop_old_name_views: tuple[str, ...] = (), + keep_in_use: bool = False, selected_target: str | None = None, as_target: str | None = None, verbose: bool = False, @@ -11986,6 +11989,8 @@ class JanitorConfig(NativeRecord): direct_state_history_versions: int delete_tracked_only: bool exclude_patterns: tuple[str, ...] + in_use_window: str + keep_in_use: bool def __init__( self, enabled: bool = ..., @@ -11994,6 +11999,8 @@ class JanitorConfig(NativeRecord): direct_state_history_versions: int = ..., delete_tracked_only: bool = ..., exclude_patterns: tuple[str, ...] = ..., + in_use_window: str = ..., + keep_in_use: bool = ..., ) -> None: ... class MigrationsConfig(NativeRecord): diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 537b5a86e5..f4d4ccbfc1 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -389,6 +389,7 @@ "JanitorRelationKey": ("sqlbuild.executor.janitor.models", "JanitorRelationKey"), "JanitorSkippedRelation": ("sqlbuild.executor.janitor.models", "JanitorSkippedRelation"), "JanitorSkippedSchema": ("sqlbuild.executor.janitor.models", "JanitorSkippedSchema"), + "JanitorUsageCheck": ("sqlbuild.executor.janitor.models", "JanitorUsageCheck"), "run_load_pipeline": ("sqlbuild.executor.load.main.run", "run_load_pipeline"), "LoadCallbacks": ("sqlbuild.executor.load.models", "LoadCallbacks"), "LoadExecutionResult": ("sqlbuild.executor.load.models", "LoadExecutionResult"), diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index db9ef689a7..97d8f9b5b4 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -40,6 +40,7 @@ ) from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -52,6 +53,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffCoverage, RowDiffPreparedRelations, RowDiffResult, @@ -366,6 +368,18 @@ def get_tables_freshness_metadata( results[request] = TableFreshnessMetadata.unavailable(message=error.message) return results + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Report that usage is not checked; adapters with a usage source override this.""" + + del connection, relations, window_seconds + return unsupported_relation_usage() + def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: """Return column names produced by a SQL query without materializing full rows.""" diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 3c1beddbe3..ec86c5acf3 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -48,6 +48,7 @@ from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -60,6 +61,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -197,6 +199,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index aa24bad368..617718f15f 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -19,7 +19,9 @@ FunctionDefinition, MigrationStagePlan, RelationGrant, + RelationInfo, RelationReadProbe, + RelationUsageReport, RowDiffTolerance, RowDiffTolerances, SnapshotChangeTarget, @@ -211,6 +213,22 @@ def get_tables_freshness_metadata( """Return one freshness outcome per requested table without failing the whole batch.""" ... + @abstractmethod + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Report which relations other queries read in the last `window_seconds`. + + Each usage names the relation, its latest read, its readers (most queries first) and how + many queries read it. SQLBuild's own queries and zero-copy clones do not count. A + warehouse that cannot report reads returns an unchecked report with a short reason. + """ + ... + @abstractmethod def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: """Return column names produced by a SQL query.""" diff --git a/src/sqlbuild/adapter/contract/constants.py b/src/sqlbuild/adapter/contract/constants.py index c5671dbfd3..9c94083584 100644 --- a/src/sqlbuild/adapter/contract/constants.py +++ b/src/sqlbuild/adapter/contract/constants.py @@ -19,3 +19,4 @@ DUCKDB_MISSING_OBJECT_MARKER: str = "does not exist" DUCKDB_INTERNAL_ERROR_CLASS_NAME: str = "InternalException" CANCELLABLE_DURATION_LIMIT_SECONDS: int = 2_147_483 +UNSUPPORTED_RELATION_USAGE_REASON: str = "unsupported adapter" diff --git a/src/sqlbuild/adapter/contract/main/unsupported_relation_usage.py b/src/sqlbuild/adapter/contract/main/unsupported_relation_usage.py new file mode 100644 index 0000000000..960363ce16 --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/unsupported_relation_usage.py @@ -0,0 +1,12 @@ +"""The relation usage report of an adapter that cannot tell who reads its relations.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.constants import UNSUPPORTED_RELATION_USAGE_REASON +from sqlbuild.adapter.contract.models import RelationUsageReport + + +def unsupported_relation_usage() -> RelationUsageReport: + """Report that usage was not checked because the adapter has no usage source.""" + + return RelationUsageReport(checked=False, reason=UNSUPPORTED_RELATION_USAGE_REASON) diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 3678afe53a..0043ef58a1 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -100,6 +100,27 @@ def identity(self) -> tuple[str | None, str | None, str]: return RelationLookup.key(database=self.database, schema=self.schema, name=self.name) +@dataclass(frozen=True) +class RelationUsage: + """Reads of one relation within a usage window: the latest, who read it and how often.""" + + database: str | None + schema: str | None + name: str + last_read_at: datetime + readers: tuple[str, ...] + query_count: int + + +@dataclass(frozen=True) +class RelationUsageReport: + """The relations read within a usage window, or why the warehouse could not tell.""" + + checked: bool + reason: str | None = None + usages: tuple[RelationUsage, ...] = () + + @dataclass(frozen=True) class RelationLookup: """Existing warehouse relations gathered once and queried in memory.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 8d90520f08..8c5702e6f7 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -54,6 +54,7 @@ ) from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -66,6 +67,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RenderedRetentionChange, RetentionRequest, RetentionState, @@ -509,6 +511,16 @@ def get_table_freshness_metadata( raise AdapterUserError(message=metadata.message or f"freshness unavailable for {name}") return metadata + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index ba9b34eaf0..fd0174dcba 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -39,6 +39,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -51,6 +52,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -2307,6 +2309,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 03ff3cbd5c..4a809a2708 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -51,6 +51,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -63,6 +64,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RenderedRetentionChange, RetentionRequest, RetentionState, @@ -608,6 +610,16 @@ def get_table_freshness_metadata( raise AdapterUserError(message=metadata.message or f"freshness unavailable for {name}") return metadata + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 7f5e14cf43..1464805c67 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -51,6 +51,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -63,6 +64,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -228,6 +230,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py new file mode 100644 index 0000000000..f8bce7cd70 --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py @@ -0,0 +1,199 @@ +"""Read who queried relations from Snowflake ACCOUNT_USAGE access history.""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +from sqlbuild._native import SQLBUILD_QUERY_TAG_APP +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage, RelationUsageReport + +USAGE_NAME_CHUNK_SIZE: int = 1_000 +USAGE_UNAVAILABLE_REASON: str = ( + "cannot read SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY " + "(needs Enterprise edition and the GOVERNANCE_VIEWER database role)" +) +SQLBUILD_QUERY_TAG_FRAGMENT: str = f'"app":"{SQLBUILD_QUERY_TAG_APP}"' +CLONE_STATEMENT_PATTERN: str = r".*\sCLONE\s.*" +_UNQUOTED_IDENTIFIER: re.Pattern[str] = re.compile(r"^[A-Z_][A-Z0-9_$]*$") +_UNAVAILABLE_ERRNOS: frozenset[int] = frozenset({2003, 3001}) +_UNAVAILABLE_MESSAGES: tuple[str, ...] = ( + "does not exist or not authorized", + "insufficient privileges", +) + + +@dataclass(frozen=True) +class UsageQuery: + """One access-history read: its SQL and bound parameters.""" + + sql: str + params: tuple[object, ...] + + +def usage_object_name(*, database: str, schema: str, name: str) -> str: + """The `objectName` access history records for a relation SQLBuild names unquoted.""" + + return ".".join(_object_name_part(part) for part in (database, schema, name)) + + +def _object_name_part(part: str) -> str: + folded: str = part.upper() + if _UNQUOTED_IDENTIFIER.match(folded): + return folded + return '"' + folded.replace('"', '""') + '"' + + +def usage_object_names( + *, relations: Iterable[RelationInfo], current_database: str | None +) -> dict[str, RelationInfo]: + """Each schema-qualified relation by its access-history object name.""" + + named: dict[str, RelationInfo] = {} + relation: RelationInfo + for relation in relations: + database: str | None = relation.database or current_database + if database is None or relation.schema is None: + continue + named[usage_object_name(database=database, schema=relation.schema, name=relation.name)] = ( + relation + ) + return named + + +def usage_queries(*, object_names: Iterable[str], window_seconds: int) -> tuple[UsageQuery, ...]: + """One access-history read per chunk of object names, oldest read within the window.""" + + names: tuple[str, ...] = tuple(sorted(object_names)) + return tuple( + usage_query( + object_names=names[start : start + USAGE_NAME_CHUNK_SIZE], + window_seconds=window_seconds, + ) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) + ) + + +def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> UsageQuery: + """Reads of the named objects per user, excluding SQLBuild-tagged queries and clones.""" + + placeholders: str = ", ".join(["%s"] * len(object_names)) + sql: str = ( + 'SELECT accessed.value:"objectName"::STRING AS object_name, ' + "history.user_name AS user_name, " + "COUNT(DISTINCT history.query_id) AS query_count, " + "MAX(history.query_start_time) AS last_read_at " + "FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY AS history, " + "LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed " + "WHERE history.query_start_time >= DATEADD('second', -%s, CURRENT_TIMESTAMP()) " + f'AND accessed.value:"objectName"::STRING IN ({placeholders}) ' + "AND NOT EXISTS (" + "SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY AS query " + "WHERE query.query_id = history.query_id " + "AND query.start_time >= DATEADD('second', -%s, CURRENT_TIMESTAMP()) " + "AND (CONTAINS(COALESCE(query.query_tag, ''), %s) " + "OR (STARTSWITH(query.query_type, 'CREATE') " + "AND REGEXP_LIKE(query.query_text, %s, 'is')))" + ") " + "GROUP BY 1, 2 " + "ORDER BY 1, 2" + ) + return UsageQuery( + sql=sql, + params=( + window_seconds, + *object_names, + window_seconds, + SQLBUILD_QUERY_TAG_FRAGMENT, + CLONE_STATEMENT_PATTERN, + ), + ) + + +def relation_usages( + *, rows: Iterable[tuple[object, ...]], relations_by_name: Mapping[str, RelationInfo] +) -> tuple[RelationUsage, ...]: + """One usage per read relation from `(object_name, user, query_count, last_read)` rows.""" + + readers: dict[str, dict[str, int]] = {} + last_read: dict[str, datetime] = {} + row: tuple[object, ...] + for row in rows: + object_name: str = str(row[0]) + if object_name not in relations_by_name: + continue + user: str = str(row[1]) + counts: dict[str, int] = readers.setdefault(object_name, {}) + counts[user] = counts.get(user, 0) + int(str(row[2])) + read_at: datetime = _utc(row[3]) + last_read[object_name] = max(read_at, last_read.get(object_name, read_at)) + usages: list[RelationUsage] = [] + object_name: str + for object_name in sorted(readers): + relation: RelationInfo = relations_by_name[object_name] + counts = readers[object_name] + usages.append( + RelationUsage( + database=relation.database, + schema=relation.schema, + name=relation.name, + last_read_at=last_read[object_name], + readers=tuple(sorted(counts, key=lambda user: (-counts[user], user))), + query_count=sum(counts.values()), + ) + ) + return tuple(usages) + + +def _utc(value: object) -> datetime: + if not isinstance(value, datetime): + return datetime.fromisoformat(str(value)).astimezone(UTC) + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def is_usage_unavailable_error(error: Exception) -> bool: + """Whether reading access history failed for edition or privilege reasons.""" + + message: str = str(error).lower() + return getattr(error, "errno", None) in _UNAVAILABLE_ERRNOS or any( + marker in message for marker in _UNAVAILABLE_MESSAGES + ) + + +def read_relation_usage( + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + current_database: Callable[[], str | None], +) -> RelationUsageReport: + """Read every relation's usage in chunked queries; no access history means not checked.""" + + database: str | None = None + if any(relation.database is None for relation in relations): + database = current_database() + relations_by_name: dict[str, RelationInfo] = usage_object_names( + relations=relations, current_database=database + ) + rows: list[tuple[object, ...]] = [] + query: UsageQuery + for query in usage_queries(object_names=relations_by_name, window_seconds=window_seconds): + cursor: Any = connection.cursor() + try: + cursor.execute(query.sql, query.params) + rows.extend(tuple(row) for row in cursor.fetchall()) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + finally: + cursor.close() + return RelationUsageReport( + checked=True, + usages=relation_usages(rows=rows, relations_by_name=relations_by_name), + ) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 1e17650c96..9e87820442 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -62,6 +62,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RenderedRetentionChange, RetentionRequest, RetentionState, @@ -130,6 +131,7 @@ information_schema_type, show_columns_type, ) +from sqlbuild.adapters.snowflake._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.snowflake._helpers.show_metadata import ( is_missing_object_error, listed_relation_from_show_table, @@ -878,6 +880,22 @@ def get_table_freshness_metadata( observed_at=row[1] if isinstance(row[1], datetime) else None, ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Read candidate reads from ACCOUNT_USAGE.ACCESS_HISTORY, which lags up to 3 hours.""" + + return read_relation_usage( + connection=connection, + relations=relations, + window_seconds=window_seconds, + current_database=lambda: self._current_database(connection=connection), + ) + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 1d648e01bd..d42b28f5d9 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -55,6 +55,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -67,6 +68,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -342,6 +344,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 0acea97875..fcbd7a3c97 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -53,6 +53,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -65,6 +66,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -183,6 +185,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 5313227c0a..ec095dd943 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -6,7 +6,7 @@ from datetime import datetime from sqlbuild._native import OldNameViewDropReason, OldNameViewHistory -from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup +from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup, RelationUsage from sqlbuild.executor.janitor.types import JanitorAction from sqlbuild.executor.janitor_events.models import JanitorEventRecord @@ -63,6 +63,7 @@ class JanitorArchiveCandidate: archive_key: JanitorRelationKey archived_at: datetime expires_at: datetime + usage: RelationUsage | None = None @dataclass(frozen=True) @@ -189,6 +190,16 @@ def display_name(self) -> str: return "" +@dataclass(frozen=True) +class JanitorUsageCheck: + """Whether recent reads of the archive candidates were checked, and how in-use ones fare.""" + + checked: bool + window: str + keep_in_use: bool = False + reason: str | None = None + + @dataclass(frozen=True) class JanitorPlan: """Complete janitor preview and execution plan.""" @@ -214,6 +225,8 @@ class JanitorPlan: scanned_schema_count: int = 0 age_metadata_supported: bool = False planned_at: datetime | None = None + usage_check: JanitorUsageCheck | None = None + kept_in_use: tuple[JanitorArchiveCandidate, ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 7c9005104e..01717a8407 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -288,3 +288,38 @@ class JanitorJsonFailedRunE2ETestCase: expected_status: str expected_error_fragment: str expected_failed_items: int + + +@dataclass(frozen=True) +class JanitorUsageUncheckedE2ETestCase: + """Test case for janitor runs on an adapter with no usage source.""" + + description: str + janitor_command: tuple[str, ...] + expected_ordered_lines: tuple[str, ...] + expected_archived: int + + +@dataclass(frozen=True) +class JanitorUsageE2ETestCase: + """Test case for janitor runs whose adapter reports which candidates are read.""" + + description: str + janitor_config: str + janitor_command: tuple[str, ...] + reads: dict[str, tuple[tuple[str, ...], int]] + expected_ordered_lines: tuple[str, ...] + unexpected_fragments: tuple[str, ...] + expected_remaining: tuple[str, ...] + expected_archived: int + + +@dataclass(frozen=True) +class JanitorUsagePromptE2ETestCase: + """Test case for the confirmation prompt's usage line.""" + + description: str + janitor_command: tuple[str, ...] + reads: dict[str, tuple[tuple[str, ...], int]] + unavailable: str | None + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index ef611e854e..e6f63b4460 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -698,3 +698,170 @@ def janitor_end_state(*, db_path: Path, schemas: tuple[str, ...]) -> tuple[tuple ) ) return (*relations, *events) + + +USAGE_JANITOR_ADAPTER_NAME: str = "usage_janitor_duckdb" +USAGE_JANITOR_SETTINGS_FILENAME: str = "janitor_usage_settings.json" +USAGE_JANITOR_CALLS_FILENAME: str = "janitor_usage_calls.json" +USAGE_JANITOR_ADAPTER_SOURCE: str = """ +import json +from dataclasses import replace +from datetime import datetime +from pathlib import Path +from typing import Any + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage, RelationUsageReport +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter + +_ROOT = Path(__file__).resolve().parent.parent + + +def _settings() -> dict[str, Any]: + return json.loads((_ROOT / "janitor_usage_settings.json").read_text(encoding="utf-8")) + + +class UsageJanitorDuckDbAdapter(DuckDbAdapter): + adapter_name = "usage_janitor_duckdb" + + def supports_relation_age_metadata(self) -> bool: + return True + + def list_relations( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[RelationInfo, ...]: + changed_at: datetime = datetime.fromisoformat(_settings()["changed_at"]) + return tuple( + replace(relation, created_at=changed_at) + if relation.name.startswith("old_") + else relation + for relation in super().list_relations( + connection=connection, database=database, schemas=schemas, names=names + ) + ) + + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + (_ROOT / "janitor_usage_calls.json").write_text( + json.dumps( + { + "relations": sorted(relation.name for relation in relations), + "window_seconds": window_seconds, + } + ), + encoding="utf-8", + ) + settings: dict[str, Any] = _settings() + if settings["unavailable"] is not None: + return RelationUsageReport(checked=False, reason=settings["unavailable"]) + reads: dict[str, dict[str, Any]] = settings["reads"] + return RelationUsageReport( + checked=True, + usages=tuple( + RelationUsage( + database=relation.database, + schema=relation.schema, + name=relation.name, + last_read_at=datetime.fromisoformat(reads[relation.name]["last_read_at"]), + readers=tuple(reads[relation.name]["readers"]), + query_count=int(reads[relation.name]["query_count"]), + ) + for relation in relations + if relation.name in reads + ), + ) +""" + + +def prepare_usage_janitor_project( + *, + tmp_path: Path, + project_name: str, + stale_tables: int, + stale_views: tuple[str, ...] = (), + janitor_config: str = "", +) -> Path: + """A built DuckDB project with stale tables and views, an expired archive, and a usage adapter. + + Every stale `old_*` relation last changed on 2026-09-01, so it is older than the retention. + """ + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name=project_name, + repo_files={ + "sqlbuild_project.toml": ( + f'name = "{project_name}"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + f'database = "{(tmp_path / project_name / "janitor.duckdb").as_posix()}"\n\n' + "[janitor]\n" + "enabled = true\n" + "retention_days = 14\n" + "archive_retention_days = 14\n" + "delete_tracked_only = false\n" + f"{dedent(janitor_config).strip()}\n\n" + "[defaults]\n" + 'materialized = "table"\n' + ), + "models/orders.sql": "MODEL (description 'Test model.');\n\nSELECT 1 AS order_id\n", + f"adapters/{USAGE_JANITOR_ADAPTER_NAME}.py": USAGE_JANITOR_ADAPTER_SOURCE, + }, + ) + _build(project_dir) + tables: str = "; ".join( + f"CREATE TABLE main.old_orders_{index:03d} AS SELECT {index} AS id" + for index in range(stale_tables) + ) + views: str = "".join(f"; CREATE VIEW main.{name} AS SELECT 1 AS id" for name in stale_views) + execute_duckdb( + db_path=project_dir / "janitor.duckdb", + sql=f'{tables}{views}; CREATE TABLE main."{EXPIRED_ARCHIVE_NAME}" AS SELECT 1 AS id', + ) + write_usage_settings(project_dir=project_dir) + return project_dir + + +def write_usage_settings( + *, + project_dir: Path, + reads: Mapping[str, tuple[tuple[str, ...], int]] = MappingProxyType({}), + unavailable: str | None = None, +) -> None: + """Run janitor on the usage adapter: who read which relation, or why usage is unavailable.""" + + (project_dir / "sqlbuild_local.toml").write_text( + f'adapter = "{USAGE_JANITOR_ADAPTER_NAME}"\n', encoding="utf-8" + ) + (project_dir / USAGE_JANITOR_SETTINGS_FILENAME).write_text( + json.dumps( + { + "changed_at": "2026-09-01T08:00:00+00:00", + "reads": { + name: { + "readers": list(readers), + "query_count": count, + "last_read_at": "2026-10-21T09:30:00+00:00", + } + for name, (readers, count) in reads.items() + }, + "unavailable": unavailable, + } + ), + encoding="utf-8", + ) + + +def read_usage_calls(*, project_dir: Path) -> dict[str, object]: + """The relations and window of the usage adapter's last `relation_usage` call.""" + + return json.loads((project_dir / USAGE_JANITOR_CALLS_FILENAME).read_text(encoding="utf-8")) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py new file mode 100644 index 0000000000..a09298331a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py @@ -0,0 +1,321 @@ +"""E2E tests for the janitor usage check: which archive candidates are in use, and keeping them.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorUsageE2ETestCase, + JanitorUsagePromptE2ETestCase, + JanitorUsageUncheckedE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( + assert_in_order, + list_archive_names, + prepare_probe_janitor_project, + prepare_usage_janitor_project, + read_usage_calls, + write_usage_settings, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + run_sqb, + run_sqb_with_pty, + table_exists, +) + +APPROVED_JANITOR: tuple[str, ...] = ("--no-color", "janitor", "--auto-approve") +FIVE_READERS: tuple[str, ...] = ("svc_reporting", "alice", "bob", "carol", "dave") + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageUncheckedE2ETestCase( + description="an adapter without a usage source says so once and archives as before", + janitor_command=APPROVED_JANITOR, + expected_ordered_lines=( + "Inspected warehouse state. (", + "Checking relation usage...", + "Relation usage not checked: unsupported adapter. (", + "Janitor preview default", + " relations to archive 3\n", + "\n usage not checked: unsupported adapter\n", + "Relations to archive", + "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + ), + expected_archived=3, + ), + JanitorUsageUncheckedE2ETestCase( + description="--keep-in-use without a usage source warns and archives everything", + janitor_command=(*APPROVED_JANITOR, "--keep-in-use"), + expected_ordered_lines=( + " usage not checked: unsupported adapter\n" + " nothing kept in use: no usage could be checked, so relations in use are " + "archived too\n", + "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + ), + expected_archived=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_duckdb_when_running_janitor_then_usage_is_reported_not_checked( + test_case: JanitorUsageUncheckedE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_unchecked", stale_tables={"main": 3} + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert_in_order(output=result.stdout, lines=test_case.expected_ordered_lines) + assert result.stdout.count("usage not checked: unsupported adapter\n") == 1 + assert "In use" not in result.stdout + archives: tuple[str, ...] = list_archive_names(db_path=project_dir / "janitor.duckdb") + assert len([name for name in archives if "__old_orders_" in name]) == ( + test_case.expected_archived + ) + + +def test_given_duckdb_when_previewing_janitor_json_then_usage_check_is_not_checked( + tmp_path: Path, +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_json", stale_tables={"main": 2} + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("janitor", "--auto-approve", "--json"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["usage_check"] == { + "status": "not_checked", + "reason": "unsupported adapter", + "window": "7d", + "keep_in_use": False, + } + assert [candidate["in_use"] for candidate in payload["plan"]["relations_to_archive"]] == [ + None, + None, + ] + assert payload["plan"]["kept_in_use"] == [] + assert "usage not checked: unsupported adapter" in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageE2ETestCase( + description="in-use relations are listed with readers and archived by default", + janitor_config="", + janitor_command=APPROVED_JANITOR, + reads={ + "old_orders_000": (FIVE_READERS, 40), + "old_orders_view": (("svc_reporting",), 14), + }, + expected_ordered_lines=( + "Checking relation usage...", + "Checked relation usage: 2 of 4 in use. (", + " relations to archive 4\n in use 2\n", + "Relations to archive", + "\nIn use\n", + " main.old_orders_000 queried 2026-10-21 by svc_reporting, alice, bob +2 more " + "(40 queries in 7 days); frozen since 2026-09-01, still in use\n", + " main.old_orders_view queried 2026-10-21 by svc_reporting " + "(14 queries in 7 days)\n", + "\u2713 Janitor completed archived 4, deleted 1, pruned 1", + ), + unexpected_fragments=( + "Kept (in use)", + "old_orders_view queried 2026-10-21 by svc_reporting (14 queries in 7 days);", + "old_orders_001 queried", + "usage not checked", + ), + expected_remaining=(), + expected_archived=4, + ), + JanitorUsageE2ETestCase( + description="--keep-in-use keeps in-use relations out of the archive and the counts", + janitor_config="", + janitor_command=(*APPROVED_JANITOR, "--keep-in-use"), + reads={ + "old_orders_000": (FIVE_READERS, 40), + "old_orders_view": (("svc_reporting",), 14), + }, + expected_ordered_lines=( + "Checked relation usage: 2 of 4 in use. (", + " relations to archive 2\n kept (in use) 2\n", + "\nKept (in use)\n main.old_orders_000 queried 2026-10-21", + "\u2713 Janitor completed archived 2, deleted 1, pruned 1", + ), + unexpected_fragments=( + "\nIn use\n", + "main.old_orders_000 ->", + "main.old_orders_view ->", + "archived main.old_orders_000", + ), + expected_remaining=("old_orders_000", "old_orders_view"), + expected_archived=2, + ), + JanitorUsageE2ETestCase( + description="keep_in_use in the project and in_use_window reach the usage source", + janitor_config='keep_in_use = true\nin_use_window = "36h"\n', + janitor_command=APPROVED_JANITOR, + reads={"old_orders_001": (("alice",), 1)}, + expected_ordered_lines=( + " kept (in use) 1\n", + " main.old_orders_001 queried 2026-10-21 by alice (1 query in 36 hours); " + "frozen since 2026-09-01, still in use\n", + "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + ), + unexpected_fragments=("main.old_orders_001 ->",), + expected_remaining=("old_orders_001",), + expected_archived=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_usage_source_when_running_janitor_then_in_use_relations_are_shown_or_kept( + test_case: JanitorUsageE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, + project_name="janitor_usage", + stale_tables=3, + stale_views=("old_orders_view",), + janitor_config=test_case.janitor_config, + ) + write_usage_settings(project_dir=project_dir, reads=test_case.reads) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + db_path: Path = project_dir / "janitor.duckdb" + assert result.returncode == 0, result.stdout + result.stderr + assert_in_order(output=result.stdout, lines=test_case.expected_ordered_lines) + fragment: str + for fragment in test_case.unexpected_fragments: + assert fragment not in result.stdout + name: str + for name in test_case.expected_remaining: + assert table_exists(db_path=db_path, table_name=name) + assert len( + [name for name in list_archive_names(db_path=db_path) if "__old_orders" in name] + ) == (test_case.expected_archived) + calls: dict[str, object] = read_usage_calls(project_dir=project_dir) + assert calls["relations"] == [ + "old_orders_000", + "old_orders_001", + "old_orders_002", + "old_orders_view", + ] + + +def test_given_usage_source_when_janitor_prints_json_then_candidates_carry_in_use( + tmp_path: Path, +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_json_shape", stale_tables=3 + ) + write_usage_settings( + project_dir=project_dir, + reads={"old_orders_000": (FIVE_READERS, 40), "old_orders_002": (("alice",), 2)}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("janitor", "--auto-approve", "--json", "--keep-in-use"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["usage_check"] == { + "status": "checked", + "reason": None, + "window": "7d", + "keep_in_use": True, + } + assert [ + (candidate["name"], candidate["in_use"]) + for candidate in payload["plan"]["relations_to_archive"] + ] == [("main.old_orders_001", False)] + assert [ + (candidate["name"], candidate["in_use"]) for candidate in payload["plan"]["kept_in_use"] + ] == [ + ( + "main.old_orders_000", + { + "last_read": "2026-10-21T09:30:00Z", + "readers": list(FIVE_READERS), + "query_count": 40, + }, + ), + ( + "main.old_orders_002", + {"last_read": "2026-10-21T09:30:00Z", "readers": ["alice"], "query_count": 2}, + ), + ] + assert payload["result"]["counts"]["archived"] == 1 + assert read_usage_calls(project_dir=project_dir)["window_seconds"] == 7 * 86_400 + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsagePromptE2ETestCase( + description="the prompt states how many relations to archive are in use", + janitor_command=("--no-color", "janitor"), + reads={"old_orders_000": (("alice",), 3)}, + unavailable=None, + expected_fragments=( + "Janitor will archive 3 and delete 2 items from default.\r\n" + "3 relations will be archived; 1 of them is in use.\r\n", + "Janitor cancelled.", + ), + ), + JanitorUsagePromptE2ETestCase( + description="the prompt repeats why usage was not checked", + janitor_command=("--no-color", "janitor"), + reads={}, + unavailable="cannot read access history", + expected_fragments=( + "\r\nUsage not checked: cannot read access history\r\nRetention: 14 days", + "Janitor cancelled.", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_usage_check_when_prompting_then_prompt_states_usage( + test_case: JanitorUsagePromptE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_prompt", stale_tables=3 + ) + write_usage_settings( + project_dir=project_dir, reads=test_case.reads, unavailable=test_case.unavailable + ) + + result: subprocess.CompletedProcess[str] = run_sqb_with_pty( + command=test_case.janitor_command, project_dir=project_dir, input_text="no\n" + ) + + assert result.returncode == 1, result.stdout + fragment: str + for fragment in test_case.expected_fragments: + assert fragment in result.stdout, result.stdout + assert len(list_archive_names(db_path=project_dir / "janitor.duckdb")) == 1 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 2ff4b3fc2f..c42a271e1b 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -294,3 +294,27 @@ class SnowflakeViewDefinitionsTestCase: read_names: tuple[str, ...] expected_definitions: tuple[str | None, ...] expected_query_count: int + + +@dataclass(frozen=True) +class SnowflakeRelationUsageTestCase: + description: str + relations: tuple[tuple[str | None, str, str], ...] + current_database: str | None + access_rows: tuple[tuple[object, ...], ...] + expected_object_names: tuple[str, ...] + expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + + +@dataclass(frozen=True) +class SnowflakeRelationUsageChunkTestCase: + description: str + relation_count: int + expected_query_names: tuple[int, ...] + + +@dataclass(frozen=True) +class SnowflakeRelationUsageErrorTestCase: + description: str + errno: int | None + message: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py new file mode 100644 index 0000000000..e109d6db1d --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py @@ -0,0 +1,277 @@ +"""Snowflake relation usage reads ACCOUNT_USAGE access history once per chunk of candidates.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.snowflake._helpers.relation_usage import ( + CLONE_STATEMENT_PATTERN, + SQLBUILD_QUERY_TAG_FRAGMENT, + USAGE_UNAVAILABLE_REASON, +) +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeRelationUsageChunkTestCase, + SnowflakeRelationUsageErrorTestCase, + SnowflakeRelationUsageTestCase, +) + +WINDOW_SECONDS: int = 7 * 86_400 +READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) +EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) + + +class _AccessHistoryError(Exception): + def __init__(self, message: str, errno: int | None) -> None: + super().__init__(message) + self.errno: int | None = errno + + +@dataclass +class _Warehouse: + """Answers CURRENT_DATABASE() and access-history reads, recording each query.""" + + current_database: str | None = None + access_rows: tuple[tuple[object, ...], ...] = () + error: Exception | None = None + queries: list[tuple[str, tuple[object, ...]]] = field(default_factory=list) + + +class _Cursor: + def __init__(self, warehouse: _Warehouse) -> None: + self.warehouse: _Warehouse = warehouse + self.rows: list[tuple[object, ...]] = [] + + def execute(self, sql: str, params: tuple[object, ...] = ()) -> None: + if sql == "SELECT CURRENT_DATABASE()": + self.rows = [(self.warehouse.current_database,)] + return + self.warehouse.queries.append((sql, params)) + if self.warehouse.error is not None: + raise self.warehouse.error + names: set[object] = set(params) + self.rows = [row for row in self.warehouse.access_rows if row[0] in names] + + def fetchone(self) -> tuple[object, ...] | None: + return self.rows[0] if self.rows else None + + def fetchall(self) -> list[tuple[object, ...]]: + return self.rows + + def close(self) -> None: + return None + + +class _Connection: + def __init__(self, warehouse: _Warehouse) -> None: + self.warehouse: _Warehouse = warehouse + + def cursor(self) -> _Cursor: + return _Cursor(self.warehouse) + + +def _relation(database: str | None, schema: str, name: str) -> RelationInfo: + return RelationInfo(database=database, schema=schema, name=name, relation_type="base table") + + +def _usage(*, connection: _Connection, relations: tuple[RelationInfo, ...]) -> RelationUsageReport: + return SnowflakeAdapter().relation_usage( + connection=connection, relations=relations, window_seconds=WINDOW_SECONDS + ) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageTestCase( + description="reads per user add up, readers rank by queries, the latest read wins", + relations=(("analytics", "mart", "orders_daily"), ("analytics", "mart", "customers")), + current_database=None, + access_rows=( + ("ANALYTICS.MART.ORDERS_DAILY", "SVC_REPORTING", 10, READ_AT), + ("ANALYTICS.MART.ORDERS_DAILY", "ALICE", 4, EARLIER_READ_AT), + ("ANALYTICS.MART.PRODUCTS", "BOB", 3, READ_AT), + ), + expected_object_names=("ANALYTICS.MART.CUSTOMERS", "ANALYTICS.MART.ORDERS_DAILY"), + expected_usages=(("orders_daily", ("SVC_REPORTING", "ALICE"), 14, READ_AT),), + ), + SnowflakeRelationUsageTestCase( + description="unqualified relations use the session database and names fold upper", + relations=((None, "mart", "orders_daily"),), + current_database="SHOP_DB", + access_rows=(("SHOP_DB.MART.ORDERS_DAILY", "SVC_REPORTING", 2, READ_AT),), + expected_object_names=("SHOP_DB.MART.ORDERS_DAILY",), + expected_usages=(("orders_daily", ("SVC_REPORTING",), 2, READ_AT),), + ), + SnowflakeRelationUsageTestCase( + description="names that need quoting match the quoted object name, case kept upper", + relations=(("analytics", "mart", "order-lines"),), + current_database=None, + access_rows=(('ANALYTICS.MART."ORDER-LINES"', "ALICE", 1, READ_AT),), + expected_object_names=('ANALYTICS.MART."ORDER-LINES"',), + expected_usages=(("order-lines", ("ALICE",), 1, READ_AT),), + ), + SnowflakeRelationUsageTestCase( + description="a mixed-case reader object does not match an unquoted relation", + relations=(("analytics", "mart", "orders_daily"),), + current_database=None, + access_rows=(('ANALYTICS.MART."orders_daily"', "ALICE", 1, READ_AT),), + expected_object_names=("ANALYTICS.MART.ORDERS_DAILY",), + expected_usages=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_access_history_when_reading_usage_then_reads_match_candidates( + test_case: SnowflakeRelationUsageTestCase, +) -> None: + warehouse: _Warehouse = _Warehouse( + current_database=test_case.current_database, access_rows=test_case.access_rows + ) + + report: RelationUsageReport = _usage( + connection=_Connection(warehouse), + relations=tuple(_relation(*relation) for relation in test_case.relations), + ) + + assert report.checked + assert report.reason is None + sql: str + params: tuple[object, ...] + ((sql, params),) = warehouse.queries + assert params[1 : 1 + len(test_case.expected_object_names)] == (test_case.expected_object_names) + assert "SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY" in sql + assert "LATERAL FLATTEN(INPUT => history.direct_objects_accessed)" in sql + assert tuple( + (usage.name, usage.readers, usage.query_count, usage.last_read_at) + for usage in report.usages + ) == (test_case.expected_usages) + + +def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_are_excluded() -> ( + None +): + warehouse: _Warehouse = _Warehouse() + + _usage( + connection=_Connection(warehouse), + relations=(_relation("analytics", "mart", "orders_daily"),), + ) + + sql: str + params: tuple[object, ...] + ((sql, params),) = warehouse.queries + assert params == ( + WINDOW_SECONDS, + "ANALYTICS.MART.ORDERS_DAILY", + WINDOW_SECONDS, + SQLBUILD_QUERY_TAG_FRAGMENT, + CLONE_STATEMENT_PATTERN, + ) + assert SQLBUILD_QUERY_TAG_FRAGMENT == '"app":"sqlbuild"' + assert "NOT EXISTS (SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY AS query" in sql + assert "CONTAINS(COALESCE(query.query_tag, ''), %s)" in sql + assert "STARTSWITH(query.query_type, 'CREATE') AND REGEXP_LIKE(query.query_text, %s" in sql + assert sql.count("DATEADD('second', -%s, CURRENT_TIMESTAMP())") == 2 + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageChunkTestCase( + description="no candidates read nothing", + relation_count=0, + expected_query_names=(), + ), + SnowflakeRelationUsageChunkTestCase( + description="a large candidate set is split into IN lists of at most 1,000 names", + relation_count=2_001, + expected_query_names=(1_000, 1_000, 1), + ), + ), + ids=lambda case: case.description, +) +def test_given_many_candidates_when_reading_usage_then_names_are_chunked( + test_case: SnowflakeRelationUsageChunkTestCase, +) -> None: + warehouse: _Warehouse = _Warehouse() + + report: RelationUsageReport = _usage( + connection=_Connection(warehouse), + relations=tuple( + _relation("analytics", "mart", f"old_orders_{index:04d}") + for index in range(test_case.relation_count) + ), + ) + + assert report.checked + assert tuple(len(params) - 4 for _, params in warehouse.queries) == ( + test_case.expected_query_names + ) + assert all(sql.count("%s") == len(params) for sql, params in warehouse.queries) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageErrorTestCase( + description="a missing view (no Enterprise edition or access) is not checked", + errno=2003, + message="Object 'SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY' does not exist or not " + "authorized.", + ), + SnowflakeRelationUsageErrorTestCase( + description="insufficient privileges are not checked", + errno=3001, + message="Insufficient privileges to operate on schema 'ACCOUNT_USAGE'", + ), + SnowflakeRelationUsageErrorTestCase( + description="a privilege message without an error number is not checked", + errno=None, + message="SQL access control error: Insufficient privileges to operate on view", + ), + ), + ids=lambda case: case.description, +) +def test_given_no_access_history_access_when_reading_usage_then_usage_is_not_checked( + test_case: SnowflakeRelationUsageErrorTestCase, +) -> None: + warehouse: _Warehouse = _Warehouse( + error=_AccessHistoryError(test_case.message, test_case.errno) + ) + + report: RelationUsageReport = _usage( + connection=_Connection(warehouse), + relations=(_relation("analytics", "mart", "orders_daily"),), + ) + + assert report == RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + + +def test_given_other_access_history_failure_when_reading_usage_then_error_propagates() -> None: + warehouse: _Warehouse = _Warehouse(error=_AccessHistoryError("SQL execution canceled", 604)) + + with pytest.raises(_AccessHistoryError, match="canceled"): + _usage( + connection=_Connection(warehouse), + relations=(_relation("analytics", "mart", "orders_daily"),), + ) + + +def test_given_other_adapter_when_reading_usage_then_it_reports_unsupported() -> None: + report: RelationUsageReport = DuckDbAdapter().relation_usage( + connection=object(), + relations=(_relation("analytics", "mart", "orders_daily"),), + window_seconds=WINDOW_SECONDS, + ) + + assert report == RelationUsageReport(checked=False, reason="unsupported adapter") + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 6ac27db06b..50c7fb9829 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -24,6 +24,7 @@ sqb --project-dir janitor [flags] | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | | `--verbose`, `-v` | List every entry of each preview section and every skipped object with its reason. Without it, each section shows its first 10 entries and skipped objects are counted by reason | | `--json` | Print the plan and the result as one JSON document on stdout; the preview, prompt and progress go to stderr | +| `--keep-in-use` | Keep relations that other queries read within `janitor.in_use_window` instead of archiving them, for this run. Same as `janitor.keep_in_use = true`. See [Relations in use](#relations-in-use) | | `--drop-old-name-view ` | Drop the [compatibility view](/docs/concepts/models/migrations#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | @@ -54,7 +55,42 @@ Skipped objects (385) 175 relation is not tracked by SQLBuild ``` -With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. +With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `usage_check`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. + +## Relations in use + +Archiving renames a relation, so anything that still reads it fails from then on. Before archiving, the janitor looks up which candidates other queries read within `in_use_window` (default `7d`) and shows them, with their readers, in an `In use` section: + +``` +In use + analytics.orders_daily queried 2026-10-21 by svc_reporting, alice (14 queries in 7 days) + analytics.old_orders queried 2026-10-20 by svc_reporting, alice, bob +2 more (40 queries in 7 days); frozen since 2026-09-01, still in use +``` + +Each line names the latest read, up to three readers (most queries first) and how many queries read the relation. A stale table that is still read is also flagged `frozen since , still in use`: its data stopped changing on that date, so its readers are getting old data. Views are not flagged, because their reads stay live. The confirmation prompt states the count, for example `14 relations will be archived; 3 of them are in use.` + +By default in-use relations are still archived, which is also how their readers find out. To keep them instead, set `keep_in_use = true` under `[janitor]` or pass `--keep-in-use`. In-use relations are then listed under `Kept (in use)` with their readers and are left out of the archive list and the counts. A later run archives them once nothing reads them within the window. + +Only relations to archive are checked. Expired archives and state-table pruning are not. + +Usage comes from the warehouse's query history: + +| Adapter | Usage source | +|---------|--------------| +| Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`) joined with `QUERY_HISTORY` | +| Other adapters | Not available | + +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: + +```sql +GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; +``` + +Access history lags by up to about 3 hours, so reads in the last few hours may not show yet. + +When usage can't be checked, because the adapter has no usage source or the role can't read access history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. + +In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers` and `query_count`. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. ## Speed @@ -164,6 +200,8 @@ retention_days = 14 archive_retention_days = 14 delete_tracked_only = true exclude_patterns = ["audit_*", "tmp_*"] +in_use_window = "7d" +keep_in_use = false ``` When `retention_days` is not set, it defaults to `14`. See [Project Configuration](/docs/concepts/project-configuration#janitor) for every janitor setting. @@ -185,6 +223,9 @@ sqb janitor --as prod # Clean up the prod target sqb janitor --target prod + +# Keep relations that are still read instead of archiving them +sqb janitor --keep-in-use ``` ## State history pruning diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index d4acab89f7..da42b449b6 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -800,6 +800,8 @@ retention_days = 14 archive_retention_days = 14 delete_tracked_only = true exclude_patterns = ["audit_*", "tmp_*"] +in_use_window = "7d" +keep_in_use = false ``` | Field | Default | Description | @@ -810,6 +812,8 @@ exclude_patterns = ["audit_*", "tmp_*"] | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | | `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` | +| `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](/docs/cli/janitor#relations-in-use) | +| `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | See [janitor](/docs/cli/janitor) for the archive and delete lifecycle. From 4448896b8d5aff4c3c48d7f70f401570c3ac7cb0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:52:17 +0100 Subject: [PATCH 08/14] refactor: satisfy native boundary and test structure rules for janitor usage --- .../src/project/_helpers/sections.rs | 19 +- .../src/janitor_text/tests/helpers.rs | 86 ++++--- .../tests/test_janitor_usage_text.rs | 132 ++++++----- .../src/janitor_text/tests/test_types.rs | 7 + .../_helpers/janitor/janitor_phases.rs | 14 +- .../_helpers/janitor/janitor_planning.rs | 3 +- .../_helpers/janitor/janitor_usage.rs | 160 ++++++------- .../contract/classes/strict_adapter.py | 7 +- .../snowflake/_helpers/relation_usage.py | 24 +- src/sqlbuild/adapters/snowflake/constants.py | 9 + src/sqlbuild/adapters/snowflake/models.py | 13 + src/sqlbuild/executor/janitor/models.py | 25 +- .../cli/commands/main/janitor/_test_types.py | 12 + .../cli/commands/main/janitor/helpers.py | 12 + .../main/janitor/test_janitor_usage.py | 121 ++++++---- .../adapters/snowflake/_test_types.py | 27 ++- .../sqlbuild/adapters/snowflake/helpers.py | 89 ++++++- .../adapters/snowflake/test_relation_usage.py | 222 +++++++++--------- 18 files changed, 602 insertions(+), 380 deletions(-) create mode 100644 src/sqlbuild/adapters/snowflake/models.py diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index fea57e01e1..5842d42869 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -568,16 +568,15 @@ pub(crate) fn load_janitor(payload: Option<&ConfigValue>, file: ConfigFile) -> L let keep_in_use = optional_bool(mapping, "keep_in_use", false)?; let in_use_window = match get(mapping, "in_use_window") { None => DEFAULT_IN_USE_WINDOW.to_owned(), - Some(_) => optional_str(mapping, "in_use_window") - .ok() - .flatten() - .filter(|window| matches!(fixed_duration_seconds(window, false), Ok(Some(_)))) - .ok_or_else(|| { - ConfigFailure::at( - file, - "janitor.in_use_window must be a positive fixed duration such as '7d'", - ) - })?, + Some(ConfigValue::String(window)) if fixed_duration_seconds(window, false)?.is_some() => { + window.clone() + } + Some(_) => { + return Err(ConfigFailure::at( + file, + "janitor.in_use_window must be a positive fixed duration such as '7d'", + )); + } }; let checks = [ (retention_days.as_ref(), "retention_days"), diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index 947d6b2693..cddf58351f 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -79,7 +79,7 @@ pub(super) fn old_name_plan(count: usize) -> JanitorPlanText { } } -/// Reads by `readers` of `count` queries, last on 2026-10-21. +/// Reads by `readers` of `query_count` queries, last on 2026-10-21. fn usage(readers: &[&str], query_count: i64) -> RelationUsageText { RelationUsageText { last_read_at: "2026-10-21 09:30:00".to_owned(), @@ -88,44 +88,42 @@ fn usage(readers: &[&str], query_count: i64) -> RelationUsageText { } } -/// Three candidates checked over `7d`: an in-use view, an in-use table read by five users, -/// and an unread table; `keep_in_use` moves the in-use two to `kept_in_use`. -pub(super) fn usage_plan(keep_in_use: bool) -> JanitorPlanText { - let view: ArchiveCandidateText = ArchiveCandidateText { - name: "dev.orders_daily".to_owned(), - archive_name: "dev._sqb_archive__20261022t101500z__orders_daily".to_owned(), - usage: Some(usage(&["svc_reporting"], 14)), - ..ArchiveCandidateText::default() - }; - let table: ArchiveCandidateText = ArchiveCandidateText { - name: "dev.old_orders".to_owned(), - archive_name: "dev._sqb_archive__20261022t101500z__old_orders".to_owned(), - is_table: true, - changed_on: Some("2026-09-01".to_owned()), - usage: Some(usage( - &["svc_reporting", "alice", "bob", "carol", "dave"], - 40, - )), - ..ArchiveCandidateText::default() - }; - let unread: ArchiveCandidateText = ArchiveCandidateText { - name: "dev.old_customers".to_owned(), - archive_name: "dev._sqb_archive__20261022t101500z__old_customers".to_owned(), - is_table: true, - ..ArchiveCandidateText::default() - }; - let (archive_candidates, kept_in_use) = if keep_in_use { - (vec![unread], vec![view, table]) - } else { - (vec![view, table, unread], Vec::new()) - }; +/// An in-use view, an in-use table read by five users, and an unread table. +fn usage_candidates() -> [ArchiveCandidateText; 3] { + [ + ArchiveCandidateText { + name: "dev.orders_daily".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__orders_daily".to_owned(), + usage: Some(usage(&["svc_reporting"], 14)), + ..ArchiveCandidateText::default() + }, + ArchiveCandidateText { + name: "dev.old_orders".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__old_orders".to_owned(), + is_table: true, + changed_on: Some("2026-09-01".to_owned()), + usage: Some(usage( + &["svc_reporting", "alice", "bob", "carol", "dave"], + 40, + )), + ..ArchiveCandidateText::default() + }, + ArchiveCandidateText { + name: "dev.old_customers".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__old_customers".to_owned(), + is_table: true, + ..ArchiveCandidateText::default() + }, + ] +} + +/// A plan for `dev` whose usage was checked over `7d`. +fn checked_plan(keep_in_use: bool) -> JanitorPlanText { JanitorPlanText { target_name: Some("dev".to_owned()), retention_days: 14, direct_mode: true, archive_retention_days: 14, - archive_candidates, - kept_in_use, usage_check: Some(UsageCheckText { checked: true, reason: None, @@ -136,7 +134,25 @@ pub(super) fn usage_plan(keep_in_use: bool) -> JanitorPlanText { } } -/// A plan whose usage could not be checked, for `reason`. +/// A checked plan archiving every usage candidate, in use or not. +pub(super) fn usage_plan() -> JanitorPlanText { + JanitorPlanText { + archive_candidates: usage_candidates().to_vec(), + ..checked_plan(false) + } +} + +/// A checked plan keeping the two in-use candidates and archiving the unread one. +pub(super) fn kept_usage_plan() -> JanitorPlanText { + let [view, table, unread] = usage_candidates(); + JanitorPlanText { + archive_candidates: vec![unread], + kept_in_use: vec![view, table], + ..checked_plan(true) + } +} + +/// A plan whose usage was not checked because the adapter has no usage source. pub(super) fn unchecked_plan(keep_in_use: bool) -> JanitorPlanText { JanitorPlanText { usage_check: Some(UsageCheckText { diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs index b951f809ef..f6f741179e 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs @@ -4,8 +4,10 @@ use crate::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation use crate::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation_prompt; use crate::janitor_text::main::render_janitor_plan::render_janitor_plan; use crate::janitor_text::models::JanitorOutcome; -use crate::janitor_text::tests::helpers::{direct_archive_plan, unchecked_plan, usage_plan}; -use crate::janitor_text::tests::test_types::{JsonTestCase, PreviewTestCase}; +use crate::janitor_text::tests::helpers::{ + direct_archive_plan, kept_usage_plan, unchecked_plan, usage_plan, +}; +use crate::janitor_text::tests::test_types::{JsonTestCase, PreviewTestCase, UsagePromptTestCase}; const PLAIN: CliStyle = CliStyle { use_color: false }; @@ -14,7 +16,7 @@ fn given_usage_check_when_rendering_preview_then_in_use_relations_show_readers() let test_cases = [ PreviewTestCase { description: "in-use candidates are listed with capped readers and archived anyway", - plan: usage_plan(false), + plan: usage_plan(), verbose: false, expected_fragments: &[ " relations to archive 3\n in use 2\n", @@ -33,7 +35,7 @@ fn given_usage_check_when_rendering_preview_then_in_use_relations_show_readers() }, PreviewTestCase { description: "keep_in_use lists in-use relations as kept and leaves them unarchived", - plan: usage_plan(true), + plan: kept_usage_plan(), verbose: false, expected_fragments: &[ " relations to archive 1\n kept (in use) 2\n", @@ -70,63 +72,72 @@ fn given_usage_check_when_rendering_preview_then_in_use_relations_show_readers() for test_case in test_cases { let text: String = render_janitor_plan(&test_case.plan, test_case.verbose, PLAIN); - for fragment in test_case.expected_fragments { - assert!( - text.contains(fragment), - "{}: missing {fragment:?} in {text}", - test_case.description - ); - } - for fragment in test_case.unexpected_fragments { - assert!( - !text.contains(fragment), - "{}: unexpected {fragment:?} in {text}", - test_case.description - ); - } + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: missing fragments in {text}", + test_case.description + ); + assert!( + !test_case + .unexpected_fragments + .iter() + .any(|fragment| text.contains(fragment)), + "{}: unexpected fragments in {text}", + test_case.description + ); } } #[test] fn given_usage_check_when_prompting_then_prompt_states_in_use_count() { - let test_cases: [(&str, crate::janitor_text::models::JanitorPlanText, &str); 4] = [ - ( - "the prompt counts in-use relations among those archived", - usage_plan(false), - "Janitor will archive 3 and delete 0 objects from dev.\n\ - 3 relations will be archived; 2 of them are in use.\nRetention: 14 days\n", - ), - ( - "keep_in_use counts the kept relations", - usage_plan(true), - "Janitor will archive 1 and delete 0 objects from dev.\n\ - 2 relations in use are kept.\nRetention: 14 days\n", - ), - ( - "an unchecked usage repeats the notice", - unchecked_plan(false), - "Janitor will archive 1 and delete 1 objects from dev.\n\ - Usage not checked: unsupported adapter\nRetention: 14 days\n", - ), - ( - "keep_in_use without a usage source repeats the warning", - unchecked_plan(true), - "Usage not checked: unsupported adapter\nNothing kept in use: no usage could be \ - checked, so relations in use are archived too\nRetention", - ), + let test_cases = [ + UsagePromptTestCase { + description: "the prompt counts in-use relations among those archived", + plan: usage_plan(), + expected_fragment: "Janitor will archive 3 and delete 0 objects from dev.\n\ + 3 relations will be archived; 2 of them are in use.\nRetention: 14 days\n", + expected_phrase: "archive 3 and delete 0 objects from dev", + }, + UsagePromptTestCase { + description: "keep_in_use counts the kept relations and leaves them out of the phrase", + plan: kept_usage_plan(), + expected_fragment: "Janitor will archive 1 and delete 0 objects from dev.\n\ + 2 relations in use are kept.\nRetention: 14 days\n", + expected_phrase: "archive 1 and delete 0 objects from dev", + }, + UsagePromptTestCase { + description: "an unchecked usage repeats the notice", + plan: unchecked_plan(false), + expected_fragment: "Janitor will archive 1 and delete 1 objects from dev.\n\ + Usage not checked: unsupported adapter\nRetention: 14 days\n", + expected_phrase: "archive 1 and delete 1 objects from dev", + }, + UsagePromptTestCase { + description: "keep_in_use without a usage source repeats the warning", + plan: unchecked_plan(true), + expected_fragment: "Usage not checked: unsupported adapter\nNothing kept in use: \ + no usage could be checked, so relations in use are archived too\nRetention", + expected_phrase: "archive 1 and delete 1 objects from dev", + }, ]; - for (description, plan, expected) in test_cases { - let prompt: String = janitor_confirmation_prompt(&plan); + for test_case in test_cases { + let prompt: String = janitor_confirmation_prompt(&test_case.plan); assert!( - prompt.contains(expected), - "{description}: missing {expected:?} in {prompt}" + prompt.contains(test_case.expected_fragment), + "{}: missing fragment in {prompt}", + test_case.description + ); + assert_eq!( + janitor_confirmation_phrase(&test_case.plan), + test_case.expected_phrase, + "{}", + test_case.description ); } - assert_eq!( - janitor_confirmation_phrase(&usage_plan(true)), - "archive 1 and delete 0 objects from dev" - ); } #[test] @@ -134,7 +145,7 @@ fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { let test_cases = [ JsonTestCase { description: "checked candidates carry their reads or false, with the check status", - plan: Some(usage_plan(false)), + plan: Some(usage_plan()), outcome: JanitorOutcome::Previewed, run: None, error: None, @@ -150,7 +161,7 @@ fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { }, JsonTestCase { description: "kept relations are listed apart from the relations to archive", - plan: Some(usage_plan(true)), + plan: Some(kept_usage_plan()), outcome: JanitorOutcome::Previewed, run: None, error: None, @@ -187,12 +198,13 @@ fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { test_case.run.as_ref(), test_case.error, ); - for fragment in test_case.expected_fragments { - assert!( - text.contains(fragment), - "{}: missing {fragment:?} in {text}", - test_case.description - ); - } + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: missing fragments in {text}", + test_case.description + ); } } diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs index ae6072dc7c..8dbd109319 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs @@ -81,3 +81,10 @@ pub(super) struct EventsUnwrittenTestCase { pub(super) error: &'static str, pub(super) expected_text: &'static str, } + +pub(super) struct UsagePromptTestCase { + pub(super) description: &'static str, + pub(super) plan: JanitorPlanText, + pub(super) expected_fragment: &'static str, + pub(super) expected_phrase: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index d06f9d89b9..9337b93031 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -15,7 +15,9 @@ use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; use crate::bindings::_helpers::build_progress::progress_reporters::{ ConnectionProgressReporter, PlanningProgressReporter, }; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::build_progress::transient_status::{ + TransientStatusReporter, status_reporter, +}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; @@ -308,7 +310,15 @@ pub(crate) fn build_janitor_execution_plan<'py>( (&compiled.adapter, connection), ( &plan, - &status_reporter(&output_stream(invocation)?, use_color(invocation)?)?, + &Bound::new( + py, + TransientStatusReporter::new( + py, + &output_stream(invocation)?, + use_color(invocation)?, + true, + )?, + )?, ), &UsageSettings { window: native_field(settings, "in_use_window")?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs index 82165954cf..44084aba45 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -63,7 +63,8 @@ fn int_field(py: Python<'_>, value: usize) -> PyResult> { integer(py, i64::try_from(value).unwrap_or(i64::MAX)) } -fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { +/// Mark an operation lifecycle completed with its item count. +pub(crate) fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { let py: Python<'_> = lifecycle.py(); let metadata: Bound<'_, PyDict> = PyDict::new(py); metadata.set_item("item_count", item_count)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs index 272a1bf06b..84db26df9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs @@ -5,12 +5,14 @@ use std::time::Instant; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::types::{PyBool, PyString}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; +use crate::bindings::_helpers::build_progress::transient_status::TransientStatusReporter; use crate::bindings::_helpers::janitor::janitor_keys::{Folded, Key, record, tuple}; +use crate::bindings::_helpers::janitor::janitor_planning::completed; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -22,83 +24,82 @@ pub(crate) struct UsageSettings { pub(crate) keep_in_use: bool, } -/// The adapter's usage report and its usages by relation. -struct UsageOutcome<'py> { - report: Bound<'py, PyAny>, - usages: HashMap>, +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() } -fn lifecycle(py: Python<'_>) -> PyResult> { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("operation_kind", "janitor")?; - options.set_item("operation_name", "janitor_usage_check")?; - host::entry(py, ExecutorHost::OperationLifecycle)?.call((), Some(&options)) +fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() } -fn replaced<'py>( - value: &Bound<'py, PyAny>, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - let py: Python<'py> = value.py(); - let options: Bound<'py, PyDict> = PyDict::new(py); - for (name, field) in fields { - options.set_item(*name, field)?; - } - imported(py, "dataclasses", "replace")?.call((value,), Some(&options)) +fn lifecycle(py: Python<'_>) -> PyResult> { + call_with( + &host::entry(py, ExecutorHost::OperationLifecycle)?, + &[ + ("operation_kind", text(py, "janitor")), + ("operation_name", text(py, "janitor_usage_check")), + ], + ) } +/// The adapter's usage report for the candidates' relations. fn read_usage<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), candidates: &[Bound<'py, PyAny>], window_seconds: i64, -) -> PyResult> { +) -> PyResult> { let py: Python<'py> = adapter.py(); let relations: Vec> = candidates .iter() .map(|candidate| native_field(candidate, "relation")) .collect::>()?; - let report: Bound<'py, PyAny> = call_with( - &native_field(adapter, "relation_usage")?, - &[ - ("connection", connection.clone()), - ("relations", tuple(py, relations)?), - ( - "window_seconds", - window_seconds.into_pyobject(py)?.into_any(), - ), - ], - )?; + within(&lifecycle(py)?, |lifecycle| { + let report: Bound<'py, PyAny> = call_with( + &native_field(adapter, "relation_usage")?, + &[ + ("connection", connection.clone()), + ("relations", tuple(py, relations)?), + ( + "window_seconds", + window_seconds.into_pyobject(py)?.into_any(), + ), + ], + )?; + completed(lifecycle, native_field(&report, "usages")?.len()?)?; + Ok(report) + }) +} + +/// Each reported usage by its relation's case-insensitive identity. +fn usages_by_relation<'py>( + report: &Bound<'py, PyAny>, +) -> PyResult>> { let mut usages: HashMap> = HashMap::new(); - if native_field(&report, "checked")?.is_truthy()? { - for usage in native_field(&report, "usages")?.try_iter()? { - let usage: Bound<'py, PyAny> = usage?; - usages.insert(Key::of(&usage)?.folded(), usage); - } + for usage in native_field(report, "usages")?.try_iter()? { + let usage: Bound<'py, PyAny> = usage?; + usages.insert(Key::of(&usage)?.folded(), usage); } - Ok(UsageOutcome { report, usages }) + Ok(usages) } fn completion_message( - report: &Bound<'_, PyAny>, + (checked, reason): (bool, Option), (in_use, total): (usize, usize), elapsed_seconds: f64, -) -> PyResult { - if !native_field(report, "checked")?.is_truthy()? { - let reason: Option = native_field(report, "reason")?.extract()?; - return Ok(format!( +) -> String { + if !checked { + return format!( "Relation usage not checked: {}. ({elapsed_seconds:.2}s)", reason.unwrap_or_default() - )); + ); } - Ok(format!( - "Checked relation usage: {in_use} of {total} in use. ({elapsed_seconds:.2}s)" - )) + format!("Checked relation usage: {in_use} of {total} in use. ({elapsed_seconds:.2}s)") } /// The plan with each candidate's reads, the usage check, and in-use candidates kept when asked. pub(crate) fn check_janitor_usage<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (plan, status): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (plan, status): (&Bound<'py, PyAny>, &Bound<'py, TransientStatusReporter>), settings: &UsageSettings, ) -> PyResult> { let py: Python<'py> = plan.py(); @@ -106,21 +107,14 @@ pub(crate) fn check_janitor_usage<'py>( .try_iter()? .collect::>()?; let window_seconds: i64 = fixed_duration_seconds(&settings.window) - .and_then(|seconds| i64::try_from(seconds).ok()) - .unwrap_or(i64::MAX); + .map_or(i64::MAX, |seconds| { + i64::try_from(seconds).unwrap_or(i64::MAX) + }); let started: Instant = Instant::now(); - status.call_method1("start", (START_MESSAGE,))?; - let outcome: UsageOutcome<'py> = within(&lifecycle(py)?, |lifecycle| { - let outcome: UsageOutcome<'py> = - read_usage((adapter, connection), &candidates, window_seconds)?; - let metadata: Bound<'py, PyDict> = PyDict::new(py); - metadata.set_item("item_count", outcome.usages.len())?; - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("metadata", metadata)?; - lifecycle.call_method("completed", (), Some(&options))?; - Ok(outcome) - })?; - let checked: bool = native_field(&outcome.report, "checked")?.is_truthy()?; + TransientStatusReporter::start(status, START_MESSAGE.to_owned())?; + let report: Bound<'py, PyAny> = read_usage((adapter, connection), &candidates, window_seconds)?; + let checked: bool = native_field(&report, "checked")?.is_truthy()?; + let usages: HashMap> = usages_by_relation(&report)?; let keep: bool = checked && settings.keep_in_use; let mut archived: Vec> = Vec::new(); let mut kept: Vec> = Vec::new(); @@ -128,12 +122,15 @@ pub(crate) fn check_janitor_usage<'py>( let mut in_use: usize = 0; for candidate in &candidates { let key: Folded = Key::of(&native_field(candidate, "relation")?)?.folded(); - let Some(usage) = outcome.usages.get(&key) else { + let Some(usage) = usages.get(&key) else { archived.push(candidate.clone()); continue; }; in_use += 1; - let read: Bound<'py, PyAny> = replaced(candidate, &[("usage", usage.clone())])?; + let read: Bound<'py, PyAny> = call_with( + &native_field(candidate, "with_usage")?, + &[("usage", usage.clone())], + )?; if keep { kept_keys.insert(Key::of(&native_field(candidate, "key")?)?.folded()); kept.push(read); @@ -149,35 +146,28 @@ pub(crate) fn check_janitor_usage<'py>( deletions.push(deletion); } } - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item( - "message", - completion_message( - &outcome.report, + let reason: Bound<'py, PyAny> = native_field(&report, "reason")?; + TransientStatusReporter::complete( + status, + &completion_message( + (checked, reason.extract()?), (in_use, candidates.len()), started.elapsed().as_secs_f64(), - )?, + ), + true, )?; - options.set_item("blank_line_after", true)?; - status.call_method("complete", (), Some(&options))?; let check: Bound<'py, PyAny> = record( py, ExecutorHost::JanitorUsageCheck, &[ - ("checked", PyBool::new(py, checked).to_owned().into_any()), - ( - "window", - settings.window.as_str().into_pyobject(py)?.into_any(), - ), - ( - "keep_in_use", - PyBool::new(py, settings.keep_in_use).to_owned().into_any(), - ), - ("reason", native_field(&outcome.report, "reason")?), + ("checked", flag(py, checked)), + ("window", text(py, &settings.window)), + ("keep_in_use", flag(py, settings.keep_in_use)), + ("reason", reason), ], )?; - replaced( - plan, + call_with( + &native_field(plan, "with_usage")?, &[ ("archive_candidates", tuple(py, archived)?), ("archive_deletion_candidates", tuple(py, deletions)?), diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 617718f15f..7db76e1936 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -221,12 +221,7 @@ def relation_usage( relations: tuple[RelationInfo, ...], window_seconds: int, ) -> RelationUsageReport: - """Report which relations other queries read in the last `window_seconds`. - - Each usage names the relation, its latest read, its readers (most queries first) and how - many queries read it. SQLBuild's own queries and zero-copy clones do not count. A - warehouse that cannot report reads returns an unchecked report with a short reason. - """ + """Report reads of the relations by other queries in the last `window_seconds`.""" ... @abstractmethod diff --git a/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py index f8bce7cd70..305164b399 100644 --- a/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py +++ b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py @@ -4,20 +4,18 @@ import re from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass from datetime import UTC, datetime from typing import Any -from sqlbuild._native import SQLBUILD_QUERY_TAG_APP from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage, RelationUsageReport - -USAGE_NAME_CHUNK_SIZE: int = 1_000 -USAGE_UNAVAILABLE_REASON: str = ( - "cannot read SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY " - "(needs Enterprise edition and the GOVERNANCE_VIEWER database role)" +from sqlbuild.adapters.snowflake.constants import ( + CLONE_STATEMENT_PATTERN, + SQLBUILD_QUERY_TAG_FRAGMENT, + USAGE_NAME_CHUNK_SIZE, + USAGE_UNAVAILABLE_REASON, ) -SQLBUILD_QUERY_TAG_FRAGMENT: str = f'"app":"{SQLBUILD_QUERY_TAG_APP}"' -CLONE_STATEMENT_PATTERN: str = r".*\sCLONE\s.*" +from sqlbuild.adapters.snowflake.models import UsageQuery + _UNQUOTED_IDENTIFIER: re.Pattern[str] = re.compile(r"^[A-Z_][A-Z0-9_$]*$") _UNAVAILABLE_ERRNOS: frozenset[int] = frozenset({2003, 3001}) _UNAVAILABLE_MESSAGES: tuple[str, ...] = ( @@ -26,14 +24,6 @@ ) -@dataclass(frozen=True) -class UsageQuery: - """One access-history read: its SQL and bound parameters.""" - - sql: str - params: tuple[object, ...] - - def usage_object_name(*, database: str, schema: str, name: str) -> str: """The `objectName` access history records for a relation SQLBuild names unquoted.""" diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index c816a0575e..323e3a878a 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -1,5 +1,7 @@ """Snowflake adapter constants.""" +from sqlbuild._native import SQLBUILD_QUERY_TAG_APP + MAX_STATEMENT_TIMEOUT_SECONDS: int = 604_800 BASE_TABLE_METADATA_TYPE: str = "BASE TABLE" @@ -21,3 +23,10 @@ STATE_TABLE_RETENTION_PARAMETER: str = "DATA_RETENTION_TIME_IN_DAYS" INVALID_PARAMETER_VALUE_ERRNO: int = 1008 TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" +USAGE_NAME_CHUNK_SIZE: int = 1_000 +USAGE_UNAVAILABLE_REASON: str = ( + "cannot read SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY " + "(needs Enterprise edition and the GOVERNANCE_VIEWER database role)" +) +SQLBUILD_QUERY_TAG_FRAGMENT: str = f'"app":"{SQLBUILD_QUERY_TAG_APP}"' +CLONE_STATEMENT_PATTERN: str = r".*\sCLONE\s.*" diff --git a/src/sqlbuild/adapters/snowflake/models.py b/src/sqlbuild/adapters/snowflake/models.py new file mode 100644 index 0000000000..f610a01d2c --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/models.py @@ -0,0 +1,13 @@ +"""Snowflake adapter models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UsageQuery: + """One access-history read: its SQL and bound parameters.""" + + sql: str + params: tuple[object, ...] diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index ec095dd943..00099e429e 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import datetime from sqlbuild._native import OldNameViewDropReason, OldNameViewHistory @@ -65,6 +65,11 @@ class JanitorArchiveCandidate: expires_at: datetime usage: RelationUsage | None = None + def with_usage(self, *, usage: RelationUsage) -> JanitorArchiveCandidate: + """Return this candidate with its reads in the usage window.""" + + return replace(self, usage=usage) + @dataclass(frozen=True) class JanitorArchivedRelation: @@ -228,6 +233,24 @@ class JanitorPlan: usage_check: JanitorUsageCheck | None = None kept_in_use: tuple[JanitorArchiveCandidate, ...] = field(default_factory=tuple) + def with_usage( + self, + *, + archive_candidates: tuple[JanitorArchiveCandidate, ...], + archive_deletion_candidates: tuple[JanitorArchivedRelation, ...], + kept_in_use: tuple[JanitorArchiveCandidate, ...], + usage_check: JanitorUsageCheck, + ) -> JanitorPlan: + """Return this plan after its usage check, with in-use candidates kept when asked.""" + + return replace( + self, + archive_candidates=archive_candidates, + archive_deletion_candidates=archive_deletion_candidates, + kept_in_use=kept_in_use, + usage_check=usage_check, + ) + @dataclass(frozen=True) class JanitorExecutionResult: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 01717a8407..b14a6bab4f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -323,3 +323,15 @@ class JanitorUsagePromptE2ETestCase: reads: dict[str, tuple[tuple[str, ...], int]] unavailable: str | None expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorUsageJsonE2ETestCase: + """Test case for the usage fields of `sqb janitor --json`.""" + + description: str + janitor_command: tuple[str, ...] + expected_usage_check: dict[str, object] + expected_archive_in_use: tuple[tuple[str, object], ...] + expected_kept_in_use: tuple[tuple[str, object], ...] + expected_archived: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index e6f63b4460..e73a994a65 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -865,3 +865,15 @@ def read_usage_calls(*, project_dir: Path) -> dict[str, object]: """The relations and window of the usage adapter's last `relation_usage` call.""" return json.loads((project_dir / USAGE_JANITOR_CALLS_FILENAME).read_text(encoding="utf-8")) + + +def count_stale_archives(*, db_path: Path) -> int: + """How many archives in `main` hold a stale `old_orders` relation.""" + + return sum("__old_orders" in name for name in list_archive_names(db_path=db_path)) + + +def in_use_entries(candidates: list[dict[str, object]]) -> tuple[tuple[object, object], ...]: + """Each `--json` candidate's name and `in_use`.""" + + return tuple((candidate["name"], candidate["in_use"]) for candidate in candidates) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py index a09298331a..50596da13a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py @@ -11,11 +11,14 @@ from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( JanitorUsageE2ETestCase, + JanitorUsageJsonE2ETestCase, JanitorUsagePromptE2ETestCase, JanitorUsageUncheckedE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( assert_in_order, + count_stale_archives, + in_use_entries, list_archive_names, prepare_probe_janitor_project, prepare_usage_janitor_project, @@ -79,36 +82,49 @@ def test_given_duckdb_when_running_janitor_then_usage_is_reported_not_checked( assert_in_order(output=result.stdout, lines=test_case.expected_ordered_lines) assert result.stdout.count("usage not checked: unsupported adapter\n") == 1 assert "In use" not in result.stdout - archives: tuple[str, ...] = list_archive_names(db_path=project_dir / "janitor.duckdb") - assert len([name for name in archives if "__old_orders_" in name]) == ( + assert count_stale_archives(db_path=project_dir / "janitor.duckdb") == ( test_case.expected_archived ) -def test_given_duckdb_when_previewing_janitor_json_then_usage_check_is_not_checked( - tmp_path: Path, +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageJsonE2ETestCase( + description="duckdb reports usage as not checked and every candidate's in_use null", + janitor_command=("janitor", "--auto-approve", "--json"), + expected_usage_check={ + "status": "not_checked", + "reason": "unsupported adapter", + "window": "7d", + "keep_in_use": False, + }, + expected_archive_in_use=(("main.old_orders_000", None), ("main.old_orders_001", None)), + expected_kept_in_use=(), + expected_archived=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( + test_case: JanitorUsageJsonE2ETestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_probe_janitor_project( tmp_path=tmp_path, project_name="janitor_usage_json", stale_tables={"main": 2} ) result: subprocess.CompletedProcess[str] = run_sqb( - command=("janitor", "--auto-approve", "--json"), project_dir=project_dir + command=test_case.janitor_command, project_dir=project_dir ) assert result.returncode == 0, result.stdout + result.stderr payload: dict[str, Any] = json.loads(result.stdout) - assert payload["usage_check"] == { - "status": "not_checked", - "reason": "unsupported adapter", - "window": "7d", - "keep_in_use": False, - } - assert [candidate["in_use"] for candidate in payload["plan"]["relations_to_archive"]] == [ - None, - None, - ] - assert payload["plan"]["kept_in_use"] == [] + assert payload["usage_check"] == test_case.expected_usage_check + assert in_use_entries(payload["plan"]["relations_to_archive"]) == ( + test_case.expected_archive_in_use + ) + assert in_use_entries(payload["plan"]["kept_in_use"]) == test_case.expected_kept_in_use + assert payload["result"]["counts"]["archived"] == test_case.expected_archived assert "usage not checked: unsupported adapter" in result.stderr @@ -210,9 +226,7 @@ def test_given_usage_source_when_running_janitor_then_in_use_relations_are_shown name: str for name in test_case.expected_remaining: assert table_exists(db_path=db_path, table_name=name) - assert len( - [name for name in list_archive_names(db_path=db_path) if "__old_orders" in name] - ) == (test_case.expected_archived) + assert count_stale_archives(db_path=db_path) == test_case.expected_archived calls: dict[str, object] = read_usage_calls(project_dir=project_dir) assert calls["relations"] == [ "old_orders_000", @@ -222,8 +236,40 @@ def test_given_usage_source_when_running_janitor_then_in_use_relations_are_shown ] +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageJsonE2ETestCase( + description="kept and archived candidates carry their reads, or false when unread", + janitor_command=("janitor", "--auto-approve", "--json", "--keep-in-use"), + expected_usage_check={ + "status": "checked", + "reason": None, + "window": "7d", + "keep_in_use": True, + }, + expected_archive_in_use=(("main.old_orders_001", False),), + expected_kept_in_use=( + ( + "main.old_orders_000", + { + "last_read": "2026-10-21T09:30:00Z", + "readers": list(FIVE_READERS), + "query_count": 40, + }, + ), + ( + "main.old_orders_002", + {"last_read": "2026-10-21T09:30:00Z", "readers": ["alice"], "query_count": 2}, + ), + ), + expected_archived=1, + ), + ], + ids=lambda case: case.description, +) def test_given_usage_source_when_janitor_prints_json_then_candidates_carry_in_use( - tmp_path: Path, + test_case: JanitorUsageJsonE2ETestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_usage_janitor_project( tmp_path=tmp_path, project_name="janitor_usage_json_shape", stale_tables=3 @@ -234,38 +280,17 @@ def test_given_usage_source_when_janitor_prints_json_then_candidates_carry_in_us ) result: subprocess.CompletedProcess[str] = run_sqb( - command=("janitor", "--auto-approve", "--json", "--keep-in-use"), project_dir=project_dir + command=test_case.janitor_command, project_dir=project_dir ) assert result.returncode == 0, result.stdout + result.stderr payload: dict[str, Any] = json.loads(result.stdout) - assert payload["usage_check"] == { - "status": "checked", - "reason": None, - "window": "7d", - "keep_in_use": True, - } - assert [ - (candidate["name"], candidate["in_use"]) - for candidate in payload["plan"]["relations_to_archive"] - ] == [("main.old_orders_001", False)] - assert [ - (candidate["name"], candidate["in_use"]) for candidate in payload["plan"]["kept_in_use"] - ] == [ - ( - "main.old_orders_000", - { - "last_read": "2026-10-21T09:30:00Z", - "readers": list(FIVE_READERS), - "query_count": 40, - }, - ), - ( - "main.old_orders_002", - {"last_read": "2026-10-21T09:30:00Z", "readers": ["alice"], "query_count": 2}, - ), - ] - assert payload["result"]["counts"]["archived"] == 1 + assert payload["usage_check"] == test_case.expected_usage_check + assert in_use_entries(payload["plan"]["relations_to_archive"]) == ( + test_case.expected_archive_in_use + ) + assert in_use_entries(payload["plan"]["kept_in_use"]) == test_case.expected_kept_in_use + assert payload["result"]["counts"]["archived"] == test_case.expected_archived assert read_usage_calls(project_dir=project_dir)["window_seconds"] == 7 * 86_400 diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index c42a271e1b..b3a653a125 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -2,7 +2,8 @@ from datetime import datetime from sqlbuild._native import InferredNullability -from sqlbuild.adapter.contract.models import FunctionInfo, SchemaDiffResult +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import FunctionInfo, RelationUsageReport, SchemaDiffResult from sqlbuild.adapter.contract.types import TableFreshnessStatus from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.types import CostStatus @@ -306,6 +307,14 @@ class SnowflakeRelationUsageTestCase: expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] +@dataclass(frozen=True) +class SnowflakeRelationUsageQueryTestCase: + description: str + window_seconds: int + expected_params: tuple[object, ...] + expected_sql_fragments: tuple[str, ...] + + @dataclass(frozen=True) class SnowflakeRelationUsageChunkTestCase: description: str @@ -318,3 +327,19 @@ class SnowflakeRelationUsageErrorTestCase: description: str errno: int | None message: str + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class SnowflakeRelationUsageFailureTestCase: + description: str + errno: int | None + message: str + expected_error_match: str + + +@dataclass(frozen=True) +class UnsupportedRelationUsageTestCase: + description: str + adapter: BaseAdapter + expected_report: RelationUsageReport diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 18478879af..1ef7659c29 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -11,7 +11,7 @@ from typing import Any from sqlbuild._native import NativeProgressProjector -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationUsageReport from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection from sqlbuild.observability import ( @@ -396,3 +396,90 @@ class CustomInitSnowflakeAdapter(SnowflakeAdapter): def __init__(self) -> None: self.custom_label: str = "orders" + + +class FakeAccessHistoryError(Exception): + """A Snowflake driver error with an error number.""" + + def __init__(self, message: str, errno: int | None) -> None: + super().__init__(message) + self.errno: int | None = errno + + +class FakeUsageWarehouse: + """Answers CURRENT_DATABASE() and access-history reads, recording each access-history read.""" + + def __init__( + self, + *, + current_database: str | None = None, + access_rows: tuple[tuple[object, ...], ...] = (), + errors: tuple[Exception, ...] = (), + ) -> None: + self.current_database: str | None = current_database + self.access_rows: tuple[tuple[object, ...], ...] = access_rows + self.errors: tuple[Exception, ...] = errors + self.queries: list[tuple[str, tuple[object, ...]]] = [] + + def current_database_rows(self) -> list[tuple[object, ...]]: + return [(self.current_database,)] + + def access_history_rows(self, sql: str, params: tuple[object, ...]) -> list[tuple[object, ...]]: + self.queries.append((sql, params)) + error: Exception + for error in self.errors: + raise error + names: frozenset[object] = frozenset(params) + return list(filter(lambda row: row[0] in names, self.access_rows)) + + +class FakeUsageCursor: + """Cursor double routing each statement to the fake usage warehouse.""" + + def __init__(self, warehouse: FakeUsageWarehouse) -> None: + self.warehouse: FakeUsageWarehouse = warehouse + self.rows: list[tuple[object, ...]] = [] + + def execute(self, sql: str, params: tuple[object, ...] = ()) -> None: + reads: dict[bool, Callable[[], list[tuple[object, ...]]]] = { + True: self.warehouse.current_database_rows, + False: lambda: self.warehouse.access_history_rows(sql, params), + } + self.rows = reads[sql == "SELECT CURRENT_DATABASE()"]() + + def fetchone(self) -> tuple[object, ...] | None: + return next(iter(self.rows), None) + + def fetchall(self) -> list[tuple[object, ...]]: + return self.rows + + def close(self) -> None: + return None + + +class FakeUsageConnection: + """Connection double handing out cursors over one fake usage warehouse.""" + + def __init__(self, warehouse: FakeUsageWarehouse) -> None: + self.warehouse: FakeUsageWarehouse = warehouse + + def cursor(self) -> FakeUsageCursor: + return FakeUsageCursor(self.warehouse) + + +def usage_relation(database: str | None, schema: str, name: str) -> RelationInfo: + """A base table relation for usage reads.""" + + return RelationInfo(database=database, schema=schema, name=name, relation_type="base table") + + +def read_snowflake_usage( + *, warehouse: FakeUsageWarehouse, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Snowflake's usage report for the relations from the fake warehouse.""" + + return SnowflakeAdapter().relation_usage( + connection=FakeUsageConnection(warehouse), + relations=relations, + window_seconds=window_seconds, + ) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py index e109d6db1d..a602a5dcd6 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py @@ -2,87 +2,37 @@ from __future__ import annotations -from dataclasses import dataclass, field from datetime import UTC, datetime import pytest from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.adapters.snowflake._helpers.relation_usage import ( +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.constants import ( CLONE_STATEMENT_PATTERN, SQLBUILD_QUERY_TAG_FRAGMENT, USAGE_UNAVAILABLE_REASON, ) -from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeRelationUsageChunkTestCase, SnowflakeRelationUsageErrorTestCase, + SnowflakeRelationUsageFailureTestCase, + SnowflakeRelationUsageQueryTestCase, SnowflakeRelationUsageTestCase, + UnsupportedRelationUsageTestCase, +) +from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( + FakeAccessHistoryError, + FakeUsageWarehouse, + read_snowflake_usage, + usage_relation, ) WINDOW_SECONDS: int = 7 * 86_400 READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) - - -class _AccessHistoryError(Exception): - def __init__(self, message: str, errno: int | None) -> None: - super().__init__(message) - self.errno: int | None = errno - - -@dataclass -class _Warehouse: - """Answers CURRENT_DATABASE() and access-history reads, recording each query.""" - - current_database: str | None = None - access_rows: tuple[tuple[object, ...], ...] = () - error: Exception | None = None - queries: list[tuple[str, tuple[object, ...]]] = field(default_factory=list) - - -class _Cursor: - def __init__(self, warehouse: _Warehouse) -> None: - self.warehouse: _Warehouse = warehouse - self.rows: list[tuple[object, ...]] = [] - - def execute(self, sql: str, params: tuple[object, ...] = ()) -> None: - if sql == "SELECT CURRENT_DATABASE()": - self.rows = [(self.warehouse.current_database,)] - return - self.warehouse.queries.append((sql, params)) - if self.warehouse.error is not None: - raise self.warehouse.error - names: set[object] = set(params) - self.rows = [row for row in self.warehouse.access_rows if row[0] in names] - - def fetchone(self) -> tuple[object, ...] | None: - return self.rows[0] if self.rows else None - - def fetchall(self) -> list[tuple[object, ...]]: - return self.rows - - def close(self) -> None: - return None - - -class _Connection: - def __init__(self, warehouse: _Warehouse) -> None: - self.warehouse: _Warehouse = warehouse - - def cursor(self) -> _Cursor: - return _Cursor(self.warehouse) - - -def _relation(database: str | None, schema: str, name: str) -> RelationInfo: - return RelationInfo(database=database, schema=schema, name=name, relation_type="base table") - - -def _usage(*, connection: _Connection, relations: tuple[RelationInfo, ...]) -> RelationUsageReport: - return SnowflakeAdapter().relation_usage( - connection=connection, relations=relations, window_seconds=WINDOW_SECONDS - ) +ORDERS_DAILY: tuple[RelationInfo, ...] = (usage_relation("analytics", "mart", "orders_daily"),) @pytest.mark.parametrize( @@ -130,54 +80,66 @@ def _usage(*, connection: _Connection, relations: tuple[RelationInfo, ...]) -> R def test_given_access_history_when_reading_usage_then_reads_match_candidates( test_case: SnowflakeRelationUsageTestCase, ) -> None: - warehouse: _Warehouse = _Warehouse( + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( current_database=test_case.current_database, access_rows=test_case.access_rows ) - report: RelationUsageReport = _usage( - connection=_Connection(warehouse), - relations=tuple(_relation(*relation) for relation in test_case.relations), + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, ) - assert report.checked - assert report.reason is None - sql: str params: tuple[object, ...] - ((sql, params),) = warehouse.queries + ((_, params),) = warehouse.queries + assert report.checked assert params[1 : 1 + len(test_case.expected_object_names)] == (test_case.expected_object_names) - assert "SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY" in sql - assert "LATERAL FLATTEN(INPUT => history.direct_objects_accessed)" in sql assert tuple( (usage.name, usage.readers, usage.query_count, usage.last_read_at) for usage in report.usages ) == (test_case.expected_usages) -def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_are_excluded() -> ( - None -): - warehouse: _Warehouse = _Warehouse() +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageQueryTestCase( + description="one flattened access-history read leaves out SQLBuild queries and clones", + window_seconds=WINDOW_SECONDS, + expected_params=( + WINDOW_SECONDS, + "ANALYTICS.MART.ORDERS_DAILY", + WINDOW_SECONDS, + '"app":"sqlbuild"', + r".*\sCLONE\s.*", + ), + expected_sql_fragments=( + "FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY AS history, " + "LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed", + "NOT EXISTS (SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY AS query", + "CONTAINS(COALESCE(query.query_tag, ''), %s)", + "STARTSWITH(query.query_type, 'CREATE') AND REGEXP_LIKE(query.query_text, %s", + "COUNT(DISTINCT history.query_id) AS query_count", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_are_excluded( + test_case: SnowflakeRelationUsageQueryTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse() - _usage( - connection=_Connection(warehouse), - relations=(_relation("analytics", "mart", "orders_daily"),), + read_snowflake_usage( + warehouse=warehouse, relations=ORDERS_DAILY, window_seconds=test_case.window_seconds ) sql: str params: tuple[object, ...] ((sql, params),) = warehouse.queries - assert params == ( - WINDOW_SECONDS, - "ANALYTICS.MART.ORDERS_DAILY", - WINDOW_SECONDS, - SQLBUILD_QUERY_TAG_FRAGMENT, - CLONE_STATEMENT_PATTERN, - ) - assert SQLBUILD_QUERY_TAG_FRAGMENT == '"app":"sqlbuild"' - assert "NOT EXISTS (SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY AS query" in sql - assert "CONTAINS(COALESCE(query.query_tag, ''), %s)" in sql - assert "STARTSWITH(query.query_type, 'CREATE') AND REGEXP_LIKE(query.query_text, %s" in sql - assert sql.count("DATEADD('second', -%s, CURRENT_TIMESTAMP())") == 2 + assert params == test_case.expected_params + assert (SQLBUILD_QUERY_TAG_FRAGMENT, CLONE_STATEMENT_PATTERN) == (test_case.expected_params[3:]) + assert all(fragment in sql for fragment in test_case.expected_sql_fragments) @pytest.mark.parametrize( @@ -199,14 +161,15 @@ def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_ def test_given_many_candidates_when_reading_usage_then_names_are_chunked( test_case: SnowflakeRelationUsageChunkTestCase, ) -> None: - warehouse: _Warehouse = _Warehouse() + warehouse: FakeUsageWarehouse = FakeUsageWarehouse() - report: RelationUsageReport = _usage( - connection=_Connection(warehouse), + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, relations=tuple( - _relation("analytics", "mart", f"old_orders_{index:04d}") + usage_relation("analytics", "mart", f"old_orders_{index:04d}") for index in range(test_case.relation_count) ), + window_seconds=WINDOW_SECONDS, ) assert report.checked @@ -224,16 +187,19 @@ def test_given_many_candidates_when_reading_usage_then_names_are_chunked( errno=2003, message="Object 'SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY' does not exist or not " "authorized.", + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), ), SnowflakeRelationUsageErrorTestCase( description="insufficient privileges are not checked", errno=3001, message="Insufficient privileges to operate on schema 'ACCOUNT_USAGE'", + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), ), SnowflakeRelationUsageErrorTestCase( description="a privilege message without an error number is not checked", errno=None, message="SQL access control error: Insufficient privileges to operate on view", + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), ), ), ids=lambda case: case.description, @@ -241,36 +207,66 @@ def test_given_many_candidates_when_reading_usage_then_names_are_chunked( def test_given_no_access_history_access_when_reading_usage_then_usage_is_not_checked( test_case: SnowflakeRelationUsageErrorTestCase, ) -> None: - warehouse: _Warehouse = _Warehouse( - error=_AccessHistoryError(test_case.message, test_case.errno) + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + errors=(FakeAccessHistoryError(test_case.message, test_case.errno),) ) - report: RelationUsageReport = _usage( - connection=_Connection(warehouse), - relations=(_relation("analytics", "mart", "orders_daily"),), + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS ) - assert report == RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + assert report == test_case.expected_report -def test_given_other_access_history_failure_when_reading_usage_then_error_propagates() -> None: - warehouse: _Warehouse = _Warehouse(error=_AccessHistoryError("SQL execution canceled", 604)) +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageFailureTestCase( + description="a cancelled read is a real error", + errno=604, + message="SQL execution canceled", + expected_error_match="canceled", + ), + ), + ids=lambda case: case.description, +) +def test_given_other_access_history_failure_when_reading_usage_then_error_propagates( + test_case: SnowflakeRelationUsageFailureTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + errors=(FakeAccessHistoryError(test_case.message, test_case.errno),) + ) - with pytest.raises(_AccessHistoryError, match="canceled"): - _usage( - connection=_Connection(warehouse), - relations=(_relation("analytics", "mart", "orders_daily"),), + with pytest.raises(FakeAccessHistoryError, match=test_case.expected_error_match): + read_snowflake_usage( + warehouse=warehouse, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS ) -def test_given_other_adapter_when_reading_usage_then_it_reports_unsupported() -> None: - report: RelationUsageReport = DuckDbAdapter().relation_usage( - connection=object(), - relations=(_relation("analytics", "mart", "orders_daily"),), - window_seconds=WINDOW_SECONDS, +@pytest.mark.parametrize( + "test_case", + ( + UnsupportedRelationUsageTestCase( + description="duckdb has no usage source", + adapter=DuckDbAdapter(), + expected_report=RelationUsageReport(checked=False, reason="unsupported adapter"), + ), + UnsupportedRelationUsageTestCase( + description="postgres has no usage source", + adapter=PostgresAdapter(), + expected_report=RelationUsageReport(checked=False, reason="unsupported adapter"), + ), + ), + ids=lambda case: case.description, +) +def test_given_other_adapter_when_reading_usage_then_it_reports_unsupported( + test_case: UnsupportedRelationUsageTestCase, +) -> None: + report: RelationUsageReport = test_case.adapter.relation_usage( + connection=object(), relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS ) - assert report == RelationUsageReport(checked=False, reason="unsupported adapter") + assert report == test_case.expected_report if __name__ == "__main__": From be05343f90c76d7b9495a1fbd8ac6f94108bde05 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:28:32 +0100 Subject: [PATCH 09/14] docs: say why the janitor ignores SQLBuild's own reads --- .../.agents/skills/sqlbuild/references/docs/cli/janitor.md | 2 +- website/src/content/docs/docs/cli/janitor.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 6c73d40bdb..0e306b1b40 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -100,7 +100,7 @@ Usage comes from the warehouse's query history: | Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`) joined with `QUERY_HISTORY` | | Other adapters | Not available | -On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: ```sql GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 50c7fb9829..3307d4b483 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -80,7 +80,7 @@ Usage comes from the warehouse's query history: | Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`) joined with `QUERY_HISTORY` | | Other adapters | Not available | -On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: ```sql GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; From cbbe0b28e7de44867e9f3f13de8661c1c2473e34 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:37:45 +0100 Subject: [PATCH 10/14] feat: read janitor relation usage on BigQuery and Databricks --- .../sqlbuild/references/docs/cli/janitor.md | 16 +- .../contract/main/aggregate_relation_usage.py | 51 ++++ src/sqlbuild/adapter/relations/constants.py | 1 + .../bigquery/_helpers/relation_usage.py | 200 ++++++++++++++ .../bigquery/classes/bigquery_adapter.py | 9 +- src/sqlbuild/adapters/bigquery/constants.py | 11 + src/sqlbuild/adapters/bigquery/models.py | 15 ++ .../databricks/_helpers/relation_usage.py | 101 +++++++ .../databricks/classes/databricks_adapter.py | 9 +- src/sqlbuild/adapters/databricks/constants.py | 15 ++ src/sqlbuild/adapters/databricks/models.py | 13 + .../sqlbuild/adapters/bigquery/_test_types.py | 40 +++ .../src/sqlbuild/adapters/bigquery/helpers.py | 104 +++++++- .../adapters/bigquery/test_relation_usage.py | 250 ++++++++++++++++++ .../adapters/databricks/_test_types.py | 42 ++- .../sqlbuild/adapters/databricks/helpers.py | 64 ++++- .../databricks/test_relation_usage.py | 223 ++++++++++++++++ website/src/content/docs/docs/cli/janitor.mdx | 16 +- 18 files changed, 1168 insertions(+), 12 deletions(-) create mode 100644 src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py create mode 100644 src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py create mode 100644 src/sqlbuild/adapters/bigquery/models.py create mode 100644 src/sqlbuild/adapters/databricks/_helpers/relation_usage.py create mode 100644 src/sqlbuild/adapters/databricks/models.py create mode 100644 tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py create mode 100644 tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 0e306b1b40..a18311989e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -98,6 +98,8 @@ Usage comes from the warehouse's query history: | Adapter | Usage source | |---------|--------------| | Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`) joined with `QUERY_HISTORY` | +| BigQuery | `` `.region-`.INFORMATION_SCHEMA.JOBS `` (`referenced_tables`) | +| Databricks | `system.access.table_lineage` (`source_table_full_name`) | | Other adapters | Not available | On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: @@ -108,7 +110,19 @@ GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; Access history lags by up to about 3 hours, so reads in the last few hours may not show yet. -When usage can't be checked, because the adapter has no usage source or the role can't read access history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. +On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the table, by `user_email`. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. + +On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`; a read through a view also counts for the tables behind it. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: + +```sql +GRANT USE CATALOG ON CATALOG system TO `janitor-principal`; +GRANT USE SCHEMA ON SCHEMA system.access TO `janitor-principal`; +GRANT SELECT ON TABLE system.access.table_lineage TO `janitor-principal`; +``` + +Lineage records only reads Databricks can capture lineage for, and can arrive with a delay. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. + +When usage can't be checked, because the adapter has no usage source or the role can't read its query history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers` and `query_count`. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. diff --git a/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py b/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py new file mode 100644 index 0000000000..772f9a1450 --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py @@ -0,0 +1,51 @@ +"""Fold per-reader usage rows into one usage per relation.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from datetime import UTC, datetime + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage +from sqlbuild.adapter.relations.constants import UNKNOWN_RELATION_READER + + +def aggregate_relation_usage( + *, rows: Iterable[tuple[object, ...]], relations_by_name: Mapping[str, RelationInfo] +) -> tuple[RelationUsage, ...]: + """One usage per read relation from `(object_name, reader, query_count, last_read)` rows.""" + + readers: dict[str, dict[str, int]] = {} + last_read: dict[str, datetime] = {} + row: tuple[object, ...] + for row in rows: + object_name: str = str(row[0]) + if object_name not in relations_by_name: + continue + reader: str = UNKNOWN_RELATION_READER if row[1] is None else str(row[1]) + counts: dict[str, int] = readers.setdefault(object_name, {}) + counts[reader] = counts.get(reader, 0) + int(str(row[2])) + read_at: datetime = _utc(row[3]) + last_read[object_name] = max(read_at, last_read.get(object_name, read_at)) + usages: list[RelationUsage] = [] + for object_name in sorted(readers): + relation: RelationInfo = relations_by_name[object_name] + counts = readers[object_name] + usages.append( + RelationUsage( + database=relation.database, + schema=relation.schema, + name=relation.name, + last_read_at=last_read[object_name], + readers=tuple(sorted(counts, key=lambda reader: (-counts[reader], reader))), + query_count=sum(counts.values()), + ) + ) + return tuple(usages) + + +def _utc(value: object) -> datetime: + if not isinstance(value, datetime): + return datetime.fromisoformat(str(value)).astimezone(UTC) + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 0c144b1164..9d1557c487 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -6,3 +6,4 @@ RELATION_COLUMNS_LOOKUP: str = "columns" RELATION_EXISTS_LOOKUP: str = "exists" RELATION_TYPE_LOOKUP: str = "type" +UNKNOWN_RELATION_READER: str = "unknown user" diff --git a/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py new file mode 100644 index 0000000000..6834a1fb6c --- /dev/null +++ b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py @@ -0,0 +1,200 @@ +"""Read who queried relations from BigQuery INFORMATION_SCHEMA.JOBS.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any + +from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.bigquery.constants import ( + CLONE_STATEMENT_PATTERN, + USAGE_NAME_CHUNK_SIZE, + USAGE_UNAVAILABLE_ERROR_CLASS_NAMES, + USAGE_UNAVAILABLE_ERROR_CODES, + USAGE_UNAVAILABLE_MESSAGES, + USAGE_UNAVAILABLE_REASON, +) +from sqlbuild.adapters.bigquery.models import UsageQuery + + +def usage_object_name(*, project: str, dataset: str, table: str) -> str: + """`project.dataset.table` as `referenced_tables` spells it; BigQuery names keep their case.""" + + return ".".join(part.replace("`", "") for part in (project, dataset, table)) + + +def usage_queries( + *, + relations: Iterable[RelationInfo], + default_project: str, + locations: Mapping[tuple[str, str], str], + window_seconds: int, +) -> tuple[tuple[UsageQuery, dict[str, RelationInfo]], ...]: + """One JOBS read per project, location and chunk of names, with the relations it covers.""" + + groups: dict[tuple[str, str], dict[str, RelationInfo]] = {} + relation: RelationInfo + for relation in relations: + if relation.schema is None: + continue + project: str = relation_project(relation=relation, default_project=default_project) + location: str = locations[(project, relation.schema)] + name: str = usage_object_name(project=project, dataset=relation.schema, table=relation.name) + groups.setdefault((project, location), {})[name] = relation + queries: list[tuple[UsageQuery, dict[str, RelationInfo]]] = [] + for (project, location), named in sorted(groups.items()): + names: tuple[str, ...] = tuple(sorted(named)) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE): + chunk: tuple[str, ...] = names[start : start + USAGE_NAME_CHUNK_SIZE] + queries.append( + ( + usage_query( + project=project, + location=location, + object_names=chunk, + window_seconds=window_seconds, + ), + {name: named[name] for name in chunk}, + ) + ) + return tuple(queries) + + +def jobs_view(*, project: str, location: str) -> str: + """`project.region-`.INFORMATION_SCHEMA.JOBS.""" + + region: str = location.strip().lower() + region = region if region.startswith("region-") else f"region-{region}" + return f"`{project}.{region}`.INFORMATION_SCHEMA.JOBS" + + +def usage_query( + *, project: str, location: str, object_names: tuple[str, ...], window_seconds: int +) -> UsageQuery: + """Reads of the named tables per user by successful query jobs, leaving out clones.""" + + referenced: str = ( + "CONCAT(referenced.project_id, '.', referenced.dataset_id, '.', referenced.table_id)" + ) + sql: str = ( + f"SELECT {referenced} AS object_name, " + "jobs.user_email AS user_name, " + "COUNT(DISTINCT jobs.job_id) AS query_count, " + "MAX(jobs.creation_time) AS last_read_at " + f"FROM {jobs_view(project=project, location=location)} AS jobs, " + "UNNEST(jobs.referenced_tables) AS referenced " + "WHERE jobs.creation_time >= " + "TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @window_seconds SECOND) " + "AND jobs.job_type = 'QUERY' " + "AND jobs.error_result IS NULL " + "AND COALESCE(jobs.statement_type, '') NOT IN ('SCRIPT', 'CREATE_SNAPSHOT_TABLE') " + "AND NOT (STARTS_WITH(COALESCE(jobs.statement_type, ''), 'CREATE_TABLE') " + "AND REGEXP_CONTAINS(jobs.query, @clone_pattern)) " + f"AND {referenced} IN UNNEST(@object_names) " + "GROUP BY 1, 2 " + "ORDER BY 1, 2" + ) + return UsageQuery( + sql=sql, location=location, window_seconds=window_seconds, object_names=object_names + ) + + +def query_parameters(query: UsageQuery) -> list[Any]: + """The query's parameters as BigQuery client parameters.""" + + from google.cloud import bigquery + + return [ + bigquery.ScalarQueryParameter("window_seconds", "INT64", query.window_seconds), + bigquery.ScalarQueryParameter("clone_pattern", "STRING", CLONE_STATEMENT_PATTERN), + bigquery.ArrayQueryParameter("object_names", "STRING", list(query.object_names)), + ] + + +def is_usage_unavailable_error(error: Exception) -> bool: + """Whether reading JOBS failed because the view or the permission to list jobs is missing.""" + + message: str = str(error).lower() + return ( + getattr(error, "code", None) in USAGE_UNAVAILABLE_ERROR_CODES + or type(error).__name__ in USAGE_UNAVAILABLE_ERROR_CLASS_NAMES + or any(marker in message for marker in USAGE_UNAVAILABLE_MESSAGES) + ) + + +def relation_project(*, relation: RelationInfo, default_project: str) -> str: + """The project holding the relation: its database, else the client's project.""" + + return (relation.database or default_project).replace("`", "") + + +def dataset_locations( + *, connection: Any, relations: Iterable[RelationInfo], default_project: str +) -> dict[tuple[str, str], str]: + """The location of each candidate's dataset, or the connection's fixed location.""" + + keys: set[tuple[str, str]] = { + (relation_project(relation=relation, default_project=default_project), relation.schema) + for relation in relations + if relation.schema is not None + } + if connection.location is not None: + return {key: str(connection.location) for key in keys} + return { + key: str(connection.client.get_dataset(f"{key[0]}.{key[1]}").location) + for key in sorted(keys) + } + + +def run_usage_query(*, connection: Any, query: UsageQuery) -> list[tuple[object, ...]]: + """Run one JOBS read with bound parameters in the query's location.""" + + from google.cloud import bigquery + + job: Any = connection.client.query( + query.sql, + location=query.location, + job_config=bigquery.QueryJobConfig(query_parameters=query_parameters(query)), + ) + return [tuple(row.values()) for row in connection.job_result(job)] + + +def read_relation_usage( + *, connection: Any, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Read every relation's usage in each dataset's location.""" + + default_project: str = str(connection.client.project) + rows: list[tuple[object, ...]] = [] + covered: dict[str, RelationInfo] = {} + query: UsageQuery + named: dict[str, RelationInfo] + for query, named in usage_queries( + relations=relations, + default_project=default_project, + locations=dataset_locations( + connection=connection, relations=relations, default_project=default_project + ), + window_seconds=window_seconds, + ): + rows.extend(run_usage_query(connection=connection, query=query)) + covered.update(named) + return RelationUsageReport( + checked=True, usages=aggregate_relation_usage(rows=rows, relations_by_name=covered) + ) + + +def bigquery_relation_usage( + *, connection: Any, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Read usage; no access to JOBS or the datasets means usage is not checked.""" + + try: + return read_relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 8c5702e6f7..7436626350 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -54,7 +54,6 @@ ) from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition -from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -122,6 +121,7 @@ render_bigquery_view_move, render_bigquery_view_revokes, ) +from sqlbuild.adapters.bigquery._helpers.relation_usage import bigquery_relation_usage from sqlbuild.adapters.bigquery._helpers.statement_telemetry import affected_rows from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection from sqlbuild.adapters.bigquery.classes.bigquery_cursor import _BigQueryCursor @@ -518,8 +518,11 @@ def relation_usage( relations: tuple[RelationInfo, ...], window_seconds: int, ) -> RelationUsageReport: - del connection, relations, window_seconds - return unsupported_relation_usage() + """Read candidate reads from the region's INFORMATION_SCHEMA.JOBS (180 days kept).""" + + return bigquery_relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) def get_tables_freshness_metadata( self, diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index 498d176b89..5e2e96a816 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -31,3 +31,14 @@ NATIVE_TABLE_TYPE: str = "TABLE" PENDING_JOB_STATE: str = "PENDING" PENDING_JOB_POLL_SECONDS: float = 1.0 +USAGE_NAME_CHUNK_SIZE: int = 1_000 +USAGE_UNAVAILABLE_REASON: str = ( + "cannot read INFORMATION_SCHEMA.JOBS " + "(needs bigquery.jobs.listAll, for example the BigQuery Resource Viewer role)" +) +USAGE_UNAVAILABLE_ERROR_CODES: frozenset[int] = frozenset({403, 404}) +USAGE_UNAVAILABLE_ERROR_CLASS_NAMES: frozenset[str] = frozenset( + {"Forbidden", "NotFound", "PermissionDenied"} +) +USAGE_UNAVAILABLE_MESSAGES: tuple[str, ...] = ("access denied", "permission") +CLONE_STATEMENT_PATTERN: str = r"(?is)\bCLONE\b" diff --git a/src/sqlbuild/adapters/bigquery/models.py b/src/sqlbuild/adapters/bigquery/models.py new file mode 100644 index 0000000000..3bfa8fd0dc --- /dev/null +++ b/src/sqlbuild/adapters/bigquery/models.py @@ -0,0 +1,15 @@ +"""BigQuery adapter models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UsageQuery: + """One INFORMATION_SCHEMA.JOBS read: its SQL, location, window and table names.""" + + sql: str + location: str + window_seconds: int + object_names: tuple[str, ...] diff --git a/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py b/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py new file mode 100644 index 0000000000..e227920850 --- /dev/null +++ b/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py @@ -0,0 +1,101 @@ +"""Read who queried relations from the Databricks system.access.table_lineage table.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.databricks.constants import ( + USAGE_NAME_CHUNK_SIZE, + USAGE_UNAVAILABLE_ERROR_CLASSES, + USAGE_UNAVAILABLE_REASON, + USAGE_UNAVAILABLE_SQLSTATES, +) +from sqlbuild.adapters.databricks.models import UsageQuery + + +def usage_object_names(*, relations: Iterable[RelationInfo]) -> dict[str, RelationInfo]: + """Each fully qualified relation by its lowercase `catalog.schema.table` lineage name.""" + + named: dict[str, RelationInfo] = {} + relation: RelationInfo + for relation in relations: + if relation.database is None or relation.schema is None: + continue + name: str = ".".join( + part.replace("`", "").lower() + for part in (relation.database, relation.schema, relation.name) + ) + named[name] = relation + return named + + +def usage_queries(*, object_names: Iterable[str], window_seconds: int) -> tuple[UsageQuery, ...]: + """One lineage read per chunk of object names.""" + + names: tuple[str, ...] = tuple(sorted(object_names)) + return tuple( + usage_query( + object_names=names[start : start + USAGE_NAME_CHUNK_SIZE], + window_seconds=window_seconds, + ) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) + ) + + +def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> UsageQuery: + """Read events of the named tables per user within the window.""" + + placeholders: str = ", ".join(f":name_{index}" for index in range(len(object_names))) + since: str = "TIMESTAMPADD(SECOND, -:window_seconds, current_timestamp())" + sql: str = ( + "SELECT lower(source_table_full_name) AS object_name, " + "created_by AS user_name, " + "COUNT(DISTINCT COALESCE(statement_id, event_id)) AS query_count, " + "MAX(event_time) AS last_read_at " + "FROM system.access.table_lineage " + f"WHERE event_date >= DATE({since}) " + f"AND event_time >= {since} " + f"AND lower(source_table_full_name) IN ({placeholders}) " + "GROUP BY 1, 2 " + "ORDER BY 1, 2" + ) + params: dict[str, object] = {"window_seconds": window_seconds} + params.update({f"name_{index}": name for index, name in enumerate(object_names)}) + return UsageQuery(sql=sql, params=params) + + +def is_usage_unavailable_error(error: Exception) -> bool: + """Whether reading lineage failed for Unity Catalog, system schema or permission reasons.""" + + message: str = str(error).upper() + return getattr(error, "sqlstate", None) in USAGE_UNAVAILABLE_SQLSTATES or any( + error_class in message for error_class in USAGE_UNAVAILABLE_ERROR_CLASSES + ) + + +def read_relation_usage( + *, connection: Any, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Read every relation's usage in chunked queries; no lineage access means not checked.""" + + relations_by_name: dict[str, RelationInfo] = usage_object_names(relations=relations) + rows: list[tuple[object, ...]] = [] + query: UsageQuery + for query in usage_queries(object_names=relations_by_name, window_seconds=window_seconds): + cursor: Any = connection.cursor() + try: + cursor.execute(query.sql, query.params) + rows.extend(tuple(row) for row in cursor.fetchall()) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + finally: + cursor.close() + return RelationUsageReport( + checked=True, + usages=aggregate_relation_usage(rows=rows, relations_by_name=relations_by_name), + ) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 4a809a2708..45890a0866 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -51,7 +51,6 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition -from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -119,6 +118,7 @@ render_databricks_view_revokes, show_grants_object_kind, ) +from sqlbuild.adapters.databricks._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.databricks.classes.databricks_connection import _DatabricksConnection from sqlbuild.adapters.databricks.constants import ( DELTA_COLUMN_MAPPING_NAME_MODE, @@ -617,8 +617,11 @@ def relation_usage( relations: tuple[RelationInfo, ...], window_seconds: int, ) -> RelationUsageReport: - del connection, relations, window_seconds - return unsupported_relation_usage() + """Read candidate reads from Unity Catalog's system.access.table_lineage.""" + + return read_relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) def get_tables_freshness_metadata( self, diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index 3ab8be030e..a43a560226 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -17,3 +17,18 @@ DETAILED_TABLE_INFORMATION_MARKER: str = "# detailed table information" DESCRIBE_DETAIL_ROW_WIDTH: int = 2 TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" +USAGE_NAME_CHUNK_SIZE: int = 1_000 +USAGE_UNAVAILABLE_REASON: str = ( + "cannot read system.access.table_lineage (needs Unity Catalog with the access system " + "schema enabled, and USE CATALOG, USE SCHEMA and SELECT on it)" +) +USAGE_UNAVAILABLE_ERROR_CLASSES: tuple[str, ...] = ( + "INSUFFICIENT_PERMISSIONS", + "PERMISSION_DENIED", + "TABLE_OR_VIEW_NOT_FOUND", + "SCHEMA_NOT_FOUND", + "CATALOG_NOT_FOUND", + "NO_SUCH_CATALOG_EXCEPTION", + "UC_NOT_ENABLED", +) +USAGE_UNAVAILABLE_SQLSTATES: frozenset[str] = frozenset({"42501", "42P01", "42704"}) diff --git a/src/sqlbuild/adapters/databricks/models.py b/src/sqlbuild/adapters/databricks/models.py new file mode 100644 index 0000000000..e4507521e4 --- /dev/null +++ b/src/sqlbuild/adapters/databricks/models.py @@ -0,0 +1,13 @@ +"""Databricks adapter models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UsageQuery: + """One system.access.table_lineage read: its SQL and named parameters.""" + + sql: str + params: dict[str, object] diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index d63ff26d46..13851b17fc 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -4,6 +4,7 @@ from sqlbuild._native import InferredNullability from sqlbuild.adapter.contract.models import ( ColumnInfo, + RelationUsageReport, RowDiffResult, RowDiffSampleRow, SchemaDiffResult, @@ -219,3 +220,42 @@ class BigQueryJobInvalidationTestCase: description: str operation: str expected_reads: dict[str, int] + + +@dataclass(frozen=True) +class BigQueryRelationUsageTestCase: + description: str + relations: tuple[tuple[str | None, str, str], ...] + dataset_locations: dict[str, str] + rows: tuple[tuple[object, ...], ...] + expected_queries: tuple[tuple[str, str, tuple[str, ...]], ...] + expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + + +@dataclass(frozen=True) +class BigQueryRelationUsageQueryTestCase: + description: str + window_seconds: int + expected_params: dict[str, object] + expected_sql_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class BigQueryRelationUsageChunkTestCase: + description: str + relation_count: int + expected_query_names: tuple[int, ...] + + +@dataclass(frozen=True) +class BigQueryRelationUsageErrorTestCase: + description: str + error: Exception + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class BigQueryRelationUsageFailureTestCase: + description: str + error: Exception + expected_error_match: str diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py index d5b7e30583..6aa9be6b56 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py @@ -3,10 +3,15 @@ from collections.abc import Callable from pathlib import Path from types import MappingProxyType -from typing import Any +from typing import Any, cast from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import ColumnInfo, RetentionRequest +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + RelationInfo, + RelationUsageReport, + RetentionRequest, +) from sqlbuild.adapter.contract.types import RetentionScope from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -254,3 +259,98 @@ def run_bigquery_job( ), } jobs[operation]() + + +class FakeBigQueryError(Exception): + """A Google API error with an HTTP status code.""" + + def __init__(self, message: str, code: int) -> None: + super().__init__(message) + self.code: int = code + + +class Forbidden(Exception): + """Stands in for google.api_core.exceptions.Forbidden, matched by class name.""" + + +class FakeUsageRow: + """A BigQuery result row.""" + + def __init__(self, values: tuple[object, ...]) -> None: + self._values: tuple[object, ...] = values + + def values(self) -> tuple[object, ...]: + return self._values + + +class FakeUsageDataset: + def __init__(self, location: str) -> None: + self.location: str = location + + +class FakeUsageClient: + """Answers dataset locations and JOBS reads, recording each read.""" + + def __init__( + self, + *, + project: str, + dataset_locations: dict[str, str], + rows: tuple[tuple[object, ...], ...] = (), + errors: tuple[Exception, ...] = (), + ) -> None: + self.project: str = project + self.dataset_locations: dict[str, str] = dataset_locations + self.rows: tuple[tuple[object, ...], ...] = rows + self.errors: tuple[Exception, ...] = errors + self.queries: list[tuple[str, str, dict[str, object]]] = [] + + def get_dataset(self, dataset_id: str) -> FakeUsageDataset: + return FakeUsageDataset(self.dataset_locations[dataset_id]) + + def query(self, sql: str, *, location: str, job_config: Any) -> list[FakeUsageRow]: + params: dict[str, object] = { + parameter.name: getattr(parameter, "values", getattr(parameter, "value", None)) + for parameter in job_config.query_parameters + } + self.queries.append((sql, location, params)) + error: Exception + for error in self.errors: + raise error + names: frozenset[object] = frozenset(cast(list[object], params["object_names"])) + return [FakeUsageRow(row) for row in filter(lambda row: row[0] in names, self.rows)] + + +class FakeUsageConnection: + """A BigQuery connection over the fake client, without a fixed location.""" + + def __init__(self, client: FakeUsageClient, location: str | None = None) -> None: + self.client: FakeUsageClient = client + self.location: str | None = location + + def job_result(self, job: list[FakeUsageRow]) -> list[FakeUsageRow]: + return job + + +def usage_relation(database: str | None, schema: str, name: str) -> RelationInfo: + """A table relation for usage reads.""" + + return RelationInfo(database=database, schema=schema, name=name, relation_type="table") + + +def read_bigquery_usage( + *, client: FakeUsageClient, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """BigQuery's usage report for the relations from the fake client.""" + + return BigQueryAdapter().relation_usage( + connection=FakeUsageConnection(client), + relations=relations, + window_seconds=window_seconds, + ) + + +def queried_names(params: dict[str, object]) -> tuple[str, ...]: + """The table names one recorded JOBS read asked about.""" + + return tuple(cast(list[str], params["object_names"])) diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py new file mode 100644 index 0000000000..d6f3f4fa13 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py @@ -0,0 +1,250 @@ +"""BigQuery relation usage reads each location's INFORMATION_SCHEMA.JOBS per chunk of tables.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.bigquery.constants import USAGE_UNAVAILABLE_REASON +from tests.unit.src.sqlbuild.adapters.bigquery._test_types import ( + BigQueryRelationUsageChunkTestCase, + BigQueryRelationUsageErrorTestCase, + BigQueryRelationUsageFailureTestCase, + BigQueryRelationUsageQueryTestCase, + BigQueryRelationUsageTestCase, +) +from tests.unit.src.sqlbuild.adapters.bigquery.helpers import ( + FakeBigQueryError, + FakeUsageClient, + Forbidden, + queried_names, + read_bigquery_usage, + usage_relation, +) + +WINDOW_SECONDS: int = 7 * 86_400 +READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) +EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) +ORDERS_DAILY: tuple[RelationInfo, ...] = (usage_relation(None, "mart", "orders_daily"),) +LOCATIONS: dict[str, str] = {"shop-analytics.mart": "EU", "shop-archive.mart": "us-east1"} + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageTestCase( + description="reads per user add up, readers rank by jobs, the latest read wins", + relations=((None, "mart", "orders_daily"), (None, "mart", "customers")), + dataset_locations=LOCATIONS, + rows=( + ("shop-analytics.mart.orders_daily", "svc-reporting@example.com", 10, READ_AT), + ("shop-analytics.mart.orders_daily", "alice@example.com", 4, EARLIER_READ_AT), + ("shop-analytics.mart.orders_daily", None, 1, EARLIER_READ_AT), + ("shop-analytics.mart.products", "bob@example.com", 3, READ_AT), + ), + expected_queries=( + ( + "`shop-analytics.region-eu`.INFORMATION_SCHEMA.JOBS", + "EU", + ("shop-analytics.mart.customers", "shop-analytics.mart.orders_daily"), + ), + ), + expected_usages=( + ( + "orders_daily", + ("svc-reporting@example.com", "alice@example.com", "unknown user"), + 15, + READ_AT, + ), + ), + ), + BigQueryRelationUsageTestCase( + description="each project and dataset location gets its own read; case is kept", + relations=(("shop-archive", "mart", "Orders_2025"), (None, "mart", "orders_daily")), + dataset_locations=LOCATIONS, + rows=(("shop-archive.mart.Orders_2025", "alice@example.com", 2, READ_AT),), + expected_queries=( + ( + "`shop-analytics.region-eu`.INFORMATION_SCHEMA.JOBS", + "EU", + ("shop-analytics.mart.orders_daily",), + ), + ( + "`shop-archive.region-us-east1`.INFORMATION_SCHEMA.JOBS", + "us-east1", + ("shop-archive.mart.Orders_2025",), + ), + ), + expected_usages=(("Orders_2025", ("alice@example.com",), 2, READ_AT),), + ), + ), + ids=lambda case: case.description, +) +def test_given_jobs_history_when_reading_usage_then_reads_match_candidates( + test_case: BigQueryRelationUsageTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient( + project="shop-analytics", dataset_locations=test_case.dataset_locations, rows=test_case.rows + ) + + report: RelationUsageReport = read_bigquery_usage( + client=client, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple( + (sql.split(" FROM ")[1].split(" AS jobs")[0], location, queried_names(params)) + for sql, location, params in client.queries + ) == (test_case.expected_queries) + assert tuple( + (usage.name, usage.readers, usage.query_count, usage.last_read_at) + for usage in report.usages + ) == (test_case.expected_usages) + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageQueryTestCase( + description="successful query jobs count; scripts, snapshots and clones do not", + window_seconds=WINDOW_SECONDS, + expected_params={ + "window_seconds": WINDOW_SECONDS, + "clone_pattern": r"(?is)\bCLONE\b", + "object_names": ["shop-analytics.mart.orders_daily"], + }, + expected_sql_fragments=( + "UNNEST(jobs.referenced_tables) AS referenced", + "INTERVAL @window_seconds SECOND", + "jobs.job_type = 'QUERY'", + "jobs.error_result IS NULL", + "NOT IN ('SCRIPT', 'CREATE_SNAPSHOT_TABLE')", + "AND NOT (STARTS_WITH(COALESCE(jobs.statement_type, ''), 'CREATE_TABLE') " + "AND REGEXP_CONTAINS(jobs.query, @clone_pattern))", + "IN UNNEST(@object_names)", + "COUNT(DISTINCT jobs.job_id) AS query_count", + "jobs.user_email AS user_name", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_jobs_query_when_built_then_parameters_are_bound_and_clones_excluded( + test_case: BigQueryRelationUsageQueryTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient(project="shop-analytics", dataset_locations=LOCATIONS) + + read_bigquery_usage( + client=client, relations=ORDERS_DAILY, window_seconds=test_case.window_seconds + ) + + sql: str + params: dict[str, object] + ((sql, _, params),) = client.queries + assert params == test_case.expected_params + assert all(fragment in sql for fragment in test_case.expected_sql_fragments) + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageChunkTestCase( + description="no candidates read nothing", + relation_count=0, + expected_query_names=(), + ), + BigQueryRelationUsageChunkTestCase( + description="a large candidate set is split into arrays of at most 1,000 names", + relation_count=2_001, + expected_query_names=(1_000, 1_000, 1), + ), + ), + ids=lambda case: case.description, +) +def test_given_many_candidates_when_reading_usage_then_names_are_chunked( + test_case: BigQueryRelationUsageChunkTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient(project="shop-analytics", dataset_locations=LOCATIONS) + + report: RelationUsageReport = read_bigquery_usage( + client=client, + relations=tuple( + usage_relation(None, "mart", f"old_orders_{index:04d}") + for index in range(test_case.relation_count) + ), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple(len(queried_names(params)) for _, _, params in client.queries) == ( + test_case.expected_query_names + ) + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageErrorTestCase( + description="a missing jobs.listAll permission is not checked", + error=FakeBigQueryError( + "Access Denied: Table shop-analytics:region-eu.INFORMATION_SCHEMA.JOBS: User " + "does not have permission bigquery.jobs.listAll", + 403, + ), + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + BigQueryRelationUsageErrorTestCase( + description="an unknown region is not checked", + error=FakeBigQueryError("Not found: Dataset shop-analytics:region-xx", 404), + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + BigQueryRelationUsageErrorTestCase( + description="a Forbidden error is not checked", + error=Forbidden("403 GET jobs"), + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_jobs_access_when_reading_usage_then_usage_is_not_checked( + test_case: BigQueryRelationUsageErrorTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient( + project="shop-analytics", dataset_locations=LOCATIONS, errors=(test_case.error,) + ) + + report: RelationUsageReport = read_bigquery_usage( + client=client, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + assert report == test_case.expected_report + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageFailureTestCase( + description="a quota failure is a real error", + error=FakeBigQueryError("Quota exceeded: too many concurrent queries", 400), + expected_error_match="Quota exceeded", + ), + ), + ids=lambda case: case.description, +) +def test_given_other_jobs_failure_when_reading_usage_then_error_propagates( + test_case: BigQueryRelationUsageFailureTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient( + project="shop-analytics", dataset_locations=LOCATIONS, errors=(test_case.error,) + ) + + with pytest.raises(FakeBigQueryError, match=test_case.expected_error_match): + read_bigquery_usage(client=client, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index f4974ee6f2..09896dfdcb 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -2,7 +2,7 @@ from datetime import datetime from sqlbuild._native import InferredNullability -from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport from tests.unit.src.sqlbuild.adapters.databricks.helpers import FakeDatabricksMetadataCursor @@ -148,3 +148,43 @@ class DatabricksDeltaDetailAgeTestCase: cursors: tuple[FakeDatabricksMetadataCursor, ...] expected_sql: tuple[str, ...] expected_timestamps: tuple[str, str] + + +@dataclass(frozen=True) +class DatabricksRelationUsageTestCase: + description: str + relations: tuple[tuple[str | None, str, str], ...] + rows: tuple[tuple[object, ...], ...] + expected_object_names: tuple[str, ...] + expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + + +@dataclass(frozen=True) +class DatabricksRelationUsageQueryTestCase: + description: str + window_seconds: int + expected_params: dict[str, object] + expected_sql_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DatabricksRelationUsageChunkTestCase: + description: str + relation_count: int + expected_query_names: tuple[int, ...] + + +@dataclass(frozen=True) +class DatabricksRelationUsageErrorTestCase: + description: str + message: str + sqlstate: str | None + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class DatabricksRelationUsageFailureTestCase: + description: str + message: str + sqlstate: str | None + expected_error_match: str diff --git a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py index 2f5ddf88fb..98643e29dd 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py @@ -4,8 +4,9 @@ from typing import Any -from sqlbuild.adapter.contract.models import RetentionRequest +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport, RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter def build_retention_request(*, desired_days: int) -> RetentionRequest: @@ -63,3 +64,64 @@ def cursor(self) -> Any: cursor: FakeDatabricksMetadataCursor = self._cursors.pop(0) self.returned_cursors.append(cursor) return cursor + + +class FakeLineageError(Exception): + """A Databricks server error with a SQLSTATE.""" + + def __init__(self, message: str, sqlstate: str | None) -> None: + super().__init__(message) + self.sqlstate: str | None = sqlstate + + +class FakeLineageCursor: + """Answers lineage reads from stored rows, recording each read.""" + + def __init__(self, connection: FakeLineageConnection) -> None: + self.connection: FakeLineageConnection = connection + self.rows: list[tuple[object, ...]] = [] + + def execute(self, sql: str, params: dict[str, object]) -> None: + self.connection.queries.append((sql, params)) + error: Exception + for error in self.connection.errors: + raise error + names: frozenset[object] = frozenset(params.values()) + self.rows = list(filter(lambda row: row[0] in names, self.connection.rows)) + + def fetchall(self) -> list[tuple[object, ...]]: + return self.rows + + def close(self) -> None: + return None + + +class FakeLineageConnection: + def __init__( + self, + *, + rows: tuple[tuple[object, ...], ...] = (), + errors: tuple[Exception, ...] = (), + ) -> None: + self.rows: tuple[tuple[object, ...], ...] = rows + self.errors: tuple[Exception, ...] = errors + self.queries: list[tuple[str, dict[str, object]]] = [] + + def cursor(self) -> FakeLineageCursor: + return FakeLineageCursor(self) + + +def usage_relation(database: str | None, schema: str, name: str) -> RelationInfo: + """A table relation for usage reads.""" + + return RelationInfo(database=database, schema=schema, name=name, relation_type="table") + + +def read_databricks_usage( + *, connection: FakeLineageConnection, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Databricks' usage report for the relations from the fake connection.""" + + return DatabricksAdapter().relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py new file mode 100644 index 0000000000..872adb42e8 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py @@ -0,0 +1,223 @@ +"""Databricks relation usage reads Unity Catalog table lineage once per chunk of candidates.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.databricks.constants import USAGE_UNAVAILABLE_REASON +from tests.unit.src.sqlbuild.adapters.databricks._test_types import ( + DatabricksRelationUsageChunkTestCase, + DatabricksRelationUsageErrorTestCase, + DatabricksRelationUsageFailureTestCase, + DatabricksRelationUsageQueryTestCase, + DatabricksRelationUsageTestCase, +) +from tests.unit.src.sqlbuild.adapters.databricks.helpers import ( + FakeLineageConnection, + FakeLineageError, + read_databricks_usage, + usage_relation, +) + +WINDOW_SECONDS: int = 7 * 86_400 +READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) +EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) +ORDERS_DAILY: tuple[RelationInfo, ...] = (usage_relation("shop", "mart", "orders_daily"),) + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageTestCase( + description="read events per user add up and names fold to lowercase", + relations=(("Shop", "Mart", "orders_daily"), ("shop", "mart", "customers")), + rows=( + ("shop.mart.orders_daily", "svc-reporting@example.com", 10, READ_AT), + ("shop.mart.orders_daily", "alice@example.com", 4, EARLIER_READ_AT), + ("shop.mart.orders_daily", None, 1, EARLIER_READ_AT), + ("shop.mart.products", "bob@example.com", 3, READ_AT), + ), + expected_object_names=("shop.mart.customers", "shop.mart.orders_daily"), + expected_usages=( + ( + "orders_daily", + ("svc-reporting@example.com", "alice@example.com", "unknown user"), + 15, + READ_AT, + ), + ), + ), + DatabricksRelationUsageTestCase( + description="a relation without a catalog is not looked up", + relations=((None, "mart", "orders_daily"), ("shop", "mart", "customers")), + rows=(), + expected_object_names=("shop.mart.customers",), + expected_usages=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_lineage_when_reading_usage_then_reads_match_candidates( + test_case: DatabricksRelationUsageTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection(rows=test_case.rows) + + report: RelationUsageReport = read_databricks_usage( + connection=connection, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + params: dict[str, object] + ((_, params),) = connection.queries + assert report.checked + assert tuple(params.values())[1:] == test_case.expected_object_names + assert tuple( + (usage.name, usage.readers, usage.query_count, usage.last_read_at) + for usage in report.usages + ) == (test_case.expected_usages) + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageQueryTestCase( + description="read events of source tables within the window, with named parameters", + window_seconds=WINDOW_SECONDS, + expected_params={"window_seconds": WINDOW_SECONDS, "name_0": "shop.mart.orders_daily"}, + expected_sql_fragments=( + "FROM system.access.table_lineage", + "lower(source_table_full_name) AS object_name", + "created_by AS user_name", + "COUNT(DISTINCT COALESCE(statement_id, event_id)) AS query_count", + "MAX(event_time) AS last_read_at", + "event_date >= DATE(TIMESTAMPADD(SECOND, -:window_seconds, current_timestamp()))", + "lower(source_table_full_name) IN (:name_0)", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_lineage_query_when_built_then_parameters_are_named( + test_case: DatabricksRelationUsageQueryTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection() + + read_databricks_usage( + connection=connection, relations=ORDERS_DAILY, window_seconds=test_case.window_seconds + ) + + sql: str + params: dict[str, object] + ((sql, params),) = connection.queries + assert params == test_case.expected_params + assert all(fragment in sql for fragment in test_case.expected_sql_fragments) + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageChunkTestCase( + description="no candidates read nothing", + relation_count=0, + expected_query_names=(), + ), + DatabricksRelationUsageChunkTestCase( + description="a large candidate set is split into IN lists of at most 1,000 names", + relation_count=2_001, + expected_query_names=(1_000, 1_000, 1), + ), + ), + ids=lambda case: case.description, +) +def test_given_many_candidates_when_reading_usage_then_names_are_chunked( + test_case: DatabricksRelationUsageChunkTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection() + + report: RelationUsageReport = read_databricks_usage( + connection=connection, + relations=tuple( + usage_relation("shop", "mart", f"old_orders_{index:04d}") + for index in range(test_case.relation_count) + ), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple(len(params) - 1 for _, params in connection.queries) == ( + test_case.expected_query_names + ) + assert all(sql.count(":name_") == len(params) - 1 for sql, params in connection.queries) + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageErrorTestCase( + description="missing permissions on the system schema are not checked", + message="[INSUFFICIENT_PERMISSIONS] Insufficient privileges: User does not have " + "USE SCHEMA on Schema 'system.access'.", + sqlstate="42501", + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + DatabricksRelationUsageErrorTestCase( + description="a disabled access schema is not checked", + message="[TABLE_OR_VIEW_NOT_FOUND] The table or view `system`.`access`.`table_lineage` " + "cannot be found.", + sqlstate=None, + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + DatabricksRelationUsageErrorTestCase( + description="a workspace without Unity Catalog is not checked", + message="[UC_NOT_ENABLED] Unity Catalog is not enabled on this cluster.", + sqlstate=None, + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_lineage_access_when_reading_usage_then_usage_is_not_checked( + test_case: DatabricksRelationUsageErrorTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection( + errors=(FakeLineageError(test_case.message, test_case.sqlstate),) + ) + + report: RelationUsageReport = read_databricks_usage( + connection=connection, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + assert report == test_case.expected_report + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageFailureTestCase( + description="a cancelled statement is a real error", + message="[OPERATION_CANCELED] The statement was cancelled.", + sqlstate="HY008", + expected_error_match="OPERATION_CANCELED", + ), + ), + ids=lambda case: case.description, +) +def test_given_other_lineage_failure_when_reading_usage_then_error_propagates( + test_case: DatabricksRelationUsageFailureTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection( + errors=(FakeLineageError(test_case.message, test_case.sqlstate),) + ) + + with pytest.raises(FakeLineageError, match=test_case.expected_error_match): + read_databricks_usage( + connection=connection, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 3307d4b483..bcd3c24495 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -78,6 +78,8 @@ Usage comes from the warehouse's query history: | Adapter | Usage source | |---------|--------------| | Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`) joined with `QUERY_HISTORY` | +| BigQuery | `` `.region-`.INFORMATION_SCHEMA.JOBS `` (`referenced_tables`) | +| Databricks | `system.access.table_lineage` (`source_table_full_name`) | | Other adapters | Not available | On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: @@ -88,7 +90,19 @@ GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; Access history lags by up to about 3 hours, so reads in the last few hours may not show yet. -When usage can't be checked, because the adapter has no usage source or the role can't read access history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. +On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the table, by `user_email`. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. + +On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`; a read through a view also counts for the tables behind it. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: + +```sql +GRANT USE CATALOG ON CATALOG system TO `janitor-principal`; +GRANT USE SCHEMA ON SCHEMA system.access TO `janitor-principal`; +GRANT SELECT ON TABLE system.access.table_lineage TO `janitor-principal`; +``` + +Lineage records only reads Databricks can capture lineage for, and can arrive with a delay. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. + +When usage can't be checked, because the adapter has no usage source or the role can't read its query history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers` and `query_count`. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. From 5eef93783a59673e3f53b45355a390f104dbb499 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:06:25 +0100 Subject: [PATCH 11/14] feat: follow view reads in janitor usage and read usage on ClickHouse --- .../src/janitor_text/_helpers/plan_json.rs | 18 +- .../src/janitor_text/_helpers/usage.rs | 9 +- .../sqlbuild-core/src/janitor_text/models.rs | 2 + .../src/janitor_text/tests/helpers.rs | 32 +++ .../tests/test_janitor_usage_text.rs | 20 +- .../build_progress/transient_status.rs | 2 +- .../bindings/_helpers/janitor/janitor_plan.rs | 5 + .../_helpers/janitor/janitor_usage.rs | 15 +- .../sqlbuild/references/docs/cli/janitor.md | 30 ++- .../adapter/contract/_helpers/array_names.py | 18 ++ .../contract/_helpers/relation_reads.py | 15 ++ .../contract/main/aggregate_relation_usage.py | 44 ++-- .../adapter/contract/main/relation_reads.py | 31 +++ src/sqlbuild/adapter/contract/models.py | 14 ++ .../bigquery/_helpers/relation_usage.py | 49 ++-- src/sqlbuild/adapters/bigquery/constants.py | 11 +- .../clickhouse/_helpers/relation_usage.py | 115 +++++++++ .../clickhouse/classes/clickhouse_adapter.py | 11 +- src/sqlbuild/adapters/clickhouse/constants.py | 10 + .../databricks/_helpers/relation_usage.py | 31 ++- src/sqlbuild/adapters/databricks/constants.py | 1 + .../snowflake/_helpers/relation_usage.py | 167 +++++++------ .../snowflake/_helpers/view_dependencies.py | 136 +++++++++++ src/sqlbuild/adapters/snowflake/constants.py | 17 +- .../commands/main/clickhouse/_test_types.py | 17 ++ .../cli/commands/main/clickhouse/conftest.py | 8 + .../cli/commands/main/clickhouse/helpers.py | 77 +++++- .../test_clickhouse_janitor_usage.py | 148 ++++++++++++ .../cli/commands/main/janitor/_test_types.py | 10 + .../cli/commands/main/janitor/helpers.py | 13 +- .../main/janitor/test_janitor_usage.py | 64 ++++- .../sqlbuild/adapters/bigquery/_test_types.py | 7 + .../adapters/bigquery/test_relation_usage.py | 68 +++++- .../adapters/clickhouse/_test_types.py | 41 ++++ .../sqlbuild/adapters/clickhouse/helpers.py | 75 ++++++ .../clickhouse/test_relation_usage.py | 227 ++++++++++++++++++ .../adapters/databricks/_test_types.py | 1 + .../databricks/test_relation_usage.py | 59 ++++- .../adapters/snowflake/_test_types.py | 17 ++ .../sqlbuild/adapters/snowflake/helpers.py | 35 ++- .../adapters/snowflake/test_relation_usage.py | 197 ++++++++++++++- website/src/content/docs/docs/cli/janitor.mdx | 30 ++- 42 files changed, 1704 insertions(+), 193 deletions(-) create mode 100644 src/sqlbuild/adapter/contract/_helpers/array_names.py create mode 100644 src/sqlbuild/adapter/contract/_helpers/relation_reads.py create mode 100644 src/sqlbuild/adapter/contract/main/relation_reads.py create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py create mode 100644 src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs index dc0a1ba8e2..3bfd6a2560 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -75,6 +75,10 @@ fn in_use(candidate: &ArchiveCandidateText, checked: bool) -> JsonValue { ("last_read", utc_timestamp(&usage.last_read_at)), ("readers", texts(&usage.readers)), ("query_count", integer(usage.query_count)), + ( + "via", + usage.via.as_deref().map_or(JsonValue::Null, texts), + ), ]) }) } @@ -90,6 +94,15 @@ fn archive_candidate(candidate: &ArchiveCandidateText, checked: bool) -> JsonVal ]) } +/// A candidate kept because it is in use: no archive name or expiry, since it stays put. +fn kept_candidate(candidate: &ArchiveCandidateText) -> JsonValue { + object(vec![ + ("name", text(&candidate.name)), + ("age_days", optional_integer(candidate.age_days)), + ("in_use", in_use(candidate, true)), + ]) +} + /// The top-level `usage_check`: its status and reason, or `null` when it did not run. pub(crate) fn usage_check_payload(plan: Option<&JanitorPlanText>) -> JsonValue { let Some(check) = plan.and_then(|plan| plan.usage_check.as_ref()) else { @@ -171,10 +184,7 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { ( "kept_in_use", JsonValue::Array( - plan.kept_in_use - .iter() - .map(|candidate| archive_candidate(candidate, checked)) - .collect(), + plan.kept_in_use.iter().map(kept_candidate).collect(), ), ), ( diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs index ca86885c3c..cc2af2d571 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs @@ -37,7 +37,7 @@ fn noun_count(count: i64, singular: &str) -> String { noun(usize::try_from(count).unwrap_or(usize::MAX), singular) } -/// `alice, bob, carol +2 more`: the first readers and how many are not named. +/// `alice, bob, carol +2 more`: the first names and how many are not shown. fn readers(names: &[String]) -> String { let shown: usize = names.len().min(READER_LIMIT); let mut text: String = names[..shown].join(", "); @@ -51,8 +51,13 @@ fn readers(names: &[String]) -> String { pub(crate) fn usage_detail(candidate: &ArchiveCandidateText, window: &str) -> Option { let usage = candidate.usage.as_ref()?; let date: &str = usage.last_read_at.split(' ').next().unwrap_or_default(); + let via: String = match usage.via.as_deref() { + None => String::new(), + Some([]) => " through a view".to_owned(), + Some(names) => format!(" via {}", readers(names)), + }; let mut text: String = format!( - "queried {date} by {} ({} in {})", + "queried {date} by {}{via} ({} in {})", readers(&usage.readers), format_count_noun(usage.query_count, "query", Some("queries")), window_label(window) diff --git a/crates/sqlbuild-core/src/janitor_text/models.rs b/crates/sqlbuild-core/src/janitor_text/models.rs index b20a159383..ad05dd346e 100644 --- a/crates/sqlbuild-core/src/janitor_text/models.rs +++ b/crates/sqlbuild-core/src/janitor_text/models.rs @@ -8,6 +8,8 @@ pub struct RelationUsageText { /// Reader user names, most queries first. pub readers: Vec, pub query_count: i64, + /// The objects it was read through when it was never read directly; empty when unnamed. + pub via: Option>, } /// A relation to archive: its name, archive name, age and expiry. diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index cddf58351f..049dc70f98 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -85,6 +85,7 @@ fn usage(readers: &[&str], query_count: i64) -> RelationUsageText { last_read_at: "2026-10-21 09:30:00".to_owned(), readers: readers.iter().map(|reader| (*reader).to_owned()).collect(), query_count, + via: None, } } @@ -142,6 +143,37 @@ pub(super) fn usage_plan() -> JanitorPlanText { } } +/// A checked plan archiving a table read through four named views and a view read through one. +pub(super) fn via_usage_plan() -> JanitorPlanText { + let views: Vec = ["daily", "weekly", "monthly", "yearly"] + .iter() + .map(|grain| format!("dev.orders_{grain}")) + .collect(); + JanitorPlanText { + archive_candidates: vec![ + ArchiveCandidateText { + name: "dev.orders_base".to_owned(), + is_table: true, + changed_on: Some("2026-09-01".to_owned()), + usage: Some(RelationUsageText { + via: Some(views), + ..usage(&["alice"], 3) + }), + ..ArchiveCandidateText::default() + }, + ArchiveCandidateText { + name: "dev.orders_view".to_owned(), + usage: Some(RelationUsageText { + via: Some(Vec::new()), + ..usage(&["alice"], 3) + }), + ..ArchiveCandidateText::default() + }, + ], + ..checked_plan(false) + } +} + /// A checked plan keeping the two in-use candidates and archiving the unread one. pub(super) fn kept_usage_plan() -> JanitorPlanText { let [view, table, unread] = usage_candidates(); diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs index f6f741179e..04a1748f9c 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs @@ -5,7 +5,7 @@ use crate::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation use crate::janitor_text::main::render_janitor_plan::render_janitor_plan; use crate::janitor_text::models::JanitorOutcome; use crate::janitor_text::tests::helpers::{ - direct_archive_plan, kept_usage_plan, unchecked_plan, usage_plan, + direct_archive_plan, kept_usage_plan, unchecked_plan, usage_plan, via_usage_plan, }; use crate::janitor_text::tests::test_types::{JsonTestCase, PreviewTestCase, UsagePromptTestCase}; @@ -44,6 +44,19 @@ fn given_usage_check_when_rendering_preview_then_in_use_relations_show_readers() ], unexpected_fragments: &["\nIn use\n", "dev.old_orders ->", "dev.orders_daily ->"], }, + PreviewTestCase { + description: "relations read only through views name the views, or say so", + plan: via_usage_plan(), + verbose: false, + expected_fragments: &[ + " dev.orders_base queried 2026-10-21 by alice via dev.orders_daily, \ + dev.orders_weekly, dev.orders_monthly +1 more (3 queries in 7 days); \ + frozen since 2026-09-01, still in use\n", + " dev.orders_view queried 2026-10-21 by alice through a view \ + (3 queries in 7 days)\n", + ], + unexpected_fragments: &["dev.orders_yearly"], + }, PreviewTestCase { description: "an unchecked usage shows one notice and no in-use rows", plan: unchecked_plan(false), @@ -152,7 +165,7 @@ fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { expected_fragments: &[ "\"in_use\": {\n \"last_read\": \"2026-10-21T09:30:00Z\",\n \ \"readers\": [\n \"svc_reporting\"\n ],\n \ - \"query_count\": 14\n }", + \"query_count\": 14,\n \"via\": null\n }", "\"in_use\": false", "\"kept_in_use\": []", "\"usage_check\": {\n \"status\": \"checked\",\n \"reason\": null,\n \ @@ -166,7 +179,8 @@ fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { run: None, error: None, expected_fragments: &[ - "\"kept_in_use\": [\n {\n \"name\": \"dev.orders_daily\"", + "\"kept_in_use\": [\n {\n \"name\": \"dev.orders_daily\",\n \ + \"age_days\": null,\n \"in_use\": {", "\"keep_in_use\": true", ], }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index abc2f86bdd..a64ef9dc5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -233,7 +233,7 @@ impl TransientStatusReporter { .write_persistent(slf.py(), &format!("{message}\n{blank}")) } - fn error(slf: &Bound<'_, Self>, message: &str) -> PyResult<()> { + pub(crate) fn error(slf: &Bound<'_, Self>, message: &str) -> PyResult<()> { Self::close(slf)?; slf.get().write_message(slf.py(), message, false) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs index 08ca8190a9..74850b1fdb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs @@ -138,6 +138,11 @@ fn relation_usage(usage: &Bound<'_, PyAny>) -> PyResult( }); let started: Instant = Instant::now(); TransientStatusReporter::start(status, START_MESSAGE.to_owned())?; - let report: Bound<'py, PyAny> = read_usage((adapter, connection), &candidates, window_seconds)?; + let report: Bound<'py, PyAny> = + match read_usage((adapter, connection), &candidates, window_seconds) { + Ok(report) => report, + Err(error) => { + TransientStatusReporter::error( + status, + &format!( + "Relation usage check failed. ({:.2}s)", + started.elapsed().as_secs_f64() + ), + )?; + return Err(error); + } + }; let checked: bool = native_field(&report, "checked")?.is_truthy()?; let usages: HashMap> = usages_by_relation(&report)?; let keep: bool = checked && settings.keep_in_use; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index a18311989e..cf2a5094b3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -85,9 +85,10 @@ Archiving renames a relation, so anything that still reads it fails from then on In use analytics.orders_daily queried 2026-10-21 by svc_reporting, alice (14 queries in 7 days) analytics.old_orders queried 2026-10-20 by svc_reporting, alice, bob +2 more (40 queries in 7 days); frozen since 2026-09-01, still in use + analytics.old_customers queried 2026-10-21 by alice via analytics.customer_report (3 queries in 7 days); frozen since 2026-09-01, still in use ``` -Each line names the latest read, up to three readers (most queries first) and how many queries read the relation. A stale table that is still read is also flagged `frozen since , still in use`: its data stopped changing on that date, so its readers are getting old data. Views are not flagged, because their reads stay live. The confirmation prompt states the count, for example `14 relations will be archived; 3 of them are in use.` +Each line names the latest read, up to three readers (most queries first) and how many queries read the relation. Reading a view also reads what it is built on, so those relations are in use too. When a relation was only read that way, the line says `via` and names the views, or says `through a view` when the warehouse doesn't record which. A stale table that is still read is also flagged `frozen since , still in use`: its data stopped changing on that date, so its readers are getting old data. Views are not flagged, because their reads stay live. The confirmation prompt states the count, for example `14 relations will be archived; 3 of them are in use.` By default in-use relations are still archived, which is also how their readers find out. To keep them instead, set `keep_in_use = true` under `[janitor]` or pass `--keep-in-use`. In-use relations are then listed under `Kept (in use)` with their readers and are left out of the archive list and the counts. A later run archives them once nothing reads them within the window. @@ -97,22 +98,23 @@ Usage comes from the warehouse's query history: | Adapter | Usage source | |---------|--------------| -| Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`) joined with `QUERY_HISTORY` | -| BigQuery | `` `.region-`.INFORMATION_SCHEMA.JOBS `` (`referenced_tables`) | -| Databricks | `system.access.table_lineage` (`source_table_full_name`) | +| Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`, `base_objects_accessed`) joined with `QUERY_HISTORY`, and `OBJECT_DEPENDENCIES` | +| BigQuery | `` `.region-`.INFORMATION_SCHEMA.JOBS `` (`referenced_tables`, `referenced_logical_views`) | +| Databricks | `system.access.table_lineage` (`source_table_full_name`, `direct_access`) | +| ClickHouse | `system.query_log` (`tables`) on the connected server | | Other adapters | Not available | -On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). A candidate listed in a query's base objects but not its direct objects was read through the objects the query named, which the line names after `via`. A second query reads `OBJECT_DEPENDENCIES` for the candidate views, so a candidate that an in-use candidate view is built on, directly or through other candidate views, is in use `via` that view, with its readers. Dependencies created by `CREATE TABLE ... AS SELECT` and similar statements are not recorded there. Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements of tables, dynamic tables, schemas and databases don't count as reads. A statement counts as a clone only when it starts with `CREATE`, after any comments, and names its source after `CLONE`; a column, string or comment containing the word doesn't make it one. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: ```sql GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; ``` -Access history lags by up to about 3 hours, so reads in the last few hours may not show yet. +Access history and object dependencies lag by up to about 3 hours, so reads in the last few hours may not show yet. -On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the table, by `user_email`. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. +On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the relation, by `user_email`. A job lists the views it reads as well as the tables behind them, so both count. Whether views nested inside other views are listed is not documented, so a candidate view that is only read through another view may not show as in use. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. -On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`; a read through a view also counts for the tables behind it. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: +On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`. Lineage records the view a query reads as a direct read and the tables behind it as reads through the view, which the line names when lineage records the view for the same statement. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: ```sql GRANT USE CATALOG ON CATALOG system TO `janitor-principal`; @@ -120,11 +122,19 @@ GRANT USE SCHEMA ON SCHEMA system.access TO `janitor-principal`; GRANT SELECT ON TABLE system.access.table_lineage TO `janitor-principal`; ``` -Lineage records only reads Databricks can capture lineage for, and can arrive with a delay. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. +Lineage records only reads Databricks can infer lineage for, and can arrive with a delay, so "not in use" is a weaker signal than on Snowflake. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. + +On ClickHouse, the janitor counts finished `SELECT` and `INSERT` queries in `system.query_log` of the server it connects to, by the user who started them. A query's `tables` lists every table and view it touched, including the views and tables a view is built on, so a whole view chain shows as in use, without `via`; queries a query starts count once, for its initial query. Queries on other replicas or shards are logged on those servers and are not seen. ClickHouse answers some queries, such as a plain `count()` of a table, from metadata without listing the table, so those reads are not seen either. SQLBuild doesn't mark its ClickHouse queries, so its own reads count. The janitor's user needs `SELECT` on `system.query_log`: + +```sql +GRANT SELECT ON system.query_log TO janitor_user; +``` + +Queries are logged when `log_queries` is on, which is the default; when it is off for the janitor's connection, usage is not checked. The log is flushed about every 7.5 seconds, and ClickHouse keeps it until a TTL configured on the server removes it. When usage can't be checked, because the adapter has no usage source or the role can't read its query history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. -In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers` and `query_count`. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. +In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers`, `query_count` and `via` (`null` when the relation was read directly, otherwise the names of the views it was read through, empty when unknown). Entries of `kept_in_use` have only `name`, `age_days` and `in_use`, since they are not archived. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. ## Speed diff --git a/src/sqlbuild/adapter/contract/_helpers/array_names.py b/src/sqlbuild/adapter/contract/_helpers/array_names.py new file mode 100644 index 0000000000..869aa993f0 --- /dev/null +++ b/src/sqlbuild/adapter/contract/_helpers/array_names.py @@ -0,0 +1,18 @@ +"""Read object names from a warehouse array value.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable + + +def array_names(value: object) -> tuple[str, ...]: + """Names from a warehouse array value: a list, an array object, or JSON text.""" + + if value is None: + return () + if isinstance(value, str): + return array_names(json.loads(value)) if value.lstrip().startswith("[") else (value,) + if not isinstance(value, Iterable): + return (str(value),) + return tuple(str(name) for name in value if name is not None) diff --git a/src/sqlbuild/adapter/contract/_helpers/relation_reads.py b/src/sqlbuild/adapter/contract/_helpers/relation_reads.py new file mode 100644 index 0000000000..81982a0b39 --- /dev/null +++ b/src/sqlbuild/adapter/contract/_helpers/relation_reads.py @@ -0,0 +1,15 @@ +"""Normalize warehouse values in relation usage rows.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def utc_datetime(value: object) -> datetime: + """A warehouse timestamp as an aware UTC datetime; naive values are taken as UTC.""" + + if not isinstance(value, datetime): + value = datetime.fromisoformat(str(value)) + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) diff --git a/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py b/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py index 772f9a1450..8ee33ce13a 100644 --- a/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py +++ b/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py @@ -1,32 +1,38 @@ -"""Fold per-reader usage rows into one usage per relation.""" +"""Fold per-reader reads into one usage per relation.""" from __future__ import annotations from collections.abc import Iterable, Mapping -from datetime import UTC, datetime +from datetime import datetime -from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage +from sqlbuild.adapter.contract._helpers.relation_reads import utc_datetime +from sqlbuild.adapter.contract.models import RelationInfo, RelationRead, RelationUsage from sqlbuild.adapter.relations.constants import UNKNOWN_RELATION_READER def aggregate_relation_usage( - *, rows: Iterable[tuple[object, ...]], relations_by_name: Mapping[str, RelationInfo] + *, reads: Iterable[RelationRead], relations_by_name: Mapping[str, RelationInfo] ) -> tuple[RelationUsage, ...]: - """One usage per read relation from `(object_name, reader, query_count, last_read)` rows.""" + """One usage per read relation; it is direct when any reader read it directly.""" readers: dict[str, dict[str, int]] = {} last_read: dict[str, datetime] = {} - row: tuple[object, ...] - for row in rows: - object_name: str = str(row[0]) - if object_name not in relations_by_name: + direct: set[str] = set() + via: dict[str, set[str]] = {} + read: RelationRead + for read in reads: + if read.object_name not in relations_by_name: continue - reader: str = UNKNOWN_RELATION_READER if row[1] is None else str(row[1]) - counts: dict[str, int] = readers.setdefault(object_name, {}) - counts[reader] = counts.get(reader, 0) + int(str(row[2])) - read_at: datetime = _utc(row[3]) - last_read[object_name] = max(read_at, last_read.get(object_name, read_at)) + reader: str = UNKNOWN_RELATION_READER if read.reader is None else read.reader + counts: dict[str, int] = readers.setdefault(read.object_name, {}) + counts[reader] = counts.get(reader, 0) + read.query_count + read_at: datetime = utc_datetime(read.last_read_at) + last_read[read.object_name] = max(read_at, last_read.get(read.object_name, read_at)) + if read.direct: + direct.add(read.object_name) + via.setdefault(read.object_name, set()).update(read.via) usages: list[RelationUsage] = [] + object_name: str for object_name in sorted(readers): relation: RelationInfo = relations_by_name[object_name] counts = readers[object_name] @@ -38,14 +44,8 @@ def aggregate_relation_usage( last_read_at=last_read[object_name], readers=tuple(sorted(counts, key=lambda reader: (-counts[reader], reader))), query_count=sum(counts.values()), + direct=object_name in direct, + via=() if object_name in direct else tuple(sorted(via[object_name])), ) ) return tuple(usages) - - -def _utc(value: object) -> datetime: - if not isinstance(value, datetime): - return datetime.fromisoformat(str(value)).astimezone(UTC) - if value.tzinfo is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) diff --git a/src/sqlbuild/adapter/contract/main/relation_reads.py b/src/sqlbuild/adapter/contract/main/relation_reads.py new file mode 100644 index 0000000000..bf249a40fd --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/relation_reads.py @@ -0,0 +1,31 @@ +"""Turn warehouse usage rows into relation reads.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable + +from sqlbuild.adapter.contract._helpers.array_names import array_names +from sqlbuild.adapter.contract.models import RelationRead + + +def relation_reads( + *, rows: Iterable[tuple[object, ...]], via_name: Callable[[str], str] = str +) -> tuple[RelationRead, ...]: + """Reads from `(object, reader, queries, last read[, direct, via names])` rows.""" + + reads: list[RelationRead] = [] + row: tuple[object, ...] + for row in rows: + direct_access, via_names = (*row[4:6], True, None)[:2] + indirect: bool = not direct_access + reads.append( + RelationRead( + object_name=str(row[0]), + reader=None if row[1] is None else str(row[1]), + query_count=int(str(row[2])), + last_read_at=row[3], + direct=not indirect, + via=tuple(map(via_name, array_names(via_names))) if indirect else (), + ) + ) + return tuple(reads) diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 0043ef58a1..1cc6e286cf 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -110,6 +110,20 @@ class RelationUsage: last_read_at: datetime readers: tuple[str, ...] query_count: int + direct: bool = True + via: tuple[str, ...] = () + + +@dataclass(frozen=True) +class RelationRead: + """One reader's reads of one object; indirect reads came through `via`, empty if unknown.""" + + object_name: str + reader: str | None + query_count: int + last_read_at: object + direct: bool = True + via: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py index 6834a1fb6c..e8c4d9351b 100644 --- a/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py +++ b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.main.relation_reads import relation_reads from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport from sqlbuild.adapters.bigquery.constants import ( CLONE_STATEMENT_PATTERN, @@ -72,26 +73,34 @@ def jobs_view(*, project: str, location: str) -> str: def usage_query( *, project: str, location: str, object_names: tuple[str, ...], window_seconds: int ) -> UsageQuery: - """Reads of the named tables per user by successful query jobs, leaving out clones.""" + """Reads of the named tables and views per user by successful query jobs, minus clones.""" + + def referenced(column: str) -> str: + return ( + "SELECT CONCAT(referenced.project_id, '.', referenced.dataset_id, '.', " + "referenced.table_id) AS object_name, " + "jobs.user_email, jobs.job_id, jobs.creation_time " + f"FROM jobs, UNNEST(jobs.{column}) AS referenced" + ) - referenced: str = ( - "CONCAT(referenced.project_id, '.', referenced.dataset_id, '.', referenced.table_id)" - ) sql: str = ( - f"SELECT {referenced} AS object_name, " - "jobs.user_email AS user_name, " - "COUNT(DISTINCT jobs.job_id) AS query_count, " - "MAX(jobs.creation_time) AS last_read_at " - f"FROM {jobs_view(project=project, location=location)} AS jobs, " - "UNNEST(jobs.referenced_tables) AS referenced " - "WHERE jobs.creation_time >= " + "WITH jobs AS (" + "SELECT job_id, user_email, creation_time, referenced_tables, referenced_logical_views " + f"FROM {jobs_view(project=project, location=location)} " + "WHERE creation_time >= " "TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @window_seconds SECOND) " - "AND jobs.job_type = 'QUERY' " - "AND jobs.error_result IS NULL " - "AND COALESCE(jobs.statement_type, '') NOT IN ('SCRIPT', 'CREATE_SNAPSHOT_TABLE') " - "AND NOT (STARTS_WITH(COALESCE(jobs.statement_type, ''), 'CREATE_TABLE') " - "AND REGEXP_CONTAINS(jobs.query, @clone_pattern)) " - f"AND {referenced} IN UNNEST(@object_names) " + "AND job_type = 'QUERY' " + "AND error_result IS NULL " + "AND COALESCE(statement_type, '') NOT IN ('SCRIPT', 'CREATE_SNAPSHOT_TABLE') " + "AND NOT (STARTS_WITH(COALESCE(statement_type, ''), 'CREATE_TABLE') " + "AND REGEXP_CONTAINS(query, @clone_pattern))), " + f"reads AS ({referenced('referenced_tables')} UNION ALL " + f"{referenced('referenced_logical_views')}) " + "SELECT object_name, user_email AS user_name, " + "COUNT(DISTINCT job_id) AS query_count, " + "MAX(creation_time) AS last_read_at " + "FROM reads " + "WHERE object_name IN UNNEST(@object_names) " "GROUP BY 1, 2 " "ORDER BY 1, 2" ) @@ -181,7 +190,11 @@ def read_relation_usage( rows.extend(run_usage_query(connection=connection, query=query)) covered.update(named) return RelationUsageReport( - checked=True, usages=aggregate_relation_usage(rows=rows, relations_by_name=covered) + checked=True, + usages=aggregate_relation_usage( + reads=relation_reads(rows=rows), + relations_by_name=covered, + ), ) diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index 5e2e96a816..d156cb40cd 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -41,4 +41,13 @@ {"Forbidden", "NotFound", "PermissionDenied"} ) USAGE_UNAVAILABLE_MESSAGES: tuple[str, ...] = ("access denied", "permission") -CLONE_STATEMENT_PATTERN: str = r"(?is)\bCLONE\b" +_CLONE_IDENTIFIER: str = r"(`[^`]*`|[A-Za-z0-9_-]+)" +_CLONE_NAME: str = _CLONE_IDENTIFIER + r"(\s*\.\s*" + _CLONE_IDENTIFIER + r"){0,2}" +CLONE_STATEMENT_PATTERN: str = ( + r"(?is)^(\s|--[^\n]*\n|#[^\n]*\n|/\*([^*]|\*+[^*/])*\*+/)*" + r"CREATE\s+(OR\s+REPLACE\s+)?(TEMP(ORARY)?\s+)?TABLE\s+(IF\s+NOT\s+EXISTS\s+)?" + + _CLONE_NAME + + r"\s+CLONE\s+" + + _CLONE_NAME + + r"([^A-Za-z0-9_`-]|$)" +) diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py b/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py new file mode 100644 index 0000000000..6b79013795 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py @@ -0,0 +1,115 @@ +"""Read who queried relations from the ClickHouse system.query_log table.""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Iterable +from datetime import UTC, datetime + +from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.models import RelationInfo, RelationRead, RelationUsageReport +from sqlbuild.adapters.clickhouse._helpers.type_text import quote_string +from sqlbuild.adapters.clickhouse.constants import ( + LOGGING_OFF_VALUES, + LOGGING_SETTING_SQL, + USAGE_ERROR_CODE_PATTERN, + USAGE_LOGGING_OFF_REASON, + USAGE_NAME_CHUNK_SIZE, + USAGE_QUERY_KINDS, + USAGE_UNAVAILABLE_ERROR_CODES, + USAGE_UNAVAILABLE_REASON, +) + + +def usage_object_names(*, relations: Iterable[RelationInfo]) -> dict[str, RelationInfo]: + """Each relation by the `database.table` name query_log records; names keep their case.""" + + return { + f"{relation.schema}.{relation.name}": relation + for relation in relations + if relation.schema is not None + } + + +def usage_queries(*, object_names: Iterable[str], window_seconds: int) -> tuple[str, ...]: + """One query_log read per chunk of object names.""" + + names: tuple[str, ...] = tuple(sorted(object_names)) + return tuple( + usage_query( + object_names=names[start : start + USAGE_NAME_CHUNK_SIZE], + window_seconds=window_seconds, + ) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) + ) + + +def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> str: + """Finished reads and inserts of the named tables per initial user within the window.""" + + since: str = f"now() - toIntervalSecond({int(window_seconds)})" + kinds: str = ", ".join(quote_string(kind) for kind in USAGE_QUERY_KINDS) + names: str = ", ".join(quote_string(name) for name in object_names) + return ( + "SELECT object_name, initial_user AS user_name, " + "uniqExact(initial_query_id) AS query_count, " + "toUnixTimestamp(max(event_time)) AS last_read_at " + "FROM system.query_log ARRAY JOIN tables AS object_name " + f"WHERE event_date >= toDate({since}) " + f"AND event_time >= {since} " + "AND type = 'QueryFinish' " + f"AND query_kind IN ({kinds}) " + f"AND object_name IN ({names}) " + "GROUP BY object_name, user_name " + "ORDER BY object_name, user_name" + ) + + +def is_usage_unavailable_error(error: Exception) -> bool: + """Whether reading query_log failed because it does not exist or may not be read.""" + + match: re.Match[str] | None = USAGE_ERROR_CODE_PATTERN.search(str(error)) + return match is not None and int(match.group(1)) in USAGE_UNAVAILABLE_ERROR_CODES + + +def usage_reads(rows: Iterable[tuple[object, ...]]) -> tuple[RelationRead, ...]: + """Reads from `(object, user, queries, last read as Unix seconds)` rows.""" + + return tuple( + RelationRead( + object_name=str(row[0]), + reader=str(row[1]) if row[1] else None, + query_count=int(str(row[2])), + last_read_at=datetime.fromtimestamp(int(str(row[3])), tz=UTC), + ) + for row in rows + ) + + +def read_relation_usage( + *, + run_query: Callable[[str], list[tuple[object, ...]]], + relations: tuple[RelationInfo, ...], + window_seconds: int, +) -> RelationUsageReport: + """Read every relation's usage; no query log or logging turned off means not checked.""" + + relations_by_name: dict[str, RelationInfo] = usage_object_names(relations=relations) + rows: list[tuple[object, ...]] = [] + try: + logging: str = str(run_query(LOGGING_SETTING_SQL)[0][0]) if relations_by_name else "" + if logging in LOGGING_OFF_VALUES: + return RelationUsageReport(checked=False, reason=USAGE_LOGGING_OFF_REASON) + sql: str + for sql in usage_queries(object_names=relations_by_name, window_seconds=window_seconds): + rows.extend(run_query(sql)) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + return RelationUsageReport( + checked=True, + usages=aggregate_relation_usage( + reads=usage_reads(rows), relations_by_name=relations_by_name + ), + ) diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index fd0174dcba..303ca95a40 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -39,7 +39,6 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition -from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -98,6 +97,7 @@ generation_comment, rewrite_partitions, ) +from sqlbuild.adapters.clickhouse._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.clickhouse._helpers.snapshot_rewrite import ( nullable_offset_windows, rewrite_snapshot_statements, @@ -2316,8 +2316,13 @@ def relation_usage( relations: tuple[RelationInfo, ...], window_seconds: int, ) -> RelationUsageReport: - del connection, relations, window_seconds - return unsupported_relation_usage() + """Read candidate reads from this node's system.query_log.""" + + return read_relation_usage( + run_query=lambda sql: list(self.execute(connection=connection, sql=sql).fetchall()), + relations=relations, + window_seconds=window_seconds, + ) def get_tables_freshness_metadata( self, diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index bc91e48c48..06c3435810 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -114,3 +114,13 @@ NULL_IN_ORDINARY_COLUMN_CODE: str = "A713" NULL_TARGET_COLUMN_PATTERN: re.Pattern[str] = re.compile(r"to destination column (\S+?):") FRESH_GENERATION_CLAUSE: str = " COMMENT ''" +USAGE_NAME_CHUNK_SIZE: int = 1_000 +USAGE_UNAVAILABLE_REASON: str = ( + "cannot read system.query_log (needs query logging on and SELECT on system.query_log)" +) +USAGE_LOGGING_OFF_REASON: str = "query logging is off for this connection (log_queries = 0)" +USAGE_UNAVAILABLE_ERROR_CODES: frozenset[int] = frozenset({60, 497}) +USAGE_QUERY_KINDS: tuple[str, ...] = ("Select", "Insert") +USAGE_ERROR_CODE_PATTERN: re.Pattern[str] = re.compile(r"Code: (\d+)\.") +LOGGING_SETTING_SQL: str = "SELECT getSetting('log_queries')" +LOGGING_OFF_VALUES: frozenset[str] = frozenset({"0", "False"}) diff --git a/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py b/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py index e227920850..bb43f73cc5 100644 --- a/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py +++ b/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py @@ -6,8 +6,10 @@ from typing import Any from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.main.relation_reads import relation_reads from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport from sqlbuild.adapters.databricks.constants import ( + LINEAGE_VIEW_SOURCE_TYPES, USAGE_NAME_CHUNK_SIZE, USAGE_UNAVAILABLE_ERROR_CLASSES, USAGE_UNAVAILABLE_REASON, @@ -46,19 +48,32 @@ def usage_queries(*, object_names: Iterable[str], window_seconds: int) -> tuple[ def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> UsageQuery: - """Read events of the named tables per user within the window.""" + """Read events of the named tables per user within the window, direct or through views.""" placeholders: str = ", ".join(f":name_{index}" for index in range(len(object_names))) since: str = "TIMESTAMPADD(SECOND, -:window_seconds, current_timestamp())" + view_types: str = ", ".join(f"'{source_type}'" for source_type in LINEAGE_VIEW_SOURCE_TYPES) sql: str = ( - "SELECT lower(source_table_full_name) AS object_name, " - "created_by AS user_name, " - "COUNT(DISTINCT COALESCE(statement_id, event_id)) AS query_count, " - "MAX(event_time) AS last_read_at " + "WITH lineage AS (" + "SELECT lower(source_table_full_name) AS object_name, source_type, created_by, " + "event_time, statement_id, event_id, COALESCE(direct_access, true) AS direct " "FROM system.access.table_lineage " f"WHERE event_date >= DATE({since}) " f"AND event_time >= {since} " - f"AND lower(source_table_full_name) IN ({placeholders}) " + "AND source_table_full_name IS NOT NULL), " + f"reads AS (SELECT * FROM lineage WHERE object_name IN ({placeholders})), " + "via AS (" + "SELECT reads.object_name, reads.created_by, collect_set(views.object_name) AS via_names " + "FROM reads JOIN lineage AS views ON views.statement_id = reads.statement_id " + f"AND views.direct AND views.source_type IN ({view_types}) " + "WHERE NOT reads.direct GROUP BY 1, 2) " + "SELECT reads.object_name, reads.created_by AS user_name, " + "COUNT(DISTINCT COALESCE(reads.statement_id, reads.event_id)) AS query_count, " + "MAX(reads.event_time) AS last_read_at, " + "bool_or(reads.direct) AS direct, " + "first(via.via_names) AS via_names " + "FROM reads LEFT JOIN via ON via.object_name = reads.object_name " + "AND via.created_by <=> reads.created_by " "GROUP BY 1, 2 " "ORDER BY 1, 2" ) @@ -97,5 +112,7 @@ def read_relation_usage( cursor.close() return RelationUsageReport( checked=True, - usages=aggregate_relation_usage(rows=rows, relations_by_name=relations_by_name), + usages=aggregate_relation_usage( + reads=relation_reads(rows=rows), relations_by_name=relations_by_name + ), ) diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index a43a560226..e10624a3c5 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -32,3 +32,4 @@ "UC_NOT_ENABLED", ) USAGE_UNAVAILABLE_SQLSTATES: frozenset[str] = frozenset({"42501", "42P01", "42704"}) +LINEAGE_VIEW_SOURCE_TYPES: tuple[str, ...] = ("VIEW", "MATERIALIZED_VIEW", "METRIC_VIEW") diff --git a/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py index 305164b399..1d70024b95 100644 --- a/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py +++ b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py @@ -3,16 +3,24 @@ from __future__ import annotations import re -from collections.abc import Callable, Iterable, Mapping -from datetime import UTC, datetime +from collections.abc import Callable, Iterable from typing import Any -from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage, RelationUsageReport +from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.main.relation_reads import relation_reads +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.snowflake._helpers.view_dependencies import ( + chained_usages, + dependency_name, + dependency_queries, + display_name, +) from sqlbuild.adapters.snowflake.constants import ( CLONE_STATEMENT_PATTERN, SQLBUILD_QUERY_TAG_FRAGMENT, USAGE_NAME_CHUNK_SIZE, USAGE_UNAVAILABLE_REASON, + VIEW_RELATION_TYPE, ) from sqlbuild.adapters.snowflake.models import UsageQuery @@ -68,26 +76,55 @@ def usage_queries(*, object_names: Iterable[str], window_seconds: int) -> tuple[ def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> UsageQuery: - """Reads of the named objects per user, excluding SQLBuild-tagged queries and clones.""" + """Reads of the named objects per user, direct or through views, minus SQLBuild and clones.""" placeholders: str = ", ".join(["%s"] * len(object_names)) + since: str = "DATEADD('second', -%s, CURRENT_TIMESTAMP())" + object_name: str = 'accessed.value:"objectName"::STRING' sql: str = ( - 'SELECT accessed.value:"objectName"::STRING AS object_name, ' - "history.user_name AS user_name, " - "COUNT(DISTINCT history.query_id) AS query_count, " - "MAX(history.query_start_time) AS last_read_at " - "FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY AS history, " - "LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed " - "WHERE history.query_start_time >= DATEADD('second', -%s, CURRENT_TIMESTAMP()) " - f'AND accessed.value:"objectName"::STRING IN ({placeholders}) ' - "AND NOT EXISTS (" + "WITH history AS (" + "SELECT query_id, user_name, query_start_time, direct_objects_accessed, " + "base_objects_accessed FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY " + f"WHERE query_start_time >= {since}), " + "direct_reads AS (" + "SELECT history.query_id, history.user_name, history.query_start_time, " + f"{object_name} AS object_name " + "FROM history, LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed " + f"WHERE {object_name} IN ({placeholders})), " + "base_reads AS (" + "SELECT history.query_id, history.user_name, history.query_start_time, " + f"{object_name} AS object_name " + "FROM history, LATERAL FLATTEN(INPUT => history.base_objects_accessed) AS accessed " + f"WHERE {object_name} IN ({placeholders})), " + "candidate_reads AS (" + "SELECT query_id, user_name, query_start_time, object_name, TRUE AS direct " + "FROM direct_reads UNION ALL " + "SELECT base.query_id, base.user_name, base.query_start_time, base.object_name, FALSE " + "FROM base_reads AS base WHERE NOT EXISTS (" + "SELECT 1 FROM direct_reads AS direct_read " + "WHERE direct_read.query_id = base.query_id " + "AND direct_read.object_name = base.object_name)), " + "reads AS (" + "SELECT * FROM candidate_reads AS candidate WHERE NOT EXISTS (" "SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY AS query " - "WHERE query.query_id = history.query_id " - "AND query.start_time >= DATEADD('second', -%s, CURRENT_TIMESTAMP()) " + "WHERE query.query_id = candidate.query_id " + f"AND query.start_time >= {since} " "AND (CONTAINS(COALESCE(query.query_tag, ''), %s) " "OR (STARTSWITH(query.query_type, 'CREATE') " - "AND REGEXP_LIKE(query.query_text, %s, 'is')))" - ") " + "AND REGEXP_LIKE(query.query_text, %s, 'is'))))), " + "via AS (" + f"SELECT reads.object_name, reads.user_name, ARRAY_AGG(DISTINCT {object_name}) " + "AS via_names " + "FROM reads JOIN history ON history.query_id = reads.query_id, " + "LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed " + "WHERE NOT reads.direct GROUP BY 1, 2) " + "SELECT reads.object_name, reads.user_name, " + "COUNT(DISTINCT reads.query_id) AS query_count, " + "MAX(reads.query_start_time) AS last_read_at, " + "BOOLOR_AGG(reads.direct) AS direct, " + "ANY_VALUE(via.via_names) AS via_names " + "FROM reads LEFT JOIN via " + "ON via.object_name = reads.object_name AND via.user_name = reads.user_name " "GROUP BY 1, 2 " "ORDER BY 1, 2" ) @@ -96,6 +133,7 @@ def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> UsageQ params=( window_seconds, *object_names, + *object_names, window_seconds, SQLBUILD_QUERY_TAG_FRAGMENT, CLONE_STATEMENT_PATTERN, @@ -103,49 +141,6 @@ def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> UsageQ ) -def relation_usages( - *, rows: Iterable[tuple[object, ...]], relations_by_name: Mapping[str, RelationInfo] -) -> tuple[RelationUsage, ...]: - """One usage per read relation from `(object_name, user, query_count, last_read)` rows.""" - - readers: dict[str, dict[str, int]] = {} - last_read: dict[str, datetime] = {} - row: tuple[object, ...] - for row in rows: - object_name: str = str(row[0]) - if object_name not in relations_by_name: - continue - user: str = str(row[1]) - counts: dict[str, int] = readers.setdefault(object_name, {}) - counts[user] = counts.get(user, 0) + int(str(row[2])) - read_at: datetime = _utc(row[3]) - last_read[object_name] = max(read_at, last_read.get(object_name, read_at)) - usages: list[RelationUsage] = [] - object_name: str - for object_name in sorted(readers): - relation: RelationInfo = relations_by_name[object_name] - counts = readers[object_name] - usages.append( - RelationUsage( - database=relation.database, - schema=relation.schema, - name=relation.name, - last_read_at=last_read[object_name], - readers=tuple(sorted(counts, key=lambda user: (-counts[user], user))), - query_count=sum(counts.values()), - ) - ) - return tuple(usages) - - -def _utc(value: object) -> datetime: - if not isinstance(value, datetime): - return datetime.fromisoformat(str(value)).astimezone(UTC) - if value.tzinfo is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) - - def is_usage_unavailable_error(error: Exception) -> bool: """Whether reading access history failed for edition or privilege reasons.""" @@ -162,7 +157,7 @@ def read_relation_usage( window_seconds: int, current_database: Callable[[], str | None], ) -> RelationUsageReport: - """Read every relation's usage in chunked queries; no access history means not checked.""" + """Read every relation's usage and view dependencies; no access history means not checked.""" database: str | None = None if any(relation.database is None for relation in relations): @@ -170,20 +165,52 @@ def read_relation_usage( relations_by_name: dict[str, RelationInfo] = usage_object_names( relations=relations, current_database=database ) + by_dependency_name: dict[str, RelationInfo] = {} + relation: RelationInfo + for relation in relations: + name: str | None = dependency_name(relation=relation, current_database=database) + if name is not None: + by_dependency_name[name] = relation + try: + rows: list[tuple[object, ...]] = _fetch( + connection=connection, + queries=usage_queries(object_names=relations_by_name, window_seconds=window_seconds), + ) + dependencies: list[tuple[object, ...]] = _fetch( + connection=connection, + queries=dependency_queries( + view_names=( + name + for name, relation in by_dependency_name.items() + if VIEW_RELATION_TYPE in relation.relation_type.lower() + ) + ), + ) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + return RelationUsageReport( + checked=True, + usages=chained_usages( + usages=aggregate_relation_usage( + reads=relation_reads(rows=rows, via_name=display_name), + relations_by_name=relations_by_name, + ), + dependencies=((str(row[0]), str(row[1])) for row in dependencies), + relations_by_dependency_name=by_dependency_name, + ), + ) + + +def _fetch(*, connection: Any, queries: tuple[UsageQuery, ...]) -> list[tuple[object, ...]]: rows: list[tuple[object, ...]] = [] query: UsageQuery - for query in usage_queries(object_names=relations_by_name, window_seconds=window_seconds): + for query in queries: cursor: Any = connection.cursor() try: cursor.execute(query.sql, query.params) rows.extend(tuple(row) for row in cursor.fetchall()) - except Exception as error: - if not is_usage_unavailable_error(error): - raise - return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) finally: cursor.close() - return RelationUsageReport( - checked=True, - usages=relation_usages(rows=rows, relations_by_name=relations_by_name), - ) + return rows diff --git a/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py b/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py new file mode 100644 index 0000000000..9cb98d4eb3 --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py @@ -0,0 +1,136 @@ +"""Follow Snowflake view dependencies from in-use candidates to the candidates behind them.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from datetime import datetime + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage +from sqlbuild.adapters.snowflake.constants import ( + DEPENDENCY_VIEW_DOMAINS, + QUOTED_IDENTIFIER_MARK, + USAGE_NAME_CHUNK_SIZE, +) +from sqlbuild.adapters.snowflake.models import UsageQuery + + +def dependency_name(*, relation: RelationInfo, current_database: str | None) -> str | None: + """The `DATABASE.SCHEMA.NAME` OBJECT_DEPENDENCIES records for an unquoted relation.""" + + database: str | None = relation.database or current_database + if database is None or relation.schema is None: + return None + return ".".join(part.upper() for part in (database, relation.schema, relation.name)) + + +def display_name(object_name: str) -> str: + """A Snowflake object name as SQLBuild prints it: unquoted names in lowercase.""" + + return object_name if QUOTED_IDENTIFIER_MARK in object_name else object_name.lower() + + +def dependency_queries(*, view_names: Iterable[str]) -> tuple[UsageQuery, ...]: + """One OBJECT_DEPENDENCIES read per chunk of candidate views.""" + + names: tuple[str, ...] = tuple(sorted(view_names)) + return tuple( + dependency_query(view_names=names[start : start + USAGE_NAME_CHUNK_SIZE]) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) + ) + + +def dependency_query(*, view_names: tuple[str, ...]) -> UsageQuery: + """What each named view references, as `(referencing, referenced)` dotted names.""" + + placeholders: str = ", ".join(["%s"] * len(view_names)) + domains: str = ", ".join(["%s"] * len(DEPENDENCY_VIEW_DOMAINS)) + referencing: str = ( + "referencing_database || '.' || referencing_schema || '.' || referencing_object_name" + ) + sql: str = ( + f"SELECT {referencing} AS referencing_name, " + "referenced_database || '.' || referenced_schema || '.' || referenced_object_name " + "AS referenced_name " + "FROM SNOWFLAKE.ACCOUNT_USAGE.OBJECT_DEPENDENCIES " + f"WHERE referencing_object_domain IN ({domains}) " + f"AND {referencing} IN ({placeholders}) " + "ORDER BY 1, 2" + ) + return UsageQuery(sql=sql, params=(*DEPENDENCY_VIEW_DOMAINS, *view_names)) + + +def _folded(relation: RelationInfo | RelationUsage) -> tuple[str, str, str]: + return ( + (relation.database or "").lower(), + (relation.schema or "").lower(), + relation.name.lower(), + ) + + +def _via_parent( + *, + relation: RelationInfo, + parent: str, + parent_usage: RelationUsage, + earlier: RelationUsage | None, +) -> RelationUsage: + via: tuple[str, ...] = (display_name(parent),) + readers: tuple[str, ...] = parent_usage.readers + last_read_at: datetime = parent_usage.last_read_at + query_count: int = parent_usage.query_count + if earlier is not None: + via = tuple(sorted({*earlier.via, *via})) + readers = tuple(dict.fromkeys((*earlier.readers, *readers))) + last_read_at = max(earlier.last_read_at, last_read_at) + query_count = max(earlier.query_count, query_count) + return RelationUsage( + database=relation.database, + schema=relation.schema, + name=relation.name, + last_read_at=last_read_at, + readers=readers, + query_count=query_count, + direct=False, + via=via, + ) + + +def chained_usages( + *, + usages: tuple[RelationUsage, ...], + dependencies: Iterable[tuple[str, str]], + relations_by_dependency_name: Mapping[str, RelationInfo], +) -> tuple[RelationUsage, ...]: + """Usages plus every candidate an in-use candidate view depends on, transitively.""" + + references: dict[str, set[str]] = {} + referencing: str + referenced: str + for referencing, referenced in dependencies: + if referenced in relations_by_dependency_name: + references.setdefault(referencing, set()).add(referenced) + names_by_key: dict[tuple[str, str, str], str] = { + _folded(relation): name for name, relation in relations_by_dependency_name.items() + } + read: dict[str, RelationUsage] = { + names_by_key[_folded(usage)]: usage for usage in usages if _folded(usage) in names_by_key + } + found: dict[str, RelationUsage] = {} + pending: list[str] = sorted(read) + while pending: + parent: str = pending.pop(0) + parent_usage: RelationUsage = read.get(parent) or found[parent] + child: str + for child in sorted(references.get(parent, ())): + if child in read: + continue + earlier: RelationUsage | None = found.get(child) + found[child] = _via_parent( + relation=relations_by_dependency_name[child], + parent=parent, + parent_usage=parent_usage, + earlier=earlier, + ) + if earlier is None: + pending.append(child) + return (*usages, *(found[name] for name in sorted(found))) diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 323e3a878a..5a2613cbe9 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -29,4 +29,19 @@ "(needs Enterprise edition and the GOVERNANCE_VIEWER database role)" ) SQLBUILD_QUERY_TAG_FRAGMENT: str = f'"app":"{SQLBUILD_QUERY_TAG_APP}"' -CLONE_STATEMENT_PATTERN: str = r".*\sCLONE\s.*" +_CLONE_IDENTIFIER: str = r'("[^"]*"|[A-Za-z_][A-Za-z0-9_$]*)' +_CLONE_NAME: str = _CLONE_IDENTIFIER + r"(\s*\.\s*" + _CLONE_IDENTIFIER + r"){0,2}" +_LEADING_COMMENTS: str = "(\\s|--[^\n]*\n|//[^\n]*\n|/\\*([^*]|\\*+[^*/])*\\*+/)*" +CLONE_STATEMENT_PATTERN: str = ( + "^" + + _LEADING_COMMENTS + + r"CREATE\s+(OR\s+REPLACE\s+)?((TRANSIENT|TEMPORARY|TEMP|VOLATILE|LOCAL\s+TEMPORARY|" + r"GLOBAL\s+TEMPORARY)\s+)?(TABLE|DYNAMIC\s+TABLE|SCHEMA|DATABASE)\s+(IF\s+NOT\s+EXISTS\s+)?" + + _CLONE_NAME + + r"\s+CLONE\s+" + + _CLONE_NAME + + r'([^A-Za-z0-9_$"].*)?$' +) +DEPENDENCY_VIEW_DOMAINS: tuple[str, ...] = ("VIEW", "MATERIALIZED VIEW", "SECURE VIEW") +VIEW_RELATION_TYPE: str = "view" +QUOTED_IDENTIFIER_MARK: str = '"' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 6283ccbede..907ae9a301 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -358,3 +358,20 @@ class ClickHouseRecallDocumentedGrantsTestCase: expected_columns: tuple[tuple[object, ...], ...] expected_stale_views: tuple[tuple[object, ...], ...] expected_version: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseJanitorUsageE2ETestCase: + description: str + janitor_command: tuple[str, ...] + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] + expected_remaining: tuple[str, ...] + + +@dataclass(frozen=True) +class ClickHouseJanitorUsageUncheckedE2ETestCase: + description: str + connection_settings: str + grants: tuple[str, ...] + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py index 62ac2d203a..bfa5d899f6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py @@ -9,6 +9,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( CLICKHOUSE_IMAGE, build_unique_schema_name, + cleanup_clickhouse_schema, execute_clickhouse_sql, ) @@ -59,3 +60,10 @@ def recall_database(clickhouse_e2e_config: dict[str, object]) -> Iterator[str]: execute_clickhouse_sql( config=clickhouse_e2e_config, statements=(f"DROP DATABASE IF EXISTS {database} SYNC",) ) + + +@pytest.fixture +def janitor_schema(clickhouse_e2e_config: dict[str, object]) -> Iterator[str]: + schema: str = build_unique_schema_name(prefix="janitor_usage") + yield schema + cleanup_clickhouse_schema(config=clickhouse_e2e_config, schema_name=schema) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index 83cb5a6a26..b1d082211d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -13,7 +13,11 @@ from typing import Any, cast from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_waffle_shop +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + prepare_waffle_shop, + run_sqb, +) from tests.integration.src.sqlbuild.cli.recall.helpers import ( killed_build, killed_during_discovery, @@ -1043,3 +1047,74 @@ def stale_views(*, config: dict[str, object], database: str) -> tuple[tuple[obje sql="SELECT countIf(position(create_table_query, 'join_use_nulls') > 0) " f"FROM system.tables WHERE database = '{database}' AND engine = 'View'", ) + + +JANITOR_SETTINGS: str = ( + "\n[janitor]\nenabled = true\nretention_days = 0\ndelete_tracked_only = false\n" +) +STALE_RELATIONS: tuple[str, ...] = ( + "CREATE TABLE {schema}.old_orders ENGINE = MergeTree ORDER BY id " + "AS SELECT toInt64(number) AS id FROM numbers(3)", + "CREATE TABLE {schema}.old_customers ENGINE = MergeTree ORDER BY id AS SELECT toInt64(1) AS id", + "CREATE VIEW {schema}.old_orders_v2 AS SELECT id FROM {schema}.old_orders", + "CREATE VIEW {schema}.old_orders_v AS SELECT id FROM {schema}.old_orders_v2", +) + + +def prepare_janitor_usage_project( + *, tmp_path: Path, config: dict[str, object], schema: str, connection_settings: str = "" +) -> Path: + """A built project in `schema` beside a stale table, an unread table and a view chain on it.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_janitor_usage", + repo_files={ + "sqlbuild_project.toml": build_clickhouse_project_toml( + project_name="clickhouse_janitor_usage", + schema_name=schema, + config=config, + extra=JANITOR_SETTINGS, + ).replace("\n[targets.dev]", connection_settings + "\n[targets.dev]"), + "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT toInt64(1) AS id\n", + }, + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + execute_clickhouse_sql( + config=config, + statements=tuple(statement.format(schema=schema) for statement in STALE_RELATIONS), + ) + return project_dir + + +def read_as(*, config: dict[str, object], sql: str) -> None: + """Run one query as the user in `config`, as any client would, outside SQLBuild.""" + + import clickhouse_connect + + client: Any = clickhouse_connect.get_client( + host=str(config["host"]), + port=int(str(config["port"])), + username=str(config["user"]), + password=str(config["password"]), + ) + try: + client.query(sql) + finally: + client.close() + + +def stale_relation_names(*, config: dict[str, object], schema: str) -> tuple[str, ...]: + """The `old_*` relations still in `schema`.""" + + return tuple( + str(row[0]) + for row in fetch_clickhouse_rows( + config=config, + sql=f"SELECT name FROM system.tables WHERE database = '{schema}' " + "AND startsWith(name, 'old_') ORDER BY name", + ) + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py new file mode 100644 index 0000000000..5189c5c77e --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py @@ -0,0 +1,148 @@ +"""ClickHouse janitor usage: system.query_log shows who reads stale tables and view chains.""" + +from __future__ import annotations + +import subprocess +from contextlib import ExitStack +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseJanitorUsageE2ETestCase, + ClickHouseJanitorUsageUncheckedE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + execute_clickhouse_sql, + limited_user, + prepare_janitor_usage_project, + read_as, + stale_relation_names, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +APPROVED_JANITOR: tuple[str, ...] = ("--no-color", "janitor", "--auto-approve") + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseJanitorUsageE2ETestCase( + description="a view chain read by another user is in use and archived by default", + janitor_command=APPROVED_JANITOR, + expected_fragments=( + "Checked relation usage: 3 of 4 in use. (", + "\nIn use\n", + ".old_orders queried ", + " by {reader} (1 query in 7 days); frozen, still in use\n", + ".old_orders_v queried ", + ".old_orders_v2 queried ", + " by {reader} (1 query in 7 days)\n", + ), + unexpected_fragments=("Kept (in use)", "old_customers queried", "usage not checked"), + expected_remaining=(), + ), + ClickHouseJanitorUsageE2ETestCase( + description="--keep-in-use keeps the read table and both views of the chain", + janitor_command=(*APPROVED_JANITOR, "--keep-in-use"), + expected_fragments=( + "Checked relation usage: 3 of 4 in use. (", + " kept (in use) 3\n", + "\nKept (in use)\n", + ".old_orders_v2 queried ", + ), + unexpected_fragments=("\nIn use\n", "old_orders_v ->", "usage not checked"), + expected_remaining=("old_orders", "old_orders_v", "old_orders_v2"), + ), + ), + ids=lambda case: case.description, +) +def test_given_query_log_when_running_janitor_then_view_chain_reads_are_in_use( + test_case: ClickHouseJanitorUsageE2ETestCase, + clickhouse_e2e_config: dict[str, object], + janitor_schema: str, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_janitor_usage_project( + tmp_path=tmp_path, config=clickhouse_e2e_config, schema=janitor_schema + ) + with limited_user( + config=clickhouse_e2e_config, database=janitor_schema, privileges="SELECT" + ) as reader: + read_as(config=reader, sql=f"SELECT sum(id) FROM {janitor_schema}.old_orders_v") + execute_clickhouse_sql(config=clickhouse_e2e_config, statements=("SYSTEM FLUSH LOGS",)) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + reader_name: str = str(reader["user"]) + assert result.returncode == 0, result.stdout + result.stderr + fragment: str + for fragment in test_case.expected_fragments: + assert fragment.format(reader=reader_name) in result.stdout, result.stdout + for fragment in test_case.unexpected_fragments: + assert fragment not in result.stdout, result.stdout + assert stale_relation_names(config=clickhouse_e2e_config, schema=janitor_schema) == ( + test_case.expected_remaining + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseJanitorUsageUncheckedE2ETestCase( + description="a janitor role that may not read system.query_log is not checked", + connection_settings="", + grants=(), + expected_fragments=( + "Relation usage not checked: cannot read system.query_log (", + " usage not checked: cannot read system.query_log (needs query logging on " + "and SELECT on system.query_log)\n", + ), + ), + ClickHouseJanitorUsageUncheckedE2ETestCase( + description="query logging turned off for the connection is not checked", + connection_settings="\n[connection.settings]\nlog_queries = 0\n", + grants=("GRANT SELECT ON system.query_log TO {user}",), + expected_fragments=( + " usage not checked: query logging is off for this connection (log_queries = 0)\n", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_query_log_when_running_janitor_then_usage_is_not_checked( + test_case: ClickHouseJanitorUsageUncheckedE2ETestCase, + clickhouse_e2e_config: dict[str, object], + janitor_schema: str, + tmp_path: Path, +) -> None: + with ExitStack() as stack: + janitor: dict[str, object] = stack.enter_context( + limited_user(config=clickhouse_e2e_config, database=janitor_schema, privileges="ALL") + ) + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=( + f"CREATE DATABASE {janitor_schema}", + *(grant.format(user=janitor["user"]) for grant in test_case.grants), + ), + ) + project_dir: Path = prepare_janitor_usage_project( + tmp_path=tmp_path, + config=janitor, + schema=janitor_schema, + connection_settings=test_case.connection_settings, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=(*APPROVED_JANITOR, "--keep-in-use"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + fragment: str + for fragment in test_case.expected_fragments: + assert fragment in result.stdout, result.stdout + assert "nothing kept in use: no usage could be checked" in result.stdout + assert stale_relation_names(config=clickhouse_e2e_config, schema=janitor_schema) == () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index b14a6bab4f..5ba61a7da0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -312,6 +312,7 @@ class JanitorUsageE2ETestCase: unexpected_fragments: tuple[str, ...] expected_remaining: tuple[str, ...] expected_archived: int + via: dict[str, tuple[str, ...]] = field(default_factory=dict) @dataclass(frozen=True) @@ -335,3 +336,12 @@ class JanitorUsageJsonE2ETestCase: expected_archive_in_use: tuple[tuple[str, object], ...] expected_kept_in_use: tuple[tuple[str, object], ...] expected_archived: int + + +@dataclass(frozen=True) +class JanitorUsageFailureE2ETestCase: + """Test case for janitor runs whose usage source fails outright.""" + + description: str + failure: str + expected_ordered_lines: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index e73a994a65..ff77f40f60 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -761,6 +761,8 @@ def relation_usage( encoding="utf-8", ) settings: dict[str, Any] = _settings() + if settings["failure"] is not None: + raise RuntimeError(settings["failure"]) if settings["unavailable"] is not None: return RelationUsageReport(checked=False, reason=settings["unavailable"]) reads: dict[str, dict[str, Any]] = settings["reads"] @@ -774,6 +776,8 @@ def relation_usage( last_read_at=datetime.fromisoformat(reads[relation.name]["last_read_at"]), readers=tuple(reads[relation.name]["readers"]), query_count=int(reads[relation.name]["query_count"]), + direct=reads[relation.name]["via"] is None, + via=tuple(reads[relation.name]["via"] or ()), ) for relation in relations if relation.name in reads @@ -836,8 +840,13 @@ def write_usage_settings( project_dir: Path, reads: Mapping[str, tuple[tuple[str, ...], int]] = MappingProxyType({}), unavailable: str | None = None, + via: Mapping[str, tuple[str, ...]] = MappingProxyType({}), + failure: str | None = None, ) -> None: - """Run janitor on the usage adapter: who read which relation, or why usage is unavailable.""" + """Run janitor on the usage adapter: who read which relation, or why usage is unavailable. + + A relation in `via` was read only through the named objects; `failure` makes the read fail. + """ (project_dir / "sqlbuild_local.toml").write_text( f'adapter = "{USAGE_JANITOR_ADAPTER_NAME}"\n', encoding="utf-8" @@ -851,10 +860,12 @@ def write_usage_settings( "readers": list(readers), "query_count": count, "last_read_at": "2026-10-21T09:30:00+00:00", + "via": via.get(name), } for name, (readers, count) in reads.items() }, "unavailable": unavailable, + "failure": failure, } ), encoding="utf-8", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py index 50596da13a..abe3a91618 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py @@ -11,6 +11,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( JanitorUsageE2ETestCase, + JanitorUsageFailureE2ETestCase, JanitorUsageJsonE2ETestCase, JanitorUsagePromptE2ETestCase, JanitorUsageUncheckedE2ETestCase, @@ -183,6 +184,22 @@ def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( expected_remaining=("old_orders_000", "old_orders_view"), expected_archived=2, ), + JanitorUsageE2ETestCase( + description="a relation read only through views names them", + janitor_config="", + janitor_command=APPROVED_JANITOR, + reads={"old_orders_002": (("alice",), 3)}, + via={"old_orders_002": ("main.orders_report", "main.orders_summary")}, + expected_ordered_lines=( + "Checked relation usage: 1 of 4 in use. (", + " main.old_orders_002 queried 2026-10-21 by alice via main.orders_report, " + "main.orders_summary (3 queries in 7 days); frozen since 2026-09-01, " + "still in use\n", + ), + unexpected_fragments=("Kept (in use)",), + expected_remaining=(), + expected_archived=4, + ), JanitorUsageE2ETestCase( description="keep_in_use in the project and in_use_window reach the usage source", janitor_config='keep_in_use = true\nin_use_window = "36h"\n', @@ -211,7 +228,7 @@ def test_given_usage_source_when_running_janitor_then_in_use_relations_are_shown stale_views=("old_orders_view",), janitor_config=test_case.janitor_config, ) - write_usage_settings(project_dir=project_dir, reads=test_case.reads) + write_usage_settings(project_dir=project_dir, reads=test_case.reads, via=test_case.via) result: subprocess.CompletedProcess[str] = run_sqb( command=test_case.janitor_command, project_dir=project_dir @@ -256,11 +273,17 @@ def test_given_usage_source_when_running_janitor_then_in_use_relations_are_shown "last_read": "2026-10-21T09:30:00Z", "readers": list(FIVE_READERS), "query_count": 40, + "via": None, }, ), ( "main.old_orders_002", - {"last_read": "2026-10-21T09:30:00Z", "readers": ["alice"], "query_count": 2}, + { + "last_read": "2026-10-21T09:30:00Z", + "readers": ["alice"], + "query_count": 2, + "via": ["main.orders_report"], + }, ), ), expected_archived=1, @@ -277,6 +300,7 @@ def test_given_usage_source_when_janitor_prints_json_then_candidates_carry_in_us write_usage_settings( project_dir=project_dir, reads={"old_orders_000": (FIVE_READERS, 40), "old_orders_002": (("alice",), 2)}, + via={"old_orders_002": ("main.orders_report",)}, ) result: subprocess.CompletedProcess[str] = run_sqb( @@ -290,6 +314,9 @@ def test_given_usage_source_when_janitor_prints_json_then_candidates_carry_in_us test_case.expected_archive_in_use ) assert in_use_entries(payload["plan"]["kept_in_use"]) == test_case.expected_kept_in_use + assert all( + set(entry) == {"name", "age_days", "in_use"} for entry in payload["plan"]["kept_in_use"] + ) assert payload["result"]["counts"]["archived"] == test_case.expected_archived assert read_usage_calls(project_dir=project_dir)["window_seconds"] == 7 * 86_400 @@ -342,5 +369,38 @@ def test_given_usage_check_when_prompting_then_prompt_states_usage( assert len(list_archive_names(db_path=project_dir / "janitor.duckdb")) == 1 +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageFailureE2ETestCase( + description="a failed usage read prints a failure line and archives nothing", + failure="usage warehouse went away", + expected_ordered_lines=( + "Checking relation usage...", + "Relation usage check failed. (", + "usage warehouse went away", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_usage_source_when_running_janitor_then_failure_line_is_printed( + test_case: JanitorUsageFailureE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_failure", stale_tables=1 + ) + write_usage_settings(project_dir=project_dir, failure=test_case.failure) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=APPROVED_JANITOR, project_dir=project_dir + ) + + assert result.returncode != 0, result.stdout + assert_in_order(output=result.stdout + result.stderr, lines=test_case.expected_ordered_lines) + assert "Janitor preview" not in result.stdout + assert len(list_archive_names(db_path=project_dir / "janitor.duckdb")) == 1 + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index 13851b17fc..68c49b8df1 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -259,3 +259,10 @@ class BigQueryRelationUsageFailureTestCase: description: str error: Exception expected_error_match: str + + +@dataclass(frozen=True) +class BigQueryCloneStatementTestCase: + description: str + statement: str + expected_clone: bool diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py index d6f3f4fa13..797eec42ec 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py @@ -2,13 +2,15 @@ from __future__ import annotations +import re from datetime import UTC, datetime import pytest from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport -from sqlbuild.adapters.bigquery.constants import USAGE_UNAVAILABLE_REASON +from sqlbuild.adapters.bigquery.constants import CLONE_STATEMENT_PATTERN, USAGE_UNAVAILABLE_REASON from tests.unit.src.sqlbuild.adapters.bigquery._test_types import ( + BigQueryCloneStatementTestCase, BigQueryRelationUsageChunkTestCase, BigQueryRelationUsageErrorTestCase, BigQueryRelationUsageFailureTestCase, @@ -97,7 +99,7 @@ def test_given_jobs_history_when_reading_usage_then_reads_match_candidates( assert report.checked assert tuple( - (sql.split(" FROM ")[1].split(" AS jobs")[0], location, queried_names(params)) + (sql.split(" FROM ")[1].split(" WHERE ")[0], location, queried_names(params)) for sql, location, params in client.queries ) == (test_case.expected_queries) assert tuple( @@ -114,20 +116,21 @@ def test_given_jobs_history_when_reading_usage_then_reads_match_candidates( window_seconds=WINDOW_SECONDS, expected_params={ "window_seconds": WINDOW_SECONDS, - "clone_pattern": r"(?is)\bCLONE\b", + "clone_pattern": CLONE_STATEMENT_PATTERN, "object_names": ["shop-analytics.mart.orders_daily"], }, expected_sql_fragments=( "UNNEST(jobs.referenced_tables) AS referenced", + "UNNEST(jobs.referenced_logical_views) AS referenced", "INTERVAL @window_seconds SECOND", - "jobs.job_type = 'QUERY'", - "jobs.error_result IS NULL", + "job_type = 'QUERY'", + "error_result IS NULL", "NOT IN ('SCRIPT', 'CREATE_SNAPSHOT_TABLE')", - "AND NOT (STARTS_WITH(COALESCE(jobs.statement_type, ''), 'CREATE_TABLE') " - "AND REGEXP_CONTAINS(jobs.query, @clone_pattern))", - "IN UNNEST(@object_names)", - "COUNT(DISTINCT jobs.job_id) AS query_count", - "jobs.user_email AS user_name", + "AND NOT (STARTS_WITH(COALESCE(statement_type, ''), 'CREATE_TABLE') " + "AND REGEXP_CONTAINS(query, @clone_pattern))", + "WHERE object_name IN UNNEST(@object_names)", + "COUNT(DISTINCT job_id) AS query_count", + "user_email AS user_name", ), ), ), @@ -246,5 +249,50 @@ def test_given_other_jobs_failure_when_reading_usage_then_error_propagates( read_bigquery_usage(client=client, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS) +@pytest.mark.parametrize( + "test_case", + ( + BigQueryCloneStatementTestCase( + description="a table clone", + statement="CREATE TABLE mart.orders_copy CLONE mart.orders", + expected_clone=True, + ), + BigQueryCloneStatementTestCase( + description="a replaced clone of a quoted name at a point in time", + statement="create or replace table `shop-analytics.mart.orders_copy` clone " + "`shop-analytics.mart.orders` FOR SYSTEM_TIME AS OF TIMESTAMP '2026-10-01'", + expected_clone=True, + ), + BigQueryCloneStatementTestCase( + description="a clone after a leading comment", + statement="# nightly copy\nCREATE TABLE IF NOT EXISTS mart.orders_copy CLONE mart.orders;", + expected_clone=True, + ), + BigQueryCloneStatementTestCase( + description="a CTAS selecting a column named clone", + statement="CREATE TABLE mart.orders_copy AS SELECT clone FROM mart.orders", + expected_clone=False, + ), + BigQueryCloneStatementTestCase( + description="a CTAS with clone inside a string", + statement="CREATE TABLE mart.notes AS SELECT 'orders CLONE copy' AS note", + expected_clone=False, + ), + BigQueryCloneStatementTestCase( + description="a CTAS whose comment mentions a clone", + statement="CREATE TABLE mart.c AS SELECT 1 AS n -- not CREATE TABLE a CLONE b", + expected_clone=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_statement_when_matching_clone_pattern_then_only_clones_match( + test_case: BigQueryCloneStatementTestCase, +) -> None: + matched: bool = re.search(CLONE_STATEMENT_PATTERN, test_case.statement) is not None + + assert matched == test_case.expected_clone + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py index 2f6d7bf201..e90367a315 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -1,4 +1,7 @@ from dataclasses import dataclass +from datetime import datetime + +from sqlbuild.adapter.contract.models import RelationUsageReport @dataclass(frozen=True) @@ -87,3 +90,41 @@ class KeyIdentifiersTestCase: description: str key_expressions: tuple[str, ...] expected_identifiers: frozenset[str] + + +@dataclass(frozen=True) +class ClickHouseRelationUsageTestCase: + description: str + relations: tuple[tuple[str, str], ...] + rows: tuple[tuple[object, ...], ...] + expected_names_sql: str + expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + + +@dataclass(frozen=True) +class ClickHouseRelationUsageQueryTestCase: + description: str + window_seconds: int + expected_sql_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ClickHouseRelationUsageChunkTestCase: + description: str + relation_count: int + expected_query_names: tuple[int, ...] + + +@dataclass(frozen=True) +class ClickHouseRelationUsageUncheckedTestCase: + description: str + log_queries: object + errors: tuple[Exception, ...] + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class ClickHouseRelationUsageFailureTestCase: + description: str + error: Exception + expected_error_match: str diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py b/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py new file mode 100644 index 0000000000..ea44197e3e --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py @@ -0,0 +1,75 @@ +"""Helpers for ClickHouse adapter unit tests.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.clickhouse._helpers.type_text import quote_string +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter + + +class FakeQueryLogError(Exception): + """A ClickHouse server error carrying its code in the message, as clickhouse-connect does.""" + + @classmethod + def raised(cls, message: str) -> tuple[Exception, ...]: + """The error as the only one a fake connection raises.""" + + return (cls(message),) + + +class FakeQueryLogCursor: + def __init__(self, rows: list[tuple[object, ...]]) -> None: + self.rows: list[tuple[object, ...]] = rows + + def fetchall(self) -> list[tuple[object, ...]]: + return self.rows + + +class FakeQueryLogConnection: + """Answers the log_queries setting and query_log reads, recording each query_log read.""" + + def __init__( + self, + *, + rows: tuple[tuple[object, ...], ...] = (), + log_queries: object = True, + errors: tuple[Exception, ...] = (), + ) -> None: + self.rows: tuple[tuple[object, ...], ...] = rows + self.log_queries: object = log_queries + self.errors: tuple[Exception, ...] = errors + self.queries: list[str] = [] + + def execute(self, sql: str) -> FakeQueryLogCursor: + reads: dict[bool, Callable[[], FakeQueryLogCursor]] = { + True: lambda: FakeQueryLogCursor([(self.log_queries,)]), + False: lambda: self.query_log_rows(sql), + } + return reads[sql == "SELECT getSetting('log_queries')"]() + + def query_log_rows(self, sql: str) -> FakeQueryLogCursor: + self.queries.append(sql) + error: Exception + for error in self.errors: + raise error + return FakeQueryLogCursor( + list(filter(lambda row: quote_string(str(row[0])) in sql.split(" IN (")[-1], self.rows)) + ) + + +def usage_relation(schema: str, name: str) -> RelationInfo: + """A ClickHouse table, whose database is its schema.""" + + return RelationInfo(database=None, schema=schema, name=name, relation_type="BASE TABLE") + + +def read_clickhouse_usage( + *, connection: FakeQueryLogConnection, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """ClickHouse's usage report for the relations from the fake connection.""" + + return ClickHouseAdapter().relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py new file mode 100644 index 0000000000..11e125ba88 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py @@ -0,0 +1,227 @@ +"""ClickHouse relation usage reads this node's system.query_log once per chunk of candidates.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.clickhouse.constants import ( + USAGE_LOGGING_OFF_REASON, + USAGE_UNAVAILABLE_REASON, +) +from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import ( + ClickHouseRelationUsageChunkTestCase, + ClickHouseRelationUsageFailureTestCase, + ClickHouseRelationUsageQueryTestCase, + ClickHouseRelationUsageTestCase, + ClickHouseRelationUsageUncheckedTestCase, +) +from tests.unit.src.sqlbuild.adapters.clickhouse.helpers import ( + FakeQueryLogConnection, + FakeQueryLogError, + read_clickhouse_usage, + usage_relation, +) + +WINDOW_SECONDS: int = 7 * 86_400 +READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) +EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) +ORDERS: tuple[RelationInfo, ...] = (usage_relation("mart", "old_orders"),) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageTestCase( + description="reads per initial user add up and the latest read wins", + relations=(("mart", "old_orders"), ("mart", "old_customers")), + rows=( + ("mart.old_orders", "svc_reporting", 10, int(READ_AT.timestamp())), + ("mart.old_orders", "alice", 4, int(EARLIER_READ_AT.timestamp())), + ("mart.old_orders", "", 1, int(EARLIER_READ_AT.timestamp())), + ("mart.products", "bob", 3, int(READ_AT.timestamp())), + ), + expected_names_sql="IN ('mart.old_customers', 'mart.old_orders')", + expected_usages=( + ("old_orders", ("svc_reporting", "alice", "unknown user"), 15, READ_AT), + ), + ), + ClickHouseRelationUsageTestCase( + description="names keep their case and quotes are escaped", + relations=(("Mart", "Old'Orders"),), + rows=( + ("Mart.Old'Orders", "alice", 1, int(READ_AT.timestamp())), + ("mart.old'orders", "bob", 1, int(READ_AT.timestamp())), + ), + expected_names_sql="IN ('Mart.Old\\'Orders')", + expected_usages=(("Old'Orders", ("alice",), 1, READ_AT),), + ), + ), + ids=lambda case: case.description, +) +def test_given_query_log_when_reading_usage_then_reads_match_candidates( + test_case: ClickHouseRelationUsageTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection(rows=test_case.rows) + + report: RelationUsageReport = read_clickhouse_usage( + connection=connection, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + sql: str + (sql,) = connection.queries + assert report.checked + assert test_case.expected_names_sql in sql + assert tuple( + (usage.name, usage.readers, usage.query_count, usage.last_read_at) + for usage in report.usages + ) == (test_case.expected_usages) + assert all(usage.direct for usage in report.usages) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageQueryTestCase( + description="finished reads and inserts of any table a query touched, once per query", + window_seconds=WINDOW_SECONDS, + expected_sql_fragments=( + "FROM system.query_log ARRAY JOIN tables AS object_name", + "initial_user AS user_name", + "uniqExact(initial_query_id) AS query_count", + "toUnixTimestamp(max(event_time)) AS last_read_at", + f"event_date >= toDate(now() - toIntervalSecond({WINDOW_SECONDS}))", + f"event_time >= now() - toIntervalSecond({WINDOW_SECONDS})", + "type = 'QueryFinish'", + "query_kind IN ('Select', 'Insert')", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_query_log_query_when_built_then_child_queries_count_once( + test_case: ClickHouseRelationUsageQueryTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection() + + read_clickhouse_usage( + connection=connection, relations=ORDERS, window_seconds=test_case.window_seconds + ) + + sql: str + (sql,) = connection.queries + assert all(fragment in sql for fragment in test_case.expected_sql_fragments) + assert "is_initial_query" not in sql + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageChunkTestCase( + description="no candidates read nothing", + relation_count=0, + expected_query_names=(), + ), + ClickHouseRelationUsageChunkTestCase( + description="a large candidate set is split into IN lists of at most 1,000 names", + relation_count=2_001, + expected_query_names=(1_000, 1_000, 1), + ), + ), + ids=lambda case: case.description, +) +def test_given_many_candidates_when_reading_usage_then_names_are_chunked( + test_case: ClickHouseRelationUsageChunkTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection() + + report: RelationUsageReport = read_clickhouse_usage( + connection=connection, + relations=tuple( + usage_relation("mart", f"old_orders_{index:04d}") + for index in range(test_case.relation_count) + ), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple(sql.count("'mart.old_orders_") for sql in connection.queries) == ( + test_case.expected_query_names + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageUncheckedTestCase( + description="no SELECT grant on system.query_log is not checked", + log_queries=True, + errors=FakeQueryLogError.raised( + "Code: 497. DB::Exception: janitor: Not enough privileges. To execute this " + "query, it's necessary to have the grant SELECT for at least one column on " + "system.query_log. (ACCESS_DENIED)" + ), + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + ClickHouseRelationUsageUncheckedTestCase( + description="a server without a query log table is not checked", + log_queries=True, + errors=FakeQueryLogError.raised( + "Code: 60. DB::Exception: Table system.query_log does not exist. (UNKNOWN_TABLE)" + ), + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + ClickHouseRelationUsageUncheckedTestCase( + description="query logging turned off is not checked", + log_queries=False, + errors=(), + expected_report=RelationUsageReport(checked=False, reason=USAGE_LOGGING_OFF_REASON), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_query_log_when_reading_usage_then_usage_is_not_checked( + test_case: ClickHouseRelationUsageUncheckedTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection( + log_queries=test_case.log_queries, + errors=test_case.errors, + ) + + report: RelationUsageReport = read_clickhouse_usage( + connection=connection, relations=ORDERS, window_seconds=WINDOW_SECONDS + ) + + assert report == test_case.expected_report + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageFailureTestCase( + description="a timeout is a real error", + error=FakeQueryLogError( + "Code: 159. DB::Exception: Timeout exceeded. (TIMEOUT_EXCEEDED)" + ), + expected_error_match="TIMEOUT_EXCEEDED", + ), + ), + ids=lambda case: case.description, +) +def test_given_other_query_log_failure_when_reading_usage_then_error_propagates( + test_case: ClickHouseRelationUsageFailureTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection(errors=(test_case.error,)) + + with pytest.raises(FakeQueryLogError, match=test_case.expected_error_match): + read_clickhouse_usage( + connection=connection, relations=ORDERS, window_seconds=WINDOW_SECONDS + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index 09896dfdcb..4f82e53f10 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -157,6 +157,7 @@ class DatabricksRelationUsageTestCase: rows: tuple[tuple[object, ...], ...] expected_object_names: tuple[str, ...] expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + expected_via: tuple[tuple[bool, tuple[str, ...]], ...] = () @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py index 872adb42e8..940370e270 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py @@ -35,10 +35,10 @@ description="read events per user add up and names fold to lowercase", relations=(("Shop", "Mart", "orders_daily"), ("shop", "mart", "customers")), rows=( - ("shop.mart.orders_daily", "svc-reporting@example.com", 10, READ_AT), - ("shop.mart.orders_daily", "alice@example.com", 4, EARLIER_READ_AT), - ("shop.mart.orders_daily", None, 1, EARLIER_READ_AT), - ("shop.mart.products", "bob@example.com", 3, READ_AT), + ("shop.mart.orders_daily", "svc-reporting@example.com", 10, READ_AT, True, None), + ("shop.mart.orders_daily", "alice@example.com", 4, EARLIER_READ_AT, True, None), + ("shop.mart.orders_daily", None, 1, EARLIER_READ_AT, True, None), + ("shop.mart.products", "bob@example.com", 3, READ_AT, True, None), ), expected_object_names=("shop.mart.customers", "shop.mart.orders_daily"), expected_usages=( @@ -49,6 +49,43 @@ READ_AT, ), ), + expected_via=((True, ()),), + ), + DatabricksRelationUsageTestCase( + description="tables read only through views name them, or say a view when unknown", + relations=( + ("shop", "mart", "orders_base"), + ("shop", "mart", "customers"), + ("shop", "mart", "products"), + ), + rows=( + ( + "shop.mart.orders_base", + "alice@example.com", + 3, + READ_AT, + False, + ["shop.mart.orders_v", "shop.mart.orders_w"], + ), + ("shop.mart.customers", "alice@example.com", 2, READ_AT, False, None), + ("shop.mart.products", "alice@example.com", 2, READ_AT, False, None), + ("shop.mart.products", "bob@example.com", 1, EARLIER_READ_AT, True, None), + ), + expected_object_names=( + "shop.mart.customers", + "shop.mart.orders_base", + "shop.mart.products", + ), + expected_usages=( + ("customers", ("alice@example.com",), 2, READ_AT), + ("orders_base", ("alice@example.com",), 3, READ_AT), + ("products", ("alice@example.com", "bob@example.com"), 3, READ_AT), + ), + expected_via=( + (False, ()), + (False, ("shop.mart.orders_v", "shop.mart.orders_w")), + (True, ()), + ), ), DatabricksRelationUsageTestCase( description="a relation without a catalog is not looked up", @@ -79,6 +116,7 @@ def test_given_lineage_when_reading_usage_then_reads_match_candidates( (usage.name, usage.readers, usage.query_count, usage.last_read_at) for usage in report.usages ) == (test_case.expected_usages) + assert tuple((usage.direct, usage.via) for usage in report.usages) == (test_case.expected_via) @pytest.mark.parametrize( @@ -91,11 +129,16 @@ def test_given_lineage_when_reading_usage_then_reads_match_candidates( expected_sql_fragments=( "FROM system.access.table_lineage", "lower(source_table_full_name) AS object_name", - "created_by AS user_name", - "COUNT(DISTINCT COALESCE(statement_id, event_id)) AS query_count", - "MAX(event_time) AS last_read_at", + "COALESCE(direct_access, true) AS direct", + "reads.created_by AS user_name", + "COUNT(DISTINCT COALESCE(reads.statement_id, reads.event_id)) AS query_count", + "MAX(reads.event_time) AS last_read_at", "event_date >= DATE(TIMESTAMPADD(SECOND, -:window_seconds, current_timestamp()))", - "lower(source_table_full_name) IN (:name_0)", + "WHERE object_name IN (:name_0)", + "JOIN lineage AS views ON views.statement_id = reads.statement_id " + "AND views.direct AND views.source_type IN " + "('VIEW', 'MATERIALIZED_VIEW', 'METRIC_VIEW')", + "bool_or(reads.direct) AS direct", ), ), ), diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index b3a653a125..32bbf9e920 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -343,3 +343,20 @@ class UnsupportedRelationUsageTestCase: description: str adapter: BaseAdapter expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class SnowflakeRelationUsageViaTestCase: + description: str + relations: tuple[tuple[str, str, str, str], ...] + access_rows: tuple[tuple[object, ...], ...] + dependency_rows: tuple[tuple[str, str], ...] + expected_dependency_params: tuple[tuple[object, ...], ...] + expected_usages: tuple[tuple[str, tuple[str, ...], int, bool, tuple[str, ...]], ...] + + +@dataclass(frozen=True) +class SnowflakeCloneStatementTestCase: + description: str + statement: str + expected_clone: bool diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 1ef7659c29..dea05da95c 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -414,12 +414,15 @@ def __init__( *, current_database: str | None = None, access_rows: tuple[tuple[object, ...], ...] = (), + dependency_rows: tuple[tuple[str, str], ...] = (), errors: tuple[Exception, ...] = (), ) -> None: self.current_database: str | None = current_database self.access_rows: tuple[tuple[object, ...], ...] = access_rows + self.dependency_rows: tuple[tuple[str, str], ...] = dependency_rows self.errors: tuple[Exception, ...] = errors self.queries: list[tuple[str, tuple[object, ...]]] = [] + self.dependency_queries: list[tuple[str, tuple[object, ...]]] = [] def current_database_rows(self) -> list[tuple[object, ...]]: return [(self.current_database,)] @@ -432,6 +435,13 @@ def access_history_rows(self, sql: str, params: tuple[object, ...]) -> list[tupl names: frozenset[object] = frozenset(params) return list(filter(lambda row: row[0] in names, self.access_rows)) + def object_dependency_rows( + self, sql: str, params: tuple[object, ...] + ) -> list[tuple[object, ...]]: + self.dependency_queries.append((sql, params)) + names: frozenset[object] = frozenset(params) + return list(filter(lambda row: row[0] in names, self.dependency_rows)) + class FakeUsageCursor: """Cursor double routing each statement to the fake usage warehouse.""" @@ -441,11 +451,12 @@ def __init__(self, warehouse: FakeUsageWarehouse) -> None: self.rows: list[tuple[object, ...]] = [] def execute(self, sql: str, params: tuple[object, ...] = ()) -> None: - reads: dict[bool, Callable[[], list[tuple[object, ...]]]] = { - True: self.warehouse.current_database_rows, - False: lambda: self.warehouse.access_history_rows(sql, params), + reads: dict[tuple[bool, bool], Callable[[], list[tuple[object, ...]]]] = { + (True, False): self.warehouse.current_database_rows, + (False, True): lambda: self.warehouse.object_dependency_rows(sql, params), + (False, False): lambda: self.warehouse.access_history_rows(sql, params), } - self.rows = reads[sql == "SELECT CURRENT_DATABASE()"]() + self.rows = reads[(sql == "SELECT CURRENT_DATABASE()", "OBJECT_DEPENDENCIES" in sql)]() def fetchone(self) -> tuple[object, ...] | None: return next(iter(self.rows), None) @@ -467,10 +478,20 @@ def cursor(self) -> FakeUsageCursor: return FakeUsageCursor(self.warehouse) -def usage_relation(database: str | None, schema: str, name: str) -> RelationInfo: - """A base table relation for usage reads.""" +def usage_relation( + database: str | None, schema: str, name: str, relation_type: str = "base table" +) -> RelationInfo: + """A relation for usage reads, a base table unless given another type.""" + + return RelationInfo(database=database, schema=schema, name=name, relation_type=relation_type) + + +def direct_read( + object_name: str, user: str, query_count: int, read_at: object +) -> tuple[object, ...]: + """An access-history result row for direct reads.""" - return RelationInfo(database=database, schema=schema, name=name, relation_type="base table") + return (object_name, user, query_count, read_at, True, None) def read_snowflake_usage( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py index a602a5dcd6..b9f434c37a 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from datetime import UTC, datetime import pytest @@ -15,16 +16,19 @@ USAGE_UNAVAILABLE_REASON, ) from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeCloneStatementTestCase, SnowflakeRelationUsageChunkTestCase, SnowflakeRelationUsageErrorTestCase, SnowflakeRelationUsageFailureTestCase, SnowflakeRelationUsageQueryTestCase, SnowflakeRelationUsageTestCase, + SnowflakeRelationUsageViaTestCase, UnsupportedRelationUsageTestCase, ) from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( FakeAccessHistoryError, FakeUsageWarehouse, + direct_read, read_snowflake_usage, usage_relation, ) @@ -43,9 +47,9 @@ relations=(("analytics", "mart", "orders_daily"), ("analytics", "mart", "customers")), current_database=None, access_rows=( - ("ANALYTICS.MART.ORDERS_DAILY", "SVC_REPORTING", 10, READ_AT), - ("ANALYTICS.MART.ORDERS_DAILY", "ALICE", 4, EARLIER_READ_AT), - ("ANALYTICS.MART.PRODUCTS", "BOB", 3, READ_AT), + direct_read("ANALYTICS.MART.ORDERS_DAILY", "SVC_REPORTING", 10, READ_AT), + direct_read("ANALYTICS.MART.ORDERS_DAILY", "ALICE", 4, EARLIER_READ_AT), + direct_read("ANALYTICS.MART.PRODUCTS", "BOB", 3, READ_AT), ), expected_object_names=("ANALYTICS.MART.CUSTOMERS", "ANALYTICS.MART.ORDERS_DAILY"), expected_usages=(("orders_daily", ("SVC_REPORTING", "ALICE"), 14, READ_AT),), @@ -54,7 +58,7 @@ description="unqualified relations use the session database and names fold upper", relations=((None, "mart", "orders_daily"),), current_database="SHOP_DB", - access_rows=(("SHOP_DB.MART.ORDERS_DAILY", "SVC_REPORTING", 2, READ_AT),), + access_rows=(direct_read("SHOP_DB.MART.ORDERS_DAILY", "SVC_REPORTING", 2, READ_AT),), expected_object_names=("SHOP_DB.MART.ORDERS_DAILY",), expected_usages=(("orders_daily", ("SVC_REPORTING",), 2, READ_AT),), ), @@ -62,7 +66,7 @@ description="names that need quoting match the quoted object name, case kept upper", relations=(("analytics", "mart", "order-lines"),), current_database=None, - access_rows=(('ANALYTICS.MART."ORDER-LINES"', "ALICE", 1, READ_AT),), + access_rows=(direct_read('ANALYTICS.MART."ORDER-LINES"', "ALICE", 1, READ_AT),), expected_object_names=('ANALYTICS.MART."ORDER-LINES"',), expected_usages=(("order-lines", ("ALICE",), 1, READ_AT),), ), @@ -70,7 +74,7 @@ description="a mixed-case reader object does not match an unquoted relation", relations=(("analytics", "mart", "orders_daily"),), current_database=None, - access_rows=(('ANALYTICS.MART."orders_daily"', "ALICE", 1, READ_AT),), + access_rows=(direct_read('ANALYTICS.MART."orders_daily"', "ALICE", 1, READ_AT),), expected_object_names=("ANALYTICS.MART.ORDERS_DAILY",), expected_usages=(), ), @@ -109,17 +113,22 @@ def test_given_access_history_when_reading_usage_then_reads_match_candidates( expected_params=( WINDOW_SECONDS, "ANALYTICS.MART.ORDERS_DAILY", + "ANALYTICS.MART.ORDERS_DAILY", WINDOW_SECONDS, '"app":"sqlbuild"', - r".*\sCLONE\s.*", + CLONE_STATEMENT_PATTERN, ), expected_sql_fragments=( - "FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY AS history, " + "FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY", "LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed", + "LATERAL FLATTEN(INPUT => history.base_objects_accessed) AS accessed", + "WHERE direct_read.query_id = base.query_id " + "AND direct_read.object_name = base.object_name", "NOT EXISTS (SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY AS query", "CONTAINS(COALESCE(query.query_tag, ''), %s)", "STARTSWITH(query.query_type, 'CREATE') AND REGEXP_LIKE(query.query_text, %s", - "COUNT(DISTINCT history.query_id) AS query_count", + "COUNT(DISTINCT reads.query_id) AS query_count", + "BOOLOR_AGG(reads.direct) AS direct", ), ), ), @@ -138,10 +147,176 @@ def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_ params: tuple[object, ...] ((sql, params),) = warehouse.queries assert params == test_case.expected_params - assert (SQLBUILD_QUERY_TAG_FRAGMENT, CLONE_STATEMENT_PATTERN) == (test_case.expected_params[3:]) + assert (SQLBUILD_QUERY_TAG_FRAGMENT, CLONE_STATEMENT_PATTERN) == (test_case.expected_params[4:]) assert all(fragment in sql for fragment in test_case.expected_sql_fragments) +VIEW_DOMAINS: tuple[str, ...] = ("VIEW", "MATERIALIZED VIEW", "SECURE VIEW") + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageViaTestCase( + description="a table read only through a view names the view; any direct read wins", + relations=( + ("analytics", "mart", "orders_base", "base table"), + ("analytics", "mart", "customers", "base table"), + ), + access_rows=( + ( + "ANALYTICS.MART.ORDERS_BASE", + "ALICE", + 3, + READ_AT, + False, + '[\n "ANALYTICS.MART.ORDERS_V",\n "ANALYTICS.MART.ORDERS_W"\n]', + ), + ("ANALYTICS.MART.CUSTOMERS", "ALICE", 2, READ_AT, False, '["ANALYTICS.MART.C_V"]'), + direct_read("ANALYTICS.MART.CUSTOMERS", "BOB", 1, EARLIER_READ_AT), + ), + dependency_rows=(), + expected_dependency_params=(), + expected_usages=( + ("customers", ("ALICE", "BOB"), 3, True, ()), + ( + "orders_base", + ("ALICE",), + 3, + False, + ("analytics.mart.orders_v", "analytics.mart.orders_w"), + ), + ), + ), + SnowflakeRelationUsageViaTestCase( + description="candidates behind an in-use candidate view are in use through it", + relations=( + ("analytics", "mart", "orders_v", "view"), + ("analytics", "mart", "orders_v2", "view"), + ("analytics", "mart", "orders", "base table"), + ("analytics", "mart", "unread_v", "view"), + ), + access_rows=( + direct_read("ANALYTICS.MART.ORDERS_V", "ALICE", 5, READ_AT), + ( + "ANALYTICS.MART.ORDERS", + "ALICE", + 5, + READ_AT, + False, + '["ANALYTICS.MART.ORDERS_V"]', + ), + ), + dependency_rows=( + ("ANALYTICS.MART.ORDERS_V", "ANALYTICS.MART.ORDERS_V2"), + ("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.ORDERS"), + ("ANALYTICS.MART.UNREAD_V", "ANALYTICS.MART.ORDERS"), + ("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.NOT_A_CANDIDATE"), + ), + expected_dependency_params=( + ( + *VIEW_DOMAINS, + "ANALYTICS.MART.ORDERS_V", + "ANALYTICS.MART.ORDERS_V2", + "ANALYTICS.MART.UNREAD_V", + ), + ), + expected_usages=( + ("orders", ("ALICE",), 5, False, ("analytics.mart.orders_v",)), + ("orders_v", ("ALICE",), 5, True, ()), + ("orders_v2", ("ALICE",), 5, False, ("analytics.mart.orders_v",)), + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_view_reads_when_reading_usage_then_indirect_reads_name_their_views( + test_case: SnowflakeRelationUsageViaTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + access_rows=test_case.access_rows, dependency_rows=test_case.dependency_rows + ) + + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple(params for _, params in warehouse.dependency_queries) == ( + test_case.expected_dependency_params + ) + assert all( + "referencing_object_domain IN (%s, %s, %s)" in sql + for sql, _ in warehouse.dependency_queries + ) + assert tuple( + sorted( + (usage.name, usage.readers, usage.query_count, usage.direct, usage.via) + for usage in report.usages + ) + ) == (test_case.expected_usages) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeCloneStatementTestCase( + description="a table clone", + statement="CREATE TABLE analytics.mart.orders_copy CLONE analytics.mart.orders", + expected_clone=True, + ), + SnowflakeCloneStatementTestCase( + description="a replaced transient clone at a point in time", + statement="create or replace transient table orders_copy clone orders " + "at(offset => -60);", + expected_clone=True, + ), + SnowflakeCloneStatementTestCase( + description="a schema clone after leading comments", + statement="-- nightly copy\n/* keep */\nCREATE SCHEMA mart_copy CLONE mart", + expected_clone=True, + ), + SnowflakeCloneStatementTestCase( + description="quoted names and a dynamic table", + statement='CREATE DYNAMIC TABLE "Orders Copy" CLONE "Orders"', + expected_clone=True, + ), + SnowflakeCloneStatementTestCase( + description="a CTAS selecting a column named clone", + statement="CREATE TABLE orders_copy AS SELECT clone FROM orders", + expected_clone=False, + ), + SnowflakeCloneStatementTestCase( + description="a view whose comment mentions a clone", + statement="CREATE VIEW orders_v AS SELECT 1 AS clone_id -- see a CLONE b", + expected_clone=False, + ), + SnowflakeCloneStatementTestCase( + description="a CTAS with clone inside a string", + statement="CREATE TABLE notes AS SELECT 'orders CLONE copy' AS note", + expected_clone=False, + ), + SnowflakeCloneStatementTestCase( + description="a select with a clone statement in a comment", + statement="SELECT * FROM orders -- CREATE TABLE a CLONE b", + expected_clone=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_statement_when_matching_clone_pattern_then_only_clones_match( + test_case: SnowflakeCloneStatementTestCase, +) -> None: + matched: bool = ( + re.fullmatch(CLONE_STATEMENT_PATTERN, test_case.statement, re.IGNORECASE | re.DOTALL) + is not None + ) + + assert matched == test_case.expected_clone + + @pytest.mark.parametrize( "test_case", ( @@ -173,7 +348,7 @@ def test_given_many_candidates_when_reading_usage_then_names_are_chunked( ) assert report.checked - assert tuple(len(params) - 4 for _, params in warehouse.queries) == ( + assert tuple((len(params) - 4) // 2 for _, params in warehouse.queries) == ( test_case.expected_query_names ) assert all(sql.count("%s") == len(params) for sql, params in warehouse.queries) diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index bcd3c24495..8cbe7cc171 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -65,9 +65,10 @@ Archiving renames a relation, so anything that still reads it fails from then on In use analytics.orders_daily queried 2026-10-21 by svc_reporting, alice (14 queries in 7 days) analytics.old_orders queried 2026-10-20 by svc_reporting, alice, bob +2 more (40 queries in 7 days); frozen since 2026-09-01, still in use + analytics.old_customers queried 2026-10-21 by alice via analytics.customer_report (3 queries in 7 days); frozen since 2026-09-01, still in use ``` -Each line names the latest read, up to three readers (most queries first) and how many queries read the relation. A stale table that is still read is also flagged `frozen since , still in use`: its data stopped changing on that date, so its readers are getting old data. Views are not flagged, because their reads stay live. The confirmation prompt states the count, for example `14 relations will be archived; 3 of them are in use.` +Each line names the latest read, up to three readers (most queries first) and how many queries read the relation. Reading a view also reads what it is built on, so those relations are in use too. When a relation was only read that way, the line says `via` and names the views, or says `through a view` when the warehouse doesn't record which. A stale table that is still read is also flagged `frozen since , still in use`: its data stopped changing on that date, so its readers are getting old data. Views are not flagged, because their reads stay live. The confirmation prompt states the count, for example `14 relations will be archived; 3 of them are in use.` By default in-use relations are still archived, which is also how their readers find out. To keep them instead, set `keep_in_use = true` under `[janitor]` or pass `--keep-in-use`. In-use relations are then listed under `Kept (in use)` with their readers and are left out of the archive list and the counts. A later run archives them once nothing reads them within the window. @@ -77,22 +78,23 @@ Usage comes from the warehouse's query history: | Adapter | Usage source | |---------|--------------| -| Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`) joined with `QUERY_HISTORY` | -| BigQuery | `` `.region-`.INFORMATION_SCHEMA.JOBS `` (`referenced_tables`) | -| Databricks | `system.access.table_lineage` (`source_table_full_name`) | +| Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`, `base_objects_accessed`) joined with `QUERY_HISTORY`, and `OBJECT_DEPENDENCIES` | +| BigQuery | `` `.region-`.INFORMATION_SCHEMA.JOBS `` (`referenced_tables`, `referenced_logical_views`) | +| Databricks | `system.access.table_lineage` (`source_table_full_name`, `direct_access`) | +| ClickHouse | `system.query_log` (`tables`) on the connected server | | Other adapters | Not available | -On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements don't count as reads. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). A candidate listed in a query's base objects but not its direct objects was read through the objects the query named, which the line names after `via`. A second query reads `OBJECT_DEPENDENCIES` for the candidate views, so a candidate that an in-use candidate view is built on, directly or through other candidate views, is in use `via` that view, with its readers. Dependencies created by `CREATE TABLE ... AS SELECT` and similar statements are not recorded there. Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements of tables, dynamic tables, schemas and databases don't count as reads. A statement counts as a clone only when it starts with `CREATE`, after any comments, and names its source after `CLONE`; a column, string or comment containing the word doesn't make it one. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: ```sql GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; ``` -Access history lags by up to about 3 hours, so reads in the last few hours may not show yet. +Access history and object dependencies lag by up to about 3 hours, so reads in the last few hours may not show yet. -On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the table, by `user_email`. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. +On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the relation, by `user_email`. A job lists the views it reads as well as the tables behind them, so both count. Whether views nested inside other views are listed is not documented, so a candidate view that is only read through another view may not show as in use. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. -On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`; a read through a view also counts for the tables behind it. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: +On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`. Lineage records the view a query reads as a direct read and the tables behind it as reads through the view, which the line names when lineage records the view for the same statement. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: ```sql GRANT USE CATALOG ON CATALOG system TO `janitor-principal`; @@ -100,11 +102,19 @@ GRANT USE SCHEMA ON SCHEMA system.access TO `janitor-principal`; GRANT SELECT ON TABLE system.access.table_lineage TO `janitor-principal`; ``` -Lineage records only reads Databricks can capture lineage for, and can arrive with a delay. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. +Lineage records only reads Databricks can infer lineage for, and can arrive with a delay, so "not in use" is a weaker signal than on Snowflake. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. + +On ClickHouse, the janitor counts finished `SELECT` and `INSERT` queries in `system.query_log` of the server it connects to, by the user who started them. A query's `tables` lists every table and view it touched, including the views and tables a view is built on, so a whole view chain shows as in use, without `via`; queries a query starts count once, for its initial query. Queries on other replicas or shards are logged on those servers and are not seen. ClickHouse answers some queries, such as a plain `count()` of a table, from metadata without listing the table, so those reads are not seen either. SQLBuild doesn't mark its ClickHouse queries, so its own reads count. The janitor's user needs `SELECT` on `system.query_log`: + +```sql +GRANT SELECT ON system.query_log TO janitor_user; +``` + +Queries are logged when `log_queries` is on, which is the default; when it is off for the janitor's connection, usage is not checked. The log is flushed about every 7.5 seconds, and ClickHouse keeps it until a TTL configured on the server removes it. When usage can't be checked, because the adapter has no usage source or the role can't read its query history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. -In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers` and `query_count`. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. +In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers`, `query_count` and `via` (`null` when the relation was read directly, otherwise the names of the views it was read through, empty when unknown). Entries of `kept_in_use` have only `name`, `age_days` and `in_use`, since they are not archived. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. ## Speed From c3b0a8078b8d7b2ebd6b17bd455add373a38a31a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:07:03 +0100 Subject: [PATCH 12/14] style: format janitor JSON --- .../sqlbuild-core/src/janitor_text/_helpers/plan_json.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs index 3bfd6a2560..c1803cbff0 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -75,10 +75,7 @@ fn in_use(candidate: &ArchiveCandidateText, checked: bool) -> JsonValue { ("last_read", utc_timestamp(&usage.last_read_at)), ("readers", texts(&usage.readers)), ("query_count", integer(usage.query_count)), - ( - "via", - usage.via.as_deref().map_or(JsonValue::Null, texts), - ), + ("via", usage.via.as_deref().map_or(JsonValue::Null, texts)), ]) }) } @@ -183,9 +180,7 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { ), ( "kept_in_use", - JsonValue::Array( - plan.kept_in_use.iter().map(kept_candidate).collect(), - ), + JsonValue::Array(plan.kept_in_use.iter().map(kept_candidate).collect()), ), ( "archives_to_delete", From 4560c674b1ba7e40349d678675b46fb300191a0e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:44:40 +0100 Subject: [PATCH 13/14] fix: follow Snowflake view chains from read objects and harden janitor usage sources --- .../src/janitor_text/_helpers/plan_json.rs | 1 + .../src/janitor_text/_helpers/usage.rs | 7 +- .../sqlbuild-core/src/janitor_text/models.rs | 2 + .../src/janitor_text/tests/helpers.rs | 18 +++ .../tests/test_janitor_usage_text.rs | 32 +++- .../bindings/_helpers/janitor/janitor_plan.rs | 1 + .../_helpers/janitor/janitor_usage.rs | 1 + .../sqlbuild/references/docs/cli/janitor.md | 13 +- src/sqlbuild/adapter/contract/models.py | 1 + .../bigquery/_helpers/relation_usage.py | 37 ++++- src/sqlbuild/adapters/bigquery/constants.py | 13 +- .../clickhouse/_helpers/relation_usage.py | 16 ++ src/sqlbuild/adapters/clickhouse/constants.py | 7 + src/sqlbuild/adapters/databricks/constants.py | 3 +- .../snowflake/_helpers/relation_usage.py | 41 +++-- .../snowflake/_helpers/view_dependencies.py | 149 ++++++++++++------ src/sqlbuild/adapters/snowflake/constants.py | 7 +- src/sqlbuild/executor/janitor/models.py | 1 + .../main/janitor/test_janitor_usage.py | 2 + .../src/sqlbuild/adapters/bigquery/helpers.py | 3 +- .../adapters/bigquery/test_relation_usage.py | 32 +++- .../adapters/clickhouse/_test_types.py | 7 + .../sqlbuild/adapters/clickhouse/helpers.py | 11 +- .../clickhouse/test_relation_usage.py | 30 ++++ .../databricks/test_relation_usage.py | 14 +- .../adapters/snowflake/_test_types.py | 9 ++ .../sqlbuild/adapters/snowflake/helpers.py | 9 +- .../adapters/snowflake/test_relation_usage.py | 86 ++++++++-- website/src/content/docs/docs/cli/janitor.mdx | 13 +- 29 files changed, 454 insertions(+), 112 deletions(-) diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs index c1803cbff0..255977ef9a 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -117,6 +117,7 @@ pub(crate) fn usage_check_payload(plan: Option<&JanitorPlanText>) -> JsonValue { ("reason", optional_text(check.reason.as_deref())), ("window", text(&check.window)), ("keep_in_use", JsonValue::Bool(check.keep_in_use)), + ("notices", texts(&check.notices)), ]) } diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs index cc2af2d571..5ea862ca81 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs @@ -73,11 +73,14 @@ pub(crate) fn usage_detail(candidate: &ArchiveCandidateText, window: &str) -> Op Some(text) } -/// The notice lines when usage could not be checked, empty otherwise. +/// The notice lines when usage could not be checked, or what a check that ran left out. pub(crate) fn usage_notices(plan: &JanitorPlanText) -> Vec { - let Some(check) = plan.usage_check.as_ref().filter(|check| !check.checked) else { + let Some(check) = plan.usage_check.as_ref() else { return Vec::new(); }; + if check.checked { + return check.notices.clone(); + } let mut notices: Vec = vec![format!( "{USAGE_NOT_CHECKED_PREFIX}{}", check.reason.as_deref().unwrap_or("unknown reason") diff --git a/crates/sqlbuild-core/src/janitor_text/models.rs b/crates/sqlbuild-core/src/janitor_text/models.rs index ad05dd346e..6011b794dd 100644 --- a/crates/sqlbuild-core/src/janitor_text/models.rs +++ b/crates/sqlbuild-core/src/janitor_text/models.rs @@ -41,6 +41,8 @@ pub struct UsageCheckText { pub window: String, /// Whether in-use relations are kept instead of archived. pub keep_in_use: bool, + /// What a check that ran could not cover, such as view chains it could not follow. + pub notices: Vec, } /// An archived relation to delete or retain. diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index 049dc70f98..a1d63fe2c1 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -118,6 +118,8 @@ fn usage_candidates() -> [ArchiveCandidateText; 3] { ] } +const CHAINS_NOT_FOLLOWED: &str = "view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER"; + /// A plan for `dev` whose usage was checked over `7d`. fn checked_plan(keep_in_use: bool) -> JanitorPlanText { JanitorPlanText { @@ -130,6 +132,7 @@ fn checked_plan(keep_in_use: bool) -> JanitorPlanText { reason: None, window: "7d".to_owned(), keep_in_use, + ..UsageCheckText::default() }), ..JanitorPlanText::default() } @@ -174,6 +177,20 @@ pub(super) fn via_usage_plan() -> JanitorPlanText { } } +/// A checked plan whose usage source could not follow view chains. +pub(super) fn chains_unfollowed_plan() -> JanitorPlanText { + JanitorPlanText { + archive_candidates: usage_candidates().to_vec(), + usage_check: Some(UsageCheckText { + checked: true, + window: "7d".to_owned(), + notices: vec![CHAINS_NOT_FOLLOWED.to_owned()], + ..UsageCheckText::default() + }), + ..checked_plan(false) + } +} + /// A checked plan keeping the two in-use candidates and archiving the unread one. pub(super) fn kept_usage_plan() -> JanitorPlanText { let [view, table, unread] = usage_candidates(); @@ -192,6 +209,7 @@ pub(super) fn unchecked_plan(keep_in_use: bool) -> JanitorPlanText { reason: Some("unsupported adapter".to_owned()), window: "7d".to_owned(), keep_in_use, + ..UsageCheckText::default() }), ..direct_archive_plan() } diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs index 04a1748f9c..23a2260f8a 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs @@ -5,7 +5,8 @@ use crate::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation use crate::janitor_text::main::render_janitor_plan::render_janitor_plan; use crate::janitor_text::models::JanitorOutcome; use crate::janitor_text::tests::helpers::{ - direct_archive_plan, kept_usage_plan, unchecked_plan, usage_plan, via_usage_plan, + chains_unfollowed_plan, direct_archive_plan, kept_usage_plan, unchecked_plan, usage_plan, + via_usage_plan, }; use crate::janitor_text::tests::test_types::{JsonTestCase, PreviewTestCase, UsagePromptTestCase}; @@ -57,6 +58,16 @@ fn given_usage_check_when_rendering_preview_then_in_use_relations_show_readers() ], unexpected_fragments: &["dev.orders_yearly"], }, + PreviewTestCase { + description: "a check that could not follow view chains says so once", + plan: chains_unfollowed_plan(), + verbose: false, + expected_fragments: &[ + "\n view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER\n", + "\nIn use\n", + ], + unexpected_fragments: &["usage not checked", "nothing kept in use"], + }, PreviewTestCase { description: "an unchecked usage shows one notice and no in-use rows", plan: unchecked_plan(false), @@ -114,6 +125,13 @@ fn given_usage_check_when_prompting_then_prompt_states_in_use_count() { 3 relations will be archived; 2 of them are in use.\nRetention: 14 days\n", expected_phrase: "archive 3 and delete 0 objects from dev", }, + UsagePromptTestCase { + description: "a check that could not follow view chains repeats its notice", + plan: chains_unfollowed_plan(), + expected_fragment: "2 of them are in use.\nView chains not followed: grant \ + SNOWFLAKE.OBJECT_VIEWER\nRetention: 14 days\n", + expected_phrase: "archive 3 and delete 0 objects from dev", + }, UsagePromptTestCase { description: "keep_in_use counts the kept relations and leaves them out of the phrase", plan: kept_usage_plan(), @@ -169,7 +187,7 @@ fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { "\"in_use\": false", "\"kept_in_use\": []", "\"usage_check\": {\n \"status\": \"checked\",\n \"reason\": null,\n \ - \"window\": \"7d\",\n \"keep_in_use\": false\n }", + \"window\": \"7d\",\n \"keep_in_use\": false,\n \"notices\": []\n }", ], }, JsonTestCase { @@ -184,6 +202,16 @@ fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { "\"keep_in_use\": true", ], }, + JsonTestCase { + description: "notices of a check that ran are listed in usage_check", + plan: Some(chains_unfollowed_plan()), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &[ + "\"notices\": [\n \"view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER\"\n ]", + ], + }, JsonTestCase { description: "an unchecked usage leaves in_use null and states the reason", plan: Some(unchecked_plan(false)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs index 74850b1fdb..36f184cd43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs @@ -187,6 +187,7 @@ fn usage_check(check: &Bound<'_, PyAny>) -> PyResult> { reason: native_field(check, "reason")?.extract()?, window: native_field(check, "window")?.extract()?, keep_in_use: native_field(check, "keep_in_use")?.is_truthy()?, + notices: native_field(check, "notices")?.extract()?, })) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs index cbddc2f3e1..bf66ebde19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs @@ -177,6 +177,7 @@ pub(crate) fn check_janitor_usage<'py>( ("window", text(py, &settings.window)), ("keep_in_use", flag(py, settings.keep_in_use)), ("reason", reason), + ("notices", native_field(&report, "notices")?), ], )?; call_with( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index cf2a5094b3..02bf74b673 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -104,15 +104,16 @@ Usage comes from the warehouse's query history: | ClickHouse | `system.query_log` (`tables`) on the connected server | | Other adapters | Not available | -On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). A candidate listed in a query's base objects but not its direct objects was read through the objects the query named, which the line names after `via`. A second query reads `OBJECT_DEPENDENCIES` for the candidate views, so a candidate that an in-use candidate view is built on, directly or through other candidate views, is in use `via` that view, with its readers. Dependencies created by `CREATE TABLE ... AS SELECT` and similar statements are not recorded there. Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements of tables, dynamic tables, schemas and databases don't count as reads. A statement counts as a clone only when it starts with `CREATE`, after any comments, and names its source after `CLONE`; a column, string or comment containing the word doesn't make it one. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). A candidate listed in a query's base objects but not its direct objects was read through the objects the query named, which the line names after `via`. When something was read, a second query reads the view dependencies in `OBJECT_DEPENDENCIES` for the databases of the candidates and of the objects queries read directly. Anything those directly read objects are built on, through any number of views, is in use: an unread candidate reached that way is in use `via` the directly read object, with its readers. So a candidate view between a non-candidate view someone reads and a candidate table is kept too. Links through views in other databases are not followed. Dependencies created by `CREATE TABLE ... AS SELECT` and similar statements are not recorded there. Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements of tables, dynamic tables, schemas and databases don't count as reads. A statement counts as a clone only when it starts with `CREATE`, after any comments, and names its source after `CLONE`; a column, string or comment containing the word doesn't make it one. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition. The janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role for access history and the `SNOWFLAKE.OBJECT_VIEWER` database role for object dependencies: ```sql GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; +GRANT DATABASE ROLE SNOWFLAKE.OBJECT_VIEWER TO ROLE janitor_role; ``` -Access history and object dependencies lag by up to about 3 hours, so reads in the last few hours may not show yet. +Without `OBJECT_VIEWER`, reads are still checked, but view chains are not followed and the preview says `view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER`. Access history and object dependencies lag by up to about 3 hours, so reads in the last few hours may not show yet. -On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the relation, by `user_email`. A job lists the views it reads as well as the tables behind them, so both count. Whether views nested inside other views are listed is not documented, so a candidate view that is only read through another view may not show as in use. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. +On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the relation, by `user_email`. A job lists the views it reads as well as the tables behind them, so both count. Whether views nested inside other views are listed is not documented, so a candidate view that is only read through another view may not show as in use. Jobs answered from cached results record no tables or views, so with an `in_use_window` under about a day a reader whose queries were all cached can be missed. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`. Lineage records the view a query reads as a direct read and the tables behind it as reads through the view, which the line names when lineage records the view for the same statement. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: @@ -124,17 +125,17 @@ GRANT SELECT ON TABLE system.access.table_lineage TO `janitor-principal`; Lineage records only reads Databricks can infer lineage for, and can arrive with a delay, so "not in use" is a weaker signal than on Snowflake. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. -On ClickHouse, the janitor counts finished `SELECT` and `INSERT` queries in `system.query_log` of the server it connects to, by the user who started them. A query's `tables` lists every table and view it touched, including the views and tables a view is built on, so a whole view chain shows as in use, without `via`; queries a query starts count once, for its initial query. Queries on other replicas or shards are logged on those servers and are not seen. ClickHouse answers some queries, such as a plain `count()` of a table, from metadata without listing the table, so those reads are not seen either. SQLBuild doesn't mark its ClickHouse queries, so its own reads count. The janitor's user needs `SELECT` on `system.query_log`: +On ClickHouse, the janitor counts finished `SELECT` and `INSERT` queries in `system.query_log` of the server it connects to, by the user who started them. A query's `tables` lists every table and view it touched, including the views and tables a view is built on, so a whole view chain shows as in use, without `via`; queries a query starts count once, for its initial query. Only the connected server's `query_log` is read: queries on other replicas or shards are logged on those servers and are not seen, which matters on ClickHouse Cloud and behind load-balanced replicas. When `system.clusters` shows other replicas of the connected server's shard, the preview says `usage read from this server only; other replicas not checked`. Writes into a relation count as use, since renaming it breaks the writer too. ClickHouse answers some queries, such as a plain `count()` of a table, from metadata without logging the table, so those reads are not seen. SQLBuild doesn't mark its ClickHouse queries, so its own reads count. The janitor's user needs `SELECT` on `system.query_log`: ```sql GRANT SELECT ON system.query_log TO janitor_user; ``` -Queries are logged when `log_queries` is on, which is the default; when it is off for the janitor's connection, usage is not checked. The log is flushed about every 7.5 seconds, and ClickHouse keeps it until a TTL configured on the server removes it. +Queries are logged when `log_queries` is on, which is the default; when it is off for the janitor's connection, usage is not checked. Readers whose settings profile turns `log_queries` off, or logs only a sample with `log_queries_probability`, are missed. The log is flushed about every 7.5 seconds, and ClickHouse keeps it until a TTL configured on the server removes it. When usage can't be checked, because the adapter has no usage source or the role can't read its query history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. -In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers`, `query_count` and `via` (`null` when the relation was read directly, otherwise the names of the views it was read through, empty when unknown). Entries of `kept_in_use` have only `name`, `age_days` and `in_use`, since they are not archived. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. +In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers`, `query_count` and `via` (`null` when the relation was read directly, otherwise the names of the views it was read through, empty when unknown). Entries of `kept_in_use` have only `name`, `age_days` and `in_use`, since they are not archived. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window`, `keep_in_use` and `notices` (what a check that ran left out, such as view chains it could not follow), or is `null` when there was nothing to check. ## Speed diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 1cc6e286cf..a66147d3ad 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -133,6 +133,7 @@ class RelationUsageReport: checked: bool reason: str | None = None usages: tuple[RelationUsage, ...] = () + notices: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py index e8c4d9351b..0fbd08c0a1 100644 --- a/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py +++ b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py @@ -10,9 +10,12 @@ from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport from sqlbuild.adapters.bigquery.constants import ( CLONE_STATEMENT_PATTERN, + FORBIDDEN_ERROR_CLASS_NAMES, + FORBIDDEN_STATUS, + NOT_FOUND_ERROR_CLASS_NAMES, + NOT_FOUND_STATUS, USAGE_NAME_CHUNK_SIZE, - USAGE_UNAVAILABLE_ERROR_CLASS_NAMES, - USAGE_UNAVAILABLE_ERROR_CODES, + USAGE_UNAVAILABLE_ERROR_REASONS, USAGE_UNAVAILABLE_MESSAGES, USAGE_UNAVAILABLE_REASON, ) @@ -124,14 +127,36 @@ def query_parameters(query: UsageQuery) -> list[Any]: def is_usage_unavailable_error(error: Exception) -> bool: """Whether reading JOBS failed because the view or the permission to list jobs is missing.""" + reasons: frozenset[str] = error_reasons(error) + if reasons: + return reasons <= USAGE_UNAVAILABLE_ERROR_REASONS + code: object = getattr(error, "code", None) + name: str = type(error).__name__ + if code == NOT_FOUND_STATUS or name in NOT_FOUND_ERROR_CLASS_NAMES: + return True message: str = str(error).lower() - return ( - getattr(error, "code", None) in USAGE_UNAVAILABLE_ERROR_CODES - or type(error).__name__ in USAGE_UNAVAILABLE_ERROR_CLASS_NAMES - or any(marker in message for marker in USAGE_UNAVAILABLE_MESSAGES) + return (code == FORBIDDEN_STATUS or name in FORBIDDEN_ERROR_CLASS_NAMES) and any( + marker in message for marker in USAGE_UNAVAILABLE_MESSAGES ) +def error_reasons(error: Exception) -> frozenset[str]: + """The `reason` of each error detail Google reports, such as `accessDenied`.""" + + details: object = getattr(error, "errors", None) + reasons: set[str] = set() + if isinstance(details, list): + reasons.update( + str(detail["reason"]) + for detail in details + if isinstance(detail, dict) and detail.get("reason") + ) + reason: object = getattr(error, "reason", None) + if isinstance(reason, str) and reason: + reasons.add(reason) + return frozenset(reasons) + + def relation_project(*, relation: RelationInfo, default_project: str) -> str: """The project holding the relation: its database, else the client's project.""" diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index d156cb40cd..52554a9caf 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -36,11 +36,16 @@ "cannot read INFORMATION_SCHEMA.JOBS " "(needs bigquery.jobs.listAll, for example the BigQuery Resource Viewer role)" ) -USAGE_UNAVAILABLE_ERROR_CODES: frozenset[int] = frozenset({403, 404}) -USAGE_UNAVAILABLE_ERROR_CLASS_NAMES: frozenset[str] = frozenset( - {"Forbidden", "NotFound", "PermissionDenied"} +FORBIDDEN_STATUS: int = 403 +NOT_FOUND_STATUS: int = 404 +FORBIDDEN_ERROR_CLASS_NAMES: frozenset[str] = frozenset({"Forbidden", "PermissionDenied"}) +NOT_FOUND_ERROR_CLASS_NAMES: frozenset[str] = frozenset({"NotFound"}) +USAGE_UNAVAILABLE_ERROR_REASONS: frozenset[str] = frozenset({"accessDenied", "notFound"}) +USAGE_UNAVAILABLE_MESSAGES: tuple[str, ...] = ( + "access denied", + "permission bigquery.jobs.listall", + "permission denied", ) -USAGE_UNAVAILABLE_MESSAGES: tuple[str, ...] = ("access denied", "permission") _CLONE_IDENTIFIER: str = r"(`[^`]*`|[A-Za-z0-9_-]+)" _CLONE_NAME: str = _CLONE_IDENTIFIER + r"(\s*\.\s*" + _CLONE_IDENTIFIER + r"){0,2}" CLONE_STATEMENT_PATTERN: str = ( diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py b/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py index 6b79013795..93e178d7cb 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py @@ -12,6 +12,8 @@ from sqlbuild.adapters.clickhouse.constants import ( LOGGING_OFF_VALUES, LOGGING_SETTING_SQL, + OTHER_REPLICAS_NOTICE, + REPLICA_COUNT_SQL, USAGE_ERROR_CODE_PATTERN, USAGE_LOGGING_OFF_REASON, USAGE_NAME_CHUNK_SIZE, @@ -112,4 +114,18 @@ def read_relation_usage( usages=aggregate_relation_usage( reads=usage_reads(rows), relations_by_name=relations_by_name ), + notices=replica_notices(run_query=run_query) if relations_by_name else (), ) + + +def replica_notices(*, run_query: Callable[[str], list[tuple[object, ...]]]) -> tuple[str, ...]: + """A notice when this server's shard has other replicas, whose query logs are not read.""" + + try: + rows: list[tuple[object, ...]] = run_query(REPLICA_COUNT_SQL) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return () + replicas: int = int(str(rows[0][0] or 0)) if rows else 0 + return (OTHER_REPLICAS_NOTICE,) if replicas > 1 else () diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index 06c3435810..783c6dfa4d 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -124,3 +124,10 @@ USAGE_ERROR_CODE_PATTERN: re.Pattern[str] = re.compile(r"Code: (\d+)\.") LOGGING_SETTING_SQL: str = "SELECT getSetting('log_queries')" LOGGING_OFF_VALUES: frozenset[str] = frozenset({"0", "False"}) +REPLICA_COUNT_SQL: str = ( + "SELECT max(replicas) FROM (SELECT count() AS replicas FROM system.clusters " + "WHERE (cluster, shard_num) IN " + "(SELECT cluster, shard_num FROM system.clusters WHERE is_local) " + "GROUP BY cluster, shard_num)" +) +OTHER_REPLICAS_NOTICE: str = "usage read from this server only; other replicas not checked" diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index e10624a3c5..1557862edf 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -17,7 +17,8 @@ DETAILED_TABLE_INFORMATION_MARKER: str = "# detailed table information" DESCRIBE_DETAIL_ROW_WIDTH: int = 2 TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" -USAGE_NAME_CHUNK_SIZE: int = 1_000 +MAX_PARAMETER_MARKERS: int = 255 +USAGE_NAME_CHUNK_SIZE: int = MAX_PARAMETER_MARKERS - 5 USAGE_UNAVAILABLE_REASON: str = ( "cannot read system.access.table_lineage (needs Unity Catalog with the access system " "schema enabled, and USE CATALOG, USE SCHEMA and SELECT on it)" diff --git a/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py index 1d70024b95..5eed12d597 100644 --- a/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py +++ b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py @@ -8,19 +8,20 @@ from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage from sqlbuild.adapter.contract.main.relation_reads import relation_reads -from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage, RelationUsageReport from sqlbuild.adapters.snowflake._helpers.view_dependencies import ( chained_usages, dependency_name, dependency_queries, display_name, + walk_seeds, ) from sqlbuild.adapters.snowflake.constants import ( + CHAINS_NOT_FOLLOWED_NOTICE, CLONE_STATEMENT_PATTERN, SQLBUILD_QUERY_TAG_FRAGMENT, USAGE_NAME_CHUNK_SIZE, USAGE_UNAVAILABLE_REASON, - VIEW_RELATION_TYPE, ) from sqlbuild.adapters.snowflake.models import UsageQuery @@ -176,33 +177,49 @@ def read_relation_usage( connection=connection, queries=usage_queries(object_names=relations_by_name, window_seconds=window_seconds), ) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + usages: tuple[RelationUsage, ...] = aggregate_relation_usage( + reads=relation_reads(rows=rows, via_name=display_name), + relations_by_name=relations_by_name, + ) + seeds: dict[str, tuple[str, RelationUsage]] = walk_seeds( + usages=usages, relations_by_dependency_name=by_dependency_name + ) + try: dependencies: list[tuple[object, ...]] = _fetch( connection=connection, queries=dependency_queries( - view_names=( - name - for name, relation in by_dependency_name.items() - if VIEW_RELATION_TYPE in relation.relation_type.lower() - ) + databases=dependency_databases(seeds=seeds, candidates=by_dependency_name) ), ) except Exception as error: if not is_usage_unavailable_error(error): raise - return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + return RelationUsageReport( + checked=True, usages=usages, notices=(CHAINS_NOT_FOLLOWED_NOTICE,) + ) return RelationUsageReport( checked=True, usages=chained_usages( - usages=aggregate_relation_usage( - reads=relation_reads(rows=rows, via_name=display_name), - relations_by_name=relations_by_name, - ), + usages=usages, dependencies=((str(row[0]), str(row[1])) for row in dependencies), relations_by_dependency_name=by_dependency_name, ), ) +def dependency_databases(*, seeds: Iterable[str], candidates: Iterable[str]) -> frozenset[str]: + """The databases of the walk's start objects and the candidates; none without a start.""" + + starts: tuple[str, ...] = tuple(seeds) + if not starts: + return frozenset() + return frozenset(name.split(".", 1)[0] for name in (*starts, *candidates)) + + def _fetch(*, connection: Any, queries: tuple[UsageQuery, ...]) -> list[tuple[object, ...]]: rows: list[tuple[object, ...]] = [] query: UsageQuery diff --git a/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py b/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py index 9cb98d4eb3..1725f598ca 100644 --- a/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py +++ b/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py @@ -1,4 +1,4 @@ -"""Follow Snowflake view dependencies from in-use candidates to the candidates behind them.""" +"""Follow Snowflake view dependencies from directly read objects to the candidates behind them.""" from __future__ import annotations @@ -8,6 +8,7 @@ from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage from sqlbuild.adapters.snowflake.constants import ( DEPENDENCY_VIEW_DOMAINS, + NAME_SEPARATOR, QUOTED_IDENTIFIER_MARK, USAGE_NAME_CHUNK_SIZE, ) @@ -29,34 +30,60 @@ def display_name(object_name: str) -> str: return object_name if QUOTED_IDENTIFIER_MARK in object_name else object_name.lower() -def dependency_queries(*, view_names: Iterable[str]) -> tuple[UsageQuery, ...]: - """One OBJECT_DEPENDENCIES read per chunk of candidate views.""" - - names: tuple[str, ...] = tuple(sorted(view_names)) +def dependency_key(object_name: str) -> str: + """An access-history or printed object name as OBJECT_DEPENDENCIES spells it.""" + + parts: list[str] = [] + current: list[str] = [] + quoted: bool = False + was_quoted: bool = False + index: int = 0 + while index < len(object_name): + character: str = object_name[index] + doubled: bool = object_name[index : index + 2] == QUOTED_IDENTIFIER_MARK * 2 + if quoted and doubled: + current.append(QUOTED_IDENTIFIER_MARK) + index += 2 + continue + if character == QUOTED_IDENTIFIER_MARK: + quoted = not quoted + was_quoted = True + elif character == NAME_SEPARATOR and not quoted: + parts.append("".join(current) if was_quoted else "".join(current).upper()) + current, was_quoted = [], False + else: + current.append(character) + index += 1 + parts.append("".join(current) if was_quoted else "".join(current).upper()) + return ".".join(parts) + + +def dependency_queries(*, databases: Iterable[str]) -> tuple[UsageQuery, ...]: + """One OBJECT_DEPENDENCIES read of the view links in the given databases.""" + + names: tuple[str, ...] = tuple(sorted(set(databases))) return tuple( - dependency_query(view_names=names[start : start + USAGE_NAME_CHUNK_SIZE]) + dependency_query(databases=names[start : start + USAGE_NAME_CHUNK_SIZE]) for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) ) -def dependency_query(*, view_names: tuple[str, ...]) -> UsageQuery: - """What each named view references, as `(referencing, referenced)` dotted names.""" +def dependency_query(*, databases: tuple[str, ...]) -> UsageQuery: + """What each view in the databases references, as `(referencing, referenced)` names.""" - placeholders: str = ", ".join(["%s"] * len(view_names)) + placeholders: str = ", ".join(["%s"] * len(databases)) domains: str = ", ".join(["%s"] * len(DEPENDENCY_VIEW_DOMAINS)) - referencing: str = ( - "referencing_database || '.' || referencing_schema || '.' || referencing_object_name" - ) sql: str = ( - f"SELECT {referencing} AS referencing_name, " + "SELECT referencing_database || '.' || referencing_schema || '.' " + "|| referencing_object_name AS referencing_name, " "referenced_database || '.' || referenced_schema || '.' || referenced_object_name " "AS referenced_name " "FROM SNOWFLAKE.ACCOUNT_USAGE.OBJECT_DEPENDENCIES " f"WHERE referencing_object_domain IN ({domains}) " - f"AND {referencing} IN ({placeholders}) " + f"AND referencing_database IN ({placeholders}) " "ORDER BY 1, 2" ) - return UsageQuery(sql=sql, params=(*DEPENDENCY_VIEW_DOMAINS, *view_names)) + return UsageQuery(sql=sql, params=(*DEPENDENCY_VIEW_DOMAINS, *databases)) def _folded(relation: RelationInfo | RelationUsage) -> tuple[str, str, str]: @@ -67,17 +94,33 @@ def _folded(relation: RelationInfo | RelationUsage) -> tuple[str, str, str]: ) -def _via_parent( - *, - relation: RelationInfo, - parent: str, - parent_usage: RelationUsage, - earlier: RelationUsage | None, +def walk_seeds( + *, usages: Iterable[RelationUsage], relations_by_dependency_name: Mapping[str, RelationInfo] +) -> dict[str, tuple[str, RelationUsage]]: + """Each directly read object to walk from, with its printed name and the usage it passes on.""" + + names_by_key: dict[tuple[str, str, str], str] = { + _folded(relation): name for name, relation in relations_by_dependency_name.items() + } + seeds: dict[str, tuple[str, RelationUsage]] = {} + usage: RelationUsage + for usage in usages: + own: str | None = names_by_key.get(_folded(usage)) + if own is not None: + seeds.setdefault(own, (display_name(own), usage)) + via: str + for via in usage.via: + seeds.setdefault(dependency_key(via), (via, usage)) + return seeds + + +def _via_seed( + *, relation: RelationInfo, label: str, seed_usage: RelationUsage, earlier: RelationUsage | None ) -> RelationUsage: - via: tuple[str, ...] = (display_name(parent),) - readers: tuple[str, ...] = parent_usage.readers - last_read_at: datetime = parent_usage.last_read_at - query_count: int = parent_usage.query_count + via: tuple[str, ...] = (label,) + readers: tuple[str, ...] = seed_usage.readers + last_read_at: datetime = seed_usage.last_read_at + query_count: int = seed_usage.query_count if earlier is not None: via = tuple(sorted({*earlier.via, *via})) readers = tuple(dict.fromkeys((*earlier.readers, *readers))) @@ -95,42 +138,56 @@ def _via_parent( ) +def _reachable(*, start: str, references: Mapping[str, set[str]]) -> list[str]: + seen: set[str] = {start} + pending: list[str] = [start] + reached: list[str] = [] + while pending: + node: str = pending.pop() + child: str + for child in sorted(references.get(node, set()) - seen): + seen.add(child) + reached.append(child) + pending.append(child) + return reached + + def chained_usages( *, usages: tuple[RelationUsage, ...], dependencies: Iterable[tuple[str, str]], relations_by_dependency_name: Mapping[str, RelationInfo], ) -> tuple[RelationUsage, ...]: - """Usages plus every candidate an in-use candidate view depends on, transitively.""" + """Usages plus every unread candidate a directly read object depends on, through any views.""" references: dict[str, set[str]] = {} referencing: str referenced: str for referencing, referenced in dependencies: - if referenced in relations_by_dependency_name: - references.setdefault(referencing, set()).add(referenced) - names_by_key: dict[tuple[str, str, str], str] = { - _folded(relation): name for name, relation in relations_by_dependency_name.items() - } - read: dict[str, RelationUsage] = { - names_by_key[_folded(usage)]: usage for usage in usages if _folded(usage) in names_by_key + references.setdefault(referencing, set()).add(referenced) + seeds: dict[str, tuple[str, RelationUsage]] = walk_seeds( + usages=usages, relations_by_dependency_name=relations_by_dependency_name + ) + read_keys: frozenset[tuple[str, str, str]] = frozenset(map(_folded, usages)) + read: set[str] = { + name + for name, relation in relations_by_dependency_name.items() + if _folded(relation) in read_keys } found: dict[str, RelationUsage] = {} - pending: list[str] = sorted(read) - while pending: - parent: str = pending.pop(0) - parent_usage: RelationUsage = read.get(parent) or found[parent] + seed: str + for seed in sorted(seeds): + label: str + seed_usage: RelationUsage + label, seed_usage = seeds[seed] child: str - for child in sorted(references.get(parent, ())): - if child in read: + for child in _reachable(start=seed, references=references): + if child in read or child not in relations_by_dependency_name: continue - earlier: RelationUsage | None = found.get(child) - found[child] = _via_parent( + found[child] = _via_seed( relation=relations_by_dependency_name[child], - parent=parent, - parent_usage=parent_usage, - earlier=earlier, + label=label, + seed_usage=seed_usage, + earlier=found.get(child), ) - if earlier is None: - pending.append(child) return (*usages, *(found[name] for name in sorted(found))) diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 5a2613cbe9..1ae48c46af 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -25,9 +25,10 @@ TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" USAGE_NAME_CHUNK_SIZE: int = 1_000 USAGE_UNAVAILABLE_REASON: str = ( - "cannot read SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY " - "(needs Enterprise edition and the GOVERNANCE_VIEWER database role)" + "cannot read SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY (needs Enterprise edition and the " + "SNOWFLAKE.GOVERNANCE_VIEWER database role; view chains also need SNOWFLAKE.OBJECT_VIEWER)" ) +CHAINS_NOT_FOLLOWED_NOTICE: str = "view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER" SQLBUILD_QUERY_TAG_FRAGMENT: str = f'"app":"{SQLBUILD_QUERY_TAG_APP}"' _CLONE_IDENTIFIER: str = r'("[^"]*"|[A-Za-z_][A-Za-z0-9_$]*)' _CLONE_NAME: str = _CLONE_IDENTIFIER + r"(\s*\.\s*" + _CLONE_IDENTIFIER + r"){0,2}" @@ -43,5 +44,5 @@ + r'([^A-Za-z0-9_$"].*)?$' ) DEPENDENCY_VIEW_DOMAINS: tuple[str, ...] = ("VIEW", "MATERIALIZED VIEW", "SECURE VIEW") -VIEW_RELATION_TYPE: str = "view" QUOTED_IDENTIFIER_MARK: str = '"' +NAME_SEPARATOR: str = "." diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 00099e429e..6f4f890f30 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -203,6 +203,7 @@ class JanitorUsageCheck: window: str keep_in_use: bool = False reason: str | None = None + notices: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py index abe3a91618..a2935276c8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py @@ -99,6 +99,7 @@ def test_given_duckdb_when_running_janitor_then_usage_is_reported_not_checked( "reason": "unsupported adapter", "window": "7d", "keep_in_use": False, + "notices": [], }, expected_archive_in_use=(("main.old_orders_000", None), ("main.old_orders_001", None)), expected_kept_in_use=(), @@ -264,6 +265,7 @@ def test_given_usage_source_when_running_janitor_then_in_use_relations_are_shown "reason": None, "window": "7d", "keep_in_use": True, + "notices": [], }, expected_archive_in_use=(("main.old_orders_001", False),), expected_kept_in_use=( diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py index 6aa9be6b56..a8ddcecf18 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py @@ -264,9 +264,10 @@ def run_bigquery_job( class FakeBigQueryError(Exception): """A Google API error with an HTTP status code.""" - def __init__(self, message: str, code: int) -> None: + def __init__(self, message: str, code: int, reasons: tuple[str, ...] = ()) -> None: super().__init__(message) self.code: int = code + self.errors: list[dict[str, str]] = [{"reason": reason} for reason in reasons] class Forbidden(Exception): diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py index 797eec42ec..5b41389a48 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py @@ -206,8 +206,13 @@ def test_given_many_candidates_when_reading_usage_then_names_are_chunked( expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), ), BigQueryRelationUsageErrorTestCase( - description="a Forbidden error is not checked", - error=Forbidden("403 GET jobs"), + description="a Forbidden access denial is not checked", + error=Forbidden("403 Access Denied: Project shop-analytics"), + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + BigQueryRelationUsageErrorTestCase( + description="an accessDenied reason is not checked", + error=FakeBigQueryError("403 GET jobs", 403, ("accessDenied",)), expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), ), ), @@ -235,6 +240,29 @@ def test_given_no_jobs_access_when_reading_usage_then_usage_is_not_checked( error=FakeBigQueryError("Quota exceeded: too many concurrent queries", 400), expected_error_match="Quota exceeded", ), + BigQueryRelationUsageFailureTestCase( + description="a 403 rate limit is a real error", + error=FakeBigQueryError( + "403 Exceeded rate limits: too many table metadata read operations", + 403, + ("rateLimitExceeded",), + ), + expected_error_match="Exceeded rate limits", + ), + BigQueryRelationUsageFailureTestCase( + description="a 403 quota denial naming a permission is a real error", + error=FakeBigQueryError( + "403 Quota exceeded: Access Denied for the project's permission budget", + 403, + ("quotaExceeded",), + ), + expected_error_match="Quota exceeded", + ), + BigQueryRelationUsageFailureTestCase( + description="a 403 without a reason or an access denial is a real error", + error=FakeBigQueryError("403 Billing has not been enabled for this project", 403), + expected_error_match="Billing", + ), ), ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py index e90367a315..64aeeaf1b4 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -128,3 +128,10 @@ class ClickHouseRelationUsageFailureTestCase: description: str error: Exception expected_error_match: str + + +@dataclass(frozen=True) +class ClickHouseRelationUsageReplicaTestCase: + description: str + replicas: int + expected_notices: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py b/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py index ea44197e3e..4fb15d4e75 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py @@ -36,18 +36,21 @@ def __init__( rows: tuple[tuple[object, ...], ...] = (), log_queries: object = True, errors: tuple[Exception, ...] = (), + replicas: int = 1, ) -> None: self.rows: tuple[tuple[object, ...], ...] = rows self.log_queries: object = log_queries self.errors: tuple[Exception, ...] = errors + self.replicas: int = replicas self.queries: list[str] = [] def execute(self, sql: str) -> FakeQueryLogCursor: - reads: dict[bool, Callable[[], FakeQueryLogCursor]] = { - True: lambda: FakeQueryLogCursor([(self.log_queries,)]), - False: lambda: self.query_log_rows(sql), + reads: dict[tuple[bool, bool], Callable[[], FakeQueryLogCursor]] = { + (True, False): lambda: FakeQueryLogCursor([(self.log_queries,)]), + (False, True): lambda: FakeQueryLogCursor([(self.replicas,)]), + (False, False): lambda: self.query_log_rows(sql), } - return reads[sql == "SELECT getSetting('log_queries')"]() + return reads[(sql == "SELECT getSetting('log_queries')", "system.clusters" in sql)]() def query_log_rows(self, sql: str) -> FakeQueryLogCursor: self.queries.append(sql) diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py index 11e125ba88..859918014f 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py @@ -15,6 +15,7 @@ ClickHouseRelationUsageChunkTestCase, ClickHouseRelationUsageFailureTestCase, ClickHouseRelationUsageQueryTestCase, + ClickHouseRelationUsageReplicaTestCase, ClickHouseRelationUsageTestCase, ClickHouseRelationUsageUncheckedTestCase, ) @@ -223,5 +224,34 @@ def test_given_other_query_log_failure_when_reading_usage_then_error_propagates( ) +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageReplicaTestCase( + description="a single server reads every query", + replicas=1, + expected_notices=(), + ), + ClickHouseRelationUsageReplicaTestCase( + description="other replicas of this server's shard are not read, and it says so", + replicas=3, + expected_notices=("usage read from this server only; other replicas not checked",), + ), + ), + ids=lambda case: case.description, +) +def test_given_replicas_when_reading_usage_then_other_replicas_are_noted( + test_case: ClickHouseRelationUsageReplicaTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection(replicas=test_case.replicas) + + report: RelationUsageReport = read_clickhouse_usage( + connection=connection, relations=ORDERS, window_seconds=WINDOW_SECONDS + ) + + assert report.checked + assert report.notices == test_case.expected_notices + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py index 940370e270..da9b8e933a 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py @@ -2,12 +2,16 @@ from __future__ import annotations +import re from datetime import UTC, datetime import pytest from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport -from sqlbuild.adapters.databricks.constants import USAGE_UNAVAILABLE_REASON +from sqlbuild.adapters.databricks.constants import ( + MAX_PARAMETER_MARKERS, + USAGE_UNAVAILABLE_REASON, +) from tests.unit.src.sqlbuild.adapters.databricks._test_types import ( DatabricksRelationUsageChunkTestCase, DatabricksRelationUsageErrorTestCase, @@ -169,9 +173,9 @@ def test_given_lineage_query_when_built_then_parameters_are_named( expected_query_names=(), ), DatabricksRelationUsageChunkTestCase( - description="a large candidate set is split into IN lists of at most 1,000 names", + description="a large candidate set is split into IN lists of at most 250 names", relation_count=2_001, - expected_query_names=(1_000, 1_000, 1), + expected_query_names=(250, 250, 250, 250, 250, 250, 250, 250, 1), ), ), ids=lambda case: case.description, @@ -195,6 +199,10 @@ def test_given_many_candidates_when_reading_usage_then_names_are_chunked( test_case.expected_query_names ) assert all(sql.count(":name_") == len(params) - 1 for sql, params in connection.queries) + assert all( + len(re.findall(r"(? None: self.current_database: str | None = current_database self.access_rows: tuple[tuple[object, ...], ...] = access_rows self.dependency_rows: tuple[tuple[str, str], ...] = dependency_rows self.errors: tuple[Exception, ...] = errors + self.dependency_errors: tuple[Exception, ...] = dependency_errors self.queries: list[tuple[str, tuple[object, ...]]] = [] self.dependency_queries: list[tuple[str, tuple[object, ...]]] = [] @@ -439,8 +441,11 @@ def object_dependency_rows( self, sql: str, params: tuple[object, ...] ) -> list[tuple[object, ...]]: self.dependency_queries.append((sql, params)) - names: frozenset[object] = frozenset(params) - return list(filter(lambda row: row[0] in names, self.dependency_rows)) + error: Exception + for error in self.dependency_errors: + raise error + databases: frozenset[object] = frozenset(params) + return list(filter(lambda row: row[0].split(".")[0] in databases, self.dependency_rows)) class FakeUsageCursor: diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py index b9f434c37a..2b91409054 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py @@ -17,6 +17,7 @@ ) from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeCloneStatementTestCase, + SnowflakeRelationUsageChainsUnfollowedTestCase, SnowflakeRelationUsageChunkTestCase, SnowflakeRelationUsageErrorTestCase, SnowflakeRelationUsageFailureTestCase, @@ -170,13 +171,13 @@ def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_ 3, READ_AT, False, - '[\n "ANALYTICS.MART.ORDERS_V",\n "ANALYTICS.MART.ORDERS_W"\n]', + '[\n "ANALYTICS.MART.ORDERS_V",\n "REPORTING.MART.ORDERS_W"\n]', ), ("ANALYTICS.MART.CUSTOMERS", "ALICE", 2, READ_AT, False, '["ANALYTICS.MART.C_V"]'), direct_read("ANALYTICS.MART.CUSTOMERS", "BOB", 1, EARLIER_READ_AT), ), dependency_rows=(), - expected_dependency_params=(), + expected_dependency_params=((*VIEW_DOMAINS, "ANALYTICS", "REPORTING"),), expected_usages=( ("customers", ("ALICE", "BOB"), 3, True, ()), ( @@ -184,7 +185,7 @@ def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_ ("ALICE",), 3, False, - ("analytics.mart.orders_v", "analytics.mart.orders_w"), + ("analytics.mart.orders_v", "reporting.mart.orders_w"), ), ), ), @@ -213,20 +214,48 @@ def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_ ("ANALYTICS.MART.UNREAD_V", "ANALYTICS.MART.ORDERS"), ("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.NOT_A_CANDIDATE"), ), - expected_dependency_params=( - ( - *VIEW_DOMAINS, - "ANALYTICS.MART.ORDERS_V", - "ANALYTICS.MART.ORDERS_V2", - "ANALYTICS.MART.UNREAD_V", - ), - ), + expected_dependency_params=((*VIEW_DOMAINS, "ANALYTICS"),), expected_usages=( ("orders", ("ALICE",), 5, False, ("analytics.mart.orders_v",)), ("orders_v", ("ALICE",), 5, True, ()), ("orders_v2", ("ALICE",), 5, False, ("analytics.mart.orders_v",)), ), ), + SnowflakeRelationUsageViaTestCase( + description="a candidate view between a read non-candidate view and a table is in use", + relations=( + ("analytics", "mart", "orders_v2", "view"), + ("analytics", "mart", "orders", "base table"), + ), + access_rows=( + ( + "ANALYTICS.MART.ORDERS", + "ALICE", + 4, + READ_AT, + False, + '["REPORTING.MART.ORDERS_W"]', + ), + ), + dependency_rows=( + ("REPORTING.MART.ORDERS_W", "REPORTING.MART.ORDERS_HELPER"), + ("REPORTING.MART.ORDERS_HELPER", "ANALYTICS.MART.ORDERS_V2"), + ("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.ORDERS"), + ), + expected_dependency_params=((*VIEW_DOMAINS, "ANALYTICS", "REPORTING"),), + expected_usages=( + ("orders", ("ALICE",), 4, False, ("reporting.mart.orders_w",)), + ("orders_v2", ("ALICE",), 4, False, ("reporting.mart.orders_w",)), + ), + ), + SnowflakeRelationUsageViaTestCase( + description="nothing read means no dependency query", + relations=(("analytics", "mart", "orders_v2", "view"),), + access_rows=(), + dependency_rows=(("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.ORDERS"),), + expected_dependency_params=(), + expected_usages=(), + ), ), ids=lambda case: case.description, ) @@ -248,7 +277,7 @@ def test_given_view_reads_when_reading_usage_then_indirect_reads_name_their_view test_case.expected_dependency_params ) assert all( - "referencing_object_domain IN (%s, %s, %s)" in sql + "referencing_object_domain IN (%s, %s, %s) AND referencing_database IN (" in sql for sql, _ in warehouse.dependency_queries ) assert tuple( @@ -393,6 +422,39 @@ def test_given_no_access_history_access_when_reading_usage_then_usage_is_not_che assert report == test_case.expected_report +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageChainsUnfollowedTestCase( + description="no OBJECT_VIEWER keeps the reads and says view chains were not followed", + errno=3001, + message="Insufficient privileges to operate on view 'OBJECT_DEPENDENCIES'", + expected_usages=(("orders_daily", ("ALICE",), 2),), + expected_notices=("view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER",), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_dependency_access_when_reading_usage_then_reads_are_kept_with_notice( + test_case: SnowflakeRelationUsageChainsUnfollowedTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + access_rows=(direct_read("ANALYTICS.MART.ORDERS_DAILY", "ALICE", 2, READ_AT),), + dependency_errors=(FakeAccessHistoryError(test_case.message, test_case.errno),), + ) + + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + assert report.checked + assert len(warehouse.dependency_queries) == 1 + assert tuple((usage.name, usage.readers, usage.query_count) for usage in report.usages) == ( + test_case.expected_usages + ) + assert report.notices == test_case.expected_notices + + @pytest.mark.parametrize( "test_case", ( diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 8cbe7cc171..4718fe60d5 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -84,15 +84,16 @@ Usage comes from the warehouse's query history: | ClickHouse | `system.query_log` (`tables`) on the connected server | | Other adapters | Not available | -On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). A candidate listed in a query's base objects but not its direct objects was read through the objects the query named, which the line names after `via`. A second query reads `OBJECT_DEPENDENCIES` for the candidate views, so a candidate that an in-use candidate view is built on, directly or through other candidate views, is in use `via` that view, with its readers. Dependencies created by `CREATE TABLE ... AS SELECT` and similar statements are not recorded there. Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements of tables, dynamic tables, schemas and databases don't count as reads. A statement counts as a clone only when it starts with `CREATE`, after any comments, and names its source after `CLONE`; a column, string or comment containing the word doesn't make it one. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition, and the janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role: +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). A candidate listed in a query's base objects but not its direct objects was read through the objects the query named, which the line names after `via`. When something was read, a second query reads the view dependencies in `OBJECT_DEPENDENCIES` for the databases of the candidates and of the objects queries read directly. Anything those directly read objects are built on, through any number of views, is in use: an unread candidate reached that way is in use `via` the directly read object, with its readers. So a candidate view between a non-candidate view someone reads and a candidate table is kept too. Links through views in other databases are not followed. Dependencies created by `CREATE TABLE ... AS SELECT` and similar statements are not recorded there. Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements of tables, dynamic tables, schemas and databases don't count as reads. A statement counts as a clone only when it starts with `CREATE`, after any comments, and names its source after `CLONE`; a column, string or comment containing the word doesn't make it one. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition. The janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role for access history and the `SNOWFLAKE.OBJECT_VIEWER` database role for object dependencies: ```sql GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; +GRANT DATABASE ROLE SNOWFLAKE.OBJECT_VIEWER TO ROLE janitor_role; ``` -Access history and object dependencies lag by up to about 3 hours, so reads in the last few hours may not show yet. +Without `OBJECT_VIEWER`, reads are still checked, but view chains are not followed and the preview says `view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER`. Access history and object dependencies lag by up to about 3 hours, so reads in the last few hours may not show yet. -On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the relation, by `user_email`. A job lists the views it reads as well as the tables behind them, so both count. Whether views nested inside other views are listed is not documented, so a candidate view that is only read through another view may not show as in use. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. +On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the relation, by `user_email`. A job lists the views it reads as well as the tables behind them, so both count. Whether views nested inside other views are listed is not documented, so a candidate view that is only read through another view may not show as in use. Jobs answered from cached results record no tables or views, so with an `in_use_window` under about a day a reader whose queries were all cached can be missed. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`. Lineage records the view a query reads as a direct read and the tables behind it as reads through the view, which the line names when lineage records the view for the same statement. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: @@ -104,17 +105,17 @@ GRANT SELECT ON TABLE system.access.table_lineage TO `janitor-principal`; Lineage records only reads Databricks can infer lineage for, and can arrive with a delay, so "not in use" is a weaker signal than on Snowflake. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. -On ClickHouse, the janitor counts finished `SELECT` and `INSERT` queries in `system.query_log` of the server it connects to, by the user who started them. A query's `tables` lists every table and view it touched, including the views and tables a view is built on, so a whole view chain shows as in use, without `via`; queries a query starts count once, for its initial query. Queries on other replicas or shards are logged on those servers and are not seen. ClickHouse answers some queries, such as a plain `count()` of a table, from metadata without listing the table, so those reads are not seen either. SQLBuild doesn't mark its ClickHouse queries, so its own reads count. The janitor's user needs `SELECT` on `system.query_log`: +On ClickHouse, the janitor counts finished `SELECT` and `INSERT` queries in `system.query_log` of the server it connects to, by the user who started them. A query's `tables` lists every table and view it touched, including the views and tables a view is built on, so a whole view chain shows as in use, without `via`; queries a query starts count once, for its initial query. Only the connected server's `query_log` is read: queries on other replicas or shards are logged on those servers and are not seen, which matters on ClickHouse Cloud and behind load-balanced replicas. When `system.clusters` shows other replicas of the connected server's shard, the preview says `usage read from this server only; other replicas not checked`. Writes into a relation count as use, since renaming it breaks the writer too. ClickHouse answers some queries, such as a plain `count()` of a table, from metadata without logging the table, so those reads are not seen. SQLBuild doesn't mark its ClickHouse queries, so its own reads count. The janitor's user needs `SELECT` on `system.query_log`: ```sql GRANT SELECT ON system.query_log TO janitor_user; ``` -Queries are logged when `log_queries` is on, which is the default; when it is off for the janitor's connection, usage is not checked. The log is flushed about every 7.5 seconds, and ClickHouse keeps it until a TTL configured on the server removes it. +Queries are logged when `log_queries` is on, which is the default; when it is off for the janitor's connection, usage is not checked. Readers whose settings profile turns `log_queries` off, or logs only a sample with `log_queries_probability`, are missed. The log is flushed about every 7.5 seconds, and ClickHouse keeps it until a TTL configured on the server removes it. When usage can't be checked, because the adapter has no usage source or the role can't read its query history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. -In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers`, `query_count` and `via` (`null` when the relation was read directly, otherwise the names of the views it was read through, empty when unknown). Entries of `kept_in_use` have only `name`, `age_days` and `in_use`, since they are not archived. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window` and `keep_in_use`, or is `null` when there was nothing to check. +In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers`, `query_count` and `via` (`null` when the relation was read directly, otherwise the names of the views it was read through, empty when unknown). Entries of `kept_in_use` have only `name`, `age_days` and `in_use`, since they are not archived. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window`, `keep_in_use` and `notices` (what a check that ran left out, such as view chains it could not follow), or is `null` when there was nothing to check. ## Speed From a9d8539b2eb4c71623bf927d1346b66dd7555149 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:18:01 +0100 Subject: [PATCH 14/14] fix: run BigQuery usage reads through the connection boundary and register its inspection module --- .../bigquery/_helpers/relation_usage.py | 21 ++++++----- .../bigquery/classes/bigquery_adapter.py | 2 +- .../bigquery/classes/bigquery_connection.py | 29 +++++++++------ src/sqlbuild/adapters/bigquery/constants.py | 4 --- src/sqlbuild/adapters/bigquery/models.py | 2 +- .../relations/test_metadata_sql_ownership.py | 1 + .../src/sqlbuild/adapters/bigquery/helpers.py | 35 ++++++------------- .../adapters/bigquery/test_relation_usage.py | 14 +++++--- 8 files changed, 53 insertions(+), 55 deletions(-) diff --git a/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py index 0fbd08c0a1..9d8a544a07 100644 --- a/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py +++ b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py @@ -17,10 +17,14 @@ USAGE_NAME_CHUNK_SIZE, USAGE_UNAVAILABLE_ERROR_REASONS, USAGE_UNAVAILABLE_MESSAGES, - USAGE_UNAVAILABLE_REASON, ) from sqlbuild.adapters.bigquery.models import UsageQuery +_USAGE_UNAVAILABLE_REASON: str = ( + "cannot read INFORMATION_SCHEMA.JOBS " + "(needs bigquery.jobs.listAll, for example the BigQuery Resource Viewer role)" +) + def usage_object_name(*, project: str, dataset: str, table: str) -> str: """`project.dataset.table` as `referenced_tables` spells it; BigQuery names keep their case.""" @@ -184,14 +188,13 @@ def dataset_locations( def run_usage_query(*, connection: Any, query: UsageQuery) -> list[tuple[object, ...]]: """Run one JOBS read with bound parameters in the query's location.""" - from google.cloud import bigquery - - job: Any = connection.client.query( - query.sql, - location=query.location, - job_config=bigquery.QueryJobConfig(query_parameters=query_parameters(query)), + return list( + connection.run_statement( + sql=query.sql, + location=query.location, + query_parameters=tuple(query_parameters(query)), + ).fetchall() ) - return [tuple(row.values()) for row in connection.job_result(job)] def read_relation_usage( @@ -235,4 +238,4 @@ def bigquery_relation_usage( except Exception as error: if not is_usage_unavailable_error(error): raise - return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + return RelationUsageReport(checked=False, reason=_USAGE_UNAVAILABLE_REASON) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 7436626350..fd35855629 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1498,7 +1498,7 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: log_sql(logger=logging.getLogger("sqlbuild.adapter.bigquery"), sql=sql) with StatementLifecycle(adapter="bigquery", sql=sql, intent="query") as lifecycle: - job: Any = connection.query_job(sql) + job: Any = connection.query_job(sql=sql) raw_job_id: object | None = getattr(job, "job_id", None) job_id: str | None = raw_job_id if isinstance(raw_job_id, str) else None lifecycle.submitted(job_id=job_id) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py index 890fdee67f..79ff3c86b5 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py @@ -32,8 +32,15 @@ def __init__(self, *, client: Any, location: str | None) -> None: self._jobs_lock: threading.Lock = threading.Lock() def execute(self, sql: str) -> _BigQueryCursor: + return self.run_statement(sql=sql) + + def run_statement( + self, *, sql: str, location: str | None = None, query_parameters: tuple[Any, ...] = () + ) -> _BigQueryCursor: + """Run one statement, optionally in another location and with bound parameters.""" + with StatementLifecycle(adapter="bigquery", sql=sql, intent="execute") as lifecycle: - job: Any = self.query_job(sql) + job: Any = self.query_job(sql=sql, location=location, query_parameters=query_parameters) job_id: str | None = _job_id(job=job) lifecycle.submitted(job_id=job_id) try: @@ -49,24 +56,24 @@ def execute(self, sql: str) -> _BigQueryCursor: lifecycle.completed(job_id=job_id, affected_rows=dml_affected_rows) return cursor - def query_job(self, sql: str) -> Any: + def query_job( + self, *, sql: str, location: str | None = None, query_parameters: tuple[Any, ...] = () + ) -> Any: """Submit a job; only the whole-build limit becomes a job timeout, as it counts queueing.""" deadline: ExecutionDeadline | ItemDeadline | None = current_execution_deadline() if deadline is not None: deadline.enforce() build: ExecutionDeadline | None = current_build_deadline() - if build is None: - return self.client.query(sql, location=self.location) + job_location: str | None = location or self.location + if build is None and not query_parameters: + return self.client.query(sql, location=job_location) from google.cloud import bigquery - return self.client.query( - sql, - location=self.location, - job_config=bigquery.QueryJobConfig( - job_timeout_ms=max(1, math.ceil(build.remaining_seconds * 1000)) - ), - ) + job_config: Any = bigquery.QueryJobConfig(query_parameters=list(query_parameters)) + if build is not None: + job_config.job_timeout_ms = max(1, math.ceil(build.remaining_seconds * 1000)) + return self.client.query(sql, location=job_location, job_config=job_config) def job_result(self, job: Any) -> Any: """Wait for a job while it is registered for cancellation at a deadline.""" diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index 52554a9caf..1e20b22b27 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -32,10 +32,6 @@ PENDING_JOB_STATE: str = "PENDING" PENDING_JOB_POLL_SECONDS: float = 1.0 USAGE_NAME_CHUNK_SIZE: int = 1_000 -USAGE_UNAVAILABLE_REASON: str = ( - "cannot read INFORMATION_SCHEMA.JOBS " - "(needs bigquery.jobs.listAll, for example the BigQuery Resource Viewer role)" -) FORBIDDEN_STATUS: int = 403 NOT_FOUND_STATUS: int = 404 FORBIDDEN_ERROR_CLASS_NAMES: frozenset[str] = frozenset({"Forbidden", "PermissionDenied"}) diff --git a/src/sqlbuild/adapters/bigquery/models.py b/src/sqlbuild/adapters/bigquery/models.py index 3bfa8fd0dc..cf83cf393d 100644 --- a/src/sqlbuild/adapters/bigquery/models.py +++ b/src/sqlbuild/adapters/bigquery/models.py @@ -7,7 +7,7 @@ @dataclass(frozen=True) class UsageQuery: - """One INFORMATION_SCHEMA.JOBS read: its SQL, location, window and table names.""" + """One read of the region's jobs view: its SQL, location, window and table names.""" sql: str location: str diff --git a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py index 1375a23885..97d0d5d5d4 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py +++ b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py @@ -15,6 +15,7 @@ { "adapter/contract/classes/base_adapter.py", "adapter/contract/classes/duckdb_backed_adapter.py", + "adapters/bigquery/_helpers/relation_usage.py", "adapters/bigquery/classes/bigquery_adapter.py", "adapters/databricks/classes/databricks_adapter.py", "adapters/postgres/classes/postgres_adapter.py", diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py index a8ddcecf18..081949c2f2 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py @@ -14,6 +14,9 @@ ) from sqlbuild.adapter.contract.types import RetentionScope from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection + +USAGE_COLUMNS: tuple[str, ...] = ("object_name", "user_name", "query_count", "last_read_at") def build_retention_request(*, desired_days: int) -> RetentionRequest: @@ -274,16 +277,6 @@ class Forbidden(Exception): """Stands in for google.api_core.exceptions.Forbidden, matched by class name.""" -class FakeUsageRow: - """A BigQuery result row.""" - - def __init__(self, values: tuple[object, ...]) -> None: - self._values: tuple[object, ...] = values - - def values(self) -> tuple[object, ...]: - return self._values - - class FakeUsageDataset: def __init__(self, location: str) -> None: self.location: str = location @@ -309,7 +302,7 @@ def __init__( def get_dataset(self, dataset_id: str) -> FakeUsageDataset: return FakeUsageDataset(self.dataset_locations[dataset_id]) - def query(self, sql: str, *, location: str, job_config: Any) -> list[FakeUsageRow]: + def query(self, sql: str, *, location: str, job_config: Any) -> FakeBigQueryJob: params: dict[str, object] = { parameter.name: getattr(parameter, "values", getattr(parameter, "value", None)) for parameter in job_config.query_parameters @@ -319,18 +312,12 @@ def query(self, sql: str, *, location: str, job_config: Any) -> list[FakeUsageRo for error in self.errors: raise error names: frozenset[object] = frozenset(cast(list[object], params["object_names"])) - return [FakeUsageRow(row) for row in filter(lambda row: row[0] in names, self.rows)] - - -class FakeUsageConnection: - """A BigQuery connection over the fake client, without a fixed location.""" - - def __init__(self, client: FakeUsageClient, location: str | None = None) -> None: - self.client: FakeUsageClient = client - self.location: str | None = location - - def job_result(self, job: list[FakeUsageRow]) -> list[FakeUsageRow]: - return job + return FakeBigQueryJob( + FakeBigQueryRows( + columns=USAGE_COLUMNS, + rows=tuple(filter(lambda row: row[0] in names, self.rows)), + ) + ) def usage_relation(database: str | None, schema: str, name: str) -> RelationInfo: @@ -345,7 +332,7 @@ def read_bigquery_usage( """BigQuery's usage report for the relations from the fake client.""" return BigQueryAdapter().relation_usage( - connection=FakeUsageConnection(client), + connection=_BigQueryConnection(client=client, location=None), relations=relations, window_seconds=window_seconds, ) diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py index 5b41389a48..d28fdb546b 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py @@ -8,7 +8,7 @@ import pytest from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport -from sqlbuild.adapters.bigquery.constants import CLONE_STATEMENT_PATTERN, USAGE_UNAVAILABLE_REASON +from sqlbuild.adapters.bigquery.constants import CLONE_STATEMENT_PATTERN from tests.unit.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryCloneStatementTestCase, BigQueryRelationUsageChunkTestCase, @@ -27,6 +27,10 @@ ) WINDOW_SECONDS: int = 7 * 86_400 +JOBS_UNAVAILABLE_REASON: str = ( + "cannot read INFORMATION_SCHEMA.JOBS " + "(needs bigquery.jobs.listAll, for example the BigQuery Resource Viewer role)" +) READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) ORDERS_DAILY: tuple[RelationInfo, ...] = (usage_relation(None, "mart", "orders_daily"),) @@ -198,22 +202,22 @@ def test_given_many_candidates_when_reading_usage_then_names_are_chunked( "does not have permission bigquery.jobs.listAll", 403, ), - expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + expected_report=RelationUsageReport(checked=False, reason=JOBS_UNAVAILABLE_REASON), ), BigQueryRelationUsageErrorTestCase( description="an unknown region is not checked", error=FakeBigQueryError("Not found: Dataset shop-analytics:region-xx", 404), - expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + expected_report=RelationUsageReport(checked=False, reason=JOBS_UNAVAILABLE_REASON), ), BigQueryRelationUsageErrorTestCase( description="a Forbidden access denial is not checked", error=Forbidden("403 Access Denied: Project shop-analytics"), - expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + expected_report=RelationUsageReport(checked=False, reason=JOBS_UNAVAILABLE_REASON), ), BigQueryRelationUsageErrorTestCase( description="an accessDenied reason is not checked", error=FakeBigQueryError("403 GET jobs", 403, ("accessDenied",)), - expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + expected_report=RelationUsageReport(checked=False, reason=JOBS_UNAVAILABLE_REASON), ), ), ids=lambda case: case.description,