diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 66c16460bf..62e396c2bf 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -5273,6 +5273,18 @@ "help": "drop the compatibility view at this old model name before it expires; repeat for several views", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--keep-in-use" + ], + "dest": "keep_in_use", + "nargs": 0, + "const": true, + "default": false, + "help": "keep relations other queries read within janitor.in_use_window instead of archiving them (overrides janitor.keep_in_use)", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 912372680d..2cd040d890 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -64,7 +64,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 179] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 180] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -139,6 +139,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 179] = [ ("retention_days", NamespaceDefault::None), ("direct_state_history_versions", NamespaceDefault::None), ("drop_old_name_views", NamespaceDefault::EmptyList), + ("keep_in_use", NamespaceDefault::Bool(false)), ("bounded", NamespaceDefault::None), ("max_column_examples", NamespaceDefault::None), ("max_row_only_examples", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index e7a5de0990..7410b3e21d 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -486,7 +486,7 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); -const JANITOR_FIELDS: [Field; 10] = [ +const JANITOR_FIELDS: [Field; 11] = [ PROJECT_DIR, NO_COLOR, same("auto_approve"), @@ -496,6 +496,7 @@ const JANITOR_FIELDS: [Field; 10] = [ "drop_old_name_views", Source::TupleOf("drop_old_name_views"), ), + same("keep_in_use"), SELECTED_TARGET, same("as_target"), same("verbose"), diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 4dc70f4a5b..5842d42869 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -12,13 +12,13 @@ use crate::project::_helpers::targets::seconds_value; use crate::project::_helpers::values::{int_below, python_equal}; use crate::project::constants::{ AUDIT_SEVERITIES, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, - DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_QUERY_ARTIFACT_TTL, - FUTURE_CURSOR_ACTIONS, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, - MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, - REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, - SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, - SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, - USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, + DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_IN_USE_WINDOW, + DEFAULT_QUERY_ARTIFACT_TTL, FUTURE_CURSOR_ACTIONS, LEGACY_CONCURRENCY_KEY, + LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, + OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY, + RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, + SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, + UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, @@ -565,6 +565,19 @@ pub(crate) fn load_janitor(payload: Option<&ConfigValue>, file: ConfigFile) -> L get(mapping, "exclude_patterns"), place(file, "janitor.exclude_patterns"), )?; + let keep_in_use = optional_bool(mapping, "keep_in_use", false)?; + let in_use_window = match get(mapping, "in_use_window") { + None => DEFAULT_IN_USE_WINDOW.to_owned(), + Some(ConfigValue::String(window)) if fixed_duration_seconds(window, false)?.is_some() => { + window.clone() + } + Some(_) => { + return Err(ConfigFailure::at( + file, + "janitor.in_use_window must be a positive fixed duration such as '7d'", + )); + } + }; let checks = [ (retention_days.as_ref(), "retention_days"), (Some(&archive), "archive_retention_days"), @@ -591,6 +604,11 @@ pub(crate) fn load_janitor(payload: Option<&ConfigValue>, file: ConfigFile) -> L Built::Value(ConfigValue::Bool(delete_tracked_only)), ), ("exclude_patterns", Built::Tuple(exclude)), + ( + "in_use_window", + Built::Value(ConfigValue::String(in_use_window)), + ), + ("keep_in_use", Built::Value(ConfigValue::Bool(keep_in_use))), ], )) } diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 4c745d97c2..7cc378c799 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -27,6 +27,7 @@ pub(crate) const MODELS_DIRECTORY_NAME: &str = "models"; pub(crate) const UNLIMITED_DIFF_FULL_ROWS: &str = "unlimited"; pub(crate) const ZERO_FIXED_DURATION: &str = "0s"; pub(crate) const DEFAULT_QUERY_ARTIFACT_TTL: &str = "24h"; +pub(crate) const DEFAULT_IN_USE_WINDOW: &str = "7d"; pub(crate) const BOUNDED_REPLAY_PREFIX: &str = "bounded-"; pub(crate) const ZERO_FLOAT: f64 = 0.0; pub(crate) const SHUTDOWN_TIMEOUT_MAX_SECONDS: u64 = 600; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 1bcced4f42..969b1b5ade 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -40,6 +40,33 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { ), ), }, + ProjectFailureTestCase { + description: "a janitor usage window and keep_in_use load", + project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nenabled = true\n\ + in_use_window = '36h'\nkeep_in_use = true\n", + local: "", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "a calendar janitor usage window is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nin_use_window = '1mo'\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "janitor.in_use_window must be a positive fixed duration such as '7d'", + None, + ), + }, + ProjectFailureTestCase { + description: "a janitor usage window that is not a duration is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nin_use_window = 7\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "janitor.in_use_window must be a positive fixed duration such as '7d'", + None, + ), + }, ProjectFailureTestCase { description: "a git base ref loads", project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n", diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs index fa60de261a..f4f92c1ed8 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod counts; pub(crate) mod phases; pub(crate) mod plan_json; pub(crate) mod plan_sections; +pub(crate) mod usage; diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs index 3a52cb0afc..255977ef9a 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -1,9 +1,10 @@ //! The JSON payload of `sqb janitor --json`. +use crate::janitor_text::_helpers::usage::checked_usage; use crate::janitor_text::constants::{DROP_EARLY, DROP_EXPIRED}; use crate::janitor_text::models::{ - JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText, OldNameViewText, - SourceSchemaText, + ArchiveCandidateText, JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText, + OldNameViewText, SourceSchemaText, }; use crate::json::models::{JsonInteger, JsonValue}; @@ -61,6 +62,65 @@ fn old_name_view(view: &OldNameViewText, planned: &str) -> JsonValue { ]) } +/// `null` when usage was not checked, `false` when unread, else the reads. +fn in_use(candidate: &ArchiveCandidateText, checked: bool) -> JsonValue { + if !checked { + return JsonValue::Null; + } + candidate + .usage + .as_ref() + .map_or(JsonValue::Bool(false), |usage| { + object(vec![ + ("last_read", utc_timestamp(&usage.last_read_at)), + ("readers", texts(&usage.readers)), + ("query_count", integer(usage.query_count)), + ("via", usage.via.as_deref().map_or(JsonValue::Null, texts)), + ]) + }) +} + +fn archive_candidate(candidate: &ArchiveCandidateText, checked: bool) -> JsonValue { + object(vec![ + ("name", text(&candidate.name)), + ("archive_name", text(&candidate.archive_name)), + ("age_days", optional_integer(candidate.age_days)), + ("expires_at", utc_timestamp(&candidate.expires_at)), + ("deleted_in_run", JsonValue::Bool(candidate.expires_now)), + ("in_use", in_use(candidate, checked)), + ]) +} + +/// A candidate kept because it is in use: no archive name or expiry, since it stays put. +fn kept_candidate(candidate: &ArchiveCandidateText) -> JsonValue { + object(vec![ + ("name", text(&candidate.name)), + ("age_days", optional_integer(candidate.age_days)), + ("in_use", in_use(candidate, true)), + ]) +} + +/// The top-level `usage_check`: its status and reason, or `null` when it did not run. +pub(crate) fn usage_check_payload(plan: Option<&JanitorPlanText>) -> JsonValue { + let Some(check) = plan.and_then(|plan| plan.usage_check.as_ref()) else { + return JsonValue::Null; + }; + object(vec![ + ( + "status", + text(if check.checked { + "checked" + } else { + "not_checked" + }), + ), + ("reason", optional_text(check.reason.as_deref())), + ("window", text(&check.window)), + ("keep_in_use", JsonValue::Bool(check.keep_in_use)), + ("notices", texts(&check.notices)), + ]) +} + fn source_schema(schema: &SourceSchemaText, blocked: bool) -> JsonValue { let mut fields: Vec<(&str, JsonValue)> = vec![ ("name", text(&schema.name)), @@ -96,6 +156,7 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { .map(|view| old_name_view(view, "missing")), ) .collect(); + let checked: bool = checked_usage(plan).is_some(); object(vec![ ("direct_mode", JsonValue::Bool(plan.direct_mode)), ("retention_days", integer(plan.retention_days)), @@ -114,18 +175,14 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { JsonValue::Array( plan.archive_candidates .iter() - .map(|candidate| { - object(vec![ - ("name", text(&candidate.name)), - ("archive_name", text(&candidate.archive_name)), - ("age_days", optional_integer(candidate.age_days)), - ("expires_at", utc_timestamp(&candidate.expires_at)), - ("deleted_in_run", JsonValue::Bool(candidate.expires_now)), - ]) - }) + .map(|candidate| archive_candidate(candidate, checked)) .collect(), ), ), + ( + "kept_in_use", + JsonValue::Array(plan.kept_in_use.iter().map(kept_candidate).collect()), + ), ( "archives_to_delete", JsonValue::Array( diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs index 776bf890c3..70283b3565 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs @@ -1,11 +1,14 @@ //! Sections of the janitor plan preview. use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::usage::{ + checked_usage, in_use_candidates, usage_detail, usage_notices, +}; use crate::janitor_text::constants::{ DROP_EARLY, DROP_EXPIRED, DROP_MISSING, LABEL_WIDTH, TREE_BLANK, TREE_BRANCH_GLYPH, TREE_CONTINUATION, TREE_LAST_GLYPH, VERBOSE_FLAG, }; -use crate::janitor_text::models::{JanitorPlanText, OldNameViewText}; +use crate::janitor_text::models::{ArchiveCandidateText, JanitorPlanText, OldNameViewText}; /// The first `limit` items, or all of them, and how many are hidden. fn visible(items: &[T], limit: Option) -> (&[T], usize) { @@ -91,6 +94,13 @@ pub(crate) fn summary(plan: &JanitorPlanText, style: CliStyle) -> String { "relations to archive", &plan.archive_candidates, )); + if let Some(check) = checked_usage(plan) { + if check.keep_in_use { + text.push_str(&count_row(style, "kept (in use)", &plan.kept_in_use)); + } else { + text.push_str(&count_row(style, "in use", &in_use_candidates(plan))); + } + } text.push_str(&count_row( style, "archives to delete", @@ -118,9 +128,38 @@ pub(crate) fn summary(plan: &JanitorPlanText, style: CliStyle) -> String { "objects skipped", &style.accent(&plan.skipped_relations.len().to_string()), )); + let notices: Vec = usage_notices(plan); + if !notices.is_empty() { + text.push('\n'); + for notice in notices { + text.push_str(&format!(" {}\n", style.warning(¬ice))); + } + } text } +/// In-use archive candidates with their readers, or the in-use relations kept instead. +fn in_use(plan: &JanitorPlanText, limit: Option, style: CliStyle) -> String { + let Some(check) = checked_usage(plan) else { + return String::new(); + }; + let (title, candidates): (&str, Vec<&ArchiveCandidateText>) = if check.keep_in_use { + ("Kept (in use)", plan.kept_in_use.iter().collect()) + } else { + ("In use", in_use_candidates(plan)) + }; + let lines: Vec<(String, Option)> = candidates + .into_iter() + .map(|candidate| { + ( + candidate.name.clone(), + usage_detail(candidate, &check.window), + ) + }) + .collect(); + listing((style, limit), title, &lines) +} + /// Schemas whose active sources block every janitor action. pub(crate) fn blocked(plan: &JanitorPlanText, limit: Option, style: CliStyle) -> String { if plan.blocked_schemas.is_empty() { @@ -190,6 +229,7 @@ pub(crate) fn archives(plan: &JanitorPlanText, limit: Option, style: CliS } text.push_str(&overflow(style, hidden, " ")); } + text.push_str(&in_use(plan, limit, style)); if !plan.archive_deletions.is_empty() { text.push_str(&format!("\n{}\n", style.success("Archives to delete"))); let (deletions, hidden) = visible(&plan.archive_deletions, limit); diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs new file mode 100644 index 0000000000..5ea862ca81 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs @@ -0,0 +1,124 @@ +//! Wording of the janitor usage check: who reads a candidate and what happens to it. + +use crate::cli_text::main::format_count_noun::format_count_noun; +use crate::janitor_text::_helpers::phases::noun; +use crate::janitor_text::constants::{ + KEEP_WITHOUT_USAGE_NOTICE, READER_LIMIT, USAGE_NOT_CHECKED_PREFIX, WINDOW_UNITS, +}; +use crate::janitor_text::models::{ArchiveCandidateText, JanitorPlanText, UsageCheckText}; + +/// The usage check when it ran and could read usage. +pub(crate) fn checked_usage(plan: &JanitorPlanText) -> Option<&UsageCheckText> { + plan.usage_check.as_ref().filter(|check| check.checked) +} + +/// Archive candidates with reads in the usage window. +pub(crate) fn in_use_candidates(plan: &JanitorPlanText) -> Vec<&ArchiveCandidateText> { + plan.archive_candidates + .iter() + .filter(|candidate| candidate.usage.is_some()) + .collect() +} + +/// `7 days` for a single-unit window such as `7d`, otherwise the window as written. +pub(crate) fn window_label(window: &str) -> String { + let digits: &str = window.trim_end_matches(|character: char| !character.is_ascii_digit()); + let unit: &str = &window[digits.len()..]; + match ( + digits.parse::(), + WINDOW_UNITS.iter().find(|(suffix, _)| *suffix == unit), + ) { + (Ok(count), Some((_, singular))) if !digits.is_empty() => noun_count(count, singular), + _ => window.to_owned(), + } +} + +fn noun_count(count: i64, singular: &str) -> String { + noun(usize::try_from(count).unwrap_or(usize::MAX), singular) +} + +/// `alice, bob, carol +2 more`: the first names and how many are not shown. +fn readers(names: &[String]) -> String { + let shown: usize = names.len().min(READER_LIMIT); + let mut text: String = names[..shown].join(", "); + if names.len() > shown { + text.push_str(&format!(" +{} more", names.len() - shown)); + } + text +} + +/// `queried by ( queries in )`, plus the frozen flag for tables. +pub(crate) fn usage_detail(candidate: &ArchiveCandidateText, window: &str) -> Option { + let usage = candidate.usage.as_ref()?; + let date: &str = usage.last_read_at.split(' ').next().unwrap_or_default(); + let via: String = match usage.via.as_deref() { + None => String::new(), + Some([]) => " through a view".to_owned(), + Some(names) => format!(" via {}", readers(names)), + }; + let mut text: String = format!( + "queried {date} by {}{via} ({} in {})", + readers(&usage.readers), + format_count_noun(usage.query_count, "query", Some("queries")), + window_label(window) + ); + if candidate.is_table { + match &candidate.changed_on { + Some(changed_on) => { + text.push_str(&format!("; frozen since {changed_on}, still in use")); + } + None => text.push_str("; frozen, still in use"), + } + } + Some(text) +} + +/// The notice lines when usage could not be checked, or what a check that ran left out. +pub(crate) fn usage_notices(plan: &JanitorPlanText) -> Vec { + let Some(check) = plan.usage_check.as_ref() else { + return Vec::new(); + }; + if check.checked { + return check.notices.clone(); + } + let mut notices: Vec = vec![format!( + "{USAGE_NOT_CHECKED_PREFIX}{}", + check.reason.as_deref().unwrap_or("unknown reason") + )]; + if check.keep_in_use { + notices.push(KEEP_WITHOUT_USAGE_NOTICE.to_owned()); + } + notices +} + +/// The prompt line stating how many archived relations are in use, or which are kept. +pub(crate) fn usage_prompt_line(plan: &JanitorPlanText) -> Option { + let check: &UsageCheckText = checked_usage(plan)?; + if check.keep_in_use && !plan.kept_in_use.is_empty() { + return Some(format!( + "{} in use {} kept.", + noun(plan.kept_in_use.len(), "relation"), + if plan.kept_in_use.len() == 1 { + "is" + } else { + "are" + } + )); + } + if plan.archive_candidates.is_empty() { + return None; + } + let in_use: usize = in_use_candidates(plan).len(); + let archived: usize = plan.archive_candidates.len(); + let in_use_text: String = match (archived, in_use) { + (1, 0) => "it is not in use".to_owned(), + (1, _) => "it is in use".to_owned(), + (_, 0) => "none of them are in use".to_owned(), + (_, 1) => "1 of them is in use".to_owned(), + _ => format!("{in_use} of them are in use"), + }; + Some(format!( + "{} will be archived; {in_use_text}.", + noun(archived, "relation") + )) +} diff --git a/crates/sqlbuild-core/src/janitor_text/constants.rs b/crates/sqlbuild-core/src/janitor_text/constants.rs index 13f8108f08..148a5f3ee0 100644 --- a/crates/sqlbuild-core/src/janitor_text/constants.rs +++ b/crates/sqlbuild-core/src/janitor_text/constants.rs @@ -22,3 +22,15 @@ pub(crate) const SECONDS_PER_MINUTE: u64 = 60; /// Durations below this many seconds show seconds with two decimals. pub(crate) const MINUTE_SECONDS: f64 = 60.0; pub(crate) const SECONDS_PER_HOUR: u64 = 3600; +/// How many readers each in-use line names before `+N more`. +pub(crate) const READER_LIMIT: usize = 3; +pub(crate) const USAGE_NOT_CHECKED_PREFIX: &str = "usage not checked: "; +pub(crate) const KEEP_WITHOUT_USAGE_NOTICE: &str = + "nothing kept in use: no usage could be checked, so relations in use are archived too"; +/// Single-unit usage windows and the noun of their unit. +pub(crate) const WINDOW_UNITS: [(&str, &str); 4] = [ + ("d", "day"), + ("h", "hour"), + ("m", "minute"), + ("s", "second"), +]; diff --git a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs index 8f644e58ab..8b626ab410 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs @@ -1,6 +1,6 @@ //! `sqb janitor --json`. -use crate::janitor_text::_helpers::plan_json::{plan_payload, run_payload}; +use crate::janitor_text::_helpers::plan_json::{plan_payload, run_payload, usage_check_payload}; use crate::janitor_text::models::{JanitorOutcome, JanitorPlanText, JanitorRunText}; use crate::json::main::dumps::dumps; use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; @@ -27,6 +27,7 @@ pub fn format_janitor_json( "plan".to_owned(), plan.map_or(JsonValue::Null, plan_payload), ), + ("usage_check".to_owned(), usage_check_payload(plan)), ( "result".to_owned(), run.map_or(JsonValue::Null, run_payload), diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs index 0db1e14284..e774f99687 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_confirmation_prompt.rs @@ -3,6 +3,7 @@ use crate::janitor_text::_helpers::counts::{ archive_prefix, environment_label, physical_deletion_count, }; +use crate::janitor_text::_helpers::usage::{usage_notices, usage_prompt_line}; use crate::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation_phrase; use crate::janitor_text::models::JanitorPlanText; @@ -22,6 +23,12 @@ pub fn janitor_confirmation_prompt(plan: &JanitorPlanText) -> String { let items: usize = physical_deletion_count(plan) + plan.state_prunes.len(); format!("Janitor will {prefix}delete {items} items from {environment}.\n") }; + if let Some(line) = usage_prompt_line(plan) { + text.push_str(&format!("{line}\n")); + } + for notice in usage_notices(plan) { + text.push_str(&format!("{}\n", capitalized(¬ice))); + } if plan.retention_days == 0 { text.push_str("Retention: disabled (0 days)\nAge metadata will not be checked.\n"); } else { @@ -39,3 +46,10 @@ pub fn janitor_confirmation_prompt(plan: &JanitorPlanText) -> String { )); text } + +fn capitalized(text: &str) -> String { + let mut characters = text.chars(); + characters.next().map_or_else(String::new, |first| { + first.to_uppercase().chain(characters).collect() + }) +} diff --git a/crates/sqlbuild-core/src/janitor_text/models.rs b/crates/sqlbuild-core/src/janitor_text/models.rs index cbd49ccd0b..6011b794dd 100644 --- a/crates/sqlbuild-core/src/janitor_text/models.rs +++ b/crates/sqlbuild-core/src/janitor_text/models.rs @@ -1,5 +1,17 @@ //! Display facts of a janitor plan and its execution, read from the executor's models. +/// Reads of one relation within the usage window. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RelationUsageText { + /// The latest read as `YYYY-MM-DD HH:MM:SS` UTC. + pub last_read_at: String, + /// Reader user names, most queries first. + pub readers: Vec, + pub query_count: i64, + /// The objects it was read through when it was never read directly; empty when unnamed. + pub via: Option>, +} + /// A relation to archive: its name, archive name, age and expiry. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ArchiveCandidateText { @@ -11,6 +23,26 @@ pub struct ArchiveCandidateText { pub expires_now: bool, /// The expiry as `YYYY-MM-DD HH:MM:SS`. pub expires_at: String, + /// Whether the relation is a table, whose data stops changing once it is stale. + pub is_table: bool, + /// The date it last changed as `YYYY-MM-DD`, or none when unknown. + pub changed_on: Option, + /// Its reads within the usage window, or none when unread or not checked. + pub usage: Option, +} + +/// Whether the janitor checked reads of its archive candidates. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct UsageCheckText { + pub checked: bool, + /// Why usage was not checked. + pub reason: Option, + /// The usage window as configured, such as `7d`. + pub window: String, + /// Whether in-use relations are kept instead of archived. + pub keep_in_use: bool, + /// What a check that ran could not cover, such as view chains it could not follow. + pub notices: Vec, } /// An archived relation to delete or retain. @@ -69,6 +101,10 @@ pub struct JanitorPlanText { pub skipped_relations: Vec<(String, String)>, pub skipped_schemas: Vec, pub blocked_schemas: Vec, + /// The usage check of the archive candidates, or none when it did not run. + pub usage_check: Option, + /// In-use relations kept instead of archived. + pub kept_in_use: Vec, } /// How many items one janitor run changed. diff --git a/crates/sqlbuild-core/src/janitor_text/tests.rs b/crates/sqlbuild-core/src/janitor_text/tests.rs index 3735ed1a78..53afbdab1c 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests.rs @@ -4,5 +4,7 @@ mod helpers; mod janitor_progress_text; #[path = "tests/test_janitor_text.rs"] mod janitor_text; +#[path = "tests/test_janitor_usage_text.rs"] +mod janitor_usage_text; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index 48c075193d..a1d63fe2c1 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -1,6 +1,6 @@ use crate::janitor_text::models::{ ArchiveCandidateText, ArchivedRelationText, JanitorItemText, JanitorPhase, JanitorPlanText, - OldNameViewText, + OldNameViewText, RelationUsageText, UsageCheckText, }; /// A direct-mode plan for `dev` that archives one relation and deletes one archive. @@ -78,3 +78,139 @@ pub(super) fn old_name_plan(count: usize) -> JanitorPlanText { ..JanitorPlanText::default() } } + +/// Reads by `readers` of `query_count` queries, last on 2026-10-21. +fn usage(readers: &[&str], query_count: i64) -> RelationUsageText { + RelationUsageText { + last_read_at: "2026-10-21 09:30:00".to_owned(), + readers: readers.iter().map(|reader| (*reader).to_owned()).collect(), + query_count, + via: None, + } +} + +/// An in-use view, an in-use table read by five users, and an unread table. +fn usage_candidates() -> [ArchiveCandidateText; 3] { + [ + ArchiveCandidateText { + name: "dev.orders_daily".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__orders_daily".to_owned(), + usage: Some(usage(&["svc_reporting"], 14)), + ..ArchiveCandidateText::default() + }, + ArchiveCandidateText { + name: "dev.old_orders".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__old_orders".to_owned(), + is_table: true, + changed_on: Some("2026-09-01".to_owned()), + usage: Some(usage( + &["svc_reporting", "alice", "bob", "carol", "dave"], + 40, + )), + ..ArchiveCandidateText::default() + }, + ArchiveCandidateText { + name: "dev.old_customers".to_owned(), + archive_name: "dev._sqb_archive__20261022t101500z__old_customers".to_owned(), + is_table: true, + ..ArchiveCandidateText::default() + }, + ] +} + +const CHAINS_NOT_FOLLOWED: &str = "view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER"; + +/// A plan for `dev` whose usage was checked over `7d`. +fn checked_plan(keep_in_use: bool) -> JanitorPlanText { + JanitorPlanText { + target_name: Some("dev".to_owned()), + retention_days: 14, + direct_mode: true, + archive_retention_days: 14, + usage_check: Some(UsageCheckText { + checked: true, + reason: None, + window: "7d".to_owned(), + keep_in_use, + ..UsageCheckText::default() + }), + ..JanitorPlanText::default() + } +} + +/// A checked plan archiving every usage candidate, in use or not. +pub(super) fn usage_plan() -> JanitorPlanText { + JanitorPlanText { + archive_candidates: usage_candidates().to_vec(), + ..checked_plan(false) + } +} + +/// A checked plan archiving a table read through four named views and a view read through one. +pub(super) fn via_usage_plan() -> JanitorPlanText { + let views: Vec = ["daily", "weekly", "monthly", "yearly"] + .iter() + .map(|grain| format!("dev.orders_{grain}")) + .collect(); + JanitorPlanText { + archive_candidates: vec![ + ArchiveCandidateText { + name: "dev.orders_base".to_owned(), + is_table: true, + changed_on: Some("2026-09-01".to_owned()), + usage: Some(RelationUsageText { + via: Some(views), + ..usage(&["alice"], 3) + }), + ..ArchiveCandidateText::default() + }, + ArchiveCandidateText { + name: "dev.orders_view".to_owned(), + usage: Some(RelationUsageText { + via: Some(Vec::new()), + ..usage(&["alice"], 3) + }), + ..ArchiveCandidateText::default() + }, + ], + ..checked_plan(false) + } +} + +/// A checked plan whose usage source could not follow view chains. +pub(super) fn chains_unfollowed_plan() -> JanitorPlanText { + JanitorPlanText { + archive_candidates: usage_candidates().to_vec(), + usage_check: Some(UsageCheckText { + checked: true, + window: "7d".to_owned(), + notices: vec![CHAINS_NOT_FOLLOWED.to_owned()], + ..UsageCheckText::default() + }), + ..checked_plan(false) + } +} + +/// A checked plan keeping the two in-use candidates and archiving the unread one. +pub(super) fn kept_usage_plan() -> JanitorPlanText { + let [view, table, unread] = usage_candidates(); + JanitorPlanText { + archive_candidates: vec![unread], + kept_in_use: vec![view, table], + ..checked_plan(true) + } +} + +/// A plan whose usage was not checked because the adapter has no usage source. +pub(super) fn unchecked_plan(keep_in_use: bool) -> JanitorPlanText { + JanitorPlanText { + usage_check: Some(UsageCheckText { + checked: false, + reason: Some("unsupported adapter".to_owned()), + window: "7d".to_owned(), + keep_in_use, + ..UsageCheckText::default() + }), + ..direct_archive_plan() + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs new file mode 100644 index 0000000000..23a2260f8a --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs @@ -0,0 +1,252 @@ +use crate::cli_text::models::CliStyle; +use crate::janitor_text::main::format_janitor_json::format_janitor_json; +use crate::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation_phrase; +use crate::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation_prompt; +use crate::janitor_text::main::render_janitor_plan::render_janitor_plan; +use crate::janitor_text::models::JanitorOutcome; +use crate::janitor_text::tests::helpers::{ + chains_unfollowed_plan, direct_archive_plan, kept_usage_plan, unchecked_plan, usage_plan, + via_usage_plan, +}; +use crate::janitor_text::tests::test_types::{JsonTestCase, PreviewTestCase, UsagePromptTestCase}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_usage_check_when_rendering_preview_then_in_use_relations_show_readers() { + let test_cases = [ + PreviewTestCase { + description: "in-use candidates are listed with capped readers and archived anyway", + plan: usage_plan(), + verbose: false, + expected_fragments: &[ + " relations to archive 3\n in use 2\n", + "\nIn use\n dev.orders_daily queried 2026-10-21 by svc_reporting \ + (14 queries in 7 days)\n", + " dev.old_orders queried 2026-10-21 by svc_reporting, alice, bob +2 more \ + (40 queries in 7 days); frozen since 2026-09-01, still in use\n", + " dev.old_customers -> dev._sqb_archive__", + ], + unexpected_fragments: &[ + "Kept (in use)", + "dev.old_customers queried", + "dev.orders_daily queried 2026-10-21 by svc_reporting (14 queries in 7 days); frozen", + "usage not checked", + ], + }, + PreviewTestCase { + description: "keep_in_use lists in-use relations as kept and leaves them unarchived", + plan: kept_usage_plan(), + verbose: false, + expected_fragments: &[ + " relations to archive 1\n kept (in use) 2\n", + "\nKept (in use)\n dev.orders_daily queried 2026-10-21", + " dev.old_orders queried 2026-10-21 by svc_reporting, alice, bob +2 more", + ], + unexpected_fragments: &["\nIn use\n", "dev.old_orders ->", "dev.orders_daily ->"], + }, + PreviewTestCase { + description: "relations read only through views name the views, or say so", + plan: via_usage_plan(), + verbose: false, + expected_fragments: &[ + " dev.orders_base queried 2026-10-21 by alice via dev.orders_daily, \ + dev.orders_weekly, dev.orders_monthly +1 more (3 queries in 7 days); \ + frozen since 2026-09-01, still in use\n", + " dev.orders_view queried 2026-10-21 by alice through a view \ + (3 queries in 7 days)\n", + ], + unexpected_fragments: &["dev.orders_yearly"], + }, + PreviewTestCase { + description: "a check that could not follow view chains says so once", + plan: chains_unfollowed_plan(), + verbose: false, + expected_fragments: &[ + "\n view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER\n", + "\nIn use\n", + ], + unexpected_fragments: &["usage not checked", "nothing kept in use"], + }, + PreviewTestCase { + description: "an unchecked usage shows one notice and no in-use rows", + plan: unchecked_plan(false), + verbose: false, + expected_fragments: &["\n usage not checked: unsupported adapter\n"], + unexpected_fragments: &["in use ", "In use", "nothing kept in use"], + }, + PreviewTestCase { + description: "keep_in_use without a usage source warns that nothing is kept", + plan: unchecked_plan(true), + verbose: false, + expected_fragments: &[ + " usage not checked: unsupported adapter\n nothing kept in use: no usage \ + could be checked, so relations in use are archived too\n", + ], + unexpected_fragments: &["Kept (in use)"], + }, + PreviewTestCase { + description: "a plan without a usage check renders as before", + plan: direct_archive_plan(), + verbose: false, + expected_fragments: &[" relations to archive 1\n archives to delete"], + unexpected_fragments: &["usage", "in use"], + }, + ]; + + for test_case in test_cases { + let text: String = render_janitor_plan(&test_case.plan, test_case.verbose, PLAIN); + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: missing fragments in {text}", + test_case.description + ); + assert!( + !test_case + .unexpected_fragments + .iter() + .any(|fragment| text.contains(fragment)), + "{}: unexpected fragments in {text}", + test_case.description + ); + } +} + +#[test] +fn given_usage_check_when_prompting_then_prompt_states_in_use_count() { + let test_cases = [ + UsagePromptTestCase { + description: "the prompt counts in-use relations among those archived", + plan: usage_plan(), + expected_fragment: "Janitor will archive 3 and delete 0 objects from dev.\n\ + 3 relations will be archived; 2 of them are in use.\nRetention: 14 days\n", + expected_phrase: "archive 3 and delete 0 objects from dev", + }, + UsagePromptTestCase { + description: "a check that could not follow view chains repeats its notice", + plan: chains_unfollowed_plan(), + expected_fragment: "2 of them are in use.\nView chains not followed: grant \ + SNOWFLAKE.OBJECT_VIEWER\nRetention: 14 days\n", + expected_phrase: "archive 3 and delete 0 objects from dev", + }, + UsagePromptTestCase { + description: "keep_in_use counts the kept relations and leaves them out of the phrase", + plan: kept_usage_plan(), + expected_fragment: "Janitor will archive 1 and delete 0 objects from dev.\n\ + 2 relations in use are kept.\nRetention: 14 days\n", + expected_phrase: "archive 1 and delete 0 objects from dev", + }, + UsagePromptTestCase { + description: "an unchecked usage repeats the notice", + plan: unchecked_plan(false), + expected_fragment: "Janitor will archive 1 and delete 1 objects from dev.\n\ + Usage not checked: unsupported adapter\nRetention: 14 days\n", + expected_phrase: "archive 1 and delete 1 objects from dev", + }, + UsagePromptTestCase { + description: "keep_in_use without a usage source repeats the warning", + plan: unchecked_plan(true), + expected_fragment: "Usage not checked: unsupported adapter\nNothing kept in use: \ + no usage could be checked, so relations in use are archived too\nRetention", + expected_phrase: "archive 1 and delete 1 objects from dev", + }, + ]; + + for test_case in test_cases { + let prompt: String = janitor_confirmation_prompt(&test_case.plan); + assert!( + prompt.contains(test_case.expected_fragment), + "{}: missing fragment in {prompt}", + test_case.description + ); + assert_eq!( + janitor_confirmation_phrase(&test_case.plan), + test_case.expected_phrase, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { + let test_cases = [ + JsonTestCase { + description: "checked candidates carry their reads or false, with the check status", + plan: Some(usage_plan()), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &[ + "\"in_use\": {\n \"last_read\": \"2026-10-21T09:30:00Z\",\n \ + \"readers\": [\n \"svc_reporting\"\n ],\n \ + \"query_count\": 14,\n \"via\": null\n }", + "\"in_use\": false", + "\"kept_in_use\": []", + "\"usage_check\": {\n \"status\": \"checked\",\n \"reason\": null,\n \ + \"window\": \"7d\",\n \"keep_in_use\": false,\n \"notices\": []\n }", + ], + }, + JsonTestCase { + description: "kept relations are listed apart from the relations to archive", + plan: Some(kept_usage_plan()), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &[ + "\"kept_in_use\": [\n {\n \"name\": \"dev.orders_daily\",\n \ + \"age_days\": null,\n \"in_use\": {", + "\"keep_in_use\": true", + ], + }, + JsonTestCase { + description: "notices of a check that ran are listed in usage_check", + plan: Some(chains_unfollowed_plan()), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &[ + "\"notices\": [\n \"view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER\"\n ]", + ], + }, + JsonTestCase { + description: "an unchecked usage leaves in_use null and states the reason", + plan: Some(unchecked_plan(false)), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &[ + "\"in_use\": null", + "\"status\": \"not_checked\",\n \"reason\": \"unsupported adapter\"", + ], + }, + JsonTestCase { + description: "no usage check is null", + plan: Some(direct_archive_plan()), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &["\"usage_check\": null", "\"in_use\": null"], + }, + ]; + + for test_case in test_cases { + let text: String = format_janitor_json( + test_case.plan.as_ref(), + test_case.outcome, + test_case.run.as_ref(), + test_case.error, + ); + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: missing fragments in {text}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs index ae6072dc7c..8dbd109319 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs @@ -81,3 +81,10 @@ pub(super) struct EventsUnwrittenTestCase { pub(super) error: &'static str, pub(super) expected_text: &'static str, } + +pub(super) struct UsagePromptTestCase { + pub(super) description: &'static str, + pub(super) plan: JanitorPlanText, + pub(super) expected_fragment: &'static str, + pub(super) expected_phrase: &'static str, +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index f6448e6691..cd6572eb5b 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -183,7 +183,7 @@ pub const OPERATION_KINDS: [&str; 10] = [ "subprocess", "warehouse", ]; -pub const OPERATION_NAMES: [&str; 56] = [ +pub const OPERATION_NAMES: [&str; 57] = [ "adbc_driver_install", "audit_evaluation", "clone_execution", @@ -208,6 +208,7 @@ pub const OPERATION_NAMES: [&str; 56] = [ "janitor_execution", "janitor_state_inspection", "janitor_target_connection", + "janitor_usage_check", "janitor_warehouse_inspection", "managed_source_load", "project_compile", diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 5e0d635827..06d7549249 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -284,6 +284,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Executor, }, + HostEntry { + name: "JanitorUsageCheck", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, HostEntry { name: "run_load_pipeline", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 85ba45b08f..531d06bd5c 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -257,6 +257,7 @@ pub enum ExecutorHost { JanitorRelationKey, JanitorSkippedRelation, JanitorSkippedSchema, + JanitorUsageCheck, RunLoadPipeline, LoadCallbacks, LoadExecutionResult, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index abc2f86bdd..a64ef9dc5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -233,7 +233,7 @@ impl TransientStatusReporter { .write_persistent(slf.py(), &format!("{message}\n{blank}")) } - fn error(slf: &Bound<'_, Self>, message: &str) -> PyResult<()> { + pub(crate) fn error(slf: &Bound<'_, Self>, message: &str) -> PyResult<()> { Self::close(slf)?; slf.get().write_message(slf.py(), message, false) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 9840e1a8f1..3ca9c5226b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -31,7 +31,7 @@ use crate::bindings::_helpers::rules::early_builtin_rules::{EarlyRulesScope, Ear use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::rules_progress; use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; -use crate::bindings::_helpers::rules::rules_config::load_rules_config; +use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; const RICH_LINEAGE: &str = "rich"; const FAST_LINEAGE: &str = "fast"; @@ -114,6 +114,11 @@ impl<'py> AnalysisOptions<'py> { .unwrap_or_else(|| py.None().into_bound(py)), }) } + + /// Whether `--no-cache` asked this compile to bypass its caches, the Rules cache included. + fn no_cache(&self) -> PyResult { + native_field(&self.analysis_selection, "no_cache")?.is_truthy() + } } /// `{seconds:.2f}`, as Python formats elapsed milliseconds over a thousand. @@ -151,7 +156,8 @@ pub(crate) fn analyze<'py>( let enabled: bool = !(native_field(selection, "select")?.is_truthy()? || native_field(selection, "exclude")?.is_truthy()?) && crate::bindings::_helpers::compile_context::delta::injection().is_none(); - let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; + let early_lint: Bound<'py, EarlySqlLint> = + Bound::new(py, EarlySqlLint::new(enabled, options.no_cache()?))?; let early_rules: EarlyRulesScope = EarlyRulesScope::new(); let analysis: PyResult> = analyzed(py, &options, (&early_lint, enabled.then_some(&early_rules))); @@ -204,6 +210,7 @@ fn analyzed<'py>( project_dir: options.project_dir.extract()?, dialect: lint_dialect.extract()?, no_sql_analysis: options.no_sql_validation.is_truthy()?, + no_cache: options.no_cache()?, }); } let (graph, graph_ms): (Bound<'py, PyAny>, u64) = @@ -289,6 +296,10 @@ fn analyzed<'py>( "skipped_type_proof_rules", native_field(&rules_result, "skipped_type_proof_rules")?, ), + ( + "custom_rule_timings", + native_field(&rules_result, "custom_rule_timings")?, + ), ], ) } @@ -481,7 +492,8 @@ impl<'py> Driver<'_, 'py> { let options: &AnalysisOptions<'py> = self.options; let project_dir: &Bound<'py, PyAny> = &options.project_dir; let rules_config: Bound<'py, PyAny> = - load_rules_config(py, project_dir.extract()?)?.into_bound(py); + load_rules_config_with_cache(py, project_dir.extract()?, !options.no_cache()?)? + .into_bound(py); if !options.prepared_artifacts.is_none() && !any_error(core_diagnostics.as_any())? { let target_dir: Bound<'py, PyAny> = project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 624495ceca..902d4fa14c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -9,6 +9,9 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_host::compile_context::main::collect_compile_timings::collect_compile_timings; use sqlbuild_host::compile_context::main::timing_milliseconds::timing_milliseconds; use sqlbuild_host::compile_context::types::CompileTimingsHandle; +use sqlbuild_rules::custom_host_pool::constants::{ + CUSTOM_RULES_GUARD_KEY, CUSTOM_RULES_INVOCATIONS_KEY, CUSTOM_RULES_MONITOR_KEY, +}; use std::rc::Rc; use crate::bindings::_helpers::boundary::python_calls::{ @@ -487,6 +490,17 @@ fn timings_ms<'py>( ] { timings.set_item(key, native_field(analysis, attribute)?)?; } + for key in [ + CUSTOM_RULES_INVOCATIONS_KEY, + CUSTOM_RULES_MONITOR_KEY, + CUSTOM_RULES_GUARD_KEY, + ] { + timings.set_item(key, 0)?; + } + for entry in native_field(analysis, "custom_rule_timings")?.try_iter()? { + let (key, value): (String, u64) = entry?.extract()?; + timings.set_item(key, value)?; + } timings.set_item("write_ms", write_ms)?; for (key, value) in timing_milliseconds(detailed_timings) { timings.set_item(key, value)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index b10e4dd7e7..07e8d1173f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; -use crate::bindings::_helpers::rules::rules_config::load_rules_config; +use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; const GENERIC_DIALECT: &str = "generic"; @@ -80,16 +80,12 @@ fn run_rules_gate<'py>( rules_error.call1(("compiler rules require a resolved project directory",))?, )); } - let mut config: Bound<'py, PyAny> = - load_rules_config(py, project_dir.extract()?)?.into_bound(py); - if native_field(options, "no_cache")?.is_truthy()? { - let cache: Bound<'py, PyAny> = new_record_with( - py, - RecordId::RulesCacheConfig, - &[("enabled", false.into_pyobject(py)?.to_owned().into_any())], - )?; - config = replaced(&config, &[("cache", cache)])?; - } + let config: Bound<'py, PyAny> = load_rules_config_with_cache( + py, + project_dir.extract()?, + !native_field(options, "no_cache")?.is_truthy()?, + )? + .into_bound(py); progress(on_progress, "Evaluating configured rules...")?; let started: f64 = monotonic(py)?; let dialect: Bound<'py, PyAny> = adapter.call_method0("sql_analysis_dialect")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 925d7e16ee..34186a05d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -360,6 +360,8 @@ const R_JANITOR_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("direct_state_history_versions", FieldDefault::Int(20)), FieldSpec::new("delete_tracked_only", FieldDefault::True), FieldSpec::new("exclude_patterns", FieldDefault::EmptyTuple), + FieldSpec::new("in_use_window", FieldDefault::Text("7d")), + FieldSpec::new("keep_in_use", FieldDefault::False), ], }; const R_MIGRATIONS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 5dba134a6c..ceea83e278 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -185,8 +185,14 @@ fn settings<'py>( "drop_old_name_views", &native_field(request, "drop_old_name_views")?, ), + ("in_use_window", &native_field(janitor, "in_use_window")?), ], )?; + options.set_item( + "keep_in_use", + native_field(request, "keep_in_use")?.is_truthy()? + || native_field(janitor, "keep_in_use")?.is_truthy()?, + )?; new_record(py, RecordId::JanitorSettings, (), Some(&options)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index ec56d16473..9337b93031 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -15,13 +15,16 @@ use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; use crate::bindings::_helpers::build_progress::progress_reporters::{ ConnectionProgressReporter, PlanningProgressReporter, }; -use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::build_progress::transient_status::{ + TransientStatusReporter, status_reporter, +}; use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; use crate::bindings::_helpers::janitor::janitor_planning::{ JanitorSettings, RelationScope, build_plan, }; +use crate::bindings::_helpers::janitor::janitor_usage::{UsageSettings, check_janitor_usage}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -284,6 +287,7 @@ pub(crate) fn build_janitor_execution_plan<'py>( Ok(plan) }, )?; + let has_candidates: bool = native_field(&plan, "archive_candidates")?.is_truthy()?; let options: Bound<'py, PyDict> = PyDict::new(py); options.set_item( "message", @@ -292,14 +296,35 @@ pub(crate) fn build_janitor_execution_plan<'py>( started.elapsed().as_secs_f64() ), )?; - options.set_item("blank_line_after", true)?; + options.set_item("blank_line_after", !has_candidates)?; status.call_method("complete", (), Some(&options))?; let unknown: Bound<'py, PyAny> = native_field(&plan, "old_name_views")?.getattr("unknown_requests")?; if unknown.is_truthy()? { return Err(unknown_view_error(py, &unknown)?); } - Ok(plan) + if !has_candidates { + return Ok(plan); + } + check_janitor_usage( + (&compiled.adapter, connection), + ( + &plan, + &Bound::new( + py, + TransientStatusReporter::new( + py, + &output_stream(invocation)?, + use_color(invocation)?, + true, + )?, + )?, + ), + &UsageSettings { + window: native_field(settings, "in_use_window")?.extract()?, + keep_in_use: native_field(settings, "keep_in_use")?.is_truthy()?, + }, + ) } /// Execute the cleanup plan on up to the command's concurrency, reporting to `progress`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs index 38a3a16c4c..36f184cd43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs @@ -8,11 +8,13 @@ use sqlbuild_core::janitor_text::main::render_janitor_completion::render_janitor use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::models::{ ArchiveCandidateText, ArchivedRelationText, JanitorPlanText, JanitorResultCounts, - OldNameViewText, SourceSchemaText, + OldNameViewText, RelationUsageText, SourceSchemaText, UsageCheckText, }; const TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S"; const DATE_FORMAT: &str = "%Y-%m-%d"; +/// Relation types containing this are views, whose reads stay live. +const VIEW_TYPE_FRAGMENT: &str = "view"; fn display_name(item: &Bound<'_, PyAny>) -> PyResult { item.call_method0("display_name")?.extract() @@ -121,21 +123,79 @@ fn source_schemas(schemas: &Bound<'_, PyAny>, blocked: bool) -> PyResult, pattern: &str) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(formatted(value, pattern)?)) +} + +fn relation_usage(usage: &Bound<'_, PyAny>) -> PyResult> { + if usage.is_none() { + return Ok(None); + } + Ok(Some(RelationUsageText { + last_read_at: formatted(&native_field(usage, "last_read_at")?, TIMESTAMP_FORMAT)?, + readers: native_field(usage, "readers")?.extract()?, + query_count: native_field(usage, "query_count")?.extract()?, + via: if native_field(usage, "direct")?.is_truthy()? { + None + } else { + Some(native_field(usage, "via")?.extract()?) + }, + })) +} + +fn archive_candidate( + candidate: &Bound<'_, PyAny>, + planned_at: &Bound<'_, PyAny>, +) -> PyResult { + let expires_at: Bound<'_, PyAny> = native_field(candidate, "expires_at")?; + let age_timestamp: Bound<'_, PyAny> = native_field(candidate, "age_timestamp")?; + let relation_type: String = + native_field(&native_field(candidate, "relation")?, "relation_type")? + .str()? + .extract()?; + Ok(ArchiveCandidateText { + name: key_name(candidate)?, + archive_name: display_name(&native_field(candidate, "archive_key")?)?, + age_days: age_days(&age_timestamp, planned_at)?, + expires_now: expires_at.le(native_field(candidate, "archived_at")?)?, + expires_at: formatted(&expires_at, TIMESTAMP_FORMAT)?, + is_table: !relation_type.to_lowercase().contains(VIEW_TYPE_FRAGMENT), + changed_on: optional_formatted(&age_timestamp, DATE_FORMAT)?, + usage: relation_usage(&native_field(candidate, "usage")?)?, + }) +} + +fn candidate_texts( + items: &Bound<'_, PyAny>, + planned_at: &Bound<'_, PyAny>, +) -> PyResult> { + items + .try_iter()? + .map(|item| archive_candidate(&item?, planned_at)) + .collect() +} + +fn usage_check(check: &Bound<'_, PyAny>) -> PyResult> { + if check.is_none() { + return Ok(None); + } + Ok(Some(UsageCheckText { + checked: native_field(check, "checked")?.is_truthy()?, + reason: native_field(check, "reason")?.extract()?, + window: native_field(check, "window")?.extract()?, + keep_in_use: native_field(check, "keep_in_use")?.is_truthy()?, + notices: native_field(check, "notices")?.extract()?, + })) +} + /// The display facts of one executor `JanitorPlan`. pub(crate) fn janitor_plan_text(plan: &Bound<'_, PyAny>) -> PyResult { let planned_at: Bound<'_, PyAny> = native_field(plan, "planned_at")?; - let mut archive_candidates: Vec = Vec::new(); - for candidate in native_field(plan, "archive_candidates")?.try_iter()? { - let candidate: Bound<'_, PyAny> = candidate?; - let expires_at: Bound<'_, PyAny> = native_field(&candidate, "expires_at")?; - archive_candidates.push(ArchiveCandidateText { - name: key_name(&candidate)?, - archive_name: display_name(&native_field(&candidate, "archive_key")?)?, - age_days: age_days(&native_field(&candidate, "age_timestamp")?, &planned_at)?, - expires_now: expires_at.le(native_field(&candidate, "archived_at")?)?, - expires_at: formatted(&expires_at, TIMESTAMP_FORMAT)?, - }); - } + let archive_candidates: Vec = + candidate_texts(&native_field(plan, "archive_candidates")?, &planned_at)?; let mut state_prunes: Vec<(String, i64)> = Vec::new(); for candidate in native_field(plan, "direct_state_prune_candidates")?.try_iter()? { let candidate: Bound<'_, PyAny> = candidate?; @@ -175,6 +235,8 @@ pub(crate) fn janitor_plan_text(plan: &Bound<'_, PyAny>) -> PyResult, value: usize) -> PyResult> { integer(py, i64::try_from(value).unwrap_or(i64::MAX)) } -fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { +/// Mark an operation lifecycle completed with its item count. +pub(crate) fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { let py: Python<'_> = lifecycle.py(); let metadata: Bound<'_, PyDict> = PyDict::new(py); metadata.set_item("item_count", item_count)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs new file mode 100644 index 0000000000..bf66ebde19 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs @@ -0,0 +1,192 @@ +//! The janitor usage check: who still reads each archive candidate, and whether to keep it. + +use std::collections::{HashMap, HashSet}; +use std::time::Instant; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyString}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; +use crate::bindings::_helpers::build_progress::transient_status::TransientStatusReporter; +use crate::bindings::_helpers::janitor::janitor_keys::{Folded, Key, record, tuple}; +use crate::bindings::_helpers::janitor::janitor_planning::completed; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; + +const START_MESSAGE: &str = "Checking relation usage..."; + +/// The usage window and whether in-use candidates are kept. +pub(crate) struct UsageSettings { + pub(crate) window: String, + pub(crate) keep_in_use: bool, +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +fn lifecycle(py: Python<'_>) -> PyResult> { + call_with( + &host::entry(py, ExecutorHost::OperationLifecycle)?, + &[ + ("operation_kind", text(py, "janitor")), + ("operation_name", text(py, "janitor_usage_check")), + ], + ) +} + +/// The adapter's usage report for the candidates' relations. +fn read_usage<'py>( + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + candidates: &[Bound<'py, PyAny>], + window_seconds: i64, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let relations: Vec> = candidates + .iter() + .map(|candidate| native_field(candidate, "relation")) + .collect::>()?; + within(&lifecycle(py)?, |lifecycle| { + let report: Bound<'py, PyAny> = call_with( + &native_field(adapter, "relation_usage")?, + &[ + ("connection", connection.clone()), + ("relations", tuple(py, relations)?), + ( + "window_seconds", + window_seconds.into_pyobject(py)?.into_any(), + ), + ], + )?; + completed(lifecycle, native_field(&report, "usages")?.len()?)?; + Ok(report) + }) +} + +/// Each reported usage by its relation's case-insensitive identity. +fn usages_by_relation<'py>( + report: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut usages: HashMap> = HashMap::new(); + for usage in native_field(report, "usages")?.try_iter()? { + let usage: Bound<'py, PyAny> = usage?; + usages.insert(Key::of(&usage)?.folded(), usage); + } + Ok(usages) +} + +fn completion_message( + (checked, reason): (bool, Option), + (in_use, total): (usize, usize), + elapsed_seconds: f64, +) -> String { + if !checked { + return format!( + "Relation usage not checked: {}. ({elapsed_seconds:.2}s)", + reason.unwrap_or_default() + ); + } + format!("Checked relation usage: {in_use} of {total} in use. ({elapsed_seconds:.2}s)") +} + +/// The plan with each candidate's reads, the usage check, and in-use candidates kept when asked. +pub(crate) fn check_janitor_usage<'py>( + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (plan, status): (&Bound<'py, PyAny>, &Bound<'py, TransientStatusReporter>), + settings: &UsageSettings, +) -> PyResult> { + let py: Python<'py> = plan.py(); + let candidates: Vec> = native_field(plan, "archive_candidates")? + .try_iter()? + .collect::>()?; + let window_seconds: i64 = fixed_duration_seconds(&settings.window) + .map_or(i64::MAX, |seconds| { + i64::try_from(seconds).unwrap_or(i64::MAX) + }); + let started: Instant = Instant::now(); + TransientStatusReporter::start(status, START_MESSAGE.to_owned())?; + let report: Bound<'py, PyAny> = + match read_usage((adapter, connection), &candidates, window_seconds) { + Ok(report) => report, + Err(error) => { + TransientStatusReporter::error( + status, + &format!( + "Relation usage check failed. ({:.2}s)", + started.elapsed().as_secs_f64() + ), + )?; + return Err(error); + } + }; + let checked: bool = native_field(&report, "checked")?.is_truthy()?; + let usages: HashMap> = usages_by_relation(&report)?; + let keep: bool = checked && settings.keep_in_use; + let mut archived: Vec> = Vec::new(); + let mut kept: Vec> = Vec::new(); + let mut kept_keys: HashSet = HashSet::new(); + let mut in_use: usize = 0; + for candidate in &candidates { + let key: Folded = Key::of(&native_field(candidate, "relation")?)?.folded(); + let Some(usage) = usages.get(&key) else { + archived.push(candidate.clone()); + continue; + }; + in_use += 1; + let read: Bound<'py, PyAny> = call_with( + &native_field(candidate, "with_usage")?, + &[("usage", usage.clone())], + )?; + if keep { + kept_keys.insert(Key::of(&native_field(candidate, "key")?)?.folded()); + kept.push(read); + } else { + archived.push(read); + } + } + let mut deletions: Vec> = Vec::new(); + for deletion in native_field(plan, "archive_deletion_candidates")?.try_iter()? { + let deletion: Bound<'py, PyAny> = deletion?; + let original: Bound<'py, PyAny> = native_field(&deletion, "original_key")?; + if original.is_none() || !kept_keys.contains(&Key::of(&original)?.folded()) { + deletions.push(deletion); + } + } + let reason: Bound<'py, PyAny> = native_field(&report, "reason")?; + TransientStatusReporter::complete( + status, + &completion_message( + (checked, reason.extract()?), + (in_use, candidates.len()), + started.elapsed().as_secs_f64(), + ), + true, + )?; + let check: Bound<'py, PyAny> = record( + py, + ExecutorHost::JanitorUsageCheck, + &[ + ("checked", flag(py, checked)), + ("window", text(py, &settings.window)), + ("keep_in_use", flag(py, settings.keep_in_use)), + ("reason", reason), + ("notices", native_field(&report, "notices")?), + ], + )?; + call_with( + &native_field(plan, "with_usage")?, + &[ + ("archive_candidates", tuple(py, archived)?), + ("archive_deletion_candidates", tuple(py, deletions)?), + ("kept_in_use", tuple(py, kept)?), + ("usage_check", check), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs index d2231d7bc5..6c12b76ff8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs @@ -7,4 +7,5 @@ pub(crate) mod janitor_phases; pub(crate) mod janitor_plan; pub(crate) mod janitor_planning; pub(crate) mod janitor_progress; +pub(crate) mod janitor_usage; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index dad59be8f8..b56fb805f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -461,6 +461,7 @@ const JANITOR_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("retention_days", FieldDefault::None), FieldSpec::new("direct_state_history_versions", FieldDefault::None), FieldSpec::new("drop_old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new("keep_in_use", FieldDefault::False), FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("as_target", FieldDefault::None), FieldSpec::new("verbose", FieldDefault::False), @@ -488,6 +489,8 @@ const JANITOR_SETTINGS: RecordSpec = RecordSpec { FieldSpec::required("direct_state_history_versions"), FieldSpec::new("archive_retention_days", FieldDefault::Int(14)), FieldSpec::new("drop_old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new("in_use_window", FieldDefault::Text("7d")), + FieldSpec::new("keep_in_use", FieldDefault::False), ], }; const LOAD_COMMAND_REQUEST: RecordSpec = RecordSpec { @@ -1024,6 +1027,7 @@ const COMPILE_ANALYSIS: RecordSpec = RecordSpec { FieldSpec::new("rule_cache_hits", FieldDefault::Int(0)), FieldSpec::new("rule_cache_misses", FieldDefault::Int(0)), FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), + FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs index 212db5027c..75fcfbcd84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -107,6 +107,7 @@ const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { FieldSpec::new("cache_hits", FieldDefault::Int(0)), FieldSpec::new("cache_misses", FieldDefault::Int(0)), FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), + FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), ], }; const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { @@ -131,6 +132,7 @@ const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { FieldSpec::required("cache_misses"), FieldSpec::required("custom_ms"), FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), ], }; const R_MODEL: RecordSpec = RecordSpec { @@ -295,6 +297,7 @@ const R_RULES_RESULT: RecordSpec = RecordSpec { FieldSpec::new("built_in_ms", FieldDefault::Int(0)), FieldSpec::new("custom_ms", FieldDefault::Int(0)), FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_rule_timings", FieldDefault::EmptyTuple), ], }; const R_RULE_FILE: RecordSpec = RecordSpec { @@ -355,6 +358,8 @@ const R_CUSTOM_RULE_RUN: RecordSpec = RecordSpec { FieldSpec::new("uncacheable_codes", FieldDefault::EmptyFrozenset), FieldSpec::new("observed", FieldDefault::EmptyTuple), FieldSpec::new("stateful_codes", FieldDefault::EmptyFrozenset), + FieldSpec::new("rule_timings", FieldDefault::EmptyDict), + FieldSpec::new("monitor_ns", FieldDefault::Int(0)), ], }; const R_CUSTOM_HOST_PARTITION: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 98eac488eb..df0e087a38 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -10,7 +10,8 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, JsonValue, OrjsonOptions}; -use sqlbuild_rules::custom_host_pool::models::{HostRuleSubject, HostRun}; +use sqlbuild_rules::custom_host_pool::main::custom_rule_timings::custom_rule_timings; +use sqlbuild_rules::custom_host_pool::models::{HostRuleSubject, HostRun, HostTimings}; use sqlbuild_rules::custom_host_pool::types::{FactKey, HostPlan}; use sqlbuild_rules::custom_rule_cache::models::CustomEvaluation; use sqlbuild_rules::custom_rule_cache::types::ReadSet; @@ -84,7 +85,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( .collect::>()?; custom.sort_by(|left, right| left.0.cmp(&right.0)); if custom.is_empty() { - return outcome(py, (Vec::new(), 0, 0), (0, 0)); + return outcome(py, (Vec::new(), 0, 0), (0, 0), &[]); } let enabled: bool = native_field(&native_field(run.config, "cache")?, "enabled")?.is_truthy()?; @@ -152,6 +153,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( readset_digests.push(reproducible(digests.combined(readset))?); } let plan: HostPlan = cache.plan(readset_digests); + let mut host_timings: HostTimings = HostTimings::default(); let delta: Option> = injection(); if let Some(injection) = &delta && let Some(reason) = delta_plan_refusal(&plan, &cache, injection) @@ -163,7 +165,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( .iter() .map(|(code, rule)| Ok((code.clone(), host_subject(rule)?))) .collect::>()?; - let host_run: HostRun = run_custom_hosts( + let mut host_run: HostRun = run_custom_hosts( py, &HostRequest { project: run.project, @@ -178,6 +180,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( }, &plan, )?; + host_timings = std::mem::take(&mut host_run.timings); let evaluations: Vec = evaluations(host_run, &mut digests, enabled)?; if let Some(injection) = &delta && !evaluations @@ -203,6 +206,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( py, (cache.findings(py)?, cache.hits(), cache.misses()), (elapsed_ms, cpu_ms), + &custom_rule_timings(&host_timings, cache.counts_by_code()), ) } @@ -285,6 +289,7 @@ pub(crate) fn outcome<'py>( py: Python<'py>, (findings, hits, misses): (Vec>, usize, usize), (custom_ms, custom_cpu_ms): (i64, i64), + timings: &[(String, u64)], ) -> PyResult> { new_record_with( py, @@ -295,6 +300,10 @@ pub(crate) fn outcome<'py>( ("cache_misses", misses.into_bound_py_any(py)?), ("custom_ms", custom_ms.into_bound_py_any(py)?), ("custom_cpu_ms", custom_cpu_ms.into_bound_py_any(py)?), + ( + "custom_rule_timings", + PyTuple::new(py, timings.iter().cloned())?.into_any(), + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index b83586f846..a387f618df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -70,6 +70,7 @@ pub(crate) struct CustomRuleCache { results: SubjectResults, hits: usize, misses: usize, + counts_by_code: BTreeMap, } impl CustomRuleCache { @@ -112,6 +113,7 @@ impl CustomRuleCache { results: SubjectResults::new(), hits: 0, misses: 0, + counts_by_code: BTreeMap::new(), } } @@ -137,6 +139,11 @@ impl CustomRuleCache { ) -> BTreeMap>> { if self.path.is_none() { self.misses = self.rules.len(); + self.counts_by_code = self + .rules + .iter() + .map(|rule| (rule.code.clone(), (0, rule.subjects.len()))) + .collect(); return self .rules .iter() @@ -154,6 +161,7 @@ impl CustomRuleCache { self.results.extend(lookup.hits); self.hits = lookup.hit_count; self.misses = lookup.miss_count; + self.counts_by_code = lookup.counts_by_code; lookup.plan } @@ -206,6 +214,11 @@ impl CustomRuleCache { self.misses } + /// Each rule's reused and evaluated subject counts; every subject misses with caching off. + pub(crate) fn counts_by_code(&self) -> &BTreeMap { + &self.counts_by_code + } + fn extend_results(&mut self, evaluations: Vec) { for evaluation in evaluations { self.results diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs index 7baeba25dd..0445d7e3a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::rules::catalogue::references_custom_rules; use crate::bindings::_helpers::rules::rule_dispatch::speculative_rules_request; use crate::bindings::_helpers::rules::rule_plan::rule_run_plan; -use crate::bindings::_helpers::rules::rules_config::load_rules_config; +use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; use crate::bindings::types::CompilerDetach; const CANCELLED: &str = "native rules evaluation was cancelled"; @@ -40,6 +40,8 @@ pub(crate) struct EarlyRulesSettings { pub(crate) project_dir: PathBuf, pub(crate) dialect: String, pub(crate) no_sql_analysis: bool, + /// Whether `--no-cache` turned the Rules cache off, as the rules phase does. + pub(crate) no_cache: bool, } /// One compile thread's early built-in rules, from arming to the rules phase taking them. @@ -186,7 +188,8 @@ fn early_request( ) -> PyResult> { let py: Python<'_> = project.py(); let config: Bound<'_, PyAny> = - load_rules_config(py, settings.project_dir.clone())?.into_bound(py); + load_rules_config_with_cache(py, settings.project_dir.clone(), !settings.no_cache)? + .into_bound(py); if references_custom_rules(&config)? { return Ok(None); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 5781e5abd8..ce9c2b3e41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -84,7 +84,11 @@ struct EarlyLintPlan { } /// The plan early SQL lint runs with, when the project selects SQL rules without a warm cache. -fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult> { +fn early_lint_plan( + inputs: &Bound<'_, PyAny>, + dialect: &str, + no_cache: bool, +) -> PyResult> { let py: Python<'_> = inputs.py(); let project_dir: Bound<'_, PyAny> = record_field(&record_field(inputs, "discovered_inputs")?, "project_dir")?; @@ -97,9 +101,10 @@ fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult bool { /// Start early SQL lint on a native thread when the project selects SQL rules without a cache. fn start_early_lint( - inputs: &Bound<'_, PyAny>, - dialect: &str, + (inputs, dialect): (&Bound<'_, PyAny>, &str), + no_cache: bool, stop: &Arc, ) -> PyResult> { - let Some(plan) = early_lint_plan(inputs, dialect)? else { + let Some(plan) = early_lint_plan(inputs, dialect, no_cache)? else { return Ok(None); }; let mut compiled: HashMap = HashMap::new(); @@ -290,6 +295,8 @@ pub(crate) struct ExpansionReuse { pub(crate) struct EarlySqlLint { #[pyo3(get)] pub(crate) enabled: bool, + /// Whether `--no-cache` turned the Rules cache off, so a warm SQL rule cache is ignored. + no_cache: bool, preparation: Option>, expansion_reuse: Option, stop_requested: Arc, @@ -297,9 +304,10 @@ pub(crate) struct EarlySqlLint { impl EarlySqlLint { /// A manager that starts speculative checks only when `enabled`. - pub(crate) fn new(enabled: bool) -> Self { + pub(crate) fn new(enabled: bool, no_cache: bool) -> Self { Self { enabled, + no_cache, preparation: None, expansion_reuse: None, stop_requested: Arc::new(AtomicBool::new(false)), @@ -337,9 +345,9 @@ impl EarlySqlLint { #[pymethods] impl EarlySqlLint { #[new] - #[pyo3(signature = (*, enabled))] - fn py_new(enabled: bool) -> Self { - Self::new(enabled) + #[pyo3(signature = (*, enabled, no_cache=false))] + fn py_new(enabled: bool, no_cache: bool) -> Self { + Self::new(enabled, no_cache) } fn __enter__(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { @@ -363,7 +371,7 @@ impl EarlySqlLint { } let py: Python<'_> = inputs.py(); let stop: Arc = Arc::clone(&slf.stop_requested); - slf.preparation = start_early_lint(inputs, dialect, &stop)? + slf.preparation = start_early_lint((inputs, dialect), slf.no_cache, &stop)? .map(|preparation| Py::new(py, preparation)) .transpose()?; let declaration_scope: Bound<'_, PyAny> = record_field(inputs, "declaration_scope")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index ff0b48d956..ef2dec1e10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -449,7 +449,7 @@ fn evaluate_native<'py>( cancel_on_interrupt(py, &evaluated, &run.cancel); let custom: PyResult> = match custom { Some(handle) => joined(py, handle, &run.cancel), - None => outcome(py, (Vec::new(), 0, 0), (0, 0)), + None => outcome(py, (Vec::new(), 0, 0), (0, 0), &[]), }; rules_result(run, defer_suppressions, evaluated, custom) } @@ -469,7 +469,7 @@ fn selectionless_result<'py>( run, true, evaluated, - outcome(py, (Vec::new(), 0, 0), (0, 0)), + outcome(py, (Vec::new(), 0, 0), (0, 0), &[]), ) } @@ -538,6 +538,10 @@ fn rules_result<'py>( ), ("custom_ms", field(&custom, "custom_ms")?), ("custom_cpu_ms", field(&custom, "custom_cpu_ms")?), + ( + "custom_rule_timings", + field(&custom, "custom_rule_timings")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index 0c59a06ca2..7bb4c3f5f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -127,6 +127,10 @@ pub(crate) fn finish_rule_run<'py>( ), ("custom_ms", record_field(result, "custom_ms")?), ("custom_cpu_ms", record_field(result, "custom_cpu_ms")?), + ( + "custom_rule_timings", + record_field(result, "custom_rule_timings")?, + ), ("cache_hits", total("cache_hits")?), ("cache_misses", total("cache_misses")?), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index c15e583ae7..a59d0e0da2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -240,10 +240,20 @@ fn rules_error(py: Python<'_>, class: ErrorId, message: String, help: Option, project_dir: PathBuf) -> PyResult> { - let config: RulesConfig = match rules_config_of(&project_dir)? { + load_rules_config_with_cache(py, project_dir, true) +} + +/// The project's rules configuration, with the Rules cache off when `cache` is `false`. +pub(crate) fn load_rules_config_with_cache( + py: Python<'_>, + project_dir: PathBuf, + cache: bool, +) -> PyResult> { + let mut config: RulesConfig = match rules_config_of(&project_dir)? { Ok(config) => config, Err(invalid) => return Err(invalid.into_pyerr(py)), }; + config.cache.enabled &= cache; Ok(config_object(py, &config)?.unbind()) } diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index f580959741..6737b8cf28 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -1,8 +1,9 @@ use crate::configuration::_helpers::removed_settings::removed_settings; use crate::configuration::errors::RulesConfigLoadError; use crate::constants::{ - ALLOWED_QUALIFIERS_OPTION, MIN_CUSTOM_RULE_TEST_CASES_KEY, MODEL_NAME_LAYERS, - PUBLISHED_LAYERS_OPTION, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, RELATION_EXCEPTION_RULES, + ALLOWED_PATHS_KEY, ALLOWED_QUALIFIERS_OPTION, MIN_CUSTOM_RULE_TEST_CASES_KEY, + MODEL_NAME_LAYERS, PUBLISHED_LAYERS_OPTION, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, + RELATION_EXCEPTION_RULES, }; use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; @@ -307,27 +308,25 @@ fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> let label: String = format!("rules.rule_options.{PUBLISHED_SHAPE_CODE}.{ALLOWED_QUALIFIERS_OPTION}"); let shape: String = format!( - "{label} entries must be {{ qualifier = \"...\", reason = \"...\" }} tables with an \ - optional domain" + "{label} entries must be {{ qualifier = \"...\", reason = \"...\" }} or \ + {{ paths = [\"...\"], reason = \"...\" }} tables with an optional domain" ); let Some(entries) = entries.as_array() else { return Err(shape); }; - let mut seen: std::collections::BTreeSet<(Option<&str>, &str)> = + let mut seen: std::collections::BTreeSet<(Option<&str>, Option<&str>, Vec<&str>)> = std::collections::BTreeSet::new(); for entry in entries { let Some(table) = entry.as_object() else { return Err(shape); }; - if table - .keys() - .any(|key| !["domain", "qualifier", "reason"].contains(&key.as_str())) - { + if table.keys().any(|key| { + !["domain", "qualifier", ALLOWED_PATHS_KEY, "reason"].contains(&key.as_str()) + }) { return Err(shape); } let text = |key: &str| table.get(key).map(|value| value.as_str().map(str::trim)); - let (Some(Some(qualifier)), Some(Some(reason))) = (text("qualifier"), text("reason")) - else { + let Some(Some(reason)) = text("reason") else { return Err(shape); }; let domain: Option<&str> = match text("domain") { @@ -335,14 +334,66 @@ fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> Some(Some(domain)) if !domain.is_empty() => Some(domain), Some(_) => return Err(shape), }; - if qualifier.is_empty() || reason.is_empty() { - return Err(format!( - "{label} entries need a non-empty qualifier and reason" - )); + let qualifier: Option<&str> = match text("qualifier") { + None => None, + Some(Some(qualifier)) => Some(qualifier), + Some(None) => return Err(shape), + }; + let paths: Option> = match table.get(ALLOWED_PATHS_KEY) { + None => None, + Some(paths) => { + let Some(paths) = paths.as_array() else { + return Err(shape); + }; + let paths: Option> = paths + .iter() + .map(|path| path.as_str().map(str::trim)) + .collect(); + let Some(mut paths) = paths else { + return Err(shape); + }; + paths.sort_unstable(); + Some(paths) + } + }; + if qualifier.is_none() && paths.is_none() { + return Err(shape); + } + if reason.is_empty() { + return Err(format!("{label} entries need a non-empty reason")); + } + if qualifier.is_some_and(str::is_empty) { + return Err(format!("{label} entries need a non-empty qualifier")); } - if !seen.insert((domain, qualifier)) { - return Err(format!("duplicate {label} entry: {qualifier}")); + if let Some(paths) = &paths { + validate_allowed_paths(paths, &label)?; } + let paths: Vec<&str> = paths.unwrap_or_default(); + let described: String = match qualifier { + Some(qualifier) => qualifier.to_owned(), + None => paths.join(", "), + }; + if !seen.insert((domain, qualifier, paths)) { + return Err(format!("duplicate {label} entry: {described}")); + } + } + Ok(()) +} + +/// One entry's sorted `paths` globs: non-empty, `/`-separated and each listed once. +fn validate_allowed_paths(paths: &[&str], label: &str) -> Result<(), String> { + if paths.is_empty() || paths.iter().any(|path| path.is_empty()) { + return Err(format!( + "{label} entries need a non-empty list of non-empty paths" + )); + } + if let Some(path) = paths.iter().find(|path| path.contains('\\')) { + return Err(format!( + "{label} paths must be project-relative globs with / separators: {path}" + )); + } + if let Some(pair) = paths.windows(2).find(|pair| pair[0] == pair[1]) { + return Err(format!("{label} entry lists path {} twice", pair[0])); } Ok(()) } diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index ad865fb4e1..6952d90a7a 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -55,5 +55,7 @@ pub(crate) const PUBLISHED_SHAPE_CODE: &str = "SQBRPROJECT107"; pub(crate) const PUBLISHED_LAYERS_OPTION: &str = "published_layers"; /// The option listing qualifiers that do not count towards a domain's name shape. pub(crate) const ALLOWED_QUALIFIERS_OPTION: &str = "allowed_qualifiers"; +/// The `allowed_qualifiers` entry key whose globs match published models by file path. +pub(crate) const ALLOWED_PATHS_KEY: &str = "paths"; /// The published layers when a project does not configure them. pub(crate) const DEFAULT_PUBLISHED_LAYERS: [&str; 2] = ["mart", "mart_v"]; diff --git a/crates/sqlbuild-rules/src/custom_host_pool/_helpers/decoding.rs b/crates/sqlbuild-rules/src/custom_host_pool/_helpers/decoding.rs index a6f3b24df4..ce3024fa23 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/_helpers/decoding.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/_helpers/decoding.rs @@ -8,7 +8,7 @@ use crate::custom_host_pool::_helpers::payload_values::{fact_key, python_text, r use crate::custom_host_pool::constants::{ CUSTOM_RULE_PROJECT_SUBJECT, INVALID_EVALUATION, INVALID_FINDINGS, INVALID_PAYLOAD, }; -use crate::custom_host_pool::models::{HostEvaluation, HostSlice}; +use crate::custom_host_pool::models::{HostEvaluation, HostSlice, HostTimings, RuleTiming}; use crate::custom_host_pool::types::FactKey; /// The slice a host response describes, or the `RulesError` message of a malformed one. @@ -51,9 +51,36 @@ pub(crate) fn decoded_slice(response: &Value) -> Result { Some(_) | None => BTreeSet::new(), }, untracked_reads, + timings: payload.get("timings").map(timings).unwrap_or_default(), }) } +/// The host's reported timings; anything malformed counts as zero. +fn timings(value: &Value) -> HostTimings { + let count = |key: &str| value.get(key).and_then(Value::as_u64).unwrap_or(0); + let rules = match value.get("rules") { + Some(Value::Object(rules)) => rules + .iter() + .map(|(code, timing)| { + let part = |index: usize| timing.get(index).and_then(Value::as_u64).unwrap_or(0); + ( + code.clone(), + RuleTiming { + invocations: part(0), + cpu_ns: part(1), + }, + ) + }) + .collect(), + Some(_) | None => BTreeMap::new(), + }; + HostTimings { + rules, + monitor_ns: count("monitor_ns"), + guard_ns: count("guard_ns"), + } +} + fn evaluation( item: &Value, readsets: &[Vec], diff --git a/crates/sqlbuild-rules/src/custom_host_pool/constants.rs b/crates/sqlbuild-rules/src/custom_host_pool/constants.rs index aedde400c8..867aaa40d8 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/constants.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/constants.rs @@ -25,3 +25,13 @@ pub(crate) const INVALID_PAYLOAD: &str = "custom rule host returned an invalid p pub(crate) const INVALID_EVALUATION: &str = "custom rule host returned an invalid evaluation"; /// The `RulesError` message of findings that are not a list. pub(crate) const INVALID_FINDINGS: &str = "custom rule host returned invalid findings"; + +/// The `compile_timings` key of every custom-rule invocation the hosts ran. +pub const CUSTOM_RULES_INVOCATIONS_KEY: &str = "custom_rules_invocations"; +/// The `compile_timings` key of the hosts' module-state monitor CPU. +pub const CUSTOM_RULES_MONITOR_KEY: &str = "custom_rules_monitor_cpu_ms"; +/// The `compile_timings` key of the hosts' runtime-guard check CPU. +pub const CUSTOM_RULES_GUARD_KEY: &str = "custom_rules_guard_cpu_ms"; +/// The prefix of one rule code's `compile_timings` keys: `custom_rules..`. +pub(crate) const CUSTOM_RULE_KEY_PREFIX: &str = "custom_rules."; +pub(crate) const NANOSECONDS_PER_MILLISECOND: u64 = 1_000_000; diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs new file mode 100644 index 0000000000..48055e2ed0 --- /dev/null +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/custom_rule_timings.rs @@ -0,0 +1,49 @@ +//! The flat `compile_timings` entries of one custom-rule evaluation. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::custom_host_pool::constants::{ + CUSTOM_RULE_KEY_PREFIX, CUSTOM_RULES_GUARD_KEY, CUSTOM_RULES_INVOCATIONS_KEY, + CUSTOM_RULES_MONITOR_KEY, NANOSECONDS_PER_MILLISECOND, +}; +use crate::custom_host_pool::models::{HostTimings, RuleTiming}; + +/// Run-wide totals, then each rule's invocations, CPU and cache counts in code order. +#[must_use] +pub fn custom_rule_timings( + timings: &HostTimings, + cache_counts: &BTreeMap, +) -> Vec<(String, u64)> { + let invocations: u64 = timings + .rules + .values() + .map(|timing| timing.invocations) + .sum(); + let mut entries: Vec<(String, u64)> = vec![ + (CUSTOM_RULES_INVOCATIONS_KEY.to_owned(), invocations), + ( + CUSTOM_RULES_MONITOR_KEY.to_owned(), + timings.monitor_ns / NANOSECONDS_PER_MILLISECOND, + ), + ( + CUSTOM_RULES_GUARD_KEY.to_owned(), + timings.guard_ns / NANOSECONDS_PER_MILLISECOND, + ), + ]; + let codes: BTreeSet<&String> = timings.rules.keys().chain(cache_counts.keys()).collect(); + for code in codes { + let timing: RuleTiming = timings.rules.get(code).copied().unwrap_or_default(); + let (hits, misses): (usize, usize) = cache_counts.get(code).copied().unwrap_or_default(); + let key = |measure: &str| format!("{CUSTOM_RULE_KEY_PREFIX}{code}.{measure}"); + entries.extend([ + (key("invocations"), timing.invocations), + (key("cpu_ms"), timing.cpu_ns / NANOSECONDS_PER_MILLISECOND), + (key("cache_hits"), u64::try_from(hits).unwrap_or(u64::MAX)), + ( + key("cache_misses"), + u64::try_from(misses).unwrap_or(u64::MAX), + ), + ]); + } + entries +} diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/merge_host_slices.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/merge_host_slices.rs index 282652d78d..f6ea3644c4 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/main/merge_host_slices.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/merge_host_slices.rs @@ -5,7 +5,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::custom_host_pool::_helpers::merging::{ collected_untracked_reads, demoted, merged_observations, ordered_evaluations, }; -use crate::custom_host_pool::models::{HostEvaluation, HostRun, HostSlice}; +use crate::custom_host_pool::models::{HostEvaluation, HostRun, HostSlice, HostTimings}; use crate::custom_host_pool::types::{FactKey, HostInvocation}; /// Merge slices in single-host invocation `order`, demoting tracking that grew too large. @@ -17,6 +17,10 @@ pub fn merge_host_slices(slices: Vec, order: &[HostInvocation]) -> Ho collected_untracked_reads(&slices, &untracked); let (evaluated, untracked_reads): (Vec, BTreeMap>) = demoted(evaluations, &untracked, untracked_reads); + let mut timings: HostTimings = HostTimings::default(); + for slice in &slices { + timings.add(&slice.timings); + } HostRun { evaluated, untracked_reads: untracked_reads @@ -28,5 +32,6 @@ pub fn merge_host_slices(slices: Vec, order: &[HostInvocation]) -> Ho .flat_map(|slice| slice.uncacheable.iter().cloned()) .collect(), observed: merged_observations(&slices), + timings, } } diff --git a/crates/sqlbuild-rules/src/custom_host_pool/main/mod.rs b/crates/sqlbuild-rules/src/custom_host_pool/main/mod.rs index 14dbd95162..c32c0ed92e 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/main/mod.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/main/mod.rs @@ -1,4 +1,5 @@ pub mod available_cores; +pub mod custom_rule_timings; pub mod decode_host_response; pub mod host_count; pub mod merge_host_slices; diff --git a/crates/sqlbuild-rules/src/custom_host_pool/models.rs b/crates/sqlbuild-rules/src/custom_host_pool/models.rs index a9cd86f651..5d944b3d90 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/models.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/models.rs @@ -30,6 +30,35 @@ pub struct HostSlice { pub observed: BTreeMap>, pub bounded: bool, pub stateful: BTreeSet, + pub timings: HostTimings, +} + +/// One rule's invocations and the CPU nanoseconds they took, excluding runtime-guard checks. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RuleTiming { + pub invocations: u64, + pub cpu_ns: u64, +} + +/// Where host CPU went: each rule, the module-state monitor and the runtime guard's checks. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HostTimings { + pub rules: BTreeMap, + pub monitor_ns: u64, + pub guard_ns: u64, +} + +impl HostTimings { + /// Add `other`'s invocations and CPU to these timings. + pub fn add(&mut self, other: &Self) { + for (code, timing) in &other.rules { + let total: &mut RuleTiming = self.rules.entry(code.clone()).or_default(); + total.invocations += timing.invocations; + total.cpu_ns += timing.cpu_ns; + } + self.monitor_ns += other.monitor_ns; + self.guard_ns += other.guard_ns; + } } /// Every host's results combined exactly as one host would have reported them. @@ -39,4 +68,6 @@ pub struct HostRun { pub untracked_reads: BTreeMap>, pub uncacheable: BTreeSet, pub observed: BTreeMap>, + /// Every host's timings summed, including hosts whose results a rerun replaced. + pub timings: HostTimings, } diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs index 1340930d24..382ac143c9 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/helpers.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use crate::custom_host_pool::models::HostEvaluation; +use crate::custom_host_pool::models::{HostEvaluation, HostTimings, RuleTiming}; use crate::custom_host_pool::types::{HostInvocation, HostPlan}; /// Write a fake `/proc/self/cgroup` and a cgroup hierarchy below `root`. @@ -54,3 +54,27 @@ pub(super) fn evaluation(code: &str, subject: &str, reads: Option<&[&[&str]]>) - }), } } + +/// Host timings with the given `(code, invocations, cpu_ns)` rules. +pub(super) fn rule_timings( + rules: &[(&str, u64, u64)], + monitor_ns: u64, + guard_ns: u64, +) -> HostTimings { + HostTimings { + rules: rules + .iter() + .map(|(code, invocations, cpu_ns)| { + ( + (*code).to_owned(), + RuleTiming { + invocations: *invocations, + cpu_ns: *cpu_ns, + }, + ) + }) + .collect(), + monitor_ns, + guard_ns, + } +} diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs index f35430e473..d6a75dd132 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_custom_host_pool.rs @@ -5,14 +5,18 @@ use serde_json::json; use crate::custom_host_pool::constants::MIN_INVOCATIONS_PER_HOST; use crate::custom_host_pool::main::available_cores::available_cores; +use crate::custom_host_pool::main::custom_rule_timings::custom_rule_timings; use crate::custom_host_pool::main::decode_host_response::decode_host_response; use crate::custom_host_pool::main::host_count::host_count; use crate::custom_host_pool::main::merge_host_slices::merge_host_slices; use crate::custom_host_pool::main::partition_host_plan::partition_host_plan; -use crate::custom_host_pool::models::HostSlice; -use crate::custom_host_pool::tests::helpers::{evaluation, invocations, plan, write_cgroup_files}; +use crate::custom_host_pool::models::{HostSlice, HostTimings, RuleTiming}; +use crate::custom_host_pool::tests::helpers::{ + evaluation, invocations, plan, rule_timings, write_cgroup_files, +}; use crate::custom_host_pool::tests::test_types::{ DecodeTestCase, HostCapacityTestCase, HostCountTestCase, HostPlanTestCase, MergeTestCase, + TimingEntriesTestCase, }; const NODE_CPUS: usize = 64; @@ -209,6 +213,7 @@ fn given_host_response_when_decoding_then_slice_or_rules_error_is_returned() { "observed": [[["tree.paths"], "abc"], [["tree.glob", "*.sql"], null]], "bounded": true, "stateful": ["R1"], + "timings": {"rules": {"R1": [2, 500], "R2": "bad"}, "monitor_ns": 7, "guard_ns": 3}, }), expected_slice: Ok(HostSlice { evaluations: vec![ @@ -229,6 +234,20 @@ fn given_host_response_when_decoding_then_slice_or_rules_error_is_returned() { ]), bounded: true, stateful: BTreeSet::from(["R1".to_owned()]), + timings: HostTimings { + rules: BTreeMap::from([ + ( + "R1".to_owned(), + RuleTiming { + invocations: 2, + cpu_ns: 500, + }, + ), + ("R2".to_owned(), RuleTiming::default()), + ]), + monitor_ns: 7, + guard_ns: 3, + }, }), }, DecodeTestCase { @@ -264,11 +283,13 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { HostSlice { evaluations: vec![evaluation("R1", "b", Some(&[&["k"]]))], observed: BTreeMap::from([(vec!["t".to_owned()], Some("1".to_owned()))]), + timings: rule_timings(&[("R1", 1, 40)], 5, 1), ..HostSlice::default() }, HostSlice { evaluations: vec![evaluation("R1", "a", Some(&[&["k"]]))], observed: BTreeMap::from([(vec!["t".to_owned()], Some("2".to_owned()))]), + timings: rule_timings(&[("R1", 1, 60), ("R2", 3, 9)], 7, 2), ..HostSlice::default() }, ], @@ -276,6 +297,7 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { expected_order: invocations(&[("R1", "a"), ("R1", "b")]), expected_untracked: vec![], expected_observed: vec![(vec!["t".to_owned()], None)], + expected_timings: rule_timings(&[("R1", 2, 100), ("R2", 3, 9)], 12, 3), }, MergeTestCase { description: "a bounded host untracks every tracked rule", @@ -288,6 +310,7 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { expected_order: invocations(&[("R1", "a")]), expected_untracked: vec!["R1".to_owned()], expected_observed: vec![], + expected_timings: HostTimings::default(), }, ]; for test_case in test_cases { @@ -311,5 +334,64 @@ fn given_host_slices_when_merging_then_results_follow_single_host_order() { "{}", test_case.description ); + assert_eq!( + run.timings, test_case.expected_timings, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_host_timings_and_cache_counts_when_reporting_then_flat_entries_cover_every_rule() { + let test_cases = [ + TimingEntriesTestCase { + description: "a fully cached rule reports zero invocations beside an evaluated one", + timings: rule_timings(&[("R2", 3, 4_500_000)], 2_900_000, 1_000_000), + cache_counts: &[("R1", (5, 0)), ("R2", (2, 3))], + expected_entries: &[ + ("custom_rules_invocations", 3), + ("custom_rules_monitor_cpu_ms", 2), + ("custom_rules_guard_cpu_ms", 1), + ("custom_rules.R1.invocations", 0), + ("custom_rules.R1.cpu_ms", 0), + ("custom_rules.R1.cache_hits", 5), + ("custom_rules.R1.cache_misses", 0), + ("custom_rules.R2.invocations", 3), + ("custom_rules.R2.cpu_ms", 4), + ("custom_rules.R2.cache_hits", 2), + ("custom_rules.R2.cache_misses", 3), + ], + }, + TimingEntriesTestCase { + description: "no custom rules report only zero totals", + timings: HostTimings::default(), + cache_counts: &[], + expected_entries: &[ + ("custom_rules_invocations", 0), + ("custom_rules_monitor_cpu_ms", 0), + ("custom_rules_guard_cpu_ms", 0), + ], + }, + ]; + for test_case in test_cases { + let cache_counts: BTreeMap = test_case + .cache_counts + .iter() + .map(|(code, counts)| ((*code).to_owned(), *counts)) + .collect(); + + let entries: Vec<(String, u64)> = custom_rule_timings(&test_case.timings, &cache_counts); + + assert_eq!( + entries, + test_case + .expected_entries + .iter() + .map(|(key, value)| ((*key).to_owned(), *value)) + .collect::>(), + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs index 16ea800bd3..10ca81d91a 100644 --- a/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/custom_host_pool/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::custom_host_pool::models::HostSlice; +use crate::custom_host_pool::models::{HostSlice, HostTimings}; use crate::custom_host_pool::types::{HostInvocation, HostPlan}; pub(super) struct HostCapacityTestCase { @@ -37,4 +37,12 @@ pub(super) struct MergeTestCase { pub(super) expected_order: Vec, pub(super) expected_untracked: Vec, pub(super) expected_observed: Vec<(Vec, Option)>, + pub(super) expected_timings: HostTimings, +} + +pub(super) struct TimingEntriesTestCase { + pub(super) description: &'static str, + pub(super) timings: HostTimings, + pub(super) cache_counts: &'static [(&'static str, (usize, usize))], + pub(super) expected_entries: &'static [(&'static str, u64)], } diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs index 7e05b58abf..1ca0843694 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/main/lookup_custom_rule_cache.rs @@ -17,7 +17,7 @@ fn reusable<'a>( (digests.get(readset) == Some(&entry.reads_digest)).then_some(entry) } -/// Hits per rule and subject, and each rule's missing subjects (`None` when all miss). +/// Hits per rule and subject, each rule's missing subjects (`None` when all miss) and counts. #[must_use] pub fn lookup_custom_rule_cache( stored: &StoredRules, @@ -28,16 +28,21 @@ pub fn lookup_custom_rule_cache( for rule in rules { let cached: Option<&StoredRule> = current_rule(stored, rule); let mut missing: Vec = Vec::new(); + let mut hits: usize = 0; for subject in &rule.subjects { match reusable(cached, (&subject.key, &subject.identity), digests) { Some(entry) => { let key = (rule.code.clone(), subject.key.clone()); lookup.hits.push((key, entry.findings.clone())); lookup.hit_count += 1; + hits += 1; } None => missing.push(subject.key.clone()), } } + lookup + .counts_by_code + .insert(rule.code.clone(), (hits, missing.len())); if missing.is_empty() { continue; } diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/models.rs b/crates/sqlbuild-rules/src/custom_rule_cache/models.rs index d464c88238..d4fef8775f 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/models.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/models.rs @@ -76,6 +76,8 @@ pub struct CustomCacheLookup { pub plan: BTreeMap>>, pub hit_count: usize, pub miss_count: usize, + /// Each rule's reused and missing subject counts. + pub counts_by_code: BTreeMap, } /// One evaluated invocation, with its read set and digest when its result may be cached. diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs index 807431d92f..d253b278c4 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/tests/test_custom_rule_cache.rs @@ -122,5 +122,11 @@ fn given_stored_results_when_looking_up_then_unchanged_subjects_and_reads_hit() "{}", test_case.description ); + assert_eq!( + lookup.counts_by_code.values().copied().collect::>(), + vec![(lookup.hit_count, lookup.miss_count)], + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs index 80da53a289..510783eee1 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs @@ -149,6 +149,78 @@ fn given_published_models_when_evaluating_layer_rules_then_reads_and_shapes_matc "allowed_qualifiers = [{ domain = \"sales\", qualifier = \"fra\"", )], }, + PublishedLayerTestCase { + description: "one qualifier glob replaces a literal entry per country", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__customers", &[]), + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__deu", &[]), + ("sales__mart__orders__ita", &[]), + ("sales__mart__returns__fra_snapshots", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"domain": "sales", "qualifier": "[a-z][a-z][a-z]", "reason": "Published once per country."}, + {"domain": "sales", "qualifier": "[a-z][a-z][a-z]_snapshots", "reason": "Daily snapshots per country."} + ]}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a qualifier glob is case-sensitive and leaves other qualifiers flagged", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__customers", &[]), + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__finance", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"qualifier": "[A-Z][A-Z][A-Z]", "reason": "Published once per country."}, + {"domain": "finance", "qualifier": "[a-z][a-z][a-z]", "reason": "Per country."} + ]}), + expected_faults: &[ + ( + "models/sales/mart/sales__mart__orders__finance.sql", + "domain sales publishes 2 models as ____ and 2 as", + ), + ( + "models/sales/mart/sales__mart__orders__fra.sql", + "sales__mart__orders__fra does not fit the majority shape", + ), + ], + }, + PublishedLayerTestCase { + description: "a paths star matches across folder separators", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__deu", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"paths": ["models/sales/*"], "reason": "Published once per country."} + ]}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a paths entry leaves out the published models in its folder", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__deu", &[]), + ("finance__mart__revenue", &[]), + ("finance__mart__revenue__fra", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"paths": ["models/sales/mart/*"], "reason": "Published once per country."} + ]}), + expected_faults: &[( + "models/finance/mart/finance__mart__revenue__fra.sql", + "domain finance publishes 1 model as ____ and 1 as", + )], + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs index 8264cd1c62..1f1b11ff0c 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs @@ -3,14 +3,15 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::Value; use crate::constants::{ - ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, INTERMEDIATE_LAYER_DIRECTORY, - MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, - STAGING_LAYER_DIRECTORY, + ALLOWED_PATHS_KEY, ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, + INTERMEDIATE_LAYER_DIRECTORY, MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, + PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, STAGING_LAYER_DIRECTORY, }; use crate::models::{EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; use crate::rules::_helpers::evaluation::{parse_name, path_fault}; use crate::rules::models::{FaultCollector, ProjectEvaluationRequest}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::text::main::glob_match::glob_match; const NO_CANONICAL_LAYER_FOLDER: &str = "no canonical layer folder"; /// Where an intermediate model that replaces a published one lives. @@ -350,10 +351,31 @@ fn intermediate_move(model: &Published<'_>) -> Option { )) } -/// One literal qualifier that does not count towards a domain's name shape. +/// One `allowed_qualifiers` entry: the published models it leaves out of the name-shape check. struct AllowedQualifier { domain: Option, - qualifier: String, + qualifier: Option, + paths: Vec, +} + +impl AllowedQualifier { + fn matches(&self, entry: &Published<'_>) -> bool { + self.domain + .as_ref() + .is_none_or(|domain| *domain == entry.name.domain) + && self.qualifier.as_ref().is_none_or(|pattern| { + entry + .name + .source + .as_ref() + .is_some_and(|qualifier| glob_match(qualifier, pattern)) + }) + && (self.paths.is_empty() + || self + .paths + .iter() + .any(|pattern| glob_match(&entry.model.relative_path, pattern))) + } } fn allowed_qualifiers(config: &RulesConfig, code: &str) -> Vec { @@ -369,27 +391,31 @@ fn allowed_qualifiers(config: &RulesConfig, code: &str) -> Vec .iter() .filter_map(Value::as_object) .filter_map(|entry| { - Some(AllowedQualifier { - domain: entry - .get("domain") - .and_then(Value::as_str) - .map(str::to_owned), - qualifier: entry.get("qualifier")?.as_str()?.to_owned(), - }) + let text = |key: &str| entry.get(key).and_then(Value::as_str).map(str::to_owned); + let allowed = AllowedQualifier { + domain: text("domain"), + qualifier: text("qualifier"), + paths: entry.get(ALLOWED_PATHS_KEY).map(texts).unwrap_or_default(), + }; + (allowed.qualifier.is_some() || !allowed.paths.is_empty()).then_some(allowed) }) .collect() } -fn allowlisted(allowed: &[AllowedQualifier], name: &PublishedName) -> bool { - name.source.as_ref().is_some_and(|qualifier| { - allowed.iter().any(|entry| { - entry.qualifier == *qualifier - && entry - .domain - .as_ref() - .is_none_or(|domain| *domain == name.domain) - }) - }) +/// The strings of a JSON list; anything else is empty. +fn texts(value: &Value) -> Vec { + let Some(items) = value.as_array() else { + return Vec::new(); + }; + items + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} + +fn allowlisted(allowed: &[AllowedQualifier], entry: &Published<'_>) -> bool { + allowed.iter().any(|allowed| allowed.matches(entry)) } /// Flag published models whose name shape differs from their domain's majority shape. @@ -405,7 +431,7 @@ pub(crate) fn published_name_shapes( let Some(entry) = published(model, &layers) else { continue; }; - if allowlisted(&allowed, &entry.name) { + if allowlisted(&allowed, &entry) { continue; } domains diff --git a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs index aa1069b965..41c222fea5 100644 --- a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs +++ b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs @@ -59,8 +59,11 @@ pub fn builtin_options(code: &str) -> Vec { BuiltinOption { kind: BuiltinOptionKind::TableList, name: ALLOWED_QUALIFIERS_OPTION, - description: "Literal qualifiers that do not count towards a domain's name \ - shape, as { qualifier, reason } tables with an optional domain.", + description: "Qualifiers that do not count towards a domain's name shape, as \ + { qualifier, reason } tables with an optional domain. qualifier \ + is a case-sensitive glob such as \"[a-z][a-z][a-z]\"; \ + { paths = [...], reason } instead leaves out published models \ + whose file path matches a glob.", }, ], "SQBRTEST203" => vec![BuiltinOption { diff --git a/fensu.toml b/fensu.toml index 66674b2c51..26113efe6e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -424,7 +424,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.executor.janitor.models"] -symbols = ["JanitorArchiveCandidate", "JanitorArchivedRelation", "JanitorBlockedSchema", "JanitorDeleteCandidate", "JanitorDirectStatePruneCandidate", "JanitorOldNameView", "JanitorOldNameViewPlanning", "JanitorPlan", "JanitorQueryDiffArtifactCandidate", "JanitorRelationKey", "JanitorSkippedRelation", "JanitorSkippedSchema"] +symbols = ["JanitorArchiveCandidate", "JanitorArchivedRelation", "JanitorBlockedSchema", "JanitorDeleteCandidate", "JanitorDirectStatePruneCandidate", "JanitorOldNameView", "JanitorOldNameViewPlanning", "JanitorPlan", "JanitorQueryDiffArtifactCandidate", "JanitorRelationKey", "JanitorSkippedRelation", "JanitorSkippedSchema", "JanitorUsageCheck"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 09030d8ff4..e567a22d60 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -34,7 +34,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | Flag | Description | |------|-------------| | `--no-sql-analysis` | Disable SQL syntax, binding, type inference, and semantic validation (`--no-sql-validation` remains an alias) | -| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | +| `--no-cache` | Bypass the reusable compile-analysis cache and the Rules cache for this invocation | | `--defer-to` | Resolve unselected model references against another target | | `--json` | Output the full compile report as JSON | | `--manifest` | Generate `target/manifest.json` with project metadata | @@ -212,6 +212,11 @@ Known limits: - In these cases, or whenever a result looks stale, run `sqb compile --no-cache` or set `SQLBUILD_DISABLE_COMPILE_REUSE=1`. +`--no-cache` also turns the Rules cache off for that compile: built-in and custom Rules evaluate +every subject again, and the compile neither reads nor writes stored Rule results, as with +`[rules.cache] enabled = false`. A `--no-cache` compile reports `rule_cache_hits` as `0`. The next +compile without the flag reuses whatever the cache held before. + A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside @@ -297,6 +302,27 @@ Returns a structured report including: - `lineage_mode` - which lineage mode was used - `artifacts` - paths to written files +Every `compile_timings` value is a non-negative integer. Besides the phase timings, it attributes +custom Rule cost: + +| Key | Meaning | +|-----|---------| +| `custom_rules_ms`, `custom_rules_cpu_ms` | Wall and CPU time of the custom Rules phase, including its host processes | +| `custom_rules_invocations` | Custom Rule invocations the hosts ran, one per Rule and subject | +| `custom_rules_monitor_cpu_ms` | Host CPU spent checking Rule modules for state one subject could leave for another | +| `custom_rules_guard_cpu_ms` | Host CPU spent by the runtime guard deciding whether an audited operation, such as opening a file or importing a module, came from Rule code | +| `custom_rules..invocations` | Invocations of one custom Rule | +| `custom_rules..cpu_ms` | Host CPU of those invocations, excluding the guard's checks | +| `custom_rules..cache_hits` | Subjects whose stored result was reused | +| `custom_rules..cache_misses` | Subjects evaluated because no stored result applied; every subject when the Rules cache is off | + +Each selected custom Rule has the four `custom_rules..*` keys; a Rule the cache answered in +full reports `0` invocations. The keys are flat so that `compile_timings` stays one level deep, for +example `jq '.compile_timings["custom_rules.XORDERS001.cpu_ms"]'`. Invocations and CPU include a +Rule SQLBuild evaluates again in one process because its module state changed, so they can exceed +the Rule's subject count. Monitor and guard time are separate from each Rule's CPU; the hosts' +remaining CPU goes to starting Python, loading Rule modules, and reading the project. + If compile stops on an error before it can build the report, such as a model header that does not parse, `--json` still prints one JSON document. It has `command`, `has_errors: true`, `stopped: true`, and a `diagnostics` list with that error's `code`, `message`, and `help`, the same diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 25074a0d76..02bf74b673 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/cli/janitor/ - Usage - Flags - Output +- Relations in use - Speed - Targets - Lifecycle @@ -43,6 +44,7 @@ sqb --project-dir janitor [flags] | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | | `--verbose`, `-v` | List every entry of each preview section and every skipped object with its reason. Without it, each section shows its first 10 entries and skipped objects are counted by reason | | `--json` | Print the plan and the result as one JSON document on stdout; the preview, prompt and progress go to stderr | +| `--keep-in-use` | Keep relations that other queries read within `janitor.in_use_window` instead of archiving them, for this run. Same as `janitor.keep_in_use = true`. See [Relations in use](#relations-in-use) | | `--drop-old-name-view ` | Drop the [compatibility view](../concepts/models/migrations.md#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | @@ -73,7 +75,67 @@ Skipped objects (385) 175 relation is not tracked by SQLBuild ``` -With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. +With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `usage_check`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. + +## Relations in use + +Archiving renames a relation, so anything that still reads it fails from then on. Before archiving, the janitor looks up which candidates other queries read within `in_use_window` (default `7d`) and shows them, with their readers, in an `In use` section: + +``` +In use + analytics.orders_daily queried 2026-10-21 by svc_reporting, alice (14 queries in 7 days) + analytics.old_orders queried 2026-10-20 by svc_reporting, alice, bob +2 more (40 queries in 7 days); frozen since 2026-09-01, still in use + analytics.old_customers queried 2026-10-21 by alice via analytics.customer_report (3 queries in 7 days); frozen since 2026-09-01, still in use +``` + +Each line names the latest read, up to three readers (most queries first) and how many queries read the relation. Reading a view also reads what it is built on, so those relations are in use too. When a relation was only read that way, the line says `via` and names the views, or says `through a view` when the warehouse doesn't record which. A stale table that is still read is also flagged `frozen since , still in use`: its data stopped changing on that date, so its readers are getting old data. Views are not flagged, because their reads stay live. The confirmation prompt states the count, for example `14 relations will be archived; 3 of them are in use.` + +By default in-use relations are still archived, which is also how their readers find out. To keep them instead, set `keep_in_use = true` under `[janitor]` or pass `--keep-in-use`. In-use relations are then listed under `Kept (in use)` with their readers and are left out of the archive list and the counts. A later run archives them once nothing reads them within the window. + +Only relations to archive are checked. Expired archives and state-table pruning are not. + +Usage comes from the warehouse's query history: + +| Adapter | Usage source | +|---------|--------------| +| Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`, `base_objects_accessed`) joined with `QUERY_HISTORY`, and `OBJECT_DEPENDENCIES` | +| BigQuery | `` `.region-`.INFORMATION_SCHEMA.JOBS `` (`referenced_tables`, `referenced_logical_views`) | +| Databricks | `system.access.table_lineage` (`source_table_full_name`, `direct_access`) | +| ClickHouse | `system.query_log` (`tables`) on the connected server | +| Other adapters | Not available | + +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). A candidate listed in a query's base objects but not its direct objects was read through the objects the query named, which the line names after `via`. When something was read, a second query reads the view dependencies in `OBJECT_DEPENDENCIES` for the databases of the candidates and of the objects queries read directly. Anything those directly read objects are built on, through any number of views, is in use: an unread candidate reached that way is in use `via` the directly read object, with its readers. So a candidate view between a non-candidate view someone reads and a candidate table is kept too. Links through views in other databases are not followed. Dependencies created by `CREATE TABLE ... AS SELECT` and similar statements are not recorded there. Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements of tables, dynamic tables, schemas and databases don't count as reads. A statement counts as a clone only when it starts with `CREATE`, after any comments, and names its source after `CLONE`; a column, string or comment containing the word doesn't make it one. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition. The janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role for access history and the `SNOWFLAKE.OBJECT_VIEWER` database role for object dependencies: + +```sql +GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; +GRANT DATABASE ROLE SNOWFLAKE.OBJECT_VIEWER TO ROLE janitor_role; +``` + +Without `OBJECT_VIEWER`, reads are still checked, but view chains are not followed and the preview says `view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER`. Access history and object dependencies lag by up to about 3 hours, so reads in the last few hours may not show yet. + +On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the relation, by `user_email`. A job lists the views it reads as well as the tables behind them, so both count. Whether views nested inside other views are listed is not documented, so a candidate view that is only read through another view may not show as in use. Jobs answered from cached results record no tables or views, so with an `in_use_window` under about a day a reader whose queries were all cached can be missed. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. + +On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`. Lineage records the view a query reads as a direct read and the tables behind it as reads through the view, which the line names when lineage records the view for the same statement. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: + +```sql +GRANT USE CATALOG ON CATALOG system TO `janitor-principal`; +GRANT USE SCHEMA ON SCHEMA system.access TO `janitor-principal`; +GRANT SELECT ON TABLE system.access.table_lineage TO `janitor-principal`; +``` + +Lineage records only reads Databricks can infer lineage for, and can arrive with a delay, so "not in use" is a weaker signal than on Snowflake. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. + +On ClickHouse, the janitor counts finished `SELECT` and `INSERT` queries in `system.query_log` of the server it connects to, by the user who started them. A query's `tables` lists every table and view it touched, including the views and tables a view is built on, so a whole view chain shows as in use, without `via`; queries a query starts count once, for its initial query. Only the connected server's `query_log` is read: queries on other replicas or shards are logged on those servers and are not seen, which matters on ClickHouse Cloud and behind load-balanced replicas. When `system.clusters` shows other replicas of the connected server's shard, the preview says `usage read from this server only; other replicas not checked`. Writes into a relation count as use, since renaming it breaks the writer too. ClickHouse answers some queries, such as a plain `count()` of a table, from metadata without logging the table, so those reads are not seen. SQLBuild doesn't mark its ClickHouse queries, so its own reads count. The janitor's user needs `SELECT` on `system.query_log`: + +```sql +GRANT SELECT ON system.query_log TO janitor_user; +``` + +Queries are logged when `log_queries` is on, which is the default; when it is off for the janitor's connection, usage is not checked. Readers whose settings profile turns `log_queries` off, or logs only a sample with `log_queries_probability`, are missed. The log is flushed about every 7.5 seconds, and ClickHouse keeps it until a TTL configured on the server removes it. + +When usage can't be checked, because the adapter has no usage source or the role can't read its query history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. + +In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers`, `query_count` and `via` (`null` when the relation was read directly, otherwise the names of the views it was read through, empty when unknown). Entries of `kept_in_use` have only `name`, `age_days` and `in_use`, since they are not archived. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window`, `keep_in_use` and `notices` (what a check that ran left out, such as view chains it could not follow), or is `null` when there was nothing to check. ## Speed @@ -183,6 +245,8 @@ retention_days = 14 archive_retention_days = 14 delete_tracked_only = true exclude_patterns = ["audit_*", "tmp_*"] +in_use_window = "7d" +keep_in_use = false ``` When `retention_days` is not set, it defaults to `14`. See [Project Configuration](../concepts/project-configuration.md#janitor) for every janitor setting. @@ -204,6 +268,9 @@ sqb janitor --as prod # Clean up the prod target sqb janitor --target prod + +# Keep relations that are still read instead of archiving them +sqb janitor --keep-in-use ``` ## State history pruning diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index b37b0055d5..00ba7a33df 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -824,6 +824,8 @@ retention_days = 14 archive_retention_days = 14 delete_tracked_only = true exclude_patterns = ["audit_*", "tmp_*"] +in_use_window = "7d" +keep_in_use = false ``` | Field | Default | Description | @@ -834,6 +836,8 @@ exclude_patterns = ["audit_*", "tmp_*"] | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | | `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` | +| `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](../cli/janitor.md#relations-in-use) | +| `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | See [janitor](../cli/janitor.md) for the archive and delete lifecycle. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 6a835eb92f..6a8d53c6da 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -72,7 +72,8 @@ reviewed exceptions. Listed tests are not flagged and still count toward `SQBRTE [Empty-input tests](../testing.md#empty-input-tests). `SQBRGRAPH103` and `SQBRPROJECT107` accept `published_layers`, and `SQBRPROJECT107` accepts -`allowed_qualifiers`, a list of `{ qualifier, reason }` tables with an optional `domain`. See +`allowed_qualifiers`, a list of `{ qualifier, reason }` or `{ paths, reason }` tables with an +optional `domain`; `qualifier` and `paths` accept globs. See [Published-layer Rules](published-layer-rules.md). Unknown Rule codes, option names, and invalid option values fail configuration. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md index ef11cb0bf5..5fafa49622 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md @@ -60,6 +60,17 @@ findings and cache entries match a single-process run. The result stays correct, longer runs in parallel and costs an extra host process. Keep each subject's findings independent of the others so Rules can run in parallel. +SQLBuild compares module state once per model, after every selected Rule has evaluated it, and +after each project-subject Rule invocation. A change still present then is reported against each +Rule whose module, or helper module it imports, holds the changed value, the same Rules a check after +every invocation would name. All of one model's Rules run in the same process, so a change undone +before the model's last Rule finishes, such as a scratch list one Rule fills and another empties for +the same model, is not detected and cannot change findings. Project Rules can run in different +processes, so they are still checked after each invocation. The same check decides which Rules the cache can track by the +facts they read: a Rule whose module state changed is cached against every project fact instead, so +any edit evaluates it again. `compile_timings` reports the time this check takes as +`custom_rules_monitor_cpu_ms`; see [`sqb compile`](../../../cli/compile.md#json-output). + ## Helpers Custom Rules can import repository-owned helpers under `rules/` and these deterministic standard diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md index 3088640fdf..3fcbc90a80 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md @@ -46,6 +46,10 @@ genuinely project-wide invariants, with correspondingly broader invalidation. Positive and negative project-tree observations are dependencies. If a Rule observes that a glob has no matches, adding a matching file invalidates that result. +`--no-cache` on `sqb compile`, `sqb plan`, and `sqb build` turns the Rules cache off for that +invocation: every built-in and custom Rule evaluates again, and stored results are neither read nor +written. Use it when a Rule finding looks stale. + ## Built-in-only execution When no custom Rule is selected, SQLBuild does not initialize the custom Python host. When every diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md index 248639b4a9..5be493a672 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md @@ -6,6 +6,13 @@ Online: https://sqlbuild.com/docs/concepts/rules/published-layer-rules/ +## Contents + +- SQBRGRAPH103: no published reads within a domain +- SQBRPROJECT107: one name shape per domain +- Published layers +- Deviating + A domain's published layer is its interface to the rest of the project. Under pressure to make something work, it is tempting to add a consumer-specific variant next to an existing published model: @@ -49,9 +56,9 @@ not fit the domain's majority shape are reported, with the remediation: example `sqb rename sales__mart__customers sales__mart__customers__core`. 3. If the qualifier is intentional and structural, allowlist it with a reason. -Allowlisted qualifiers are literal and do not count towards the shape: models that use them are left -out of the check. Each entry needs a reason, and an entry with a `domain` applies to that domain -only: +Allowlisted qualifiers do not count towards the shape: models that use them are left out of the +check and out of their domain's counts. Each entry needs a reason, and an entry with a `domain` +applies to that domain only: ```toml [rules.rule_options.SQBRPROJECT107] @@ -61,6 +68,34 @@ allowed_qualifiers = [ ] ``` +`qualifier` is a case-sensitive glob with the same grammar as selectors: `*` matches any run of +characters, `?` one character, and `[...]` one character from a set, with `[!...]` negating it. A +qualifier without wildcards matches only itself. One entry can cover a whole family of +qualifiers, such as a domain published once per country: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { domain = "sales", qualifier = "[a-z][a-z][a-z]", reason = "Published once per country." }, + { domain = "sales", qualifier = "[a-z][a-z][a-z]_snapshots", reason = "Daily snapshots per country." }, +] +``` + +An entry with `paths` instead leaves out the published models whose file path, relative to the +project and written with `/` separators, matches any of its globs. As in selectors and janitor +`exclude_patterns`, `*` also matches `/`, so `models/sales/mart/*` covers nested folders too. A +family published once per qualifier usually lives in one folder: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { paths = ["models/sales/mart/regional/*"], reason = "Published once per region." }, +] +``` + +An entry with both `qualifier` and `paths` leaves out only the models that match both. Published +models that match no entry are checked as before. + ## Published layers Both Rules treat `mart` and `mart_v` as the published layers. Set `published_layers` on either Rule diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f22d759fa1..bf81edc288 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3130,6 +3130,7 @@ class CliNamespace: retention_days: int | None direct_state_history_versions: int | None drop_old_name_views: list[str] + keep_in_use: bool bounded: str | None max_column_examples: int | None max_row_only_examples: int | None @@ -5800,7 +5801,7 @@ def run_planned_sql_rules( class EarlySqlLint: """Own speculative SQL checks for one compilation invocation.""" - def __init__(self, *, enabled: bool) -> None: ... + def __init__(self, *, enabled: bool, no_cache: bool = False) -> None: ... @property def enabled(self) -> bool: ... def __enter__(self) -> EarlySqlLint: ... @@ -6676,6 +6677,7 @@ class JanitorCommandRequest(NativeRecord): retention_days: int | None direct_state_history_versions: int | None drop_old_name_views: tuple[str, ...] + keep_in_use: bool selected_target: str | None as_target: str | None verbose: bool @@ -6688,6 +6690,7 @@ class JanitorCommandRequest(NativeRecord): retention_days: int | None = None, direct_state_history_versions: int | None = None, drop_old_name_views: tuple[str, ...] = (), + keep_in_use: bool = False, selected_target: str | None = None, as_target: str | None = None, verbose: bool = False, @@ -10704,6 +10707,7 @@ class RulesRunResult(NativeRecord): cache_hits: int cache_misses: int skipped_type_proof_rules: tuple[str, ...] + custom_rule_timings: tuple[tuple[str, int], ...] def __init__( self, findings: tuple[Finding, ...], @@ -10715,6 +10719,7 @@ class RulesRunResult(NativeRecord): cache_hits: int = ..., cache_misses: int = ..., skipped_type_proof_rules: tuple[str, ...] = ..., + custom_rule_timings: tuple[tuple[str, int], ...] = ..., ) -> None: ... class CustomRulesOutcome(NativeRecord): @@ -10725,6 +10730,7 @@ class CustomRulesOutcome(NativeRecord): cache_misses: int custom_ms: int custom_cpu_ms: int + custom_rule_timings: tuple[tuple[str, int], ...] def __init__( self, findings: tuple[Finding, ...], @@ -10732,6 +10738,7 @@ class CustomRulesOutcome(NativeRecord): cache_misses: int, custom_ms: int, custom_cpu_ms: int = ..., + custom_rule_timings: tuple[tuple[str, int], ...] = ..., ) -> None: ... class ResourceKind(StrEnum): @@ -11982,6 +11989,8 @@ class JanitorConfig(NativeRecord): direct_state_history_versions: int delete_tracked_only: bool exclude_patterns: tuple[str, ...] + in_use_window: str + keep_in_use: bool def __init__( self, enabled: bool = ..., @@ -11990,6 +11999,8 @@ class JanitorConfig(NativeRecord): direct_state_history_versions: int = ..., delete_tracked_only: bool = ..., exclude_patterns: tuple[str, ...] = ..., + in_use_window: str = ..., + keep_in_use: bool = ..., ) -> None: ... class MigrationsConfig(NativeRecord): @@ -15835,6 +15846,7 @@ class RulesResult(NativeRecord): built_in_ms: int custom_ms: int custom_cpu_ms: int + custom_rule_timings: tuple[tuple[str, int], ...] def __init__( self, findings: tuple[Finding, ...], @@ -15845,6 +15857,7 @@ class RulesResult(NativeRecord): built_in_ms: int = ..., custom_ms: int = ..., custom_cpu_ms: int = ..., + custom_rule_timings: tuple[tuple[str, int], ...] = ..., ) -> None: ... class RuleFile(NativeRecord): @@ -15929,6 +15942,8 @@ class CustomRuleRun(NativeRecord): uncacheable_codes: frozenset[str] observed: tuple[tuple[tuple[str, ...], str | None], ...] stateful_codes: frozenset[str] + rule_timings: dict[str, tuple[int, int]] + monitor_ns: int def __init__( self, evaluations: tuple[CustomRuleEvaluation, ...], @@ -15936,6 +15951,8 @@ class CustomRuleRun(NativeRecord): uncacheable_codes: frozenset[str] = ..., observed: tuple[tuple[tuple[str, ...], str | None], ...] = ..., stateful_codes: frozenset[str] = ..., + rule_timings: dict[str, tuple[int, int]] = ..., + monitor_ns: int = ..., ) -> None: ... class CustomHostPartition(NativeRecord): diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 537b5a86e5..f4d4ccbfc1 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -389,6 +389,7 @@ "JanitorRelationKey": ("sqlbuild.executor.janitor.models", "JanitorRelationKey"), "JanitorSkippedRelation": ("sqlbuild.executor.janitor.models", "JanitorSkippedRelation"), "JanitorSkippedSchema": ("sqlbuild.executor.janitor.models", "JanitorSkippedSchema"), + "JanitorUsageCheck": ("sqlbuild.executor.janitor.models", "JanitorUsageCheck"), "run_load_pipeline": ("sqlbuild.executor.load.main.run", "run_load_pipeline"), "LoadCallbacks": ("sqlbuild.executor.load.models", "LoadCallbacks"), "LoadExecutionResult": ("sqlbuild.executor.load.models", "LoadExecutionResult"), diff --git a/src/sqlbuild/adapter/contract/_helpers/array_names.py b/src/sqlbuild/adapter/contract/_helpers/array_names.py new file mode 100644 index 0000000000..869aa993f0 --- /dev/null +++ b/src/sqlbuild/adapter/contract/_helpers/array_names.py @@ -0,0 +1,18 @@ +"""Read object names from a warehouse array value.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable + + +def array_names(value: object) -> tuple[str, ...]: + """Names from a warehouse array value: a list, an array object, or JSON text.""" + + if value is None: + return () + if isinstance(value, str): + return array_names(json.loads(value)) if value.lstrip().startswith("[") else (value,) + if not isinstance(value, Iterable): + return (str(value),) + return tuple(str(name) for name in value if name is not None) diff --git a/src/sqlbuild/adapter/contract/_helpers/relation_reads.py b/src/sqlbuild/adapter/contract/_helpers/relation_reads.py new file mode 100644 index 0000000000..81982a0b39 --- /dev/null +++ b/src/sqlbuild/adapter/contract/_helpers/relation_reads.py @@ -0,0 +1,15 @@ +"""Normalize warehouse values in relation usage rows.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def utc_datetime(value: object) -> datetime: + """A warehouse timestamp as an aware UTC datetime; naive values are taken as UTC.""" + + if not isinstance(value, datetime): + value = datetime.fromisoformat(str(value)) + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index db9ef689a7..97d8f9b5b4 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -40,6 +40,7 @@ ) from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -52,6 +53,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffCoverage, RowDiffPreparedRelations, RowDiffResult, @@ -366,6 +368,18 @@ def get_tables_freshness_metadata( results[request] = TableFreshnessMetadata.unavailable(message=error.message) return results + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Report that usage is not checked; adapters with a usage source override this.""" + + del connection, relations, window_seconds + return unsupported_relation_usage() + def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: """Return column names produced by a SQL query without materializing full rows.""" diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 3c1beddbe3..ec86c5acf3 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -48,6 +48,7 @@ from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -60,6 +61,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -197,6 +199,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index aa24bad368..7db76e1936 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -19,7 +19,9 @@ FunctionDefinition, MigrationStagePlan, RelationGrant, + RelationInfo, RelationReadProbe, + RelationUsageReport, RowDiffTolerance, RowDiffTolerances, SnapshotChangeTarget, @@ -211,6 +213,17 @@ def get_tables_freshness_metadata( """Return one freshness outcome per requested table without failing the whole batch.""" ... + @abstractmethod + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Report reads of the relations by other queries in the last `window_seconds`.""" + ... + @abstractmethod def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: """Return column names produced by a SQL query.""" diff --git a/src/sqlbuild/adapter/contract/constants.py b/src/sqlbuild/adapter/contract/constants.py index c5671dbfd3..9c94083584 100644 --- a/src/sqlbuild/adapter/contract/constants.py +++ b/src/sqlbuild/adapter/contract/constants.py @@ -19,3 +19,4 @@ DUCKDB_MISSING_OBJECT_MARKER: str = "does not exist" DUCKDB_INTERNAL_ERROR_CLASS_NAME: str = "InternalException" CANCELLABLE_DURATION_LIMIT_SECONDS: int = 2_147_483 +UNSUPPORTED_RELATION_USAGE_REASON: str = "unsupported adapter" diff --git a/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py b/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py new file mode 100644 index 0000000000..8ee33ce13a --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/aggregate_relation_usage.py @@ -0,0 +1,51 @@ +"""Fold per-reader reads into one usage per relation.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from datetime import datetime + +from sqlbuild.adapter.contract._helpers.relation_reads import utc_datetime +from sqlbuild.adapter.contract.models import RelationInfo, RelationRead, RelationUsage +from sqlbuild.adapter.relations.constants import UNKNOWN_RELATION_READER + + +def aggregate_relation_usage( + *, reads: Iterable[RelationRead], relations_by_name: Mapping[str, RelationInfo] +) -> tuple[RelationUsage, ...]: + """One usage per read relation; it is direct when any reader read it directly.""" + + readers: dict[str, dict[str, int]] = {} + last_read: dict[str, datetime] = {} + direct: set[str] = set() + via: dict[str, set[str]] = {} + read: RelationRead + for read in reads: + if read.object_name not in relations_by_name: + continue + reader: str = UNKNOWN_RELATION_READER if read.reader is None else read.reader + counts: dict[str, int] = readers.setdefault(read.object_name, {}) + counts[reader] = counts.get(reader, 0) + read.query_count + read_at: datetime = utc_datetime(read.last_read_at) + last_read[read.object_name] = max(read_at, last_read.get(read.object_name, read_at)) + if read.direct: + direct.add(read.object_name) + via.setdefault(read.object_name, set()).update(read.via) + usages: list[RelationUsage] = [] + object_name: str + for object_name in sorted(readers): + relation: RelationInfo = relations_by_name[object_name] + counts = readers[object_name] + usages.append( + RelationUsage( + database=relation.database, + schema=relation.schema, + name=relation.name, + last_read_at=last_read[object_name], + readers=tuple(sorted(counts, key=lambda reader: (-counts[reader], reader))), + query_count=sum(counts.values()), + direct=object_name in direct, + via=() if object_name in direct else tuple(sorted(via[object_name])), + ) + ) + return tuple(usages) diff --git a/src/sqlbuild/adapter/contract/main/relation_reads.py b/src/sqlbuild/adapter/contract/main/relation_reads.py new file mode 100644 index 0000000000..bf249a40fd --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/relation_reads.py @@ -0,0 +1,31 @@ +"""Turn warehouse usage rows into relation reads.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable + +from sqlbuild.adapter.contract._helpers.array_names import array_names +from sqlbuild.adapter.contract.models import RelationRead + + +def relation_reads( + *, rows: Iterable[tuple[object, ...]], via_name: Callable[[str], str] = str +) -> tuple[RelationRead, ...]: + """Reads from `(object, reader, queries, last read[, direct, via names])` rows.""" + + reads: list[RelationRead] = [] + row: tuple[object, ...] + for row in rows: + direct_access, via_names = (*row[4:6], True, None)[:2] + indirect: bool = not direct_access + reads.append( + RelationRead( + object_name=str(row[0]), + reader=None if row[1] is None else str(row[1]), + query_count=int(str(row[2])), + last_read_at=row[3], + direct=not indirect, + via=tuple(map(via_name, array_names(via_names))) if indirect else (), + ) + ) + return tuple(reads) diff --git a/src/sqlbuild/adapter/contract/main/unsupported_relation_usage.py b/src/sqlbuild/adapter/contract/main/unsupported_relation_usage.py new file mode 100644 index 0000000000..960363ce16 --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/unsupported_relation_usage.py @@ -0,0 +1,12 @@ +"""The relation usage report of an adapter that cannot tell who reads its relations.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.constants import UNSUPPORTED_RELATION_USAGE_REASON +from sqlbuild.adapter.contract.models import RelationUsageReport + + +def unsupported_relation_usage() -> RelationUsageReport: + """Report that usage was not checked because the adapter has no usage source.""" + + return RelationUsageReport(checked=False, reason=UNSUPPORTED_RELATION_USAGE_REASON) diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 3678afe53a..a66147d3ad 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -100,6 +100,42 @@ def identity(self) -> tuple[str | None, str | None, str]: return RelationLookup.key(database=self.database, schema=self.schema, name=self.name) +@dataclass(frozen=True) +class RelationUsage: + """Reads of one relation within a usage window: the latest, who read it and how often.""" + + database: str | None + schema: str | None + name: str + last_read_at: datetime + readers: tuple[str, ...] + query_count: int + direct: bool = True + via: tuple[str, ...] = () + + +@dataclass(frozen=True) +class RelationRead: + """One reader's reads of one object; indirect reads came through `via`, empty if unknown.""" + + object_name: str + reader: str | None + query_count: int + last_read_at: object + direct: bool = True + via: tuple[str, ...] = () + + +@dataclass(frozen=True) +class RelationUsageReport: + """The relations read within a usage window, or why the warehouse could not tell.""" + + checked: bool + reason: str | None = None + usages: tuple[RelationUsage, ...] = () + notices: tuple[str, ...] = () + + @dataclass(frozen=True) class RelationLookup: """Existing warehouse relations gathered once and queried in memory.""" diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 0c144b1164..9d1557c487 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -6,3 +6,4 @@ RELATION_COLUMNS_LOOKUP: str = "columns" RELATION_EXISTS_LOOKUP: str = "exists" RELATION_TYPE_LOOKUP: str = "type" +UNKNOWN_RELATION_READER: str = "unknown user" diff --git a/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py new file mode 100644 index 0000000000..9d8a544a07 --- /dev/null +++ b/src/sqlbuild/adapters/bigquery/_helpers/relation_usage.py @@ -0,0 +1,241 @@ +"""Read who queried relations from BigQuery INFORMATION_SCHEMA.JOBS.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any + +from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.main.relation_reads import relation_reads +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.bigquery.constants import ( + CLONE_STATEMENT_PATTERN, + FORBIDDEN_ERROR_CLASS_NAMES, + FORBIDDEN_STATUS, + NOT_FOUND_ERROR_CLASS_NAMES, + NOT_FOUND_STATUS, + USAGE_NAME_CHUNK_SIZE, + USAGE_UNAVAILABLE_ERROR_REASONS, + USAGE_UNAVAILABLE_MESSAGES, +) +from sqlbuild.adapters.bigquery.models import UsageQuery + +_USAGE_UNAVAILABLE_REASON: str = ( + "cannot read INFORMATION_SCHEMA.JOBS " + "(needs bigquery.jobs.listAll, for example the BigQuery Resource Viewer role)" +) + + +def usage_object_name(*, project: str, dataset: str, table: str) -> str: + """`project.dataset.table` as `referenced_tables` spells it; BigQuery names keep their case.""" + + return ".".join(part.replace("`", "") for part in (project, dataset, table)) + + +def usage_queries( + *, + relations: Iterable[RelationInfo], + default_project: str, + locations: Mapping[tuple[str, str], str], + window_seconds: int, +) -> tuple[tuple[UsageQuery, dict[str, RelationInfo]], ...]: + """One JOBS read per project, location and chunk of names, with the relations it covers.""" + + groups: dict[tuple[str, str], dict[str, RelationInfo]] = {} + relation: RelationInfo + for relation in relations: + if relation.schema is None: + continue + project: str = relation_project(relation=relation, default_project=default_project) + location: str = locations[(project, relation.schema)] + name: str = usage_object_name(project=project, dataset=relation.schema, table=relation.name) + groups.setdefault((project, location), {})[name] = relation + queries: list[tuple[UsageQuery, dict[str, RelationInfo]]] = [] + for (project, location), named in sorted(groups.items()): + names: tuple[str, ...] = tuple(sorted(named)) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE): + chunk: tuple[str, ...] = names[start : start + USAGE_NAME_CHUNK_SIZE] + queries.append( + ( + usage_query( + project=project, + location=location, + object_names=chunk, + window_seconds=window_seconds, + ), + {name: named[name] for name in chunk}, + ) + ) + return tuple(queries) + + +def jobs_view(*, project: str, location: str) -> str: + """`project.region-`.INFORMATION_SCHEMA.JOBS.""" + + region: str = location.strip().lower() + region = region if region.startswith("region-") else f"region-{region}" + return f"`{project}.{region}`.INFORMATION_SCHEMA.JOBS" + + +def usage_query( + *, project: str, location: str, object_names: tuple[str, ...], window_seconds: int +) -> UsageQuery: + """Reads of the named tables and views per user by successful query jobs, minus clones.""" + + def referenced(column: str) -> str: + return ( + "SELECT CONCAT(referenced.project_id, '.', referenced.dataset_id, '.', " + "referenced.table_id) AS object_name, " + "jobs.user_email, jobs.job_id, jobs.creation_time " + f"FROM jobs, UNNEST(jobs.{column}) AS referenced" + ) + + sql: str = ( + "WITH jobs AS (" + "SELECT job_id, user_email, creation_time, referenced_tables, referenced_logical_views " + f"FROM {jobs_view(project=project, location=location)} " + "WHERE creation_time >= " + "TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @window_seconds SECOND) " + "AND job_type = 'QUERY' " + "AND error_result IS NULL " + "AND COALESCE(statement_type, '') NOT IN ('SCRIPT', 'CREATE_SNAPSHOT_TABLE') " + "AND NOT (STARTS_WITH(COALESCE(statement_type, ''), 'CREATE_TABLE') " + "AND REGEXP_CONTAINS(query, @clone_pattern))), " + f"reads AS ({referenced('referenced_tables')} UNION ALL " + f"{referenced('referenced_logical_views')}) " + "SELECT object_name, user_email AS user_name, " + "COUNT(DISTINCT job_id) AS query_count, " + "MAX(creation_time) AS last_read_at " + "FROM reads " + "WHERE object_name IN UNNEST(@object_names) " + "GROUP BY 1, 2 " + "ORDER BY 1, 2" + ) + return UsageQuery( + sql=sql, location=location, window_seconds=window_seconds, object_names=object_names + ) + + +def query_parameters(query: UsageQuery) -> list[Any]: + """The query's parameters as BigQuery client parameters.""" + + from google.cloud import bigquery + + return [ + bigquery.ScalarQueryParameter("window_seconds", "INT64", query.window_seconds), + bigquery.ScalarQueryParameter("clone_pattern", "STRING", CLONE_STATEMENT_PATTERN), + bigquery.ArrayQueryParameter("object_names", "STRING", list(query.object_names)), + ] + + +def is_usage_unavailable_error(error: Exception) -> bool: + """Whether reading JOBS failed because the view or the permission to list jobs is missing.""" + + reasons: frozenset[str] = error_reasons(error) + if reasons: + return reasons <= USAGE_UNAVAILABLE_ERROR_REASONS + code: object = getattr(error, "code", None) + name: str = type(error).__name__ + if code == NOT_FOUND_STATUS or name in NOT_FOUND_ERROR_CLASS_NAMES: + return True + message: str = str(error).lower() + return (code == FORBIDDEN_STATUS or name in FORBIDDEN_ERROR_CLASS_NAMES) and any( + marker in message for marker in USAGE_UNAVAILABLE_MESSAGES + ) + + +def error_reasons(error: Exception) -> frozenset[str]: + """The `reason` of each error detail Google reports, such as `accessDenied`.""" + + details: object = getattr(error, "errors", None) + reasons: set[str] = set() + if isinstance(details, list): + reasons.update( + str(detail["reason"]) + for detail in details + if isinstance(detail, dict) and detail.get("reason") + ) + reason: object = getattr(error, "reason", None) + if isinstance(reason, str) and reason: + reasons.add(reason) + return frozenset(reasons) + + +def relation_project(*, relation: RelationInfo, default_project: str) -> str: + """The project holding the relation: its database, else the client's project.""" + + return (relation.database or default_project).replace("`", "") + + +def dataset_locations( + *, connection: Any, relations: Iterable[RelationInfo], default_project: str +) -> dict[tuple[str, str], str]: + """The location of each candidate's dataset, or the connection's fixed location.""" + + keys: set[tuple[str, str]] = { + (relation_project(relation=relation, default_project=default_project), relation.schema) + for relation in relations + if relation.schema is not None + } + if connection.location is not None: + return {key: str(connection.location) for key in keys} + return { + key: str(connection.client.get_dataset(f"{key[0]}.{key[1]}").location) + for key in sorted(keys) + } + + +def run_usage_query(*, connection: Any, query: UsageQuery) -> list[tuple[object, ...]]: + """Run one JOBS read with bound parameters in the query's location.""" + + return list( + connection.run_statement( + sql=query.sql, + location=query.location, + query_parameters=tuple(query_parameters(query)), + ).fetchall() + ) + + +def read_relation_usage( + *, connection: Any, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Read every relation's usage in each dataset's location.""" + + default_project: str = str(connection.client.project) + rows: list[tuple[object, ...]] = [] + covered: dict[str, RelationInfo] = {} + query: UsageQuery + named: dict[str, RelationInfo] + for query, named in usage_queries( + relations=relations, + default_project=default_project, + locations=dataset_locations( + connection=connection, relations=relations, default_project=default_project + ), + window_seconds=window_seconds, + ): + rows.extend(run_usage_query(connection=connection, query=query)) + covered.update(named) + return RelationUsageReport( + checked=True, + usages=aggregate_relation_usage( + reads=relation_reads(rows=rows), + relations_by_name=covered, + ), + ) + + +def bigquery_relation_usage( + *, connection: Any, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Read usage; no access to JOBS or the datasets means usage is not checked.""" + + try: + return read_relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=_USAGE_UNAVAILABLE_REASON) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 8d90520f08..fd35855629 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -66,6 +66,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RenderedRetentionChange, RetentionRequest, RetentionState, @@ -120,6 +121,7 @@ render_bigquery_view_move, render_bigquery_view_revokes, ) +from sqlbuild.adapters.bigquery._helpers.relation_usage import bigquery_relation_usage from sqlbuild.adapters.bigquery._helpers.statement_telemetry import affected_rows from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection from sqlbuild.adapters.bigquery.classes.bigquery_cursor import _BigQueryCursor @@ -509,6 +511,19 @@ def get_table_freshness_metadata( raise AdapterUserError(message=metadata.message or f"freshness unavailable for {name}") return metadata + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Read candidate reads from the region's INFORMATION_SCHEMA.JOBS (180 days kept).""" + + return bigquery_relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) + def get_tables_freshness_metadata( self, *, @@ -1483,7 +1498,7 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: log_sql(logger=logging.getLogger("sqlbuild.adapter.bigquery"), sql=sql) with StatementLifecycle(adapter="bigquery", sql=sql, intent="query") as lifecycle: - job: Any = connection.query_job(sql) + job: Any = connection.query_job(sql=sql) raw_job_id: object | None = getattr(job, "job_id", None) job_id: str | None = raw_job_id if isinstance(raw_job_id, str) else None lifecycle.submitted(job_id=job_id) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py index 890fdee67f..79ff3c86b5 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py @@ -32,8 +32,15 @@ def __init__(self, *, client: Any, location: str | None) -> None: self._jobs_lock: threading.Lock = threading.Lock() def execute(self, sql: str) -> _BigQueryCursor: + return self.run_statement(sql=sql) + + def run_statement( + self, *, sql: str, location: str | None = None, query_parameters: tuple[Any, ...] = () + ) -> _BigQueryCursor: + """Run one statement, optionally in another location and with bound parameters.""" + with StatementLifecycle(adapter="bigquery", sql=sql, intent="execute") as lifecycle: - job: Any = self.query_job(sql) + job: Any = self.query_job(sql=sql, location=location, query_parameters=query_parameters) job_id: str | None = _job_id(job=job) lifecycle.submitted(job_id=job_id) try: @@ -49,24 +56,24 @@ def execute(self, sql: str) -> _BigQueryCursor: lifecycle.completed(job_id=job_id, affected_rows=dml_affected_rows) return cursor - def query_job(self, sql: str) -> Any: + def query_job( + self, *, sql: str, location: str | None = None, query_parameters: tuple[Any, ...] = () + ) -> Any: """Submit a job; only the whole-build limit becomes a job timeout, as it counts queueing.""" deadline: ExecutionDeadline | ItemDeadline | None = current_execution_deadline() if deadline is not None: deadline.enforce() build: ExecutionDeadline | None = current_build_deadline() - if build is None: - return self.client.query(sql, location=self.location) + job_location: str | None = location or self.location + if build is None and not query_parameters: + return self.client.query(sql, location=job_location) from google.cloud import bigquery - return self.client.query( - sql, - location=self.location, - job_config=bigquery.QueryJobConfig( - job_timeout_ms=max(1, math.ceil(build.remaining_seconds * 1000)) - ), - ) + job_config: Any = bigquery.QueryJobConfig(query_parameters=list(query_parameters)) + if build is not None: + job_config.job_timeout_ms = max(1, math.ceil(build.remaining_seconds * 1000)) + return self.client.query(sql, location=job_location, job_config=job_config) def job_result(self, job: Any) -> Any: """Wait for a job while it is registered for cancellation at a deadline.""" diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index 498d176b89..1e20b22b27 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -31,3 +31,24 @@ NATIVE_TABLE_TYPE: str = "TABLE" PENDING_JOB_STATE: str = "PENDING" PENDING_JOB_POLL_SECONDS: float = 1.0 +USAGE_NAME_CHUNK_SIZE: int = 1_000 +FORBIDDEN_STATUS: int = 403 +NOT_FOUND_STATUS: int = 404 +FORBIDDEN_ERROR_CLASS_NAMES: frozenset[str] = frozenset({"Forbidden", "PermissionDenied"}) +NOT_FOUND_ERROR_CLASS_NAMES: frozenset[str] = frozenset({"NotFound"}) +USAGE_UNAVAILABLE_ERROR_REASONS: frozenset[str] = frozenset({"accessDenied", "notFound"}) +USAGE_UNAVAILABLE_MESSAGES: tuple[str, ...] = ( + "access denied", + "permission bigquery.jobs.listall", + "permission denied", +) +_CLONE_IDENTIFIER: str = r"(`[^`]*`|[A-Za-z0-9_-]+)" +_CLONE_NAME: str = _CLONE_IDENTIFIER + r"(\s*\.\s*" + _CLONE_IDENTIFIER + r"){0,2}" +CLONE_STATEMENT_PATTERN: str = ( + r"(?is)^(\s|--[^\n]*\n|#[^\n]*\n|/\*([^*]|\*+[^*/])*\*+/)*" + r"CREATE\s+(OR\s+REPLACE\s+)?(TEMP(ORARY)?\s+)?TABLE\s+(IF\s+NOT\s+EXISTS\s+)?" + + _CLONE_NAME + + r"\s+CLONE\s+" + + _CLONE_NAME + + r"([^A-Za-z0-9_`-]|$)" +) diff --git a/src/sqlbuild/adapters/bigquery/models.py b/src/sqlbuild/adapters/bigquery/models.py new file mode 100644 index 0000000000..cf83cf393d --- /dev/null +++ b/src/sqlbuild/adapters/bigquery/models.py @@ -0,0 +1,15 @@ +"""BigQuery adapter models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UsageQuery: + """One read of the region's jobs view: its SQL, location, window and table names.""" + + sql: str + location: str + window_seconds: int + object_names: tuple[str, ...] diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py b/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py new file mode 100644 index 0000000000..93e178d7cb --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/relation_usage.py @@ -0,0 +1,131 @@ +"""Read who queried relations from the ClickHouse system.query_log table.""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Iterable +from datetime import UTC, datetime + +from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.models import RelationInfo, RelationRead, RelationUsageReport +from sqlbuild.adapters.clickhouse._helpers.type_text import quote_string +from sqlbuild.adapters.clickhouse.constants import ( + LOGGING_OFF_VALUES, + LOGGING_SETTING_SQL, + OTHER_REPLICAS_NOTICE, + REPLICA_COUNT_SQL, + USAGE_ERROR_CODE_PATTERN, + USAGE_LOGGING_OFF_REASON, + USAGE_NAME_CHUNK_SIZE, + USAGE_QUERY_KINDS, + USAGE_UNAVAILABLE_ERROR_CODES, + USAGE_UNAVAILABLE_REASON, +) + + +def usage_object_names(*, relations: Iterable[RelationInfo]) -> dict[str, RelationInfo]: + """Each relation by the `database.table` name query_log records; names keep their case.""" + + return { + f"{relation.schema}.{relation.name}": relation + for relation in relations + if relation.schema is not None + } + + +def usage_queries(*, object_names: Iterable[str], window_seconds: int) -> tuple[str, ...]: + """One query_log read per chunk of object names.""" + + names: tuple[str, ...] = tuple(sorted(object_names)) + return tuple( + usage_query( + object_names=names[start : start + USAGE_NAME_CHUNK_SIZE], + window_seconds=window_seconds, + ) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) + ) + + +def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> str: + """Finished reads and inserts of the named tables per initial user within the window.""" + + since: str = f"now() - toIntervalSecond({int(window_seconds)})" + kinds: str = ", ".join(quote_string(kind) for kind in USAGE_QUERY_KINDS) + names: str = ", ".join(quote_string(name) for name in object_names) + return ( + "SELECT object_name, initial_user AS user_name, " + "uniqExact(initial_query_id) AS query_count, " + "toUnixTimestamp(max(event_time)) AS last_read_at " + "FROM system.query_log ARRAY JOIN tables AS object_name " + f"WHERE event_date >= toDate({since}) " + f"AND event_time >= {since} " + "AND type = 'QueryFinish' " + f"AND query_kind IN ({kinds}) " + f"AND object_name IN ({names}) " + "GROUP BY object_name, user_name " + "ORDER BY object_name, user_name" + ) + + +def is_usage_unavailable_error(error: Exception) -> bool: + """Whether reading query_log failed because it does not exist or may not be read.""" + + match: re.Match[str] | None = USAGE_ERROR_CODE_PATTERN.search(str(error)) + return match is not None and int(match.group(1)) in USAGE_UNAVAILABLE_ERROR_CODES + + +def usage_reads(rows: Iterable[tuple[object, ...]]) -> tuple[RelationRead, ...]: + """Reads from `(object, user, queries, last read as Unix seconds)` rows.""" + + return tuple( + RelationRead( + object_name=str(row[0]), + reader=str(row[1]) if row[1] else None, + query_count=int(str(row[2])), + last_read_at=datetime.fromtimestamp(int(str(row[3])), tz=UTC), + ) + for row in rows + ) + + +def read_relation_usage( + *, + run_query: Callable[[str], list[tuple[object, ...]]], + relations: tuple[RelationInfo, ...], + window_seconds: int, +) -> RelationUsageReport: + """Read every relation's usage; no query log or logging turned off means not checked.""" + + relations_by_name: dict[str, RelationInfo] = usage_object_names(relations=relations) + rows: list[tuple[object, ...]] = [] + try: + logging: str = str(run_query(LOGGING_SETTING_SQL)[0][0]) if relations_by_name else "" + if logging in LOGGING_OFF_VALUES: + return RelationUsageReport(checked=False, reason=USAGE_LOGGING_OFF_REASON) + sql: str + for sql in usage_queries(object_names=relations_by_name, window_seconds=window_seconds): + rows.extend(run_query(sql)) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + return RelationUsageReport( + checked=True, + usages=aggregate_relation_usage( + reads=usage_reads(rows), relations_by_name=relations_by_name + ), + notices=replica_notices(run_query=run_query) if relations_by_name else (), + ) + + +def replica_notices(*, run_query: Callable[[str], list[tuple[object, ...]]]) -> tuple[str, ...]: + """A notice when this server's shard has other replicas, whose query logs are not read.""" + + try: + rows: list[tuple[object, ...]] = run_query(REPLICA_COUNT_SQL) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return () + replicas: int = int(str(rows[0][0] or 0)) if rows else 0 + return (OTHER_REPLICAS_NOTICE,) if replicas > 1 else () diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index ba9b34eaf0..303ca95a40 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -51,6 +51,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -96,6 +97,7 @@ generation_comment, rewrite_partitions, ) +from sqlbuild.adapters.clickhouse._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.clickhouse._helpers.snapshot_rewrite import ( nullable_offset_windows, rewrite_snapshot_statements, @@ -2307,6 +2309,21 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Read candidate reads from this node's system.query_log.""" + + return read_relation_usage( + run_query=lambda sql: list(self.execute(connection=connection, sql=sql).fetchall()), + relations=relations, + window_seconds=window_seconds, + ) + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index bc91e48c48..783c6dfa4d 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -114,3 +114,20 @@ NULL_IN_ORDINARY_COLUMN_CODE: str = "A713" NULL_TARGET_COLUMN_PATTERN: re.Pattern[str] = re.compile(r"to destination column (\S+?):") FRESH_GENERATION_CLAUSE: str = " COMMENT ''" +USAGE_NAME_CHUNK_SIZE: int = 1_000 +USAGE_UNAVAILABLE_REASON: str = ( + "cannot read system.query_log (needs query logging on and SELECT on system.query_log)" +) +USAGE_LOGGING_OFF_REASON: str = "query logging is off for this connection (log_queries = 0)" +USAGE_UNAVAILABLE_ERROR_CODES: frozenset[int] = frozenset({60, 497}) +USAGE_QUERY_KINDS: tuple[str, ...] = ("Select", "Insert") +USAGE_ERROR_CODE_PATTERN: re.Pattern[str] = re.compile(r"Code: (\d+)\.") +LOGGING_SETTING_SQL: str = "SELECT getSetting('log_queries')" +LOGGING_OFF_VALUES: frozenset[str] = frozenset({"0", "False"}) +REPLICA_COUNT_SQL: str = ( + "SELECT max(replicas) FROM (SELECT count() AS replicas FROM system.clusters " + "WHERE (cluster, shard_num) IN " + "(SELECT cluster, shard_num FROM system.clusters WHERE is_local) " + "GROUP BY cluster, shard_num)" +) +OTHER_REPLICAS_NOTICE: str = "usage read from this server only; other replicas not checked" diff --git a/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py b/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py new file mode 100644 index 0000000000..bb43f73cc5 --- /dev/null +++ b/src/sqlbuild/adapters/databricks/_helpers/relation_usage.py @@ -0,0 +1,118 @@ +"""Read who queried relations from the Databricks system.access.table_lineage table.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.main.relation_reads import relation_reads +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.databricks.constants import ( + LINEAGE_VIEW_SOURCE_TYPES, + USAGE_NAME_CHUNK_SIZE, + USAGE_UNAVAILABLE_ERROR_CLASSES, + USAGE_UNAVAILABLE_REASON, + USAGE_UNAVAILABLE_SQLSTATES, +) +from sqlbuild.adapters.databricks.models import UsageQuery + + +def usage_object_names(*, relations: Iterable[RelationInfo]) -> dict[str, RelationInfo]: + """Each fully qualified relation by its lowercase `catalog.schema.table` lineage name.""" + + named: dict[str, RelationInfo] = {} + relation: RelationInfo + for relation in relations: + if relation.database is None or relation.schema is None: + continue + name: str = ".".join( + part.replace("`", "").lower() + for part in (relation.database, relation.schema, relation.name) + ) + named[name] = relation + return named + + +def usage_queries(*, object_names: Iterable[str], window_seconds: int) -> tuple[UsageQuery, ...]: + """One lineage read per chunk of object names.""" + + names: tuple[str, ...] = tuple(sorted(object_names)) + return tuple( + usage_query( + object_names=names[start : start + USAGE_NAME_CHUNK_SIZE], + window_seconds=window_seconds, + ) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) + ) + + +def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> UsageQuery: + """Read events of the named tables per user within the window, direct or through views.""" + + placeholders: str = ", ".join(f":name_{index}" for index in range(len(object_names))) + since: str = "TIMESTAMPADD(SECOND, -:window_seconds, current_timestamp())" + view_types: str = ", ".join(f"'{source_type}'" for source_type in LINEAGE_VIEW_SOURCE_TYPES) + sql: str = ( + "WITH lineage AS (" + "SELECT lower(source_table_full_name) AS object_name, source_type, created_by, " + "event_time, statement_id, event_id, COALESCE(direct_access, true) AS direct " + "FROM system.access.table_lineage " + f"WHERE event_date >= DATE({since}) " + f"AND event_time >= {since} " + "AND source_table_full_name IS NOT NULL), " + f"reads AS (SELECT * FROM lineage WHERE object_name IN ({placeholders})), " + "via AS (" + "SELECT reads.object_name, reads.created_by, collect_set(views.object_name) AS via_names " + "FROM reads JOIN lineage AS views ON views.statement_id = reads.statement_id " + f"AND views.direct AND views.source_type IN ({view_types}) " + "WHERE NOT reads.direct GROUP BY 1, 2) " + "SELECT reads.object_name, reads.created_by AS user_name, " + "COUNT(DISTINCT COALESCE(reads.statement_id, reads.event_id)) AS query_count, " + "MAX(reads.event_time) AS last_read_at, " + "bool_or(reads.direct) AS direct, " + "first(via.via_names) AS via_names " + "FROM reads LEFT JOIN via ON via.object_name = reads.object_name " + "AND via.created_by <=> reads.created_by " + "GROUP BY 1, 2 " + "ORDER BY 1, 2" + ) + params: dict[str, object] = {"window_seconds": window_seconds} + params.update({f"name_{index}": name for index, name in enumerate(object_names)}) + return UsageQuery(sql=sql, params=params) + + +def is_usage_unavailable_error(error: Exception) -> bool: + """Whether reading lineage failed for Unity Catalog, system schema or permission reasons.""" + + message: str = str(error).upper() + return getattr(error, "sqlstate", None) in USAGE_UNAVAILABLE_SQLSTATES or any( + error_class in message for error_class in USAGE_UNAVAILABLE_ERROR_CLASSES + ) + + +def read_relation_usage( + *, connection: Any, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Read every relation's usage in chunked queries; no lineage access means not checked.""" + + relations_by_name: dict[str, RelationInfo] = usage_object_names(relations=relations) + rows: list[tuple[object, ...]] = [] + query: UsageQuery + for query in usage_queries(object_names=relations_by_name, window_seconds=window_seconds): + cursor: Any = connection.cursor() + try: + cursor.execute(query.sql, query.params) + rows.extend(tuple(row) for row in cursor.fetchall()) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + finally: + cursor.close() + return RelationUsageReport( + checked=True, + usages=aggregate_relation_usage( + reads=relation_reads(rows=rows), relations_by_name=relations_by_name + ), + ) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 03ff3cbd5c..45890a0866 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -63,6 +63,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RenderedRetentionChange, RetentionRequest, RetentionState, @@ -117,6 +118,7 @@ render_databricks_view_revokes, show_grants_object_kind, ) +from sqlbuild.adapters.databricks._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.databricks.classes.databricks_connection import _DatabricksConnection from sqlbuild.adapters.databricks.constants import ( DELTA_COLUMN_MAPPING_NAME_MODE, @@ -608,6 +610,19 @@ def get_table_freshness_metadata( raise AdapterUserError(message=metadata.message or f"freshness unavailable for {name}") return metadata + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Read candidate reads from Unity Catalog's system.access.table_lineage.""" + + return read_relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index 3ab8be030e..1557862edf 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -17,3 +17,20 @@ DETAILED_TABLE_INFORMATION_MARKER: str = "# detailed table information" DESCRIBE_DETAIL_ROW_WIDTH: int = 2 TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" +MAX_PARAMETER_MARKERS: int = 255 +USAGE_NAME_CHUNK_SIZE: int = MAX_PARAMETER_MARKERS - 5 +USAGE_UNAVAILABLE_REASON: str = ( + "cannot read system.access.table_lineage (needs Unity Catalog with the access system " + "schema enabled, and USE CATALOG, USE SCHEMA and SELECT on it)" +) +USAGE_UNAVAILABLE_ERROR_CLASSES: tuple[str, ...] = ( + "INSUFFICIENT_PERMISSIONS", + "PERMISSION_DENIED", + "TABLE_OR_VIEW_NOT_FOUND", + "SCHEMA_NOT_FOUND", + "CATALOG_NOT_FOUND", + "NO_SUCH_CATALOG_EXCEPTION", + "UC_NOT_ENABLED", +) +USAGE_UNAVAILABLE_SQLSTATES: frozenset[str] = frozenset({"42501", "42P01", "42704"}) +LINEAGE_VIEW_SOURCE_TYPES: tuple[str, ...] = ("VIEW", "MATERIALIZED_VIEW", "METRIC_VIEW") diff --git a/src/sqlbuild/adapters/databricks/models.py b/src/sqlbuild/adapters/databricks/models.py new file mode 100644 index 0000000000..e4507521e4 --- /dev/null +++ b/src/sqlbuild/adapters/databricks/models.py @@ -0,0 +1,13 @@ +"""Databricks adapter models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UsageQuery: + """One system.access.table_lineage read: its SQL and named parameters.""" + + sql: str + params: dict[str, object] diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 7f5e14cf43..1464805c67 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -51,6 +51,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -63,6 +64,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -228,6 +230,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py new file mode 100644 index 0000000000..5eed12d597 --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/relation_usage.py @@ -0,0 +1,233 @@ +"""Read who queried relations from Snowflake ACCOUNT_USAGE access history.""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Iterable +from typing import Any + +from sqlbuild.adapter.contract.main.aggregate_relation_usage import aggregate_relation_usage +from sqlbuild.adapter.contract.main.relation_reads import relation_reads +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage, RelationUsageReport +from sqlbuild.adapters.snowflake._helpers.view_dependencies import ( + chained_usages, + dependency_name, + dependency_queries, + display_name, + walk_seeds, +) +from sqlbuild.adapters.snowflake.constants import ( + CHAINS_NOT_FOLLOWED_NOTICE, + CLONE_STATEMENT_PATTERN, + SQLBUILD_QUERY_TAG_FRAGMENT, + USAGE_NAME_CHUNK_SIZE, + USAGE_UNAVAILABLE_REASON, +) +from sqlbuild.adapters.snowflake.models import UsageQuery + +_UNQUOTED_IDENTIFIER: re.Pattern[str] = re.compile(r"^[A-Z_][A-Z0-9_$]*$") +_UNAVAILABLE_ERRNOS: frozenset[int] = frozenset({2003, 3001}) +_UNAVAILABLE_MESSAGES: tuple[str, ...] = ( + "does not exist or not authorized", + "insufficient privileges", +) + + +def usage_object_name(*, database: str, schema: str, name: str) -> str: + """The `objectName` access history records for a relation SQLBuild names unquoted.""" + + return ".".join(_object_name_part(part) for part in (database, schema, name)) + + +def _object_name_part(part: str) -> str: + folded: str = part.upper() + if _UNQUOTED_IDENTIFIER.match(folded): + return folded + return '"' + folded.replace('"', '""') + '"' + + +def usage_object_names( + *, relations: Iterable[RelationInfo], current_database: str | None +) -> dict[str, RelationInfo]: + """Each schema-qualified relation by its access-history object name.""" + + named: dict[str, RelationInfo] = {} + relation: RelationInfo + for relation in relations: + database: str | None = relation.database or current_database + if database is None or relation.schema is None: + continue + named[usage_object_name(database=database, schema=relation.schema, name=relation.name)] = ( + relation + ) + return named + + +def usage_queries(*, object_names: Iterable[str], window_seconds: int) -> tuple[UsageQuery, ...]: + """One access-history read per chunk of object names, oldest read within the window.""" + + names: tuple[str, ...] = tuple(sorted(object_names)) + return tuple( + usage_query( + object_names=names[start : start + USAGE_NAME_CHUNK_SIZE], + window_seconds=window_seconds, + ) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) + ) + + +def usage_query(*, object_names: tuple[str, ...], window_seconds: int) -> UsageQuery: + """Reads of the named objects per user, direct or through views, minus SQLBuild and clones.""" + + placeholders: str = ", ".join(["%s"] * len(object_names)) + since: str = "DATEADD('second', -%s, CURRENT_TIMESTAMP())" + object_name: str = 'accessed.value:"objectName"::STRING' + sql: str = ( + "WITH history AS (" + "SELECT query_id, user_name, query_start_time, direct_objects_accessed, " + "base_objects_accessed FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY " + f"WHERE query_start_time >= {since}), " + "direct_reads AS (" + "SELECT history.query_id, history.user_name, history.query_start_time, " + f"{object_name} AS object_name " + "FROM history, LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed " + f"WHERE {object_name} IN ({placeholders})), " + "base_reads AS (" + "SELECT history.query_id, history.user_name, history.query_start_time, " + f"{object_name} AS object_name " + "FROM history, LATERAL FLATTEN(INPUT => history.base_objects_accessed) AS accessed " + f"WHERE {object_name} IN ({placeholders})), " + "candidate_reads AS (" + "SELECT query_id, user_name, query_start_time, object_name, TRUE AS direct " + "FROM direct_reads UNION ALL " + "SELECT base.query_id, base.user_name, base.query_start_time, base.object_name, FALSE " + "FROM base_reads AS base WHERE NOT EXISTS (" + "SELECT 1 FROM direct_reads AS direct_read " + "WHERE direct_read.query_id = base.query_id " + "AND direct_read.object_name = base.object_name)), " + "reads AS (" + "SELECT * FROM candidate_reads AS candidate WHERE NOT EXISTS (" + "SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY AS query " + "WHERE query.query_id = candidate.query_id " + f"AND query.start_time >= {since} " + "AND (CONTAINS(COALESCE(query.query_tag, ''), %s) " + "OR (STARTSWITH(query.query_type, 'CREATE') " + "AND REGEXP_LIKE(query.query_text, %s, 'is'))))), " + "via AS (" + f"SELECT reads.object_name, reads.user_name, ARRAY_AGG(DISTINCT {object_name}) " + "AS via_names " + "FROM reads JOIN history ON history.query_id = reads.query_id, " + "LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed " + "WHERE NOT reads.direct GROUP BY 1, 2) " + "SELECT reads.object_name, reads.user_name, " + "COUNT(DISTINCT reads.query_id) AS query_count, " + "MAX(reads.query_start_time) AS last_read_at, " + "BOOLOR_AGG(reads.direct) AS direct, " + "ANY_VALUE(via.via_names) AS via_names " + "FROM reads LEFT JOIN via " + "ON via.object_name = reads.object_name AND via.user_name = reads.user_name " + "GROUP BY 1, 2 " + "ORDER BY 1, 2" + ) + return UsageQuery( + sql=sql, + params=( + window_seconds, + *object_names, + *object_names, + window_seconds, + SQLBUILD_QUERY_TAG_FRAGMENT, + CLONE_STATEMENT_PATTERN, + ), + ) + + +def is_usage_unavailable_error(error: Exception) -> bool: + """Whether reading access history failed for edition or privilege reasons.""" + + message: str = str(error).lower() + return getattr(error, "errno", None) in _UNAVAILABLE_ERRNOS or any( + marker in message for marker in _UNAVAILABLE_MESSAGES + ) + + +def read_relation_usage( + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + current_database: Callable[[], str | None], +) -> RelationUsageReport: + """Read every relation's usage and view dependencies; no access history means not checked.""" + + database: str | None = None + if any(relation.database is None for relation in relations): + database = current_database() + relations_by_name: dict[str, RelationInfo] = usage_object_names( + relations=relations, current_database=database + ) + by_dependency_name: dict[str, RelationInfo] = {} + relation: RelationInfo + for relation in relations: + name: str | None = dependency_name(relation=relation, current_database=database) + if name is not None: + by_dependency_name[name] = relation + try: + rows: list[tuple[object, ...]] = _fetch( + connection=connection, + queries=usage_queries(object_names=relations_by_name, window_seconds=window_seconds), + ) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON) + usages: tuple[RelationUsage, ...] = aggregate_relation_usage( + reads=relation_reads(rows=rows, via_name=display_name), + relations_by_name=relations_by_name, + ) + seeds: dict[str, tuple[str, RelationUsage]] = walk_seeds( + usages=usages, relations_by_dependency_name=by_dependency_name + ) + try: + dependencies: list[tuple[object, ...]] = _fetch( + connection=connection, + queries=dependency_queries( + databases=dependency_databases(seeds=seeds, candidates=by_dependency_name) + ), + ) + except Exception as error: + if not is_usage_unavailable_error(error): + raise + return RelationUsageReport( + checked=True, usages=usages, notices=(CHAINS_NOT_FOLLOWED_NOTICE,) + ) + return RelationUsageReport( + checked=True, + usages=chained_usages( + usages=usages, + dependencies=((str(row[0]), str(row[1])) for row in dependencies), + relations_by_dependency_name=by_dependency_name, + ), + ) + + +def dependency_databases(*, seeds: Iterable[str], candidates: Iterable[str]) -> frozenset[str]: + """The databases of the walk's start objects and the candidates; none without a start.""" + + starts: tuple[str, ...] = tuple(seeds) + if not starts: + return frozenset() + return frozenset(name.split(".", 1)[0] for name in (*starts, *candidates)) + + +def _fetch(*, connection: Any, queries: tuple[UsageQuery, ...]) -> list[tuple[object, ...]]: + rows: list[tuple[object, ...]] = [] + query: UsageQuery + for query in queries: + cursor: Any = connection.cursor() + try: + cursor.execute(query.sql, query.params) + rows.extend(tuple(row) for row in cursor.fetchall()) + finally: + cursor.close() + return rows diff --git a/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py b/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py new file mode 100644 index 0000000000..1725f598ca --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/view_dependencies.py @@ -0,0 +1,193 @@ +"""Follow Snowflake view dependencies from directly read objects to the candidates behind them.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from datetime import datetime + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage +from sqlbuild.adapters.snowflake.constants import ( + DEPENDENCY_VIEW_DOMAINS, + NAME_SEPARATOR, + QUOTED_IDENTIFIER_MARK, + USAGE_NAME_CHUNK_SIZE, +) +from sqlbuild.adapters.snowflake.models import UsageQuery + + +def dependency_name(*, relation: RelationInfo, current_database: str | None) -> str | None: + """The `DATABASE.SCHEMA.NAME` OBJECT_DEPENDENCIES records for an unquoted relation.""" + + database: str | None = relation.database or current_database + if database is None or relation.schema is None: + return None + return ".".join(part.upper() for part in (database, relation.schema, relation.name)) + + +def display_name(object_name: str) -> str: + """A Snowflake object name as SQLBuild prints it: unquoted names in lowercase.""" + + return object_name if QUOTED_IDENTIFIER_MARK in object_name else object_name.lower() + + +def dependency_key(object_name: str) -> str: + """An access-history or printed object name as OBJECT_DEPENDENCIES spells it.""" + + parts: list[str] = [] + current: list[str] = [] + quoted: bool = False + was_quoted: bool = False + index: int = 0 + while index < len(object_name): + character: str = object_name[index] + doubled: bool = object_name[index : index + 2] == QUOTED_IDENTIFIER_MARK * 2 + if quoted and doubled: + current.append(QUOTED_IDENTIFIER_MARK) + index += 2 + continue + if character == QUOTED_IDENTIFIER_MARK: + quoted = not quoted + was_quoted = True + elif character == NAME_SEPARATOR and not quoted: + parts.append("".join(current) if was_quoted else "".join(current).upper()) + current, was_quoted = [], False + else: + current.append(character) + index += 1 + parts.append("".join(current) if was_quoted else "".join(current).upper()) + return ".".join(parts) + + +def dependency_queries(*, databases: Iterable[str]) -> tuple[UsageQuery, ...]: + """One OBJECT_DEPENDENCIES read of the view links in the given databases.""" + + names: tuple[str, ...] = tuple(sorted(set(databases))) + return tuple( + dependency_query(databases=names[start : start + USAGE_NAME_CHUNK_SIZE]) + for start in range(0, len(names), USAGE_NAME_CHUNK_SIZE) + ) + + +def dependency_query(*, databases: tuple[str, ...]) -> UsageQuery: + """What each view in the databases references, as `(referencing, referenced)` names.""" + + placeholders: str = ", ".join(["%s"] * len(databases)) + domains: str = ", ".join(["%s"] * len(DEPENDENCY_VIEW_DOMAINS)) + sql: str = ( + "SELECT referencing_database || '.' || referencing_schema || '.' " + "|| referencing_object_name AS referencing_name, " + "referenced_database || '.' || referenced_schema || '.' || referenced_object_name " + "AS referenced_name " + "FROM SNOWFLAKE.ACCOUNT_USAGE.OBJECT_DEPENDENCIES " + f"WHERE referencing_object_domain IN ({domains}) " + f"AND referencing_database IN ({placeholders}) " + "ORDER BY 1, 2" + ) + return UsageQuery(sql=sql, params=(*DEPENDENCY_VIEW_DOMAINS, *databases)) + + +def _folded(relation: RelationInfo | RelationUsage) -> tuple[str, str, str]: + return ( + (relation.database or "").lower(), + (relation.schema or "").lower(), + relation.name.lower(), + ) + + +def walk_seeds( + *, usages: Iterable[RelationUsage], relations_by_dependency_name: Mapping[str, RelationInfo] +) -> dict[str, tuple[str, RelationUsage]]: + """Each directly read object to walk from, with its printed name and the usage it passes on.""" + + names_by_key: dict[tuple[str, str, str], str] = { + _folded(relation): name for name, relation in relations_by_dependency_name.items() + } + seeds: dict[str, tuple[str, RelationUsage]] = {} + usage: RelationUsage + for usage in usages: + own: str | None = names_by_key.get(_folded(usage)) + if own is not None: + seeds.setdefault(own, (display_name(own), usage)) + via: str + for via in usage.via: + seeds.setdefault(dependency_key(via), (via, usage)) + return seeds + + +def _via_seed( + *, relation: RelationInfo, label: str, seed_usage: RelationUsage, earlier: RelationUsage | None +) -> RelationUsage: + via: tuple[str, ...] = (label,) + readers: tuple[str, ...] = seed_usage.readers + last_read_at: datetime = seed_usage.last_read_at + query_count: int = seed_usage.query_count + if earlier is not None: + via = tuple(sorted({*earlier.via, *via})) + readers = tuple(dict.fromkeys((*earlier.readers, *readers))) + last_read_at = max(earlier.last_read_at, last_read_at) + query_count = max(earlier.query_count, query_count) + return RelationUsage( + database=relation.database, + schema=relation.schema, + name=relation.name, + last_read_at=last_read_at, + readers=readers, + query_count=query_count, + direct=False, + via=via, + ) + + +def _reachable(*, start: str, references: Mapping[str, set[str]]) -> list[str]: + seen: set[str] = {start} + pending: list[str] = [start] + reached: list[str] = [] + while pending: + node: str = pending.pop() + child: str + for child in sorted(references.get(node, set()) - seen): + seen.add(child) + reached.append(child) + pending.append(child) + return reached + + +def chained_usages( + *, + usages: tuple[RelationUsage, ...], + dependencies: Iterable[tuple[str, str]], + relations_by_dependency_name: Mapping[str, RelationInfo], +) -> tuple[RelationUsage, ...]: + """Usages plus every unread candidate a directly read object depends on, through any views.""" + + references: dict[str, set[str]] = {} + referencing: str + referenced: str + for referencing, referenced in dependencies: + references.setdefault(referencing, set()).add(referenced) + seeds: dict[str, tuple[str, RelationUsage]] = walk_seeds( + usages=usages, relations_by_dependency_name=relations_by_dependency_name + ) + read_keys: frozenset[tuple[str, str, str]] = frozenset(map(_folded, usages)) + read: set[str] = { + name + for name, relation in relations_by_dependency_name.items() + if _folded(relation) in read_keys + } + found: dict[str, RelationUsage] = {} + seed: str + for seed in sorted(seeds): + label: str + seed_usage: RelationUsage + label, seed_usage = seeds[seed] + child: str + for child in _reachable(start=seed, references=references): + if child in read or child not in relations_by_dependency_name: + continue + found[child] = _via_seed( + relation=relations_by_dependency_name[child], + label=label, + seed_usage=seed_usage, + earlier=found.get(child), + ) + return (*usages, *(found[name] for name in sorted(found))) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 1e17650c96..9e87820442 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -62,6 +62,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RenderedRetentionChange, RetentionRequest, RetentionState, @@ -130,6 +131,7 @@ information_schema_type, show_columns_type, ) +from sqlbuild.adapters.snowflake._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.snowflake._helpers.show_metadata import ( is_missing_object_error, listed_relation_from_show_table, @@ -878,6 +880,22 @@ def get_table_freshness_metadata( observed_at=row[1] if isinstance(row[1], datetime) else None, ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + """Read candidate reads from ACCOUNT_USAGE.ACCESS_HISTORY, which lags up to 3 hours.""" + + return read_relation_usage( + connection=connection, + relations=relations, + window_seconds=window_seconds, + current_database=lambda: self._current_database(connection=connection), + ) + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index c816a0575e..1ae48c46af 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -1,5 +1,7 @@ """Snowflake adapter constants.""" +from sqlbuild._native import SQLBUILD_QUERY_TAG_APP + MAX_STATEMENT_TIMEOUT_SECONDS: int = 604_800 BASE_TABLE_METADATA_TYPE: str = "BASE TABLE" @@ -21,3 +23,26 @@ STATE_TABLE_RETENTION_PARAMETER: str = "DATA_RETENTION_TIME_IN_DAYS" INVALID_PARAMETER_VALUE_ERRNO: int = 1008 TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" +USAGE_NAME_CHUNK_SIZE: int = 1_000 +USAGE_UNAVAILABLE_REASON: str = ( + "cannot read SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY (needs Enterprise edition and the " + "SNOWFLAKE.GOVERNANCE_VIEWER database role; view chains also need SNOWFLAKE.OBJECT_VIEWER)" +) +CHAINS_NOT_FOLLOWED_NOTICE: str = "view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER" +SQLBUILD_QUERY_TAG_FRAGMENT: str = f'"app":"{SQLBUILD_QUERY_TAG_APP}"' +_CLONE_IDENTIFIER: str = r'("[^"]*"|[A-Za-z_][A-Za-z0-9_$]*)' +_CLONE_NAME: str = _CLONE_IDENTIFIER + r"(\s*\.\s*" + _CLONE_IDENTIFIER + r"){0,2}" +_LEADING_COMMENTS: str = "(\\s|--[^\n]*\n|//[^\n]*\n|/\\*([^*]|\\*+[^*/])*\\*+/)*" +CLONE_STATEMENT_PATTERN: str = ( + "^" + + _LEADING_COMMENTS + + r"CREATE\s+(OR\s+REPLACE\s+)?((TRANSIENT|TEMPORARY|TEMP|VOLATILE|LOCAL\s+TEMPORARY|" + r"GLOBAL\s+TEMPORARY)\s+)?(TABLE|DYNAMIC\s+TABLE|SCHEMA|DATABASE)\s+(IF\s+NOT\s+EXISTS\s+)?" + + _CLONE_NAME + + r"\s+CLONE\s+" + + _CLONE_NAME + + r'([^A-Za-z0-9_$"].*)?$' +) +DEPENDENCY_VIEW_DOMAINS: tuple[str, ...] = ("VIEW", "MATERIALIZED VIEW", "SECURE VIEW") +QUOTED_IDENTIFIER_MARK: str = '"' +NAME_SEPARATOR: str = "." diff --git a/src/sqlbuild/adapters/snowflake/models.py b/src/sqlbuild/adapters/snowflake/models.py new file mode 100644 index 0000000000..f610a01d2c --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/models.py @@ -0,0 +1,13 @@ +"""Snowflake adapter models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UsageQuery: + """One access-history read: its SQL and bound parameters.""" + + sql: str + params: tuple[object, ...] diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 1d648e01bd..d42b28f5d9 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -55,6 +55,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -67,6 +68,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -342,6 +344,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 0acea97875..fcbd7a3c97 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -53,6 +53,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -65,6 +66,7 @@ RelationInfo, RelationReadProbe, RelationRowCountEstimate, + RelationUsageReport, RowDiffColumnResult, RowDiffCoverage, RowDiffPreparedRelations, @@ -183,6 +185,16 @@ def get_table_freshness_metadata( message=f"adapter '{self.adapter_name}' does not support table freshness metadata" ) + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + del connection, relations, window_seconds + return unsupported_relation_usage() + def get_tables_freshness_metadata( self, *, diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 5313227c0a..6f4f890f30 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -2,11 +2,11 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import datetime from sqlbuild._native import OldNameViewDropReason, OldNameViewHistory -from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup +from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup, RelationUsage from sqlbuild.executor.janitor.types import JanitorAction from sqlbuild.executor.janitor_events.models import JanitorEventRecord @@ -63,6 +63,12 @@ class JanitorArchiveCandidate: archive_key: JanitorRelationKey archived_at: datetime expires_at: datetime + usage: RelationUsage | None = None + + def with_usage(self, *, usage: RelationUsage) -> JanitorArchiveCandidate: + """Return this candidate with its reads in the usage window.""" + + return replace(self, usage=usage) @dataclass(frozen=True) @@ -189,6 +195,17 @@ def display_name(self) -> str: return "" +@dataclass(frozen=True) +class JanitorUsageCheck: + """Whether recent reads of the archive candidates were checked, and how in-use ones fare.""" + + checked: bool + window: str + keep_in_use: bool = False + reason: str | None = None + notices: tuple[str, ...] = () + + @dataclass(frozen=True) class JanitorPlan: """Complete janitor preview and execution plan.""" @@ -214,6 +231,26 @@ class JanitorPlan: scanned_schema_count: int = 0 age_metadata_supported: bool = False planned_at: datetime | None = None + usage_check: JanitorUsageCheck | None = None + kept_in_use: tuple[JanitorArchiveCandidate, ...] = field(default_factory=tuple) + + def with_usage( + self, + *, + archive_candidates: tuple[JanitorArchiveCandidate, ...], + archive_deletion_candidates: tuple[JanitorArchivedRelation, ...], + kept_in_use: tuple[JanitorArchiveCandidate, ...], + usage_check: JanitorUsageCheck, + ) -> JanitorPlan: + """Return this plan after its usage check, with in-use candidates kept when asked.""" + + return replace( + self, + archive_candidates=archive_candidates, + archive_deletion_candidates=archive_deletion_candidates, + kept_in_use=kept_in_use, + usage_check=usage_check, + ) @dataclass(frozen=True) diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 355cf43f44..132ae77ba9 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -5,9 +5,10 @@ from collections import Counter from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext -from dataclasses import dataclass +from dataclasses import dataclass, field from itertools import chain from pathlib import Path +from time import thread_time_ns from sqlbuild._native import ( CUSTOM_RULE_PROJECT_SUBJECT, @@ -41,6 +42,18 @@ type _FindingKey = tuple[str, str, int, int, str, str] +@dataclass +class _RuleTimings: + """Invocations and CPU nanoseconds per rule code, excluding the runtime guard's checks.""" + + by_code: dict[str, list[int]] = field(default_factory=dict) + + def record(self, *, code: str, cpu_ns: int) -> None: + totals: list[int] = self.by_code.setdefault(code, [0, 0]) + totals[0] += 1 + totals[1] += max(cpu_ns, 0) + + @dataclass(frozen=True) class _PassInputs: project: RuleFactProject @@ -52,6 +65,7 @@ class _PassInputs: plan: CustomRulePlan | None guard: RuntimeGuard | None cancelled: Callable[[], bool] | None + timings: _RuleTimings def evaluate_custom_rules( @@ -98,6 +112,7 @@ def evaluate_custom_rules( plan=plan, guard=guard, cancelled=None if partition is None else partition.cancelled, + timings=_RuleTimings(), ) def run_pass( @@ -108,12 +123,15 @@ def run_pass( evaluations: list[CustomRuleEvaluation] = run_pass( rules=custom_rules, subjects=models, active=tracker ) + monitor_ns: int = tracker.monitor_ns if verify_determinism: + repeat_tracker: _ReadTracker = _ReadTracker( + reads=None, rules=(), rules_root=project_dir / "rules" + ) repeated: list[CustomRuleEvaluation] = run_pass( - rules=custom_rules[::-1], - subjects=models[::-1], - active=_ReadTracker(reads=None, rules=(), rules_root=project_dir / "rules"), + rules=custom_rules[::-1], subjects=models[::-1], active=repeat_tracker ) + monitor_ns += repeat_tracker.monitor_ns _verify_repeatable( rules=custom_rules, first=list(chain.from_iterable(item.findings for item in evaluations)), @@ -125,6 +143,11 @@ def run_pass( uncacheable_codes=frozenset(tracker.uncacheable_codes), observed=() if reads is None else tuple(reads.observed.items()), stateful_codes=frozenset(tracker.stateful_codes), + rule_timings={ + code: (invocations, cpu_ns) + for code, (invocations, cpu_ns) in inputs.timings.by_code.items() + }, + monitor_ns=monitor_ns, ) @@ -143,6 +166,8 @@ def __init__( self.untracked_codes: set[str] = set() self.uncacheable_codes: set[str] = set() self.stateful_codes: set[str] = set() + self.monitor_ns: int = 0 + started: int = thread_time_ns() self._state: ModuleStateMonitor | None = ( ModuleStateMonitor( namespaces={rule.code: rule_namespaces((rule.check,)) for rule in rules}, @@ -151,6 +176,7 @@ def __init__( if reads is not None or detect_state else None ) + self.monitor_ns += thread_time_ns() - started def start(self) -> None: if self.reads is not None: @@ -168,11 +194,19 @@ def finish(self, *, code: str) -> frozenset[tuple[object, ...]] | None: self.untracked_codes.add(code) if reads.uncacheable: self.uncacheable_codes.add(code) - changed: tuple[str, ...] = () if self._state is None else self._state.changed_codes() + return None if observed is None else frozenset(observed) + + def finish_subject(self) -> None: + """Mark codes whose module state changed since the last check; project rules check each.""" + + if self._state is None: + return + started: int = thread_time_ns() + changed: tuple[str, ...] = self._state.changed_codes() + self.monitor_ns += thread_time_ns() - started self.stateful_codes.update(changed) - if reads is not None: + if self.reads is not None: self.untracked_codes.update(changed) - return None if observed is None else frozenset(observed) def _evaluate_pass( @@ -230,6 +264,7 @@ def context(rule: Rule) -> EvaluationRuleContext: ), ctx=ctx, guard=guard, + timings=inputs.timings, ) except NonHermeticRuleError: raise @@ -243,16 +278,21 @@ def context(rule: Rule) -> EvaluationRuleContext: reads=tracker.finish(code=rule.code), ) ) + tracker.finish_subject() for model in models: subject: Model = public_model(model) path: str = model.relative_path.as_posix() + evaluated: bool = False for rule, ctx, planned in model_contexts: if planned is not None and path not in planned: continue + evaluated = True _stop_if_cancelled(inputs) tracker.start() try: - findings = _invoke(rule=rule, subject=subject, ctx=ctx, guard=guard) + findings = _invoke( + rule=rule, subject=subject, ctx=ctx, guard=guard, timings=inputs.timings + ) except NonHermeticRuleError: raise except Exception as error: @@ -267,6 +307,8 @@ def context(rule: Rule) -> EvaluationRuleContext: reads=tracker.finish(code=rule.code), ) ) + if evaluated: + tracker.finish_subject() return evaluations @@ -278,18 +320,29 @@ def _stop_if_cancelled(inputs: _PassInputs) -> None: def _invoke( - *, rule: Rule, subject: object, ctx: EvaluationRuleContext, guard: RuntimeGuard | None + *, + rule: Rule, + subject: object, + ctx: EvaluationRuleContext, + guard: RuntimeGuard | None, + timings: _RuleTimings, ) -> list[Finding]: if rule.subject_parameter is None or rule.context_parameter is None: raise RulesError(f"custom rule {rule.code} has no resolved typed signature") scope: AbstractContextManager[None] = nullcontext() if guard is None else guard.rule(rule.code) - with scope: - result: object = rule.check( - **{ - rule.subject_parameter: subject, - rule.context_parameter: ctx, - } - ) + guard_started: int = 0 if guard is None else guard.check_ns + started: int = thread_time_ns() + try: + with scope: + result: object = rule.check( + **{ + rule.subject_parameter: subject, + rule.context_parameter: ctx, + } + ) + finally: + guard_ns: int = 0 if guard is None else guard.check_ns - guard_started + timings.record(code=rule.code, cpu_ns=thread_time_ns() - started - guard_ns) if not isinstance(result, list) or any(not isinstance(finding, Finding) for finding in result): raise RulesError(f"custom rule {rule.code} must return list[Finding]") return result diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index fbc81eb9c7..4b1dad7dd5 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -108,7 +108,7 @@ def main() -> int: "protocol": CUSTOM_HOST_PROTOCOL_VERSION, "runtime_version": CUSTOM_HOST_RUNTIME_VERSION, "error": None, - "payload": _run_payload(run), + "payload": _run_payload(run=run, guard_ns=0 if guard is None else guard.check_ns), "messages": messages.getvalue().splitlines(), } sys.stdout.write(json.dumps(response, sort_keys=True)) @@ -193,7 +193,7 @@ def _decode_plan(value: object) -> CustomRulePlan: return plan -def _run_payload(run: CustomRuleRun) -> dict[str, object]: +def _run_payload(*, run: CustomRuleRun, guard_ns: int) -> dict[str, object]: """Encode invocations with read sets deduplicated across subjects and bounded in size.""" readsets: dict[frozenset[tuple[object, ...]], int] = {} @@ -231,6 +231,11 @@ def _run_payload(run: CustomRuleRun) -> dict[str, object]: "observed": sorted([list(key), digest] for key, digest in run.observed), "bounded": bounded, "stateful": sorted(run.stateful_codes), + "timings": { + "rules": {code: list(timing) for code, timing in sorted(run.rule_timings.items())}, + "monitor_ns": run.monitor_ns, + "guard_ns": guard_ns, + }, } diff --git a/src/sqlbuild/rule_engine/classes/runtime_guard.py b/src/sqlbuild/rule_engine/classes/runtime_guard.py index 03fc03ba1e..fa59971da5 100644 --- a/src/sqlbuild/rule_engine/classes/runtime_guard.py +++ b/src/sqlbuild/rule_engine/classes/runtime_guard.py @@ -10,6 +10,7 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager from pathlib import Path +from time import thread_time_ns from types import CodeType, FrameType, FunctionType, ModuleType from sqlbuild._native import NonHermeticRuleError, allowed_custom_rule_import @@ -82,11 +83,18 @@ def __init__(self, *, project_dir: Path) -> None: self._metadata_scope: threading.local = threading.local() self.active_rule: str | None = None self.violation: NonHermeticRuleError | None = None + self.check_ns: int = 0 def __call__(self, event: str, arguments: tuple[object, ...]) -> None: if event not in _GUARDED_EVENTS and not event.startswith(_GUARDED_PREFIXES): return - frame: FrameType = sys._getframe(1) + started: int = thread_time_ns() + try: + self._check(event=event, arguments=arguments, frame=sys._getframe(1)) + finally: + self.check_ns += thread_time_ns() - started + + def _check(self, *, event: str, arguments: tuple[object, ...], frame: FrameType) -> None: rule_frame: FrameType | None = ( self._importing_rule_frame(frame) if event == _IMPORT_EVENT @@ -127,12 +135,16 @@ def _call_metadata( function: Callable[..., object] = guarded_function if getattr(self._metadata_scope, "trusted", False): return function(*arguments, **keywords) - paths: tuple[object, ...] = tuple(map(_plain_path, arguments)) - named: dict[str, object] = {key: _plain_path(value) for key, value in keywords.items()} - rule_frame: FrameType | None = self._responsible_rule_frame(sys._getframe(1)) - if rule_frame is not None: - target: str = f" for {paths[0]!r}" if paths else "" - self._reject(rule_frame=rule_frame, action=f"{_METADATA_ACTION}{target}") + started: int = thread_time_ns() + try: + paths: tuple[object, ...] = tuple(map(_plain_path, arguments)) + named: dict[str, object] = {key: _plain_path(value) for key, value in keywords.items()} + rule_frame: FrameType | None = self._responsible_rule_frame(sys._getframe(1)) + if rule_frame is not None: + target: str = f" for {paths[0]!r}" if paths else "" + self._reject(rule_frame=rule_frame, action=f"{_METADATA_ACTION}{target}") + finally: + self.check_ns += thread_time_ns() - started self._metadata_scope.trusted = True try: return function(*paths, **named) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 0107e1a80e..0947504e0d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -72,6 +72,9 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "built_in_rules_ms", "custom_rules_ms", "custom_rules_cpu_ms", + "custom_rules_invocations", + "custom_rules_monitor_cpu_ms", + "custom_rules_guard_cpu_ms", "early_lint_wait_ms", "model_analysis_ms", "model_analysis_cpu_ms", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 6283ccbede..907ae9a301 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -358,3 +358,20 @@ class ClickHouseRecallDocumentedGrantsTestCase: expected_columns: tuple[tuple[object, ...], ...] expected_stale_views: tuple[tuple[object, ...], ...] expected_version: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseJanitorUsageE2ETestCase: + description: str + janitor_command: tuple[str, ...] + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] + expected_remaining: tuple[str, ...] + + +@dataclass(frozen=True) +class ClickHouseJanitorUsageUncheckedE2ETestCase: + description: str + connection_settings: str + grants: tuple[str, ...] + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py index 62ac2d203a..bfa5d899f6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py @@ -9,6 +9,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( CLICKHOUSE_IMAGE, build_unique_schema_name, + cleanup_clickhouse_schema, execute_clickhouse_sql, ) @@ -59,3 +60,10 @@ def recall_database(clickhouse_e2e_config: dict[str, object]) -> Iterator[str]: execute_clickhouse_sql( config=clickhouse_e2e_config, statements=(f"DROP DATABASE IF EXISTS {database} SYNC",) ) + + +@pytest.fixture +def janitor_schema(clickhouse_e2e_config: dict[str, object]) -> Iterator[str]: + schema: str = build_unique_schema_name(prefix="janitor_usage") + yield schema + cleanup_clickhouse_schema(config=clickhouse_e2e_config, schema_name=schema) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index 83cb5a6a26..b1d082211d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -13,7 +13,11 @@ from typing import Any, cast from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_waffle_shop +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + prepare_waffle_shop, + run_sqb, +) from tests.integration.src.sqlbuild.cli.recall.helpers import ( killed_build, killed_during_discovery, @@ -1043,3 +1047,74 @@ def stale_views(*, config: dict[str, object], database: str) -> tuple[tuple[obje sql="SELECT countIf(position(create_table_query, 'join_use_nulls') > 0) " f"FROM system.tables WHERE database = '{database}' AND engine = 'View'", ) + + +JANITOR_SETTINGS: str = ( + "\n[janitor]\nenabled = true\nretention_days = 0\ndelete_tracked_only = false\n" +) +STALE_RELATIONS: tuple[str, ...] = ( + "CREATE TABLE {schema}.old_orders ENGINE = MergeTree ORDER BY id " + "AS SELECT toInt64(number) AS id FROM numbers(3)", + "CREATE TABLE {schema}.old_customers ENGINE = MergeTree ORDER BY id AS SELECT toInt64(1) AS id", + "CREATE VIEW {schema}.old_orders_v2 AS SELECT id FROM {schema}.old_orders", + "CREATE VIEW {schema}.old_orders_v AS SELECT id FROM {schema}.old_orders_v2", +) + + +def prepare_janitor_usage_project( + *, tmp_path: Path, config: dict[str, object], schema: str, connection_settings: str = "" +) -> Path: + """A built project in `schema` beside a stale table, an unread table and a view chain on it.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_janitor_usage", + repo_files={ + "sqlbuild_project.toml": build_clickhouse_project_toml( + project_name="clickhouse_janitor_usage", + schema_name=schema, + config=config, + extra=JANITOR_SETTINGS, + ).replace("\n[targets.dev]", connection_settings + "\n[targets.dev]"), + "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT toInt64(1) AS id\n", + }, + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + execute_clickhouse_sql( + config=config, + statements=tuple(statement.format(schema=schema) for statement in STALE_RELATIONS), + ) + return project_dir + + +def read_as(*, config: dict[str, object], sql: str) -> None: + """Run one query as the user in `config`, as any client would, outside SQLBuild.""" + + import clickhouse_connect + + client: Any = clickhouse_connect.get_client( + host=str(config["host"]), + port=int(str(config["port"])), + username=str(config["user"]), + password=str(config["password"]), + ) + try: + client.query(sql) + finally: + client.close() + + +def stale_relation_names(*, config: dict[str, object], schema: str) -> tuple[str, ...]: + """The `old_*` relations still in `schema`.""" + + return tuple( + str(row[0]) + for row in fetch_clickhouse_rows( + config=config, + sql=f"SELECT name FROM system.tables WHERE database = '{schema}' " + "AND startsWith(name, 'old_') ORDER BY name", + ) + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py new file mode 100644 index 0000000000..5189c5c77e --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py @@ -0,0 +1,148 @@ +"""ClickHouse janitor usage: system.query_log shows who reads stale tables and view chains.""" + +from __future__ import annotations + +import subprocess +from contextlib import ExitStack +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseJanitorUsageE2ETestCase, + ClickHouseJanitorUsageUncheckedE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + execute_clickhouse_sql, + limited_user, + prepare_janitor_usage_project, + read_as, + stale_relation_names, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +APPROVED_JANITOR: tuple[str, ...] = ("--no-color", "janitor", "--auto-approve") + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseJanitorUsageE2ETestCase( + description="a view chain read by another user is in use and archived by default", + janitor_command=APPROVED_JANITOR, + expected_fragments=( + "Checked relation usage: 3 of 4 in use. (", + "\nIn use\n", + ".old_orders queried ", + " by {reader} (1 query in 7 days); frozen, still in use\n", + ".old_orders_v queried ", + ".old_orders_v2 queried ", + " by {reader} (1 query in 7 days)\n", + ), + unexpected_fragments=("Kept (in use)", "old_customers queried", "usage not checked"), + expected_remaining=(), + ), + ClickHouseJanitorUsageE2ETestCase( + description="--keep-in-use keeps the read table and both views of the chain", + janitor_command=(*APPROVED_JANITOR, "--keep-in-use"), + expected_fragments=( + "Checked relation usage: 3 of 4 in use. (", + " kept (in use) 3\n", + "\nKept (in use)\n", + ".old_orders_v2 queried ", + ), + unexpected_fragments=("\nIn use\n", "old_orders_v ->", "usage not checked"), + expected_remaining=("old_orders", "old_orders_v", "old_orders_v2"), + ), + ), + ids=lambda case: case.description, +) +def test_given_query_log_when_running_janitor_then_view_chain_reads_are_in_use( + test_case: ClickHouseJanitorUsageE2ETestCase, + clickhouse_e2e_config: dict[str, object], + janitor_schema: str, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_janitor_usage_project( + tmp_path=tmp_path, config=clickhouse_e2e_config, schema=janitor_schema + ) + with limited_user( + config=clickhouse_e2e_config, database=janitor_schema, privileges="SELECT" + ) as reader: + read_as(config=reader, sql=f"SELECT sum(id) FROM {janitor_schema}.old_orders_v") + execute_clickhouse_sql(config=clickhouse_e2e_config, statements=("SYSTEM FLUSH LOGS",)) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + reader_name: str = str(reader["user"]) + assert result.returncode == 0, result.stdout + result.stderr + fragment: str + for fragment in test_case.expected_fragments: + assert fragment.format(reader=reader_name) in result.stdout, result.stdout + for fragment in test_case.unexpected_fragments: + assert fragment not in result.stdout, result.stdout + assert stale_relation_names(config=clickhouse_e2e_config, schema=janitor_schema) == ( + test_case.expected_remaining + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseJanitorUsageUncheckedE2ETestCase( + description="a janitor role that may not read system.query_log is not checked", + connection_settings="", + grants=(), + expected_fragments=( + "Relation usage not checked: cannot read system.query_log (", + " usage not checked: cannot read system.query_log (needs query logging on " + "and SELECT on system.query_log)\n", + ), + ), + ClickHouseJanitorUsageUncheckedE2ETestCase( + description="query logging turned off for the connection is not checked", + connection_settings="\n[connection.settings]\nlog_queries = 0\n", + grants=("GRANT SELECT ON system.query_log TO {user}",), + expected_fragments=( + " usage not checked: query logging is off for this connection (log_queries = 0)\n", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_query_log_when_running_janitor_then_usage_is_not_checked( + test_case: ClickHouseJanitorUsageUncheckedE2ETestCase, + clickhouse_e2e_config: dict[str, object], + janitor_schema: str, + tmp_path: Path, +) -> None: + with ExitStack() as stack: + janitor: dict[str, object] = stack.enter_context( + limited_user(config=clickhouse_e2e_config, database=janitor_schema, privileges="ALL") + ) + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=( + f"CREATE DATABASE {janitor_schema}", + *(grant.format(user=janitor["user"]) for grant in test_case.grants), + ), + ) + project_dir: Path = prepare_janitor_usage_project( + tmp_path=tmp_path, + config=janitor, + schema=janitor_schema, + connection_settings=test_case.connection_settings, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=(*APPROVED_JANITOR, "--keep-in-use"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + fragment: str + for fragment in test_case.expected_fragments: + assert fragment in result.stdout, result.stdout + assert "nothing kept in use: no usage could be checked" in result.stdout + assert stale_relation_names(config=clickhouse_e2e_config, schema=janitor_schema) == () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 7c9005104e..5ba61a7da0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -288,3 +288,60 @@ class JanitorJsonFailedRunE2ETestCase: expected_status: str expected_error_fragment: str expected_failed_items: int + + +@dataclass(frozen=True) +class JanitorUsageUncheckedE2ETestCase: + """Test case for janitor runs on an adapter with no usage source.""" + + description: str + janitor_command: tuple[str, ...] + expected_ordered_lines: tuple[str, ...] + expected_archived: int + + +@dataclass(frozen=True) +class JanitorUsageE2ETestCase: + """Test case for janitor runs whose adapter reports which candidates are read.""" + + description: str + janitor_config: str + janitor_command: tuple[str, ...] + reads: dict[str, tuple[tuple[str, ...], int]] + expected_ordered_lines: tuple[str, ...] + unexpected_fragments: tuple[str, ...] + expected_remaining: tuple[str, ...] + expected_archived: int + via: dict[str, tuple[str, ...]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class JanitorUsagePromptE2ETestCase: + """Test case for the confirmation prompt's usage line.""" + + description: str + janitor_command: tuple[str, ...] + reads: dict[str, tuple[tuple[str, ...], int]] + unavailable: str | None + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorUsageJsonE2ETestCase: + """Test case for the usage fields of `sqb janitor --json`.""" + + description: str + janitor_command: tuple[str, ...] + expected_usage_check: dict[str, object] + expected_archive_in_use: tuple[tuple[str, object], ...] + expected_kept_in_use: tuple[tuple[str, object], ...] + expected_archived: int + + +@dataclass(frozen=True) +class JanitorUsageFailureE2ETestCase: + """Test case for janitor runs whose usage source fails outright.""" + + description: str + failure: str + expected_ordered_lines: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index ef611e854e..ff77f40f60 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -698,3 +698,193 @@ def janitor_end_state(*, db_path: Path, schemas: tuple[str, ...]) -> tuple[tuple ) ) return (*relations, *events) + + +USAGE_JANITOR_ADAPTER_NAME: str = "usage_janitor_duckdb" +USAGE_JANITOR_SETTINGS_FILENAME: str = "janitor_usage_settings.json" +USAGE_JANITOR_CALLS_FILENAME: str = "janitor_usage_calls.json" +USAGE_JANITOR_ADAPTER_SOURCE: str = """ +import json +from dataclasses import replace +from datetime import datetime +from pathlib import Path +from typing import Any + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsage, RelationUsageReport +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter + +_ROOT = Path(__file__).resolve().parent.parent + + +def _settings() -> dict[str, Any]: + return json.loads((_ROOT / "janitor_usage_settings.json").read_text(encoding="utf-8")) + + +class UsageJanitorDuckDbAdapter(DuckDbAdapter): + adapter_name = "usage_janitor_duckdb" + + def supports_relation_age_metadata(self) -> bool: + return True + + def list_relations( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[RelationInfo, ...]: + changed_at: datetime = datetime.fromisoformat(_settings()["changed_at"]) + return tuple( + replace(relation, created_at=changed_at) + if relation.name.startswith("old_") + else relation + for relation in super().list_relations( + connection=connection, database=database, schemas=schemas, names=names + ) + ) + + def relation_usage( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + window_seconds: int, + ) -> RelationUsageReport: + (_ROOT / "janitor_usage_calls.json").write_text( + json.dumps( + { + "relations": sorted(relation.name for relation in relations), + "window_seconds": window_seconds, + } + ), + encoding="utf-8", + ) + settings: dict[str, Any] = _settings() + if settings["failure"] is not None: + raise RuntimeError(settings["failure"]) + if settings["unavailable"] is not None: + return RelationUsageReport(checked=False, reason=settings["unavailable"]) + reads: dict[str, dict[str, Any]] = settings["reads"] + return RelationUsageReport( + checked=True, + usages=tuple( + RelationUsage( + database=relation.database, + schema=relation.schema, + name=relation.name, + last_read_at=datetime.fromisoformat(reads[relation.name]["last_read_at"]), + readers=tuple(reads[relation.name]["readers"]), + query_count=int(reads[relation.name]["query_count"]), + direct=reads[relation.name]["via"] is None, + via=tuple(reads[relation.name]["via"] or ()), + ) + for relation in relations + if relation.name in reads + ), + ) +""" + + +def prepare_usage_janitor_project( + *, + tmp_path: Path, + project_name: str, + stale_tables: int, + stale_views: tuple[str, ...] = (), + janitor_config: str = "", +) -> Path: + """A built DuckDB project with stale tables and views, an expired archive, and a usage adapter. + + Every stale `old_*` relation last changed on 2026-09-01, so it is older than the retention. + """ + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name=project_name, + repo_files={ + "sqlbuild_project.toml": ( + f'name = "{project_name}"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + f'database = "{(tmp_path / project_name / "janitor.duckdb").as_posix()}"\n\n' + "[janitor]\n" + "enabled = true\n" + "retention_days = 14\n" + "archive_retention_days = 14\n" + "delete_tracked_only = false\n" + f"{dedent(janitor_config).strip()}\n\n" + "[defaults]\n" + 'materialized = "table"\n' + ), + "models/orders.sql": "MODEL (description 'Test model.');\n\nSELECT 1 AS order_id\n", + f"adapters/{USAGE_JANITOR_ADAPTER_NAME}.py": USAGE_JANITOR_ADAPTER_SOURCE, + }, + ) + _build(project_dir) + tables: str = "; ".join( + f"CREATE TABLE main.old_orders_{index:03d} AS SELECT {index} AS id" + for index in range(stale_tables) + ) + views: str = "".join(f"; CREATE VIEW main.{name} AS SELECT 1 AS id" for name in stale_views) + execute_duckdb( + db_path=project_dir / "janitor.duckdb", + sql=f'{tables}{views}; CREATE TABLE main."{EXPIRED_ARCHIVE_NAME}" AS SELECT 1 AS id', + ) + write_usage_settings(project_dir=project_dir) + return project_dir + + +def write_usage_settings( + *, + project_dir: Path, + reads: Mapping[str, tuple[tuple[str, ...], int]] = MappingProxyType({}), + unavailable: str | None = None, + via: Mapping[str, tuple[str, ...]] = MappingProxyType({}), + failure: str | None = None, +) -> None: + """Run janitor on the usage adapter: who read which relation, or why usage is unavailable. + + A relation in `via` was read only through the named objects; `failure` makes the read fail. + """ + + (project_dir / "sqlbuild_local.toml").write_text( + f'adapter = "{USAGE_JANITOR_ADAPTER_NAME}"\n', encoding="utf-8" + ) + (project_dir / USAGE_JANITOR_SETTINGS_FILENAME).write_text( + json.dumps( + { + "changed_at": "2026-09-01T08:00:00+00:00", + "reads": { + name: { + "readers": list(readers), + "query_count": count, + "last_read_at": "2026-10-21T09:30:00+00:00", + "via": via.get(name), + } + for name, (readers, count) in reads.items() + }, + "unavailable": unavailable, + "failure": failure, + } + ), + encoding="utf-8", + ) + + +def read_usage_calls(*, project_dir: Path) -> dict[str, object]: + """The relations and window of the usage adapter's last `relation_usage` call.""" + + return json.loads((project_dir / USAGE_JANITOR_CALLS_FILENAME).read_text(encoding="utf-8")) + + +def count_stale_archives(*, db_path: Path) -> int: + """How many archives in `main` hold a stale `old_orders` relation.""" + + return sum("__old_orders" in name for name in list_archive_names(db_path=db_path)) + + +def in_use_entries(candidates: list[dict[str, object]]) -> tuple[tuple[object, object], ...]: + """Each `--json` candidate's name and `in_use`.""" + + return tuple((candidate["name"], candidate["in_use"]) for candidate in candidates) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py new file mode 100644 index 0000000000..a2935276c8 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py @@ -0,0 +1,408 @@ +"""E2E tests for the janitor usage check: which archive candidates are in use, and keeping them.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorUsageE2ETestCase, + JanitorUsageFailureE2ETestCase, + JanitorUsageJsonE2ETestCase, + JanitorUsagePromptE2ETestCase, + JanitorUsageUncheckedE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( + assert_in_order, + count_stale_archives, + in_use_entries, + list_archive_names, + prepare_probe_janitor_project, + prepare_usage_janitor_project, + read_usage_calls, + write_usage_settings, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + run_sqb, + run_sqb_with_pty, + table_exists, +) + +APPROVED_JANITOR: tuple[str, ...] = ("--no-color", "janitor", "--auto-approve") +FIVE_READERS: tuple[str, ...] = ("svc_reporting", "alice", "bob", "carol", "dave") + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageUncheckedE2ETestCase( + description="an adapter without a usage source says so once and archives as before", + janitor_command=APPROVED_JANITOR, + expected_ordered_lines=( + "Inspected warehouse state. (", + "Checking relation usage...", + "Relation usage not checked: unsupported adapter. (", + "Janitor preview default", + " relations to archive 3\n", + "\n usage not checked: unsupported adapter\n", + "Relations to archive", + "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + ), + expected_archived=3, + ), + JanitorUsageUncheckedE2ETestCase( + description="--keep-in-use without a usage source warns and archives everything", + janitor_command=(*APPROVED_JANITOR, "--keep-in-use"), + expected_ordered_lines=( + " usage not checked: unsupported adapter\n" + " nothing kept in use: no usage could be checked, so relations in use are " + "archived too\n", + "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + ), + expected_archived=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_duckdb_when_running_janitor_then_usage_is_reported_not_checked( + test_case: JanitorUsageUncheckedE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_unchecked", stale_tables={"main": 3} + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert_in_order(output=result.stdout, lines=test_case.expected_ordered_lines) + assert result.stdout.count("usage not checked: unsupported adapter\n") == 1 + assert "In use" not in result.stdout + assert count_stale_archives(db_path=project_dir / "janitor.duckdb") == ( + test_case.expected_archived + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageJsonE2ETestCase( + description="duckdb reports usage as not checked and every candidate's in_use null", + janitor_command=("janitor", "--auto-approve", "--json"), + expected_usage_check={ + "status": "not_checked", + "reason": "unsupported adapter", + "window": "7d", + "keep_in_use": False, + "notices": [], + }, + expected_archive_in_use=(("main.old_orders_000", None), ("main.old_orders_001", None)), + expected_kept_in_use=(), + expected_archived=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( + test_case: JanitorUsageJsonE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_json", stale_tables={"main": 2} + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["usage_check"] == test_case.expected_usage_check + assert in_use_entries(payload["plan"]["relations_to_archive"]) == ( + test_case.expected_archive_in_use + ) + assert in_use_entries(payload["plan"]["kept_in_use"]) == test_case.expected_kept_in_use + assert payload["result"]["counts"]["archived"] == test_case.expected_archived + assert "usage not checked: unsupported adapter" in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageE2ETestCase( + description="in-use relations are listed with readers and archived by default", + janitor_config="", + janitor_command=APPROVED_JANITOR, + reads={ + "old_orders_000": (FIVE_READERS, 40), + "old_orders_view": (("svc_reporting",), 14), + }, + expected_ordered_lines=( + "Checking relation usage...", + "Checked relation usage: 2 of 4 in use. (", + " relations to archive 4\n in use 2\n", + "Relations to archive", + "\nIn use\n", + " main.old_orders_000 queried 2026-10-21 by svc_reporting, alice, bob +2 more " + "(40 queries in 7 days); frozen since 2026-09-01, still in use\n", + " main.old_orders_view queried 2026-10-21 by svc_reporting " + "(14 queries in 7 days)\n", + "\u2713 Janitor completed archived 4, deleted 1, pruned 1", + ), + unexpected_fragments=( + "Kept (in use)", + "old_orders_view queried 2026-10-21 by svc_reporting (14 queries in 7 days);", + "old_orders_001 queried", + "usage not checked", + ), + expected_remaining=(), + expected_archived=4, + ), + JanitorUsageE2ETestCase( + description="--keep-in-use keeps in-use relations out of the archive and the counts", + janitor_config="", + janitor_command=(*APPROVED_JANITOR, "--keep-in-use"), + reads={ + "old_orders_000": (FIVE_READERS, 40), + "old_orders_view": (("svc_reporting",), 14), + }, + expected_ordered_lines=( + "Checked relation usage: 2 of 4 in use. (", + " relations to archive 2\n kept (in use) 2\n", + "\nKept (in use)\n main.old_orders_000 queried 2026-10-21", + "\u2713 Janitor completed archived 2, deleted 1, pruned 1", + ), + unexpected_fragments=( + "\nIn use\n", + "main.old_orders_000 ->", + "main.old_orders_view ->", + "archived main.old_orders_000", + ), + expected_remaining=("old_orders_000", "old_orders_view"), + expected_archived=2, + ), + JanitorUsageE2ETestCase( + description="a relation read only through views names them", + janitor_config="", + janitor_command=APPROVED_JANITOR, + reads={"old_orders_002": (("alice",), 3)}, + via={"old_orders_002": ("main.orders_report", "main.orders_summary")}, + expected_ordered_lines=( + "Checked relation usage: 1 of 4 in use. (", + " main.old_orders_002 queried 2026-10-21 by alice via main.orders_report, " + "main.orders_summary (3 queries in 7 days); frozen since 2026-09-01, " + "still in use\n", + ), + unexpected_fragments=("Kept (in use)",), + expected_remaining=(), + expected_archived=4, + ), + JanitorUsageE2ETestCase( + description="keep_in_use in the project and in_use_window reach the usage source", + janitor_config='keep_in_use = true\nin_use_window = "36h"\n', + janitor_command=APPROVED_JANITOR, + reads={"old_orders_001": (("alice",), 1)}, + expected_ordered_lines=( + " kept (in use) 1\n", + " main.old_orders_001 queried 2026-10-21 by alice (1 query in 36 hours); " + "frozen since 2026-09-01, still in use\n", + "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + ), + unexpected_fragments=("main.old_orders_001 ->",), + expected_remaining=("old_orders_001",), + expected_archived=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_usage_source_when_running_janitor_then_in_use_relations_are_shown_or_kept( + test_case: JanitorUsageE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, + project_name="janitor_usage", + stale_tables=3, + stale_views=("old_orders_view",), + janitor_config=test_case.janitor_config, + ) + write_usage_settings(project_dir=project_dir, reads=test_case.reads, via=test_case.via) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + db_path: Path = project_dir / "janitor.duckdb" + assert result.returncode == 0, result.stdout + result.stderr + assert_in_order(output=result.stdout, lines=test_case.expected_ordered_lines) + fragment: str + for fragment in test_case.unexpected_fragments: + assert fragment not in result.stdout + name: str + for name in test_case.expected_remaining: + assert table_exists(db_path=db_path, table_name=name) + assert count_stale_archives(db_path=db_path) == test_case.expected_archived + calls: dict[str, object] = read_usage_calls(project_dir=project_dir) + assert calls["relations"] == [ + "old_orders_000", + "old_orders_001", + "old_orders_002", + "old_orders_view", + ] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageJsonE2ETestCase( + description="kept and archived candidates carry their reads, or false when unread", + janitor_command=("janitor", "--auto-approve", "--json", "--keep-in-use"), + expected_usage_check={ + "status": "checked", + "reason": None, + "window": "7d", + "keep_in_use": True, + "notices": [], + }, + expected_archive_in_use=(("main.old_orders_001", False),), + expected_kept_in_use=( + ( + "main.old_orders_000", + { + "last_read": "2026-10-21T09:30:00Z", + "readers": list(FIVE_READERS), + "query_count": 40, + "via": None, + }, + ), + ( + "main.old_orders_002", + { + "last_read": "2026-10-21T09:30:00Z", + "readers": ["alice"], + "query_count": 2, + "via": ["main.orders_report"], + }, + ), + ), + expected_archived=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_usage_source_when_janitor_prints_json_then_candidates_carry_in_use( + test_case: JanitorUsageJsonE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_json_shape", stale_tables=3 + ) + write_usage_settings( + project_dir=project_dir, + reads={"old_orders_000": (FIVE_READERS, 40), "old_orders_002": (("alice",), 2)}, + via={"old_orders_002": ("main.orders_report",)}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["usage_check"] == test_case.expected_usage_check + assert in_use_entries(payload["plan"]["relations_to_archive"]) == ( + test_case.expected_archive_in_use + ) + assert in_use_entries(payload["plan"]["kept_in_use"]) == test_case.expected_kept_in_use + assert all( + set(entry) == {"name", "age_days", "in_use"} for entry in payload["plan"]["kept_in_use"] + ) + assert payload["result"]["counts"]["archived"] == test_case.expected_archived + assert read_usage_calls(project_dir=project_dir)["window_seconds"] == 7 * 86_400 + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsagePromptE2ETestCase( + description="the prompt states how many relations to archive are in use", + janitor_command=("--no-color", "janitor"), + reads={"old_orders_000": (("alice",), 3)}, + unavailable=None, + expected_fragments=( + "Janitor will archive 3 and delete 2 items from default.\r\n" + "3 relations will be archived; 1 of them is in use.\r\n", + "Janitor cancelled.", + ), + ), + JanitorUsagePromptE2ETestCase( + description="the prompt repeats why usage was not checked", + janitor_command=("--no-color", "janitor"), + reads={}, + unavailable="cannot read access history", + expected_fragments=( + "\r\nUsage not checked: cannot read access history\r\nRetention: 14 days", + "Janitor cancelled.", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_usage_check_when_prompting_then_prompt_states_usage( + test_case: JanitorUsagePromptE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_prompt", stale_tables=3 + ) + write_usage_settings( + project_dir=project_dir, reads=test_case.reads, unavailable=test_case.unavailable + ) + + result: subprocess.CompletedProcess[str] = run_sqb_with_pty( + command=test_case.janitor_command, project_dir=project_dir, input_text="no\n" + ) + + assert result.returncode == 1, result.stdout + fragment: str + for fragment in test_case.expected_fragments: + assert fragment in result.stdout, result.stdout + assert len(list_archive_names(db_path=project_dir / "janitor.duckdb")) == 1 + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUsageFailureE2ETestCase( + description="a failed usage read prints a failure line and archives nothing", + failure="usage warehouse went away", + expected_ordered_lines=( + "Checking relation usage...", + "Relation usage check failed. (", + "usage warehouse went away", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_usage_source_when_running_janitor_then_failure_line_is_printed( + test_case: JanitorUsageFailureE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, project_name="janitor_usage_failure", stale_tables=1 + ) + write_usage_settings(project_dir=project_dir, failure=test_case.failure) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=APPROVED_JANITOR, project_dir=project_dir + ) + + assert result.returncode != 0, result.stdout + assert_in_order(output=result.stdout + result.stderr, lines=test_case.expected_ordered_lines) + assert "Janitor preview" not in result.stdout + assert len(list_archive_names(db_path=project_dir / "janitor.duckdb")) == 1 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index a8a6b4ddf9..654eaa77cc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -315,3 +315,24 @@ class SelectedGraphRuleCase: expected_compile_findings: tuple[tuple[str, str], ...] expected_plan_findings: tuple[tuple[str, str], ...] expected_plan_exit: int + + +@dataclass(frozen=True) +class CustomRuleTimingCase: + """Custom Rules compiled cold, warm, with `--no-cache`, and warm again.""" + + description: str + selected_rules: tuple[str, ...] + files: tuple[tuple[str, str], ...] + expected_subjects: int + + +@dataclass(frozen=True) +class ModuleStateAttributionCase: + """Rules whose module state changes persist or are undone within one model.""" + + description: str + selected_rules: tuple[str, ...] + files: tuple[tuple[str, str], ...] + edited_model: str + expected_misses_after_edit: dict[str, int] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index f8aaf7541f..f9cf7dd094 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -114,6 +114,7 @@ def run_compile_cli( environment: tuple[tuple[str, str], ...] = (), working_directory: Path | None = None, preexec: Callable[[], None] | None = None, + arguments: tuple[str, ...] = (), ) -> subprocess.CompletedProcess[str]: """Run the real `sqb compile --json` command from an optional invocation directory.""" @@ -125,6 +126,7 @@ def run_compile_cli( str(project_dir), "compile", "--json", + *arguments, ], capture_output=True, text=True, @@ -206,6 +208,44 @@ def rule_cache_hits(result: subprocess.CompletedProcess[str]) -> int: return int(json.loads(result.stdout)["compile_timings"]["rule_cache_hits"]) +def custom_rule_timings( + result: subprocess.CompletedProcess[str], codes: tuple[str, ...] +) -> dict[str, tuple[int, int, int, int]]: + """Map each custom Rule code to its invocations, CPU ms, cache hits and cache misses.""" + + timings: dict[str, int] = json.loads(result.stdout)["compile_timings"] + return { + code: ( + timings[f"custom_rules.{code}.invocations"], + timings[f"custom_rules.{code}.cpu_ms"], + timings[f"custom_rules.{code}.cache_hits"], + timings[f"custom_rules.{code}.cache_misses"], + ) + for code in codes + } + + +def custom_rule_cache_counts( + result: subprocess.CompletedProcess[str], codes: tuple[str, ...] +) -> dict[str, tuple[int, int, int]]: + """Map each custom Rule code to its invocations, cache hits and cache misses.""" + + return { + code: (invocations, hits, misses) + for code, (invocations, _, hits, misses) in custom_rule_timings(result, codes).items() + } + + +def named_custom_rule_source( + *, code: str, body: str, function: str, header: str = CUSTOM_RULE_HEADER +) -> str: + """Return one model-subject custom Rule whose check function has the given name.""" + + return custom_rule_source(code=code, header=header, body=body).replace( + "def check(", f"def {function}(", 1 + ) + + def string_set_order(*, names: tuple[str, ...], hash_seed: str) -> str: """Return the comma-joined iteration order of a string set under one hash seed.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py new file mode 100644 index 0000000000..b47ec59e4b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rule_timings.py @@ -0,0 +1,176 @@ +"""Real CLI coverage of per-Rule custom-rule timings, `--no-cache` and module-state attribution.""" + +import json +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( + CustomRuleTimingCase, + ModuleStateAttributionCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + custom_rule_cache_counts, + custom_rule_timings, + named_custom_rule_source, + rule_cache_hits, + run_compile_cli, + split_model_files, + write_custom_rule_project, +) + +_FULL_COMPILE: tuple[tuple[str, str], ...] = ( + ("SQLBUILD_DISABLE_COMPILE_REUSE", "1"), + ("SQLBUILD_DISABLE_DELTA_COMPILE", "1"), +) +_MODEL_COUNT: int = 4 +_RULE_HEADER: str = "from sqlbuild.rules import Finding, Model, RuleContext, rule\n" +_PURE_BODY: str = " return [] if model.name else []\n" + + +@pytest.mark.parametrize( + "test_case", + [ + CustomRuleTimingCase( + description="two model rules in separate files", + selected_rules=("XSQBRTIME001", "XSQBRTIME002"), + files=( + ( + "rules/names.py", + named_custom_rule_source( + code="XSQBRTIME001", body=_PURE_BODY, function="names" + ), + ), + ( + "rules/paths.py", + named_custom_rule_source( + code="XSQBRTIME002", + body=" return [] if model.path.suffix == '.sql' else []\n", + function="paths", + ), + ), + *split_model_files(_MODEL_COUNT), + ), + expected_subjects=_MODEL_COUNT + 1, + ) + ], + ids=lambda case: case.description, +) +def test_given_custom_rules_when_compiling_then_timings_and_no_cache_report_each_rule( + tmp_path: Path, test_case: CustomRuleTimingCase +) -> None: + write_custom_rule_project( + project_dir=tmp_path, selected_rules=test_case.selected_rules, files=test_case.files + ) + codes: tuple[str, ...] = test_case.selected_rules + evaluated: dict[str, tuple[int, int, int]] = dict.fromkeys( + codes, (test_case.expected_subjects, 0, test_case.expected_subjects) + ) + reused: dict[str, tuple[int, int, int]] = dict.fromkeys( + codes, (0, test_case.expected_subjects, 0) + ) + + cold: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) + warm: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) + uncached: subprocess.CompletedProcess[str] = run_compile_cli( + tmp_path, environment=_FULL_COMPILE, arguments=("--no-cache",) + ) + rewarmed: subprocess.CompletedProcess[str] = run_compile_cli( + tmp_path, environment=_FULL_COMPILE + ) + + assert [result.returncode for result in (cold, warm, uncached, rewarmed)] == [0, 0, 0, 0], ( + cold.stderr + uncached.stderr + ) + assert custom_rule_cache_counts(cold, codes) == evaluated + assert custom_rule_cache_counts(warm, codes) == reused + assert custom_rule_cache_counts(uncached, codes) == evaluated + assert rule_cache_hits(uncached) == 0 + assert custom_rule_cache_counts(rewarmed, codes) == reused + cold_timings: dict[str, int] = json.loads(cold.stdout)["compile_timings"] + assert cold_timings["custom_rules_invocations"] == test_case.expected_subjects * len(codes) + assert ( + min(cold_timings["custom_rules_monitor_cpu_ms"], cold_timings["custom_rules_guard_cpu_ms"]) + >= 0 + ) + assert min(cpu_ms for _, cpu_ms, _, _ in custom_rule_timings(cold, codes).values()) >= 0 + assert json.loads(warm.stdout)["compile_timings"]["custom_rules_invocations"] == 0 + + +@pytest.mark.parametrize( + "test_case", + [ + ModuleStateAttributionCase( + description="a persistent mutation is attributed, one undone within a model is not", + selected_rules=("XSQBRSTATE001", "XSQBRSTATE002", "XSQBRSTATE003", "XSQBRSTATE004"), + files=( + ( + "rules/persistent.py", + named_custom_rule_source( + code="XSQBRSTATE001", + header=_RULE_HEADER + "SEEN: list[str] = []\n", + body=" SEEN.append(model.name)\n return []\n", + function="remember", + ), + ), + ( + "rules/transient.py", + named_custom_rule_source( + code="XSQBRSTATE002", + header=_RULE_HEADER + "SCRATCH: list[str] = []\n", + body=" SCRATCH.append(model.name)\n return []\n", + function="stage", + ) + + "\n\n" + + named_custom_rule_source( + code="XSQBRSTATE003", + header="", + body=" SCRATCH.clear()\n return []\n", + function="restore", + ), + ), + ( + "rules/clean.py", + named_custom_rule_source( + code="XSQBRSTATE004", body=_PURE_BODY, function="names" + ), + ), + *split_model_files(_MODEL_COUNT), + ), + edited_model="models/orders_001.sql", + expected_misses_after_edit={ + "XSQBRSTATE001": _MODEL_COUNT + 1, + "XSQBRSTATE002": 0, + "XSQBRSTATE003": 0, + "XSQBRSTATE004": 0, + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_rule_module_state_when_recompiling_an_edit_then_only_persistent_state_is_untracked( + tmp_path: Path, test_case: ModuleStateAttributionCase +) -> None: + write_custom_rule_project( + project_dir=tmp_path, selected_rules=test_case.selected_rules, files=test_case.files + ) + + cold: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) + (tmp_path / test_case.edited_model).write_text( + 'MODEL (description "Orders");\nSELECT 2 AS order_id\n', encoding="utf-8" + ) + edited: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path, environment=_FULL_COMPILE) + + assert cold.returncode == 0, cold.stdout + cold.stderr + assert edited.returncode == 0, edited.stdout + edited.stderr + assert { + code: misses + for code, (_, _, misses) in custom_rule_cache_counts( + edited, test_case.selected_rules + ).items() + } == test_case.expected_misses_after_edit + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py index 72112b7c1e..6e71507791 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py @@ -57,6 +57,30 @@ "MODEL (description 'Finance view of orders.');\n" 'SELECT o.order_id FROM __ref("sales__mart__orders") AS o\n', ) +_COUNTRY_GLOB: str = ( + "\n[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ domain = "sales", qualifier = "[a-z][a-z][a-z]", ' + 'reason = "Published once per country." }]\n' +) +_REGIONAL_PATHS: str = ( + "\n[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = ["models/sales/mart/regional/*"], ' + 'reason = "Published once per country." }]\n' +) +_COUNTRIES: tuple[tuple[str, str], ...] = tuple( + ( + f"models/sales/mart/regional/sales__mart__orders__{country}.sql", + f"MODEL (description 'Orders published for {country}.');\n" + 'SELECT o.order_id FROM __ref("sales__int_clean__orders") AS o\n', + ) + for country in ("deu", "fra", "ita") +) +_FINANCE_FROM_INTERMEDIATE: tuple[str, str] = ( + _FINANCE_PATH, + "MODEL (description 'Orders reshaped for finance.');\n" + 'SELECT o.order_id FROM __ref("sales__int_clean__orders") AS o\n', +) +_PER_COUNTRY: tuple[tuple[str, str], ...] = (_INTERMEDIATE, _ORDERS, _CUSTOMERS, *_COUNTRIES) _ORDERS_FINANCE: tuple[tuple[str, str], ...] = ( _INTERMEDIATE, _ORDERS, @@ -83,6 +107,37 @@ expected_findings=(("SQBRGRAPH103", _FINANCE_PATH),), expected_exit=1, ), + PublishedLayerCase( + description="a per-country family without an entry outnumbers the unqualified models", + files=_PER_COUNTRY, + rules_toml="", + expected_findings=( + ("SQBRPROJECT107", "models/sales/mart/sales__mart__customers.sql"), + ("SQBRPROJECT107", "models/sales/mart/sales__mart__orders.sql"), + ), + expected_exit=1, + ), + PublishedLayerCase( + description="one qualifier glob replaces a literal entry per country", + files=_PER_COUNTRY, + rules_toml=_COUNTRY_GLOB, + expected_findings=(), + expected_exit=0, + ), + PublishedLayerCase( + description="a paths entry leaves out the per-country folder", + files=_PER_COUNTRY, + rules_toml=_REGIONAL_PATHS, + expected_findings=(), + expected_exit=0, + ), + PublishedLayerCase( + description="a qualifier the glob does not match is still flagged", + files=(*_PER_COUNTRY, _FINANCE_FROM_INTERMEDIATE), + rules_toml=_COUNTRY_GLOB, + expected_findings=(("SQBRPROJECT107", _FINANCE_PATH),), + expected_exit=1, + ), PublishedLayerCase( description="shared logic in intermediate and a view of its own entity", files=(_INTERMEDIATE, _ORDERS, _CUSTOMERS, _ORDERS_VIEW, _FINANCE_FROM_PUBLISHED), diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 8eaebda0bf..598e87a79b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -186,6 +186,16 @@ class CustomHostSplitIntegrationTestCase: expected_code: str +@dataclass(frozen=True) +class ProjectRuleScratchStateTestCase: + """Project rules sharing module state that one fills, one reads and one empties.""" + + description: str + hosts: int + expected_findings: tuple[tuple[str, str], ...] + expected_launches: int + + @dataclass(frozen=True) class CustomHostFailureIntegrationTestCase: """A compile whose custom rule fails for some models while its hosts are split.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 3d401d105a..d118f061f7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -845,11 +845,14 @@ def write_project_files(*, project_dir: Path, files: dict[str, str]) -> None: def compile_findings( - *, project_dir: Path, capsys: pytest.CaptureFixture[str] + *, + project_dir: Path, + capsys: pytest.CaptureFixture[str], + flags: tuple[str, ...] = ("--no-cache",), ) -> tuple[tuple[str, str, int | None, int | None], ...]: - """Compile through the CLI with no cache and return each finding's code and location.""" + """Compile through the CLI, by default with no cache, and return each finding's location.""" - _ = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + _ = main(["--project-dir", str(project_dir), "compile", "--json", *flags]) payload: dict[str, object] = json.loads(capsys.readouterr().out) diagnostics: list[dict[str, Any]] = cast(list[dict[str, Any]], payload["diagnostics"]) return tuple( @@ -1143,3 +1146,46 @@ def rebuild_formatter(_project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> No """Pretend another SQLBuild build formats the project.""" rebuild_identity(monkeypatch=monkeypatch) + + +PROJECT_SCRATCH_RULES: str = """from sqlbuild.rules import Finding, Project, RuleContext, rule + +SCRATCH: list[str] = [] + + +@rule(code="XSQBRSCR001", message="Stage names", remediation="Keep rules pure.") +def stage(*, project: Project, ctx: RuleContext) -> list[Finding]: + del project + SCRATCH.extend(model.name for model in ctx.project.models) + return [] + + +@rule(code="XSQBRSCR002", message="Staged names are present", remediation="Keep rules pure.") +def read(*, project: Project, ctx: RuleContext) -> list[Finding]: + del project + return [ctx.finding(subject=ctx.project.models[0])] * int(bool(SCRATCH)) + + +@rule(code="XSQBRSCR003", message="Clear staged names", remediation="Keep rules pure.") +def clear(*, project: Project, ctx: RuleContext) -> list[Finding]: + del project + SCRATCH.clear() + return [] +""" + + +def write_project_scratch_rules(project_dir: Path) -> None: + """Write two order models and three project rules sharing one scratch list.""" + + write_project_files( + project_dir=project_dir, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["XSQBRSCR"]\n\n' + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" + ), + "models/orders.sql": 'MODEL (description "Orders");\nSELECT 1 AS order_id\n', + "models/customers.sql": ('MODEL (description "Customers");\nSELECT 1 AS customer_id\n'), + "rules/scratch.py": PROJECT_SCRATCH_RULES, + }, + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 20ed28a8e4..d2d435fb76 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -35,6 +35,7 @@ ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, NumericRangeDecisionIntegrationTestCase, + ProjectRuleScratchStateTestCase, RulePassIntegrationTestCase, RulesIntegrationTestCase, SetOperationStarRuleTestCase, @@ -50,6 +51,7 @@ observe_hosts, write_custom_host_project, write_project_files, + write_project_scratch_rules, ) _CACHE_RULES_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' @@ -1054,21 +1056,23 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ test_case: EarlyLintInputsTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_CACHE", "1") write_project_files(project_dir=tmp_path, files=EARLY_LINT_PROJECT_FILES) previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( - project_dir=tmp_path, capsys=capsys + project_dir=tmp_path, capsys=capsys, flags=() ) cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="sql.json") stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) stored["build_identity"] = "previous-build" cache_path.write_text(json.dumps(stored), encoding="utf-8") upgraded_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( - project_dir=tmp_path, capsys=capsys + project_dir=tmp_path, capsys=capsys, flags=() ) assert previous_findings == upgraded_findings == test_case.expected_findings - assert compile_findings(project_dir=tmp_path, capsys=capsys) == upgraded_findings + assert compile_findings(project_dir=tmp_path, capsys=capsys, flags=()) == (upgraded_findings) @pytest.mark.parametrize( @@ -1189,6 +1193,41 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: } <= cold +@pytest.mark.parametrize( + "test_case", + [ + ProjectRuleScratchStateTestCase( + description="one host runs the project rules in order", + hosts=1, + expected_findings=(("XSQBRSCR002", "models/customers.sql"),), + expected_launches=1, + ), + ProjectRuleScratchStateTestCase( + description="split project rules rerun in one host after the scratch list fills", + hosts=2, + expected_findings=(("XSQBRSCR002", "models/customers.sql"),), + expected_launches=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_rules_sharing_scratch_state_when_splitting_hosts_then_findings_match_one_host( + test_case: ProjectRuleScratchStateTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_project_scratch_rules(tmp_path) + hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=test_case.hosts) + + findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( + project_dir=tmp_path, capsys=capsys + ) + + assert tuple((code, path) for code, path, _, _ in findings) == test_case.expected_findings + assert len(hosts.launches()) == test_case.expected_launches + + @pytest.mark.parametrize( "test_case", [CustomHostSplitIntegrationTestCase("custom rules split across hosts", 9, 3, "XSQBRARCH001")], @@ -1231,10 +1270,11 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: ) def compile_payload() -> dict[str, Any]: - exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) assert exit_code == 1 return json.loads(capsys.readouterr().out) + monkeypatch.setenv("SQLBUILD_DISABLE_COMPILE_CACHE", "1") single_hosts: CustomHostObserver = observe_hosts(monkeypatch=monkeypatch, cores=1) single: dict[str, Any] = compile_payload() rules_bulk_cache_path(project_dir=tmp_path, file_name="custom-rules.json").parent.parent.rename( diff --git a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py index 1375a23885..97d0d5d5d4 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py +++ b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py @@ -15,6 +15,7 @@ { "adapter/contract/classes/base_adapter.py", "adapter/contract/classes/duckdb_backed_adapter.py", + "adapters/bigquery/_helpers/relation_usage.py", "adapters/bigquery/classes/bigquery_adapter.py", "adapters/databricks/classes/databricks_adapter.py", "adapters/postgres/classes/postgres_adapter.py", diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index d63ff26d46..68c49b8df1 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -4,6 +4,7 @@ from sqlbuild._native import InferredNullability from sqlbuild.adapter.contract.models import ( ColumnInfo, + RelationUsageReport, RowDiffResult, RowDiffSampleRow, SchemaDiffResult, @@ -219,3 +220,49 @@ class BigQueryJobInvalidationTestCase: description: str operation: str expected_reads: dict[str, int] + + +@dataclass(frozen=True) +class BigQueryRelationUsageTestCase: + description: str + relations: tuple[tuple[str | None, str, str], ...] + dataset_locations: dict[str, str] + rows: tuple[tuple[object, ...], ...] + expected_queries: tuple[tuple[str, str, tuple[str, ...]], ...] + expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + + +@dataclass(frozen=True) +class BigQueryRelationUsageQueryTestCase: + description: str + window_seconds: int + expected_params: dict[str, object] + expected_sql_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class BigQueryRelationUsageChunkTestCase: + description: str + relation_count: int + expected_query_names: tuple[int, ...] + + +@dataclass(frozen=True) +class BigQueryRelationUsageErrorTestCase: + description: str + error: Exception + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class BigQueryRelationUsageFailureTestCase: + description: str + error: Exception + expected_error_match: str + + +@dataclass(frozen=True) +class BigQueryCloneStatementTestCase: + description: str + statement: str + expected_clone: bool diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py index d5b7e30583..081949c2f2 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py @@ -3,12 +3,20 @@ from collections.abc import Callable from pathlib import Path from types import MappingProxyType -from typing import Any +from typing import Any, cast from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import ColumnInfo, RetentionRequest +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + RelationInfo, + RelationUsageReport, + RetentionRequest, +) from sqlbuild.adapter.contract.types import RetentionScope from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection + +USAGE_COLUMNS: tuple[str, ...] = ("object_name", "user_name", "query_count", "last_read_at") def build_retention_request(*, desired_days: int) -> RetentionRequest: @@ -254,3 +262,83 @@ def run_bigquery_job( ), } jobs[operation]() + + +class FakeBigQueryError(Exception): + """A Google API error with an HTTP status code.""" + + def __init__(self, message: str, code: int, reasons: tuple[str, ...] = ()) -> None: + super().__init__(message) + self.code: int = code + self.errors: list[dict[str, str]] = [{"reason": reason} for reason in reasons] + + +class Forbidden(Exception): + """Stands in for google.api_core.exceptions.Forbidden, matched by class name.""" + + +class FakeUsageDataset: + def __init__(self, location: str) -> None: + self.location: str = location + + +class FakeUsageClient: + """Answers dataset locations and JOBS reads, recording each read.""" + + def __init__( + self, + *, + project: str, + dataset_locations: dict[str, str], + rows: tuple[tuple[object, ...], ...] = (), + errors: tuple[Exception, ...] = (), + ) -> None: + self.project: str = project + self.dataset_locations: dict[str, str] = dataset_locations + self.rows: tuple[tuple[object, ...], ...] = rows + self.errors: tuple[Exception, ...] = errors + self.queries: list[tuple[str, str, dict[str, object]]] = [] + + def get_dataset(self, dataset_id: str) -> FakeUsageDataset: + return FakeUsageDataset(self.dataset_locations[dataset_id]) + + def query(self, sql: str, *, location: str, job_config: Any) -> FakeBigQueryJob: + params: dict[str, object] = { + parameter.name: getattr(parameter, "values", getattr(parameter, "value", None)) + for parameter in job_config.query_parameters + } + self.queries.append((sql, location, params)) + error: Exception + for error in self.errors: + raise error + names: frozenset[object] = frozenset(cast(list[object], params["object_names"])) + return FakeBigQueryJob( + FakeBigQueryRows( + columns=USAGE_COLUMNS, + rows=tuple(filter(lambda row: row[0] in names, self.rows)), + ) + ) + + +def usage_relation(database: str | None, schema: str, name: str) -> RelationInfo: + """A table relation for usage reads.""" + + return RelationInfo(database=database, schema=schema, name=name, relation_type="table") + + +def read_bigquery_usage( + *, client: FakeUsageClient, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """BigQuery's usage report for the relations from the fake client.""" + + return BigQueryAdapter().relation_usage( + connection=_BigQueryConnection(client=client, location=None), + relations=relations, + window_seconds=window_seconds, + ) + + +def queried_names(params: dict[str, object]) -> tuple[str, ...]: + """The table names one recorded JOBS read asked about.""" + + return tuple(cast(list[str], params["object_names"])) diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py new file mode 100644 index 0000000000..d28fdb546b --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_relation_usage.py @@ -0,0 +1,330 @@ +"""BigQuery relation usage reads each location's INFORMATION_SCHEMA.JOBS per chunk of tables.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.bigquery.constants import CLONE_STATEMENT_PATTERN +from tests.unit.src.sqlbuild.adapters.bigquery._test_types import ( + BigQueryCloneStatementTestCase, + BigQueryRelationUsageChunkTestCase, + BigQueryRelationUsageErrorTestCase, + BigQueryRelationUsageFailureTestCase, + BigQueryRelationUsageQueryTestCase, + BigQueryRelationUsageTestCase, +) +from tests.unit.src.sqlbuild.adapters.bigquery.helpers import ( + FakeBigQueryError, + FakeUsageClient, + Forbidden, + queried_names, + read_bigquery_usage, + usage_relation, +) + +WINDOW_SECONDS: int = 7 * 86_400 +JOBS_UNAVAILABLE_REASON: str = ( + "cannot read INFORMATION_SCHEMA.JOBS " + "(needs bigquery.jobs.listAll, for example the BigQuery Resource Viewer role)" +) +READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) +EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) +ORDERS_DAILY: tuple[RelationInfo, ...] = (usage_relation(None, "mart", "orders_daily"),) +LOCATIONS: dict[str, str] = {"shop-analytics.mart": "EU", "shop-archive.mart": "us-east1"} + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageTestCase( + description="reads per user add up, readers rank by jobs, the latest read wins", + relations=((None, "mart", "orders_daily"), (None, "mart", "customers")), + dataset_locations=LOCATIONS, + rows=( + ("shop-analytics.mart.orders_daily", "svc-reporting@example.com", 10, READ_AT), + ("shop-analytics.mart.orders_daily", "alice@example.com", 4, EARLIER_READ_AT), + ("shop-analytics.mart.orders_daily", None, 1, EARLIER_READ_AT), + ("shop-analytics.mart.products", "bob@example.com", 3, READ_AT), + ), + expected_queries=( + ( + "`shop-analytics.region-eu`.INFORMATION_SCHEMA.JOBS", + "EU", + ("shop-analytics.mart.customers", "shop-analytics.mart.orders_daily"), + ), + ), + expected_usages=( + ( + "orders_daily", + ("svc-reporting@example.com", "alice@example.com", "unknown user"), + 15, + READ_AT, + ), + ), + ), + BigQueryRelationUsageTestCase( + description="each project and dataset location gets its own read; case is kept", + relations=(("shop-archive", "mart", "Orders_2025"), (None, "mart", "orders_daily")), + dataset_locations=LOCATIONS, + rows=(("shop-archive.mart.Orders_2025", "alice@example.com", 2, READ_AT),), + expected_queries=( + ( + "`shop-analytics.region-eu`.INFORMATION_SCHEMA.JOBS", + "EU", + ("shop-analytics.mart.orders_daily",), + ), + ( + "`shop-archive.region-us-east1`.INFORMATION_SCHEMA.JOBS", + "us-east1", + ("shop-archive.mart.Orders_2025",), + ), + ), + expected_usages=(("Orders_2025", ("alice@example.com",), 2, READ_AT),), + ), + ), + ids=lambda case: case.description, +) +def test_given_jobs_history_when_reading_usage_then_reads_match_candidates( + test_case: BigQueryRelationUsageTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient( + project="shop-analytics", dataset_locations=test_case.dataset_locations, rows=test_case.rows + ) + + report: RelationUsageReport = read_bigquery_usage( + client=client, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple( + (sql.split(" FROM ")[1].split(" WHERE ")[0], location, queried_names(params)) + for sql, location, params in client.queries + ) == (test_case.expected_queries) + assert tuple( + (usage.name, usage.readers, usage.query_count, usage.last_read_at) + for usage in report.usages + ) == (test_case.expected_usages) + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageQueryTestCase( + description="successful query jobs count; scripts, snapshots and clones do not", + window_seconds=WINDOW_SECONDS, + expected_params={ + "window_seconds": WINDOW_SECONDS, + "clone_pattern": CLONE_STATEMENT_PATTERN, + "object_names": ["shop-analytics.mart.orders_daily"], + }, + expected_sql_fragments=( + "UNNEST(jobs.referenced_tables) AS referenced", + "UNNEST(jobs.referenced_logical_views) AS referenced", + "INTERVAL @window_seconds SECOND", + "job_type = 'QUERY'", + "error_result IS NULL", + "NOT IN ('SCRIPT', 'CREATE_SNAPSHOT_TABLE')", + "AND NOT (STARTS_WITH(COALESCE(statement_type, ''), 'CREATE_TABLE') " + "AND REGEXP_CONTAINS(query, @clone_pattern))", + "WHERE object_name IN UNNEST(@object_names)", + "COUNT(DISTINCT job_id) AS query_count", + "user_email AS user_name", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_jobs_query_when_built_then_parameters_are_bound_and_clones_excluded( + test_case: BigQueryRelationUsageQueryTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient(project="shop-analytics", dataset_locations=LOCATIONS) + + read_bigquery_usage( + client=client, relations=ORDERS_DAILY, window_seconds=test_case.window_seconds + ) + + sql: str + params: dict[str, object] + ((sql, _, params),) = client.queries + assert params == test_case.expected_params + assert all(fragment in sql for fragment in test_case.expected_sql_fragments) + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageChunkTestCase( + description="no candidates read nothing", + relation_count=0, + expected_query_names=(), + ), + BigQueryRelationUsageChunkTestCase( + description="a large candidate set is split into arrays of at most 1,000 names", + relation_count=2_001, + expected_query_names=(1_000, 1_000, 1), + ), + ), + ids=lambda case: case.description, +) +def test_given_many_candidates_when_reading_usage_then_names_are_chunked( + test_case: BigQueryRelationUsageChunkTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient(project="shop-analytics", dataset_locations=LOCATIONS) + + report: RelationUsageReport = read_bigquery_usage( + client=client, + relations=tuple( + usage_relation(None, "mart", f"old_orders_{index:04d}") + for index in range(test_case.relation_count) + ), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple(len(queried_names(params)) for _, _, params in client.queries) == ( + test_case.expected_query_names + ) + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageErrorTestCase( + description="a missing jobs.listAll permission is not checked", + error=FakeBigQueryError( + "Access Denied: Table shop-analytics:region-eu.INFORMATION_SCHEMA.JOBS: User " + "does not have permission bigquery.jobs.listAll", + 403, + ), + expected_report=RelationUsageReport(checked=False, reason=JOBS_UNAVAILABLE_REASON), + ), + BigQueryRelationUsageErrorTestCase( + description="an unknown region is not checked", + error=FakeBigQueryError("Not found: Dataset shop-analytics:region-xx", 404), + expected_report=RelationUsageReport(checked=False, reason=JOBS_UNAVAILABLE_REASON), + ), + BigQueryRelationUsageErrorTestCase( + description="a Forbidden access denial is not checked", + error=Forbidden("403 Access Denied: Project shop-analytics"), + expected_report=RelationUsageReport(checked=False, reason=JOBS_UNAVAILABLE_REASON), + ), + BigQueryRelationUsageErrorTestCase( + description="an accessDenied reason is not checked", + error=FakeBigQueryError("403 GET jobs", 403, ("accessDenied",)), + expected_report=RelationUsageReport(checked=False, reason=JOBS_UNAVAILABLE_REASON), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_jobs_access_when_reading_usage_then_usage_is_not_checked( + test_case: BigQueryRelationUsageErrorTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient( + project="shop-analytics", dataset_locations=LOCATIONS, errors=(test_case.error,) + ) + + report: RelationUsageReport = read_bigquery_usage( + client=client, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + assert report == test_case.expected_report + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryRelationUsageFailureTestCase( + description="a quota failure is a real error", + error=FakeBigQueryError("Quota exceeded: too many concurrent queries", 400), + expected_error_match="Quota exceeded", + ), + BigQueryRelationUsageFailureTestCase( + description="a 403 rate limit is a real error", + error=FakeBigQueryError( + "403 Exceeded rate limits: too many table metadata read operations", + 403, + ("rateLimitExceeded",), + ), + expected_error_match="Exceeded rate limits", + ), + BigQueryRelationUsageFailureTestCase( + description="a 403 quota denial naming a permission is a real error", + error=FakeBigQueryError( + "403 Quota exceeded: Access Denied for the project's permission budget", + 403, + ("quotaExceeded",), + ), + expected_error_match="Quota exceeded", + ), + BigQueryRelationUsageFailureTestCase( + description="a 403 without a reason or an access denial is a real error", + error=FakeBigQueryError("403 Billing has not been enabled for this project", 403), + expected_error_match="Billing", + ), + ), + ids=lambda case: case.description, +) +def test_given_other_jobs_failure_when_reading_usage_then_error_propagates( + test_case: BigQueryRelationUsageFailureTestCase, +) -> None: + client: FakeUsageClient = FakeUsageClient( + project="shop-analytics", dataset_locations=LOCATIONS, errors=(test_case.error,) + ) + + with pytest.raises(FakeBigQueryError, match=test_case.expected_error_match): + read_bigquery_usage(client=client, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS) + + +@pytest.mark.parametrize( + "test_case", + ( + BigQueryCloneStatementTestCase( + description="a table clone", + statement="CREATE TABLE mart.orders_copy CLONE mart.orders", + expected_clone=True, + ), + BigQueryCloneStatementTestCase( + description="a replaced clone of a quoted name at a point in time", + statement="create or replace table `shop-analytics.mart.orders_copy` clone " + "`shop-analytics.mart.orders` FOR SYSTEM_TIME AS OF TIMESTAMP '2026-10-01'", + expected_clone=True, + ), + BigQueryCloneStatementTestCase( + description="a clone after a leading comment", + statement="# nightly copy\nCREATE TABLE IF NOT EXISTS mart.orders_copy CLONE mart.orders;", + expected_clone=True, + ), + BigQueryCloneStatementTestCase( + description="a CTAS selecting a column named clone", + statement="CREATE TABLE mart.orders_copy AS SELECT clone FROM mart.orders", + expected_clone=False, + ), + BigQueryCloneStatementTestCase( + description="a CTAS with clone inside a string", + statement="CREATE TABLE mart.notes AS SELECT 'orders CLONE copy' AS note", + expected_clone=False, + ), + BigQueryCloneStatementTestCase( + description="a CTAS whose comment mentions a clone", + statement="CREATE TABLE mart.c AS SELECT 1 AS n -- not CREATE TABLE a CLONE b", + expected_clone=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_statement_when_matching_clone_pattern_then_only_clones_match( + test_case: BigQueryCloneStatementTestCase, +) -> None: + matched: bool = re.search(CLONE_STATEMENT_PATTERN, test_case.statement) is not None + + assert matched == test_case.expected_clone + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py index 2f6d7bf201..64aeeaf1b4 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -1,4 +1,7 @@ from dataclasses import dataclass +from datetime import datetime + +from sqlbuild.adapter.contract.models import RelationUsageReport @dataclass(frozen=True) @@ -87,3 +90,48 @@ class KeyIdentifiersTestCase: description: str key_expressions: tuple[str, ...] expected_identifiers: frozenset[str] + + +@dataclass(frozen=True) +class ClickHouseRelationUsageTestCase: + description: str + relations: tuple[tuple[str, str], ...] + rows: tuple[tuple[object, ...], ...] + expected_names_sql: str + expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + + +@dataclass(frozen=True) +class ClickHouseRelationUsageQueryTestCase: + description: str + window_seconds: int + expected_sql_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ClickHouseRelationUsageChunkTestCase: + description: str + relation_count: int + expected_query_names: tuple[int, ...] + + +@dataclass(frozen=True) +class ClickHouseRelationUsageUncheckedTestCase: + description: str + log_queries: object + errors: tuple[Exception, ...] + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class ClickHouseRelationUsageFailureTestCase: + description: str + error: Exception + expected_error_match: str + + +@dataclass(frozen=True) +class ClickHouseRelationUsageReplicaTestCase: + description: str + replicas: int + expected_notices: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py b/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py new file mode 100644 index 0000000000..4fb15d4e75 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/helpers.py @@ -0,0 +1,78 @@ +"""Helpers for ClickHouse adapter unit tests.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.clickhouse._helpers.type_text import quote_string +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter + + +class FakeQueryLogError(Exception): + """A ClickHouse server error carrying its code in the message, as clickhouse-connect does.""" + + @classmethod + def raised(cls, message: str) -> tuple[Exception, ...]: + """The error as the only one a fake connection raises.""" + + return (cls(message),) + + +class FakeQueryLogCursor: + def __init__(self, rows: list[tuple[object, ...]]) -> None: + self.rows: list[tuple[object, ...]] = rows + + def fetchall(self) -> list[tuple[object, ...]]: + return self.rows + + +class FakeQueryLogConnection: + """Answers the log_queries setting and query_log reads, recording each query_log read.""" + + def __init__( + self, + *, + rows: tuple[tuple[object, ...], ...] = (), + log_queries: object = True, + errors: tuple[Exception, ...] = (), + replicas: int = 1, + ) -> None: + self.rows: tuple[tuple[object, ...], ...] = rows + self.log_queries: object = log_queries + self.errors: tuple[Exception, ...] = errors + self.replicas: int = replicas + self.queries: list[str] = [] + + def execute(self, sql: str) -> FakeQueryLogCursor: + reads: dict[tuple[bool, bool], Callable[[], FakeQueryLogCursor]] = { + (True, False): lambda: FakeQueryLogCursor([(self.log_queries,)]), + (False, True): lambda: FakeQueryLogCursor([(self.replicas,)]), + (False, False): lambda: self.query_log_rows(sql), + } + return reads[(sql == "SELECT getSetting('log_queries')", "system.clusters" in sql)]() + + def query_log_rows(self, sql: str) -> FakeQueryLogCursor: + self.queries.append(sql) + error: Exception + for error in self.errors: + raise error + return FakeQueryLogCursor( + list(filter(lambda row: quote_string(str(row[0])) in sql.split(" IN (")[-1], self.rows)) + ) + + +def usage_relation(schema: str, name: str) -> RelationInfo: + """A ClickHouse table, whose database is its schema.""" + + return RelationInfo(database=None, schema=schema, name=name, relation_type="BASE TABLE") + + +def read_clickhouse_usage( + *, connection: FakeQueryLogConnection, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """ClickHouse's usage report for the relations from the fake connection.""" + + return ClickHouseAdapter().relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py new file mode 100644 index 0000000000..859918014f --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_relation_usage.py @@ -0,0 +1,257 @@ +"""ClickHouse relation usage reads this node's system.query_log once per chunk of candidates.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.clickhouse.constants import ( + USAGE_LOGGING_OFF_REASON, + USAGE_UNAVAILABLE_REASON, +) +from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import ( + ClickHouseRelationUsageChunkTestCase, + ClickHouseRelationUsageFailureTestCase, + ClickHouseRelationUsageQueryTestCase, + ClickHouseRelationUsageReplicaTestCase, + ClickHouseRelationUsageTestCase, + ClickHouseRelationUsageUncheckedTestCase, +) +from tests.unit.src.sqlbuild.adapters.clickhouse.helpers import ( + FakeQueryLogConnection, + FakeQueryLogError, + read_clickhouse_usage, + usage_relation, +) + +WINDOW_SECONDS: int = 7 * 86_400 +READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) +EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) +ORDERS: tuple[RelationInfo, ...] = (usage_relation("mart", "old_orders"),) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageTestCase( + description="reads per initial user add up and the latest read wins", + relations=(("mart", "old_orders"), ("mart", "old_customers")), + rows=( + ("mart.old_orders", "svc_reporting", 10, int(READ_AT.timestamp())), + ("mart.old_orders", "alice", 4, int(EARLIER_READ_AT.timestamp())), + ("mart.old_orders", "", 1, int(EARLIER_READ_AT.timestamp())), + ("mart.products", "bob", 3, int(READ_AT.timestamp())), + ), + expected_names_sql="IN ('mart.old_customers', 'mart.old_orders')", + expected_usages=( + ("old_orders", ("svc_reporting", "alice", "unknown user"), 15, READ_AT), + ), + ), + ClickHouseRelationUsageTestCase( + description="names keep their case and quotes are escaped", + relations=(("Mart", "Old'Orders"),), + rows=( + ("Mart.Old'Orders", "alice", 1, int(READ_AT.timestamp())), + ("mart.old'orders", "bob", 1, int(READ_AT.timestamp())), + ), + expected_names_sql="IN ('Mart.Old\\'Orders')", + expected_usages=(("Old'Orders", ("alice",), 1, READ_AT),), + ), + ), + ids=lambda case: case.description, +) +def test_given_query_log_when_reading_usage_then_reads_match_candidates( + test_case: ClickHouseRelationUsageTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection(rows=test_case.rows) + + report: RelationUsageReport = read_clickhouse_usage( + connection=connection, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + sql: str + (sql,) = connection.queries + assert report.checked + assert test_case.expected_names_sql in sql + assert tuple( + (usage.name, usage.readers, usage.query_count, usage.last_read_at) + for usage in report.usages + ) == (test_case.expected_usages) + assert all(usage.direct for usage in report.usages) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageQueryTestCase( + description="finished reads and inserts of any table a query touched, once per query", + window_seconds=WINDOW_SECONDS, + expected_sql_fragments=( + "FROM system.query_log ARRAY JOIN tables AS object_name", + "initial_user AS user_name", + "uniqExact(initial_query_id) AS query_count", + "toUnixTimestamp(max(event_time)) AS last_read_at", + f"event_date >= toDate(now() - toIntervalSecond({WINDOW_SECONDS}))", + f"event_time >= now() - toIntervalSecond({WINDOW_SECONDS})", + "type = 'QueryFinish'", + "query_kind IN ('Select', 'Insert')", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_query_log_query_when_built_then_child_queries_count_once( + test_case: ClickHouseRelationUsageQueryTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection() + + read_clickhouse_usage( + connection=connection, relations=ORDERS, window_seconds=test_case.window_seconds + ) + + sql: str + (sql,) = connection.queries + assert all(fragment in sql for fragment in test_case.expected_sql_fragments) + assert "is_initial_query" not in sql + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageChunkTestCase( + description="no candidates read nothing", + relation_count=0, + expected_query_names=(), + ), + ClickHouseRelationUsageChunkTestCase( + description="a large candidate set is split into IN lists of at most 1,000 names", + relation_count=2_001, + expected_query_names=(1_000, 1_000, 1), + ), + ), + ids=lambda case: case.description, +) +def test_given_many_candidates_when_reading_usage_then_names_are_chunked( + test_case: ClickHouseRelationUsageChunkTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection() + + report: RelationUsageReport = read_clickhouse_usage( + connection=connection, + relations=tuple( + usage_relation("mart", f"old_orders_{index:04d}") + for index in range(test_case.relation_count) + ), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple(sql.count("'mart.old_orders_") for sql in connection.queries) == ( + test_case.expected_query_names + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageUncheckedTestCase( + description="no SELECT grant on system.query_log is not checked", + log_queries=True, + errors=FakeQueryLogError.raised( + "Code: 497. DB::Exception: janitor: Not enough privileges. To execute this " + "query, it's necessary to have the grant SELECT for at least one column on " + "system.query_log. (ACCESS_DENIED)" + ), + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + ClickHouseRelationUsageUncheckedTestCase( + description="a server without a query log table is not checked", + log_queries=True, + errors=FakeQueryLogError.raised( + "Code: 60. DB::Exception: Table system.query_log does not exist. (UNKNOWN_TABLE)" + ), + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + ClickHouseRelationUsageUncheckedTestCase( + description="query logging turned off is not checked", + log_queries=False, + errors=(), + expected_report=RelationUsageReport(checked=False, reason=USAGE_LOGGING_OFF_REASON), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_query_log_when_reading_usage_then_usage_is_not_checked( + test_case: ClickHouseRelationUsageUncheckedTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection( + log_queries=test_case.log_queries, + errors=test_case.errors, + ) + + report: RelationUsageReport = read_clickhouse_usage( + connection=connection, relations=ORDERS, window_seconds=WINDOW_SECONDS + ) + + assert report == test_case.expected_report + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageFailureTestCase( + description="a timeout is a real error", + error=FakeQueryLogError( + "Code: 159. DB::Exception: Timeout exceeded. (TIMEOUT_EXCEEDED)" + ), + expected_error_match="TIMEOUT_EXCEEDED", + ), + ), + ids=lambda case: case.description, +) +def test_given_other_query_log_failure_when_reading_usage_then_error_propagates( + test_case: ClickHouseRelationUsageFailureTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection(errors=(test_case.error,)) + + with pytest.raises(FakeQueryLogError, match=test_case.expected_error_match): + read_clickhouse_usage( + connection=connection, relations=ORDERS, window_seconds=WINDOW_SECONDS + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseRelationUsageReplicaTestCase( + description="a single server reads every query", + replicas=1, + expected_notices=(), + ), + ClickHouseRelationUsageReplicaTestCase( + description="other replicas of this server's shard are not read, and it says so", + replicas=3, + expected_notices=("usage read from this server only; other replicas not checked",), + ), + ), + ids=lambda case: case.description, +) +def test_given_replicas_when_reading_usage_then_other_replicas_are_noted( + test_case: ClickHouseRelationUsageReplicaTestCase, +) -> None: + connection: FakeQueryLogConnection = FakeQueryLogConnection(replicas=test_case.replicas) + + report: RelationUsageReport = read_clickhouse_usage( + connection=connection, relations=ORDERS, window_seconds=WINDOW_SECONDS + ) + + assert report.checked + assert report.notices == test_case.expected_notices + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index f4974ee6f2..4f82e53f10 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -2,7 +2,7 @@ from datetime import datetime from sqlbuild._native import InferredNullability -from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport from tests.unit.src.sqlbuild.adapters.databricks.helpers import FakeDatabricksMetadataCursor @@ -148,3 +148,44 @@ class DatabricksDeltaDetailAgeTestCase: cursors: tuple[FakeDatabricksMetadataCursor, ...] expected_sql: tuple[str, ...] expected_timestamps: tuple[str, str] + + +@dataclass(frozen=True) +class DatabricksRelationUsageTestCase: + description: str + relations: tuple[tuple[str | None, str, str], ...] + rows: tuple[tuple[object, ...], ...] + expected_object_names: tuple[str, ...] + expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + expected_via: tuple[tuple[bool, tuple[str, ...]], ...] = () + + +@dataclass(frozen=True) +class DatabricksRelationUsageQueryTestCase: + description: str + window_seconds: int + expected_params: dict[str, object] + expected_sql_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DatabricksRelationUsageChunkTestCase: + description: str + relation_count: int + expected_query_names: tuple[int, ...] + + +@dataclass(frozen=True) +class DatabricksRelationUsageErrorTestCase: + description: str + message: str + sqlstate: str | None + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class DatabricksRelationUsageFailureTestCase: + description: str + message: str + sqlstate: str | None + expected_error_match: str diff --git a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py index 2f5ddf88fb..98643e29dd 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py @@ -4,8 +4,9 @@ from typing import Any -from sqlbuild.adapter.contract.models import RetentionRequest +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport, RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter def build_retention_request(*, desired_days: int) -> RetentionRequest: @@ -63,3 +64,64 @@ def cursor(self) -> Any: cursor: FakeDatabricksMetadataCursor = self._cursors.pop(0) self.returned_cursors.append(cursor) return cursor + + +class FakeLineageError(Exception): + """A Databricks server error with a SQLSTATE.""" + + def __init__(self, message: str, sqlstate: str | None) -> None: + super().__init__(message) + self.sqlstate: str | None = sqlstate + + +class FakeLineageCursor: + """Answers lineage reads from stored rows, recording each read.""" + + def __init__(self, connection: FakeLineageConnection) -> None: + self.connection: FakeLineageConnection = connection + self.rows: list[tuple[object, ...]] = [] + + def execute(self, sql: str, params: dict[str, object]) -> None: + self.connection.queries.append((sql, params)) + error: Exception + for error in self.connection.errors: + raise error + names: frozenset[object] = frozenset(params.values()) + self.rows = list(filter(lambda row: row[0] in names, self.connection.rows)) + + def fetchall(self) -> list[tuple[object, ...]]: + return self.rows + + def close(self) -> None: + return None + + +class FakeLineageConnection: + def __init__( + self, + *, + rows: tuple[tuple[object, ...], ...] = (), + errors: tuple[Exception, ...] = (), + ) -> None: + self.rows: tuple[tuple[object, ...], ...] = rows + self.errors: tuple[Exception, ...] = errors + self.queries: list[tuple[str, dict[str, object]]] = [] + + def cursor(self) -> FakeLineageCursor: + return FakeLineageCursor(self) + + +def usage_relation(database: str | None, schema: str, name: str) -> RelationInfo: + """A table relation for usage reads.""" + + return RelationInfo(database=database, schema=schema, name=name, relation_type="table") + + +def read_databricks_usage( + *, connection: FakeLineageConnection, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Databricks' usage report for the relations from the fake connection.""" + + return DatabricksAdapter().relation_usage( + connection=connection, relations=relations, window_seconds=window_seconds + ) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py new file mode 100644 index 0000000000..da9b8e933a --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_relation_usage.py @@ -0,0 +1,274 @@ +"""Databricks relation usage reads Unity Catalog table lineage once per chunk of candidates.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.databricks.constants import ( + MAX_PARAMETER_MARKERS, + USAGE_UNAVAILABLE_REASON, +) +from tests.unit.src.sqlbuild.adapters.databricks._test_types import ( + DatabricksRelationUsageChunkTestCase, + DatabricksRelationUsageErrorTestCase, + DatabricksRelationUsageFailureTestCase, + DatabricksRelationUsageQueryTestCase, + DatabricksRelationUsageTestCase, +) +from tests.unit.src.sqlbuild.adapters.databricks.helpers import ( + FakeLineageConnection, + FakeLineageError, + read_databricks_usage, + usage_relation, +) + +WINDOW_SECONDS: int = 7 * 86_400 +READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) +EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) +ORDERS_DAILY: tuple[RelationInfo, ...] = (usage_relation("shop", "mart", "orders_daily"),) + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageTestCase( + description="read events per user add up and names fold to lowercase", + relations=(("Shop", "Mart", "orders_daily"), ("shop", "mart", "customers")), + rows=( + ("shop.mart.orders_daily", "svc-reporting@example.com", 10, READ_AT, True, None), + ("shop.mart.orders_daily", "alice@example.com", 4, EARLIER_READ_AT, True, None), + ("shop.mart.orders_daily", None, 1, EARLIER_READ_AT, True, None), + ("shop.mart.products", "bob@example.com", 3, READ_AT, True, None), + ), + expected_object_names=("shop.mart.customers", "shop.mart.orders_daily"), + expected_usages=( + ( + "orders_daily", + ("svc-reporting@example.com", "alice@example.com", "unknown user"), + 15, + READ_AT, + ), + ), + expected_via=((True, ()),), + ), + DatabricksRelationUsageTestCase( + description="tables read only through views name them, or say a view when unknown", + relations=( + ("shop", "mart", "orders_base"), + ("shop", "mart", "customers"), + ("shop", "mart", "products"), + ), + rows=( + ( + "shop.mart.orders_base", + "alice@example.com", + 3, + READ_AT, + False, + ["shop.mart.orders_v", "shop.mart.orders_w"], + ), + ("shop.mart.customers", "alice@example.com", 2, READ_AT, False, None), + ("shop.mart.products", "alice@example.com", 2, READ_AT, False, None), + ("shop.mart.products", "bob@example.com", 1, EARLIER_READ_AT, True, None), + ), + expected_object_names=( + "shop.mart.customers", + "shop.mart.orders_base", + "shop.mart.products", + ), + expected_usages=( + ("customers", ("alice@example.com",), 2, READ_AT), + ("orders_base", ("alice@example.com",), 3, READ_AT), + ("products", ("alice@example.com", "bob@example.com"), 3, READ_AT), + ), + expected_via=( + (False, ()), + (False, ("shop.mart.orders_v", "shop.mart.orders_w")), + (True, ()), + ), + ), + DatabricksRelationUsageTestCase( + description="a relation without a catalog is not looked up", + relations=((None, "mart", "orders_daily"), ("shop", "mart", "customers")), + rows=(), + expected_object_names=("shop.mart.customers",), + expected_usages=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_lineage_when_reading_usage_then_reads_match_candidates( + test_case: DatabricksRelationUsageTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection(rows=test_case.rows) + + report: RelationUsageReport = read_databricks_usage( + connection=connection, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + params: dict[str, object] + ((_, params),) = connection.queries + assert report.checked + assert tuple(params.values())[1:] == test_case.expected_object_names + assert tuple( + (usage.name, usage.readers, usage.query_count, usage.last_read_at) + for usage in report.usages + ) == (test_case.expected_usages) + assert tuple((usage.direct, usage.via) for usage in report.usages) == (test_case.expected_via) + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageQueryTestCase( + description="read events of source tables within the window, with named parameters", + window_seconds=WINDOW_SECONDS, + expected_params={"window_seconds": WINDOW_SECONDS, "name_0": "shop.mart.orders_daily"}, + expected_sql_fragments=( + "FROM system.access.table_lineage", + "lower(source_table_full_name) AS object_name", + "COALESCE(direct_access, true) AS direct", + "reads.created_by AS user_name", + "COUNT(DISTINCT COALESCE(reads.statement_id, reads.event_id)) AS query_count", + "MAX(reads.event_time) AS last_read_at", + "event_date >= DATE(TIMESTAMPADD(SECOND, -:window_seconds, current_timestamp()))", + "WHERE object_name IN (:name_0)", + "JOIN lineage AS views ON views.statement_id = reads.statement_id " + "AND views.direct AND views.source_type IN " + "('VIEW', 'MATERIALIZED_VIEW', 'METRIC_VIEW')", + "bool_or(reads.direct) AS direct", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_lineage_query_when_built_then_parameters_are_named( + test_case: DatabricksRelationUsageQueryTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection() + + read_databricks_usage( + connection=connection, relations=ORDERS_DAILY, window_seconds=test_case.window_seconds + ) + + sql: str + params: dict[str, object] + ((sql, params),) = connection.queries + assert params == test_case.expected_params + assert all(fragment in sql for fragment in test_case.expected_sql_fragments) + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageChunkTestCase( + description="no candidates read nothing", + relation_count=0, + expected_query_names=(), + ), + DatabricksRelationUsageChunkTestCase( + description="a large candidate set is split into IN lists of at most 250 names", + relation_count=2_001, + expected_query_names=(250, 250, 250, 250, 250, 250, 250, 250, 1), + ), + ), + ids=lambda case: case.description, +) +def test_given_many_candidates_when_reading_usage_then_names_are_chunked( + test_case: DatabricksRelationUsageChunkTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection() + + report: RelationUsageReport = read_databricks_usage( + connection=connection, + relations=tuple( + usage_relation("shop", "mart", f"old_orders_{index:04d}") + for index in range(test_case.relation_count) + ), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple(len(params) - 1 for _, params in connection.queries) == ( + test_case.expected_query_names + ) + assert all(sql.count(":name_") == len(params) - 1 for sql, params in connection.queries) + assert all( + len(re.findall(r"(? None: + connection: FakeLineageConnection = FakeLineageConnection( + errors=(FakeLineageError(test_case.message, test_case.sqlstate),) + ) + + report: RelationUsageReport = read_databricks_usage( + connection=connection, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + assert report == test_case.expected_report + + +@pytest.mark.parametrize( + "test_case", + ( + DatabricksRelationUsageFailureTestCase( + description="a cancelled statement is a real error", + message="[OPERATION_CANCELED] The statement was cancelled.", + sqlstate="HY008", + expected_error_match="OPERATION_CANCELED", + ), + ), + ids=lambda case: case.description, +) +def test_given_other_lineage_failure_when_reading_usage_then_error_propagates( + test_case: DatabricksRelationUsageFailureTestCase, +) -> None: + connection: FakeLineageConnection = FakeLineageConnection( + errors=(FakeLineageError(test_case.message, test_case.sqlstate),) + ) + + with pytest.raises(FakeLineageError, match=test_case.expected_error_match): + read_databricks_usage( + connection=connection, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 2ff4b3fc2f..be6cfb3c96 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -2,7 +2,8 @@ from datetime import datetime from sqlbuild._native import InferredNullability -from sqlbuild.adapter.contract.models import FunctionInfo, SchemaDiffResult +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import FunctionInfo, RelationUsageReport, SchemaDiffResult from sqlbuild.adapter.contract.types import TableFreshnessStatus from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.types import CostStatus @@ -294,3 +295,77 @@ class SnowflakeViewDefinitionsTestCase: read_names: tuple[str, ...] expected_definitions: tuple[str | None, ...] expected_query_count: int + + +@dataclass(frozen=True) +class SnowflakeRelationUsageTestCase: + description: str + relations: tuple[tuple[str | None, str, str], ...] + current_database: str | None + access_rows: tuple[tuple[object, ...], ...] + expected_object_names: tuple[str, ...] + expected_usages: tuple[tuple[str, tuple[str, ...], int, datetime], ...] + + +@dataclass(frozen=True) +class SnowflakeRelationUsageQueryTestCase: + description: str + window_seconds: int + expected_params: tuple[object, ...] + expected_sql_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeRelationUsageChunkTestCase: + description: str + relation_count: int + expected_query_names: tuple[int, ...] + + +@dataclass(frozen=True) +class SnowflakeRelationUsageErrorTestCase: + description: str + errno: int | None + message: str + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class SnowflakeRelationUsageFailureTestCase: + description: str + errno: int | None + message: str + expected_error_match: str + + +@dataclass(frozen=True) +class UnsupportedRelationUsageTestCase: + description: str + adapter: BaseAdapter + expected_report: RelationUsageReport + + +@dataclass(frozen=True) +class SnowflakeRelationUsageViaTestCase: + description: str + relations: tuple[tuple[str, str, str, str], ...] + access_rows: tuple[tuple[object, ...], ...] + dependency_rows: tuple[tuple[str, str], ...] + expected_dependency_params: tuple[tuple[object, ...], ...] + expected_usages: tuple[tuple[str, tuple[str, ...], int, bool, tuple[str, ...]], ...] + + +@dataclass(frozen=True) +class SnowflakeCloneStatementTestCase: + description: str + statement: str + expected_clone: bool + + +@dataclass(frozen=True) +class SnowflakeRelationUsageChainsUnfollowedTestCase: + description: str + errno: int | None + message: str + expected_usages: tuple[tuple[str, tuple[str, ...], int], ...] + expected_notices: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 18478879af..dd72a9d08d 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -11,7 +11,7 @@ from typing import Any from sqlbuild._native import NativeProgressProjector -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationUsageReport from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection from sqlbuild.observability import ( @@ -396,3 +396,116 @@ class CustomInitSnowflakeAdapter(SnowflakeAdapter): def __init__(self) -> None: self.custom_label: str = "orders" + + +class FakeAccessHistoryError(Exception): + """A Snowflake driver error with an error number.""" + + def __init__(self, message: str, errno: int | None) -> None: + super().__init__(message) + self.errno: int | None = errno + + +class FakeUsageWarehouse: + """Answers CURRENT_DATABASE() and access-history reads, recording each access-history read.""" + + def __init__( + self, + *, + current_database: str | None = None, + access_rows: tuple[tuple[object, ...], ...] = (), + dependency_rows: tuple[tuple[str, str], ...] = (), + errors: tuple[Exception, ...] = (), + dependency_errors: tuple[Exception, ...] = (), + ) -> None: + self.current_database: str | None = current_database + self.access_rows: tuple[tuple[object, ...], ...] = access_rows + self.dependency_rows: tuple[tuple[str, str], ...] = dependency_rows + self.errors: tuple[Exception, ...] = errors + self.dependency_errors: tuple[Exception, ...] = dependency_errors + self.queries: list[tuple[str, tuple[object, ...]]] = [] + self.dependency_queries: list[tuple[str, tuple[object, ...]]] = [] + + def current_database_rows(self) -> list[tuple[object, ...]]: + return [(self.current_database,)] + + def access_history_rows(self, sql: str, params: tuple[object, ...]) -> list[tuple[object, ...]]: + self.queries.append((sql, params)) + error: Exception + for error in self.errors: + raise error + names: frozenset[object] = frozenset(params) + return list(filter(lambda row: row[0] in names, self.access_rows)) + + def object_dependency_rows( + self, sql: str, params: tuple[object, ...] + ) -> list[tuple[object, ...]]: + self.dependency_queries.append((sql, params)) + error: Exception + for error in self.dependency_errors: + raise error + databases: frozenset[object] = frozenset(params) + return list(filter(lambda row: row[0].split(".")[0] in databases, self.dependency_rows)) + + +class FakeUsageCursor: + """Cursor double routing each statement to the fake usage warehouse.""" + + def __init__(self, warehouse: FakeUsageWarehouse) -> None: + self.warehouse: FakeUsageWarehouse = warehouse + self.rows: list[tuple[object, ...]] = [] + + def execute(self, sql: str, params: tuple[object, ...] = ()) -> None: + reads: dict[tuple[bool, bool], Callable[[], list[tuple[object, ...]]]] = { + (True, False): self.warehouse.current_database_rows, + (False, True): lambda: self.warehouse.object_dependency_rows(sql, params), + (False, False): lambda: self.warehouse.access_history_rows(sql, params), + } + self.rows = reads[(sql == "SELECT CURRENT_DATABASE()", "OBJECT_DEPENDENCIES" in sql)]() + + def fetchone(self) -> tuple[object, ...] | None: + return next(iter(self.rows), None) + + def fetchall(self) -> list[tuple[object, ...]]: + return self.rows + + def close(self) -> None: + return None + + +class FakeUsageConnection: + """Connection double handing out cursors over one fake usage warehouse.""" + + def __init__(self, warehouse: FakeUsageWarehouse) -> None: + self.warehouse: FakeUsageWarehouse = warehouse + + def cursor(self) -> FakeUsageCursor: + return FakeUsageCursor(self.warehouse) + + +def usage_relation( + database: str | None, schema: str, name: str, relation_type: str = "base table" +) -> RelationInfo: + """A relation for usage reads, a base table unless given another type.""" + + return RelationInfo(database=database, schema=schema, name=name, relation_type=relation_type) + + +def direct_read( + object_name: str, user: str, query_count: int, read_at: object +) -> tuple[object, ...]: + """An access-history result row for direct reads.""" + + return (object_name, user, query_count, read_at, True, None) + + +def read_snowflake_usage( + *, warehouse: FakeUsageWarehouse, relations: tuple[RelationInfo, ...], window_seconds: int +) -> RelationUsageReport: + """Snowflake's usage report for the relations from the fake warehouse.""" + + return SnowflakeAdapter().relation_usage( + connection=FakeUsageConnection(warehouse), + relations=relations, + window_seconds=window_seconds, + ) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py new file mode 100644 index 0000000000..2b91409054 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_relation_usage.py @@ -0,0 +1,510 @@ +"""Snowflake relation usage reads ACCOUNT_USAGE access history once per chunk of candidates.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo, RelationUsageReport +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.constants import ( + CLONE_STATEMENT_PATTERN, + SQLBUILD_QUERY_TAG_FRAGMENT, + USAGE_UNAVAILABLE_REASON, +) +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeCloneStatementTestCase, + SnowflakeRelationUsageChainsUnfollowedTestCase, + SnowflakeRelationUsageChunkTestCase, + SnowflakeRelationUsageErrorTestCase, + SnowflakeRelationUsageFailureTestCase, + SnowflakeRelationUsageQueryTestCase, + SnowflakeRelationUsageTestCase, + SnowflakeRelationUsageViaTestCase, + UnsupportedRelationUsageTestCase, +) +from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( + FakeAccessHistoryError, + FakeUsageWarehouse, + direct_read, + read_snowflake_usage, + usage_relation, +) + +WINDOW_SECONDS: int = 7 * 86_400 +READ_AT: datetime = datetime(2026, 10, 21, 9, 30, tzinfo=UTC) +EARLIER_READ_AT: datetime = datetime(2026, 10, 19, 14, 0, tzinfo=UTC) +ORDERS_DAILY: tuple[RelationInfo, ...] = (usage_relation("analytics", "mart", "orders_daily"),) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageTestCase( + description="reads per user add up, readers rank by queries, the latest read wins", + relations=(("analytics", "mart", "orders_daily"), ("analytics", "mart", "customers")), + current_database=None, + access_rows=( + direct_read("ANALYTICS.MART.ORDERS_DAILY", "SVC_REPORTING", 10, READ_AT), + direct_read("ANALYTICS.MART.ORDERS_DAILY", "ALICE", 4, EARLIER_READ_AT), + direct_read("ANALYTICS.MART.PRODUCTS", "BOB", 3, READ_AT), + ), + expected_object_names=("ANALYTICS.MART.CUSTOMERS", "ANALYTICS.MART.ORDERS_DAILY"), + expected_usages=(("orders_daily", ("SVC_REPORTING", "ALICE"), 14, READ_AT),), + ), + SnowflakeRelationUsageTestCase( + description="unqualified relations use the session database and names fold upper", + relations=((None, "mart", "orders_daily"),), + current_database="SHOP_DB", + access_rows=(direct_read("SHOP_DB.MART.ORDERS_DAILY", "SVC_REPORTING", 2, READ_AT),), + expected_object_names=("SHOP_DB.MART.ORDERS_DAILY",), + expected_usages=(("orders_daily", ("SVC_REPORTING",), 2, READ_AT),), + ), + SnowflakeRelationUsageTestCase( + description="names that need quoting match the quoted object name, case kept upper", + relations=(("analytics", "mart", "order-lines"),), + current_database=None, + access_rows=(direct_read('ANALYTICS.MART."ORDER-LINES"', "ALICE", 1, READ_AT),), + expected_object_names=('ANALYTICS.MART."ORDER-LINES"',), + expected_usages=(("order-lines", ("ALICE",), 1, READ_AT),), + ), + SnowflakeRelationUsageTestCase( + description="a mixed-case reader object does not match an unquoted relation", + relations=(("analytics", "mart", "orders_daily"),), + current_database=None, + access_rows=(direct_read('ANALYTICS.MART."orders_daily"', "ALICE", 1, READ_AT),), + expected_object_names=("ANALYTICS.MART.ORDERS_DAILY",), + expected_usages=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_access_history_when_reading_usage_then_reads_match_candidates( + test_case: SnowflakeRelationUsageTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + current_database=test_case.current_database, access_rows=test_case.access_rows + ) + + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + params: tuple[object, ...] + ((_, params),) = warehouse.queries + assert report.checked + assert params[1 : 1 + len(test_case.expected_object_names)] == (test_case.expected_object_names) + assert tuple( + (usage.name, usage.readers, usage.query_count, usage.last_read_at) + for usage in report.usages + ) == (test_case.expected_usages) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageQueryTestCase( + description="one flattened access-history read leaves out SQLBuild queries and clones", + window_seconds=WINDOW_SECONDS, + expected_params=( + WINDOW_SECONDS, + "ANALYTICS.MART.ORDERS_DAILY", + "ANALYTICS.MART.ORDERS_DAILY", + WINDOW_SECONDS, + '"app":"sqlbuild"', + CLONE_STATEMENT_PATTERN, + ), + expected_sql_fragments=( + "FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY", + "LATERAL FLATTEN(INPUT => history.direct_objects_accessed) AS accessed", + "LATERAL FLATTEN(INPUT => history.base_objects_accessed) AS accessed", + "WHERE direct_read.query_id = base.query_id " + "AND direct_read.object_name = base.object_name", + "NOT EXISTS (SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY AS query", + "CONTAINS(COALESCE(query.query_tag, ''), %s)", + "STARTSWITH(query.query_type, 'CREATE') AND REGEXP_LIKE(query.query_text, %s", + "COUNT(DISTINCT reads.query_id) AS query_count", + "BOOLOR_AGG(reads.direct) AS direct", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_access_history_query_when_built_then_sqlbuild_queries_and_clones_are_excluded( + test_case: SnowflakeRelationUsageQueryTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse() + + read_snowflake_usage( + warehouse=warehouse, relations=ORDERS_DAILY, window_seconds=test_case.window_seconds + ) + + sql: str + params: tuple[object, ...] + ((sql, params),) = warehouse.queries + assert params == test_case.expected_params + assert (SQLBUILD_QUERY_TAG_FRAGMENT, CLONE_STATEMENT_PATTERN) == (test_case.expected_params[4:]) + assert all(fragment in sql for fragment in test_case.expected_sql_fragments) + + +VIEW_DOMAINS: tuple[str, ...] = ("VIEW", "MATERIALIZED VIEW", "SECURE VIEW") + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageViaTestCase( + description="a table read only through a view names the view; any direct read wins", + relations=( + ("analytics", "mart", "orders_base", "base table"), + ("analytics", "mart", "customers", "base table"), + ), + access_rows=( + ( + "ANALYTICS.MART.ORDERS_BASE", + "ALICE", + 3, + READ_AT, + False, + '[\n "ANALYTICS.MART.ORDERS_V",\n "REPORTING.MART.ORDERS_W"\n]', + ), + ("ANALYTICS.MART.CUSTOMERS", "ALICE", 2, READ_AT, False, '["ANALYTICS.MART.C_V"]'), + direct_read("ANALYTICS.MART.CUSTOMERS", "BOB", 1, EARLIER_READ_AT), + ), + dependency_rows=(), + expected_dependency_params=((*VIEW_DOMAINS, "ANALYTICS", "REPORTING"),), + expected_usages=( + ("customers", ("ALICE", "BOB"), 3, True, ()), + ( + "orders_base", + ("ALICE",), + 3, + False, + ("analytics.mart.orders_v", "reporting.mart.orders_w"), + ), + ), + ), + SnowflakeRelationUsageViaTestCase( + description="candidates behind an in-use candidate view are in use through it", + relations=( + ("analytics", "mart", "orders_v", "view"), + ("analytics", "mart", "orders_v2", "view"), + ("analytics", "mart", "orders", "base table"), + ("analytics", "mart", "unread_v", "view"), + ), + access_rows=( + direct_read("ANALYTICS.MART.ORDERS_V", "ALICE", 5, READ_AT), + ( + "ANALYTICS.MART.ORDERS", + "ALICE", + 5, + READ_AT, + False, + '["ANALYTICS.MART.ORDERS_V"]', + ), + ), + dependency_rows=( + ("ANALYTICS.MART.ORDERS_V", "ANALYTICS.MART.ORDERS_V2"), + ("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.ORDERS"), + ("ANALYTICS.MART.UNREAD_V", "ANALYTICS.MART.ORDERS"), + ("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.NOT_A_CANDIDATE"), + ), + expected_dependency_params=((*VIEW_DOMAINS, "ANALYTICS"),), + expected_usages=( + ("orders", ("ALICE",), 5, False, ("analytics.mart.orders_v",)), + ("orders_v", ("ALICE",), 5, True, ()), + ("orders_v2", ("ALICE",), 5, False, ("analytics.mart.orders_v",)), + ), + ), + SnowflakeRelationUsageViaTestCase( + description="a candidate view between a read non-candidate view and a table is in use", + relations=( + ("analytics", "mart", "orders_v2", "view"), + ("analytics", "mart", "orders", "base table"), + ), + access_rows=( + ( + "ANALYTICS.MART.ORDERS", + "ALICE", + 4, + READ_AT, + False, + '["REPORTING.MART.ORDERS_W"]', + ), + ), + dependency_rows=( + ("REPORTING.MART.ORDERS_W", "REPORTING.MART.ORDERS_HELPER"), + ("REPORTING.MART.ORDERS_HELPER", "ANALYTICS.MART.ORDERS_V2"), + ("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.ORDERS"), + ), + expected_dependency_params=((*VIEW_DOMAINS, "ANALYTICS", "REPORTING"),), + expected_usages=( + ("orders", ("ALICE",), 4, False, ("reporting.mart.orders_w",)), + ("orders_v2", ("ALICE",), 4, False, ("reporting.mart.orders_w",)), + ), + ), + SnowflakeRelationUsageViaTestCase( + description="nothing read means no dependency query", + relations=(("analytics", "mart", "orders_v2", "view"),), + access_rows=(), + dependency_rows=(("ANALYTICS.MART.ORDERS_V2", "ANALYTICS.MART.ORDERS"),), + expected_dependency_params=(), + expected_usages=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_view_reads_when_reading_usage_then_indirect_reads_name_their_views( + test_case: SnowflakeRelationUsageViaTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + access_rows=test_case.access_rows, dependency_rows=test_case.dependency_rows + ) + + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, + relations=tuple(usage_relation(*relation) for relation in test_case.relations), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple(params for _, params in warehouse.dependency_queries) == ( + test_case.expected_dependency_params + ) + assert all( + "referencing_object_domain IN (%s, %s, %s) AND referencing_database IN (" in sql + for sql, _ in warehouse.dependency_queries + ) + assert tuple( + sorted( + (usage.name, usage.readers, usage.query_count, usage.direct, usage.via) + for usage in report.usages + ) + ) == (test_case.expected_usages) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeCloneStatementTestCase( + description="a table clone", + statement="CREATE TABLE analytics.mart.orders_copy CLONE analytics.mart.orders", + expected_clone=True, + ), + SnowflakeCloneStatementTestCase( + description="a replaced transient clone at a point in time", + statement="create or replace transient table orders_copy clone orders " + "at(offset => -60);", + expected_clone=True, + ), + SnowflakeCloneStatementTestCase( + description="a schema clone after leading comments", + statement="-- nightly copy\n/* keep */\nCREATE SCHEMA mart_copy CLONE mart", + expected_clone=True, + ), + SnowflakeCloneStatementTestCase( + description="quoted names and a dynamic table", + statement='CREATE DYNAMIC TABLE "Orders Copy" CLONE "Orders"', + expected_clone=True, + ), + SnowflakeCloneStatementTestCase( + description="a CTAS selecting a column named clone", + statement="CREATE TABLE orders_copy AS SELECT clone FROM orders", + expected_clone=False, + ), + SnowflakeCloneStatementTestCase( + description="a view whose comment mentions a clone", + statement="CREATE VIEW orders_v AS SELECT 1 AS clone_id -- see a CLONE b", + expected_clone=False, + ), + SnowflakeCloneStatementTestCase( + description="a CTAS with clone inside a string", + statement="CREATE TABLE notes AS SELECT 'orders CLONE copy' AS note", + expected_clone=False, + ), + SnowflakeCloneStatementTestCase( + description="a select with a clone statement in a comment", + statement="SELECT * FROM orders -- CREATE TABLE a CLONE b", + expected_clone=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_statement_when_matching_clone_pattern_then_only_clones_match( + test_case: SnowflakeCloneStatementTestCase, +) -> None: + matched: bool = ( + re.fullmatch(CLONE_STATEMENT_PATTERN, test_case.statement, re.IGNORECASE | re.DOTALL) + is not None + ) + + assert matched == test_case.expected_clone + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageChunkTestCase( + description="no candidates read nothing", + relation_count=0, + expected_query_names=(), + ), + SnowflakeRelationUsageChunkTestCase( + description="a large candidate set is split into IN lists of at most 1,000 names", + relation_count=2_001, + expected_query_names=(1_000, 1_000, 1), + ), + ), + ids=lambda case: case.description, +) +def test_given_many_candidates_when_reading_usage_then_names_are_chunked( + test_case: SnowflakeRelationUsageChunkTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse() + + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, + relations=tuple( + usage_relation("analytics", "mart", f"old_orders_{index:04d}") + for index in range(test_case.relation_count) + ), + window_seconds=WINDOW_SECONDS, + ) + + assert report.checked + assert tuple((len(params) - 4) // 2 for _, params in warehouse.queries) == ( + test_case.expected_query_names + ) + assert all(sql.count("%s") == len(params) for sql, params in warehouse.queries) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageErrorTestCase( + description="a missing view (no Enterprise edition or access) is not checked", + errno=2003, + message="Object 'SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY' does not exist or not " + "authorized.", + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + SnowflakeRelationUsageErrorTestCase( + description="insufficient privileges are not checked", + errno=3001, + message="Insufficient privileges to operate on schema 'ACCOUNT_USAGE'", + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + SnowflakeRelationUsageErrorTestCase( + description="a privilege message without an error number is not checked", + errno=None, + message="SQL access control error: Insufficient privileges to operate on view", + expected_report=RelationUsageReport(checked=False, reason=USAGE_UNAVAILABLE_REASON), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_access_history_access_when_reading_usage_then_usage_is_not_checked( + test_case: SnowflakeRelationUsageErrorTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + errors=(FakeAccessHistoryError(test_case.message, test_case.errno),) + ) + + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + assert report == test_case.expected_report + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageChainsUnfollowedTestCase( + description="no OBJECT_VIEWER keeps the reads and says view chains were not followed", + errno=3001, + message="Insufficient privileges to operate on view 'OBJECT_DEPENDENCIES'", + expected_usages=(("orders_daily", ("ALICE",), 2),), + expected_notices=("view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER",), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_dependency_access_when_reading_usage_then_reads_are_kept_with_notice( + test_case: SnowflakeRelationUsageChainsUnfollowedTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + access_rows=(direct_read("ANALYTICS.MART.ORDERS_DAILY", "ALICE", 2, READ_AT),), + dependency_errors=(FakeAccessHistoryError(test_case.message, test_case.errno),), + ) + + report: RelationUsageReport = read_snowflake_usage( + warehouse=warehouse, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + assert report.checked + assert len(warehouse.dependency_queries) == 1 + assert tuple((usage.name, usage.readers, usage.query_count) for usage in report.usages) == ( + test_case.expected_usages + ) + assert report.notices == test_case.expected_notices + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeRelationUsageFailureTestCase( + description="a cancelled read is a real error", + errno=604, + message="SQL execution canceled", + expected_error_match="canceled", + ), + ), + ids=lambda case: case.description, +) +def test_given_other_access_history_failure_when_reading_usage_then_error_propagates( + test_case: SnowflakeRelationUsageFailureTestCase, +) -> None: + warehouse: FakeUsageWarehouse = FakeUsageWarehouse( + errors=(FakeAccessHistoryError(test_case.message, test_case.errno),) + ) + + with pytest.raises(FakeAccessHistoryError, match=test_case.expected_error_match): + read_snowflake_usage( + warehouse=warehouse, relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + +@pytest.mark.parametrize( + "test_case", + ( + UnsupportedRelationUsageTestCase( + description="duckdb has no usage source", + adapter=DuckDbAdapter(), + expected_report=RelationUsageReport(checked=False, reason="unsupported adapter"), + ), + UnsupportedRelationUsageTestCase( + description="postgres has no usage source", + adapter=PostgresAdapter(), + expected_report=RelationUsageReport(checked=False, reason="unsupported adapter"), + ), + ), + ids=lambda case: case.description, +) +def test_given_other_adapter_when_reading_usage_then_it_reports_unsupported( + test_case: UnsupportedRelationUsageTestCase, +) -> None: + report: RelationUsageReport = test_case.adapter.relation_usage( + connection=object(), relations=ORDERS_DAILY, window_seconds=WINDOW_SECONDS + ) + + assert report == test_case.expected_report + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index ad73e6c27f..8a4696fe6b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -194,6 +194,74 @@ ), expected_error_pattern="allowed_qualifiers entries must be", ), + RulesConfigErrorTestCase( + description="allowed qualifier entry with neither qualifier nor paths", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ domain = "sales", reason = "Regional exports." }]\n' + ), + expected_error_pattern="allowed_qualifiers entries must be", + ), + RulesConfigErrorTestCase( + description="allowed qualifier paths that are not a list", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = "models/sales/*", reason = "Exports." }]\n' + ), + expected_error_pattern="allowed_qualifiers entries must be", + ), + RulesConfigErrorTestCase( + description="allowed qualifier with an empty paths list", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = [], reason = "Exports." }]\n' + ), + expected_error_pattern="need a non-empty list of non-empty paths", + ), + RulesConfigErrorTestCase( + description="allowed paths entry with an empty reason", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = ["models/sales/*"], reason = " " }]\n' + ), + expected_error_pattern=r"allowed_qualifiers entries need a non-empty reason$", + ), + RulesConfigErrorTestCase( + description="allowed qualifier that is empty", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ qualifier = "", reason = "Exports." }]\n' + ), + expected_error_pattern=r"allowed_qualifiers entries need a non-empty qualifier$", + ), + RulesConfigErrorTestCase( + description="allowed path with backslash separators", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + "allowed_qualifiers = [{ paths = ['models\\sales\\*'], reason = \"Exports.\" }]\n" + ), + expected_error_pattern=r"paths must be project-relative globs with / separators", + ), + RulesConfigErrorTestCase( + description="allowed path listed twice in one entry", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ paths = ["models/sales/*", "models/sales/*"], ' + 'reason = "Exports." }]\n' + ), + expected_error_pattern=r"allowed_qualifiers entry lists path models/sales/\* twice", + ), + RulesConfigErrorTestCase( + description="duplicate allowed qualifier glob", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + "allowed_qualifiers = [\n" + ' { qualifier = "[a-z][a-z][a-z]", reason = "Per country." },\n' + ' { qualifier = "[a-z][a-z][a-z]", reason = "Per region." },\n' + "]\n" + ), + expected_error_pattern=r"duplicate .*allowed_qualifiers entry: \[a-z\]\[a-z\]\[a-z\]", + ), RulesConfigErrorTestCase( description="unknown SQL test rules key", source='[rules.sql_tests]\npipeline_directory = "pipelines"\nroot = "tests"\n', diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 0973b91ad2..a0ac52bf6b 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -22,7 +22,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | Flag | Description | |------|-------------| | `--no-sql-analysis` | Disable SQL syntax, binding, type inference, and semantic validation (`--no-sql-validation` remains an alias) | -| `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | +| `--no-cache` | Bypass the reusable compile-analysis cache and the Rules cache for this invocation | | `--defer-to` | Resolve unselected model references against another target | | `--json` | Output the full compile report as JSON | | `--manifest` | Generate `target/manifest.json` with project metadata | @@ -202,6 +202,11 @@ Known limits: - In these cases, or whenever a result looks stale, run `sqb compile --no-cache` or set `SQLBUILD_DISABLE_COMPILE_REUSE=1`. +`--no-cache` also turns the Rules cache off for that compile: built-in and custom Rules evaluate +every subject again, and the compile neither reads nor writes stored Rule results, as with +`[rules.cache] enabled = false`. A `--no-cache` compile reports `rule_cache_hits` as `0`. The next +compile without the flag reuses whatever the cache held before. + A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, @@ -291,6 +296,27 @@ Returns a structured report including: - `lineage_mode` - which lineage mode was used - `artifacts` - paths to written files +Every `compile_timings` value is a non-negative integer. Besides the phase timings, it attributes +custom Rule cost: + +| Key | Meaning | +|-----|---------| +| `custom_rules_ms`, `custom_rules_cpu_ms` | Wall and CPU time of the custom Rules phase, including its host processes | +| `custom_rules_invocations` | Custom Rule invocations the hosts ran, one per Rule and subject | +| `custom_rules_monitor_cpu_ms` | Host CPU spent checking Rule modules for state one subject could leave for another | +| `custom_rules_guard_cpu_ms` | Host CPU spent by the runtime guard deciding whether an audited operation, such as opening a file or importing a module, came from Rule code | +| `custom_rules..invocations` | Invocations of one custom Rule | +| `custom_rules..cpu_ms` | Host CPU of those invocations, excluding the guard's checks | +| `custom_rules..cache_hits` | Subjects whose stored result was reused | +| `custom_rules..cache_misses` | Subjects evaluated because no stored result applied; every subject when the Rules cache is off | + +Each selected custom Rule has the four `custom_rules..*` keys; a Rule the cache answered in +full reports `0` invocations. The keys are flat so that `compile_timings` stays one level deep, for +example `jq '.compile_timings["custom_rules.XORDERS001.cpu_ms"]'`. Invocations and CPU include a +Rule SQLBuild evaluates again in one process because its module state changed, so they can exceed +the Rule's subject count. Monitor and guard time are separate from each Rule's CPU; the hosts' +remaining CPU goes to starting Python, loading Rule modules, and reading the project. + If compile stops on an error before it can build the report, such as a model header that does not parse, `--json` still prints one JSON document. It has `command`, `has_errors: true`, `stopped: true`, and a `diagnostics` list with that error's `code`, `message`, and `help`, the same diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 6ac27db06b..4718fe60d5 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -24,6 +24,7 @@ sqb --project-dir janitor [flags] | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | | `--verbose`, `-v` | List every entry of each preview section and every skipped object with its reason. Without it, each section shows its first 10 entries and skipped objects are counted by reason | | `--json` | Print the plan and the result as one JSON document on stdout; the preview, prompt and progress go to stderr | +| `--keep-in-use` | Keep relations that other queries read within `janitor.in_use_window` instead of archiving them, for this run. Same as `janitor.keep_in_use = true`. See [Relations in use](#relations-in-use) | | `--drop-old-name-view ` | Drop the [compatibility view](/docs/concepts/models/migrations#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | @@ -54,7 +55,67 @@ Skipped objects (385) 175 relation is not tracked by SQLBuild ``` -With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. +With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `usage_check`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. + +## Relations in use + +Archiving renames a relation, so anything that still reads it fails from then on. Before archiving, the janitor looks up which candidates other queries read within `in_use_window` (default `7d`) and shows them, with their readers, in an `In use` section: + +``` +In use + analytics.orders_daily queried 2026-10-21 by svc_reporting, alice (14 queries in 7 days) + analytics.old_orders queried 2026-10-20 by svc_reporting, alice, bob +2 more (40 queries in 7 days); frozen since 2026-09-01, still in use + analytics.old_customers queried 2026-10-21 by alice via analytics.customer_report (3 queries in 7 days); frozen since 2026-09-01, still in use +``` + +Each line names the latest read, up to three readers (most queries first) and how many queries read the relation. Reading a view also reads what it is built on, so those relations are in use too. When a relation was only read that way, the line says `via` and names the views, or says `through a view` when the warehouse doesn't record which. A stale table that is still read is also flagged `frozen since , still in use`: its data stopped changing on that date, so its readers are getting old data. Views are not flagged, because their reads stay live. The confirmation prompt states the count, for example `14 relations will be archived; 3 of them are in use.` + +By default in-use relations are still archived, which is also how their readers find out. To keep them instead, set `keep_in_use = true` under `[janitor]` or pass `--keep-in-use`. In-use relations are then listed under `Kept (in use)` with their readers and are left out of the archive list and the counts. A later run archives them once nothing reads them within the window. + +Only relations to archive are checked. Expired archives and state-table pruning are not. + +Usage comes from the warehouse's query history: + +| Adapter | Usage source | +|---------|--------------| +| Snowflake | `SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY` (`direct_objects_accessed`, `base_objects_accessed`) joined with `QUERY_HISTORY`, and `OBJECT_DEPENDENCIES` | +| BigQuery | `` `.region-`.INFORMATION_SCHEMA.JOBS `` (`referenced_tables`, `referenced_logical_views`) | +| Databricks | `system.access.table_lineage` (`source_table_full_name`, `direct_access`) | +| ClickHouse | `system.query_log` (`tables`) on the connected server | +| Other adapters | Not available | + +On Snowflake, one query reads the access history of all candidates (up to 1,000 names per query). A candidate listed in a query's base objects but not its direct objects was read through the objects the query named, which the line names after `via`. When something was read, a second query reads the view dependencies in `OBJECT_DEPENDENCIES` for the databases of the candidates and of the objects queries read directly. Anything those directly read objects are built on, through any number of views, is in use: an unread candidate reached that way is in use `via` the directly read object, with its readers. So a candidate view between a non-candidate view someone reads and a candidate table is kept too. Links through views in other databases are not followed. Dependencies created by `CREATE TABLE ... AS SELECT` and similar statements are not recorded there. Queries SQLBuild tags as its own (`QUERY_TAG` with `"app":"sqlbuild"`, which builds, audits, loads and hooks set) and `CREATE ... CLONE` statements of tables, dynamic tables, schemas and databases don't count as reads. A statement counts as a clone only when it starts with `CREATE`, after any comments, and names its source after `CLONE`; a column, string or comment containing the word doesn't make it one. SQLBuild's own reads don't need this check: schemas holding the project's sources are skipped, so the project graph already protects what the project reads. Access history needs Enterprise edition. The janitor's role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, or the `SNOWFLAKE.GOVERNANCE_VIEWER` database role for access history and the `SNOWFLAKE.OBJECT_VIEWER` database role for object dependencies: + +```sql +GRANT DATABASE ROLE SNOWFLAKE.GOVERNANCE_VIEWER TO ROLE janitor_role; +GRANT DATABASE ROLE SNOWFLAKE.OBJECT_VIEWER TO ROLE janitor_role; +``` + +Without `OBJECT_VIEWER`, reads are still checked, but view chains are not followed and the preview says `view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER`. Access history and object dependencies lag by up to about 3 hours, so reads in the last few hours may not show yet. + +On BigQuery, the janitor reads the `INFORMATION_SCHEMA.JOBS` view of each candidate's project, in the location of its dataset, and counts successful query jobs that reference the relation, by `user_email`. A job lists the views it reads as well as the tables behind them, so both count. Whether views nested inside other views are listed is not documented, so a candidate view that is only read through another view may not show as in use. Jobs answered from cached results record no tables or views, so with an `in_use_window` under about a day a reader whose queries were all cached can be missed. Jobs that run in another project are not in that view, so their reads are not seen. Script parent jobs, `CREATE SNAPSHOT TABLE` and `CREATE TABLE ... CLONE` don't count. SQLBuild sets no job labels on BigQuery, so its own jobs count like any other. The janitor's role needs `bigquery.jobs.listAll` and `bigquery.jobs.create` on each project, for example through the BigQuery Resource Viewer and BigQuery User roles. The view keeps 180 days of jobs, so a longer `in_use_window` sees only the last 180 days. + +On Databricks, the janitor counts read events in Unity Catalog's table lineage, by `created_by`. Lineage records the view a query reads as a direct read and the tables behind it as reads through the view, which the line names when lineage records the view for the same statement. It needs Unity Catalog with the `access` system schema enabled, and `USE CATALOG` on `system`, `USE SCHEMA` on `system.access` and `SELECT` on `system.access.table_lineage`: + +```sql +GRANT USE CATALOG ON CATALOG system TO `janitor-principal`; +GRANT USE SCHEMA ON SCHEMA system.access TO `janitor-principal`; +GRANT SELECT ON TABLE system.access.table_lineage TO `janitor-principal`; +``` + +Lineage records only reads Databricks can infer lineage for, and can arrive with a delay, so "not in use" is a weaker signal than on Snowflake. Lineage has no query text, so clones and SQLBuild's own reads can't be told apart and count as reads. The table keeps one year of events. + +On ClickHouse, the janitor counts finished `SELECT` and `INSERT` queries in `system.query_log` of the server it connects to, by the user who started them. A query's `tables` lists every table and view it touched, including the views and tables a view is built on, so a whole view chain shows as in use, without `via`; queries a query starts count once, for its initial query. Only the connected server's `query_log` is read: queries on other replicas or shards are logged on those servers and are not seen, which matters on ClickHouse Cloud and behind load-balanced replicas. When `system.clusters` shows other replicas of the connected server's shard, the preview says `usage read from this server only; other replicas not checked`. Writes into a relation count as use, since renaming it breaks the writer too. ClickHouse answers some queries, such as a plain `count()` of a table, from metadata without logging the table, so those reads are not seen. SQLBuild doesn't mark its ClickHouse queries, so its own reads count. The janitor's user needs `SELECT` on `system.query_log`: + +```sql +GRANT SELECT ON system.query_log TO janitor_user; +``` + +Queries are logged when `log_queries` is on, which is the default; when it is off for the janitor's connection, usage is not checked. Readers whose settings profile turns `log_queries` off, or logs only a sample with `log_queries_probability`, are missed. The log is flushed about every 7.5 seconds, and ClickHouse keeps it until a TTL configured on the server removes it. + +When usage can't be checked, because the adapter has no usage source or the role can't read its query history, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that relations in use are archived too. + +In the `--json` plan, each entry of `relations_to_archive` and `kept_in_use` has `in_use`: `null` when usage was not checked, `false` when nothing read the relation within the window, or an object with `last_read`, `readers`, `query_count` and `via` (`null` when the relation was read directly, otherwise the names of the views it was read through, empty when unknown). Entries of `kept_in_use` have only `name`, `age_days` and `in_use`, since they are not archived. The top-level `usage_check` has `status` (`checked` or `not_checked`), `reason`, `window`, `keep_in_use` and `notices` (what a check that ran left out, such as view chains it could not follow), or is `null` when there was nothing to check. ## Speed @@ -164,6 +225,8 @@ retention_days = 14 archive_retention_days = 14 delete_tracked_only = true exclude_patterns = ["audit_*", "tmp_*"] +in_use_window = "7d" +keep_in_use = false ``` When `retention_days` is not set, it defaults to `14`. See [Project Configuration](/docs/concepts/project-configuration#janitor) for every janitor setting. @@ -185,6 +248,9 @@ sqb janitor --as prod # Clean up the prod target sqb janitor --target prod + +# Keep relations that are still read instead of archiving them +sqb janitor --keep-in-use ``` ## State history pruning diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index d4acab89f7..da42b449b6 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -800,6 +800,8 @@ retention_days = 14 archive_retention_days = 14 delete_tracked_only = true exclude_patterns = ["audit_*", "tmp_*"] +in_use_window = "7d" +keep_in_use = false ``` | Field | Default | Description | @@ -810,6 +812,8 @@ exclude_patterns = ["audit_*", "tmp_*"] | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | | `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` | +| `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](/docs/cli/janitor#relations-in-use) | +| `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | See [janitor](/docs/cli/janitor) for the archive and delete lifecycle. diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 070b4e38f4..fdd98992e3 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -62,7 +62,8 @@ reviewed exceptions. Listed tests are not flagged and still count toward `SQBRTE [Empty-input tests](/docs/concepts/testing#empty-input-tests). `SQBRGRAPH103` and `SQBRPROJECT107` accept `published_layers`, and `SQBRPROJECT107` accepts -`allowed_qualifiers`, a list of `{ qualifier, reason }` tables with an optional `domain`. See +`allowed_qualifiers`, a list of `{ qualifier, reason }` or `{ paths, reason }` tables with an +optional `domain`; `qualifier` and `paths` accept globs. See [Published-layer Rules](/docs/concepts/rules/published-layer-rules). Unknown Rule codes, option names, and invalid option values fail configuration. diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx index e0c011bbdc..972fb6bd07 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx @@ -50,6 +50,17 @@ findings and cache entries match a single-process run. The result stays correct, longer runs in parallel and costs an extra host process. Keep each subject's findings independent of the others so Rules can run in parallel. +SQLBuild compares module state once per model, after every selected Rule has evaluated it, and +after each project-subject Rule invocation. A change still present then is reported against each +Rule whose module, or helper module it imports, holds the changed value, the same Rules a check after +every invocation would name. All of one model's Rules run in the same process, so a change undone +before the model's last Rule finishes, such as a scratch list one Rule fills and another empties for +the same model, is not detected and cannot change findings. Project Rules can run in different +processes, so they are still checked after each invocation. The same check decides which Rules the cache can track by the +facts they read: a Rule whose module state changed is cached against every project fact instead, so +any edit evaluates it again. `compile_timings` reports the time this check takes as +`custom_rules_monitor_cpu_ms`; see [`sqb compile`](/docs/cli/compile#json-output). + ## Helpers Custom Rules can import repository-owned helpers under `rules/` and these deterministic standard diff --git a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx index 7d9170ac54..3c9f1d8983 100644 --- a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx +++ b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx @@ -43,6 +43,10 @@ genuinely project-wide invariants, with correspondingly broader invalidation. Positive and negative project-tree observations are dependencies. If a Rule observes that a glob has no matches, adding a matching file invalidates that result. +`--no-cache` on `sqb compile`, `sqb plan`, and `sqb build` turns the Rules cache off for that +invocation: every built-in and custom Rule evaluates again, and stored results are neither read nor +written. Use it when a Rule finding looks stale. + ## Built-in-only execution When no custom Rule is selected, SQLBuild does not initialize the custom Python host. When every diff --git a/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx index a7fcb6472b..4b5ec4d085 100644 --- a/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx @@ -46,9 +46,9 @@ not fit the domain's majority shape are reported, with the remediation: example `sqb rename sales__mart__customers sales__mart__customers__core`. 3. If the qualifier is intentional and structural, allowlist it with a reason. -Allowlisted qualifiers are literal and do not count towards the shape: models that use them are left -out of the check. Each entry needs a reason, and an entry with a `domain` applies to that domain -only: +Allowlisted qualifiers do not count towards the shape: models that use them are left out of the +check and out of their domain's counts. Each entry needs a reason, and an entry with a `domain` +applies to that domain only: ```toml [rules.rule_options.SQBRPROJECT107] @@ -58,6 +58,34 @@ allowed_qualifiers = [ ] ``` +`qualifier` is a case-sensitive glob with the same grammar as selectors: `*` matches any run of +characters, `?` one character, and `[...]` one character from a set, with `[!...]` negating it. A +qualifier without wildcards matches only itself. One entry can cover a whole family of +qualifiers, such as a domain published once per country: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { domain = "sales", qualifier = "[a-z][a-z][a-z]", reason = "Published once per country." }, + { domain = "sales", qualifier = "[a-z][a-z][a-z]_snapshots", reason = "Daily snapshots per country." }, +] +``` + +An entry with `paths` instead leaves out the published models whose file path, relative to the +project and written with `/` separators, matches any of its globs. As in selectors and janitor +`exclude_patterns`, `*` also matches `/`, so `models/sales/mart/*` covers nested folders too. A +family published once per qualifier usually lives in one folder: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { paths = ["models/sales/mart/regional/*"], reason = "Published once per region." }, +] +``` + +An entry with both `qualifier` and `paths` leaves out only the models that match both. Published +models that match no entry are checked as before. + ## Published layers Both Rules treat `mart` and `mart_v` as the published layers. Set `published_layers` on either Rule