diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 95a778e52a..6ba7934b09 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -5198,6 +5198,7 @@ "nargs": 0, "const": true, "default": false, + "help": "skip the confirmation prompt and run the planned archives, deletes and prunes", "group": 1 }, { @@ -5207,6 +5208,7 @@ ], "dest": "retention_days", "type": "int", + "help": "days a stale relation must be unchanged before it is archived (overrides janitor.retention_days)", "group": 1 }, { @@ -5216,6 +5218,7 @@ ], "dest": "direct_state_history_versions", "type": "int", + "help": "state history rows to keep per identity when pruning direct state tables (overrides janitor.direct_state_history_versions)", "group": 1 }, { @@ -5226,6 +5229,32 @@ "dest": "drop_old_name_views", "default": [], "metavar": "NAME", + "help": "drop the compatibility view at this old model name before it expires; repeat for several views", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--verbose", + "-v" + ], + "dest": "verbose", + "nargs": 0, + "const": true, + "default": false, + "help": "list every entry of each preview section and every skipped object", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--json" + ], + "dest": "json", + "nargs": 0, + "const": true, + "default": false, + "help": "print the plan and result as JSON on stdout; progress goes to stderr", "group": 1 }, { @@ -5234,6 +5263,7 @@ "--target" ], "dest": "target", + "help": "clean up this configured target instead of the active target", "group": 1 }, { diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index da56abafec..9070e10680 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -484,7 +484,7 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); -const JANITOR_FIELDS: [Field; 8] = [ +const JANITOR_FIELDS: [Field; 10] = [ PROJECT_DIR, NO_COLOR, same("auto_approve"), @@ -496,6 +496,8 @@ const JANITOR_FIELDS: [Field; 8] = [ ), SELECTED_TARGET, same("as_target"), + same("verbose"), + JSON_OUTPUT, ]; const JANITOR_ARGUMENTS: [Field; 1] = [request(class("JanitorCommandRequest"), &JANITOR_FIELDS)]; pub(crate) const JANITOR_ROUTE: Route = routed( diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs index c6213b2bdb..fa60de261a 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod counts; +pub(crate) mod phases; +pub(crate) mod plan_json; pub(crate) mod plan_sections; diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs new file mode 100644 index 0000000000..98a0026abd --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs @@ -0,0 +1,47 @@ +//! Wording and durations of janitor apply phases. + +use crate::cli_text::main::format_count_noun::format_count_noun; +use crate::janitor_text::constants::{MINUTE_SECONDS, SECONDS_PER_HOUR, SECONDS_PER_MINUTE}; +use crate::janitor_text::models::JanitorPhase; + +/// The phase's present participle, past tense and singular noun. +pub(crate) fn phase_words(phase: JanitorPhase) -> (&'static str, &'static str, &'static str) { + match phase { + JanitorPhase::DeleteRelations => ("Deleting", "Deleted", "relation"), + JanitorPhase::Archive => ("Archiving", "Archived", "relation"), + JanitorPhase::DeleteArchives => ("Deleting", "Deleted", "expired archive"), + JanitorPhase::DropOldNameViews => ("Dropping", "Dropped", "old-name view"), + JanitorPhase::DropQueryArtifacts => ("Dropping", "Dropped", "query artifact"), + JanitorPhase::PruneDirectState => ("Pruning", "Pruned", "direct state table"), + } +} + +/// `count` with the singular noun for one and its plural otherwise. +pub(crate) fn noun(count: usize, singular: &str) -> String { + format_count_noun(i64::try_from(count).unwrap_or(i64::MAX), singular, None) +} + +/// Seconds with two decimals under a minute, then whole minutes and seconds. +pub(crate) fn duration(seconds: f64) -> String { + if seconds < MINUTE_SECONDS { + return format!("{:.2}s", seconds.max(0.0)); + } + let whole: u64 = seconds.round() as u64; + let (hours, minutes, rest) = ( + whole / SECONDS_PER_HOUR, + whole % SECONDS_PER_HOUR / SECONDS_PER_MINUTE, + whole % SECONDS_PER_MINUTE, + ); + if hours > 0 { + return format!("{hours}h {minutes:02}m {rest:02}s"); + } + format!("{minutes}m {rest:02}s") +} + +/// `name -> destination`, or the name alone. +pub(crate) fn flow(name: &str, destination: Option<&str>) -> String { + destination.map_or_else( + || name.to_owned(), + |destination| format!("{name} -> {destination}"), + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs new file mode 100644 index 0000000000..3a52cb0afc --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -0,0 +1,255 @@ +//! The JSON payload of `sqb janitor --json`. + +use crate::janitor_text::constants::{DROP_EARLY, DROP_EXPIRED}; +use crate::janitor_text::models::{ + JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText, OldNameViewText, + SourceSchemaText, +}; +use crate::json::models::{JsonInteger, JsonValue}; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn optional_text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, text) +} + +fn texts(values: &[String]) -> JsonValue { + JsonValue::Array(values.iter().map(|value| text(value)).collect()) +} + +fn integer(value: impl ToString) -> JsonValue { + JsonInteger::parse(&value.to_string()).map_or(JsonValue::Null, JsonValue::Integer) +} + +fn optional_integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, integer) +} + +fn object(fields: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + fields + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +/// A preview `YYYY-MM-DD HH:MM:SS` UTC time as ISO 8601. +fn utc_timestamp(value: &str) -> JsonValue { + if value.is_empty() { + return JsonValue::Null; + } + text(&format!("{}Z", value.replacen(' ', "T", 1))) +} + +fn old_name_view(view: &OldNameViewText, planned: &str) -> JsonValue { + let action: &str = match (planned, view.drop_reason.as_deref()) { + ("drop", Some(DROP_EARLY | DROP_EXPIRED)) => "drop", + ("live", _) => "keep", + _ => "record_dropped", + }; + object(vec![ + ("name", text(&view.name)), + ("destination_model", text(&view.destination_model)), + ("expires_on", optional_text(view.expires_on.as_deref())), + ("action", text(action)), + ("drop_reason", optional_text(view.drop_reason.as_deref())), + ("occupied", JsonValue::Bool(view.occupied)), + ("claimed_by", optional_text(view.claimed_by.as_deref())), + ]) +} + +fn source_schema(schema: &SourceSchemaText, blocked: bool) -> JsonValue { + let mut fields: Vec<(&str, JsonValue)> = vec![ + ("name", text(&schema.name)), + ("source_names", texts(&schema.source_names)), + ]; + if blocked { + fields.push(( + "suppressed_candidates", + texts(&schema.suppressed_candidates), + )); + fields.push(( + "suppressed_archive_deletions", + texts(&schema.suppressed_archive_deletions), + )); + } + object(fields) +} + +/// Every planned action, retained archive and skipped object of the plan. +pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { + let views: Vec = plan + .old_name_drops + .iter() + .map(|view| old_name_view(view, "drop")) + .chain( + plan.old_name_live + .iter() + .map(|view| old_name_view(view, "live")), + ) + .chain( + plan.old_name_missing + .iter() + .map(|view| old_name_view(view, "missing")), + ) + .collect(); + object(vec![ + ("direct_mode", JsonValue::Bool(plan.direct_mode)), + ("retention_days", integer(plan.retention_days)), + ( + "archive_retention_days", + integer(plan.archive_retention_days), + ), + ( + "age_metadata_supported", + JsonValue::Bool(plan.age_metadata_supported), + ), + ("schemas_scanned", integer(plan.scanned_schema_count)), + ("eligible_for_deletion", texts(&plan.candidates)), + ( + "relations_to_archive", + JsonValue::Array( + plan.archive_candidates + .iter() + .map(|candidate| { + object(vec![ + ("name", text(&candidate.name)), + ("archive_name", text(&candidate.archive_name)), + ("age_days", optional_integer(candidate.age_days)), + ("expires_at", utc_timestamp(&candidate.expires_at)), + ("deleted_in_run", JsonValue::Bool(candidate.expires_now)), + ]) + }) + .collect(), + ), + ), + ( + "archives_to_delete", + JsonValue::Array( + plan.archive_deletions + .iter() + .map(|archive| { + object(vec![ + ("name", text(&archive.name)), + ("archived_in_run", JsonValue::Bool(archive.archived_in_run)), + ("archived_at", utc_timestamp(&archive.archived_at)), + ("age_days", optional_integer(archive.age_days)), + ("expires_at", utc_timestamp(&archive.expires_at)), + ]) + }) + .collect(), + ), + ), + ( + "retained_archives", + JsonValue::Array( + plan.retained_archives + .iter() + .map(|archive| { + object(vec![ + ("name", text(&archive.name)), + ("archived_at", utc_timestamp(&archive.archived_at)), + ("expires_at", utc_timestamp(&archive.expires_at)), + ]) + }) + .collect(), + ), + ), + ("old_name_views", JsonValue::Array(views)), + ("query_artifacts", texts(&plan.query_artifacts)), + ( + "direct_state_prunes", + JsonValue::Array( + plan.state_prunes + .iter() + .map(|(name, versions)| { + object(vec![ + ("name", text(name)), + ("retain_versions", integer(*versions)), + ]) + }) + .collect(), + ), + ), + ( + "skipped_objects", + JsonValue::Array( + plan.skipped_relations + .iter() + .map(|(name, reason)| { + object(vec![("name", text(name)), ("reason", text(reason))]) + }) + .collect(), + ), + ), + ( + "skipped_schemas", + JsonValue::Array( + plan.skipped_schemas + .iter() + .map(|schema| source_schema(schema, false)) + .collect(), + ), + ), + ( + "blocked_schemas", + JsonValue::Array( + plan.blocked_schemas + .iter() + .map(|schema| source_schema(schema, true)) + .collect(), + ), + ), + ]) +} + +fn counts_payload(counts: &JanitorResultCounts) -> JsonValue { + object(vec![ + ("archived", integer(counts.archived)), + ("deleted", integer(counts.deleted)), + ("deleted_archives", integer(counts.deleted_archives)), + ( + "deleted_query_artifacts", + integer(counts.deleted_query_artifacts), + ), + ( + "dropped_old_name_views", + integer(counts.dropped_old_name_views), + ), + ("pruned_direct_state", integer(counts.pruned_direct_state)), + ]) +} + +fn item_payload(item: &JanitorItemText) -> JsonValue { + object(vec![ + ("phase", text(item.phase.value())), + ("action", text(&item.action)), + ("name", text(&item.name)), + ("destination", optional_text(item.destination.as_deref())), + ( + "status", + text(if item.error.is_some() { "failed" } else { "ok" }), + ), + ("error", optional_text(item.error.as_deref())), + ( + "duration_seconds", + item.duration_seconds + .map_or(JsonValue::Null, JsonValue::Float), + ), + ]) +} + +/// The counts, actions and duration of a run. +pub(crate) fn run_payload(run: &JanitorRunText) -> JsonValue { + object(vec![ + ("counts", counts_payload(&run.counts)), + ("elapsed_seconds", JsonValue::Float(run.elapsed_seconds)), + ( + "items", + JsonValue::Array(run.items.iter().map(item_payload).collect()), + ), + ]) +} diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs index 87d0aeb42f..776bf890c3 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs @@ -3,10 +3,29 @@ use crate::cli_text::models::CliStyle; use crate::janitor_text::constants::{ DROP_EARLY, DROP_EXPIRED, DROP_MISSING, LABEL_WIDTH, TREE_BLANK, TREE_BRANCH_GLYPH, - TREE_CONTINUATION, TREE_LAST_GLYPH, + TREE_CONTINUATION, TREE_LAST_GLYPH, VERBOSE_FLAG, }; use crate::janitor_text::models::{JanitorPlanText, OldNameViewText}; +/// The first `limit` items, or all of them, and how many are hidden. +fn visible(items: &[T], limit: Option) -> (&[T], usize) { + let shown: usize = limit.map_or(items.len(), |limit| items.len().min(limit)); + (&items[..shown], items.len() - shown) +} + +/// The dim `... and N more` line of a section that hides entries. +fn overflow(style: CliStyle, hidden: usize, indent: &str) -> String { + if hidden == 0 { + return String::new(); + } + format!( + "{}\n", + style.muted(&format!( + "{indent}... and {hidden} more (use {VERBOSE_FLAG} to show all)" + )) + ) +} + fn row(label: &str, rendered: &str) -> String { format!(" {label: String { } /// Schemas whose active sources block every janitor action. -pub(crate) fn blocked(plan: &JanitorPlanText, style: CliStyle) -> String { +pub(crate) fn blocked(plan: &JanitorPlanText, limit: Option, style: CliStyle) -> String { if plan.blocked_schemas.is_empty() { return String::new(); } @@ -126,18 +145,22 @@ pub(crate) fn blocked(plan: &JanitorPlanText, style: CliStyle) -> String { schema.source_names.join(", ") )) )); - for name in &schema.suppressed_candidates { + let (candidates, hidden) = visible(&schema.suppressed_candidates, limit); + for name in candidates { text.push_str(&format!( " {}\n", style.error_muted(&format!("suppressed {action}: {name}")) )); } - for name in &schema.suppressed_archive_deletions { + text.push_str(&overflow(style, hidden, " ")); + let (deletions, hidden) = visible(&schema.suppressed_archive_deletions, limit); + for name in deletions { text.push_str(&format!( " {}\n", style.error_muted(&format!("suppressed archive deletion: {name}")) )); } + text.push_str(&overflow(style, hidden, " ")); } text.push_str(&format!( " {}\n", @@ -147,11 +170,12 @@ pub(crate) fn blocked(plan: &JanitorPlanText, style: CliStyle) -> String { } /// Relations to archive, archives to delete, and retained archives. -pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { +pub(crate) fn archives(plan: &JanitorPlanText, limit: Option, style: CliStyle) -> String { let mut text: String = String::new(); if !plan.archive_candidates.is_empty() { text.push_str(&format!("\n{}\n", style.success("Relations to archive"))); - for candidate in &plan.archive_candidates { + let (candidates, hidden) = visible(&plan.archive_candidates, limit); + for candidate in candidates { let expiry: String = if candidate.expires_now { "deleted in this run".to_owned() } else { @@ -164,10 +188,12 @@ pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { style.muted(&format!("{}, {expiry}", age(candidate.age_days))) )); } + text.push_str(&overflow(style, hidden, " ")); } if !plan.archive_deletions.is_empty() { text.push_str(&format!("\n{}\n", style.success("Archives to delete"))); - for deletion in &plan.archive_deletions { + let (deletions, hidden) = visible(&plan.archive_deletions, limit); + for deletion in deletions { let detail: String = if deletion.archived_in_run { "archived in this run".to_owned() } else { @@ -184,10 +210,12 @@ pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { style.muted(&detail) )); } + text.push_str(&overflow(style, hidden, " ")); } if !plan.retained_archives.is_empty() { text.push_str(&format!("\n{}\n", style.success("Retained archives"))); - for retained in &plan.retained_archives { + let (retained_archives, hidden) = visible(&plan.retained_archives, limit); + for retained in retained_archives { text.push_str(&format!( " {} {}\n", style.object_name(&retained.name), @@ -197,6 +225,7 @@ pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { )) )); } + text.push_str(&overflow(style, hidden, " ")); } text } @@ -228,7 +257,11 @@ fn old_name_detail(view: &OldNameViewText) -> (&'static str, String) { } /// Compatibility views at old model names and what happens to each. -pub(crate) fn old_name_views(plan: &JanitorPlanText, style: CliStyle) -> String { +pub(crate) fn old_name_views( + plan: &JanitorPlanText, + limit: Option, + style: CliStyle, +) -> String { let listed: Vec<&OldNameViewText> = plan .old_name_drops .iter() @@ -243,7 +276,8 @@ pub(crate) fn old_name_views(plan: &JanitorPlanText, style: CliStyle) -> String style.success("Old name views"), style.muted(&format!("({})", listed.len())) ); - for (index, view) in listed.iter().enumerate() { + let (shown, hidden) = visible(&listed, limit); + for (index, view) in shown.iter().enumerate() { let last: bool = index + 1 == listed.len(); text.push_str(&format!( "{} {} {}\n", @@ -263,16 +297,28 @@ pub(crate) fn old_name_views(plan: &JanitorPlanText, style: CliStyle) -> String style.muted(label) )); } + if hidden > 0 { + text.push_str(&format!( + "{} {}", + tree_connector(style, true), + overflow(style, hidden, "") + )); + } text } /// A titled list of ` name` or ` name detail` lines, or nothing when empty. -pub(crate) fn listing(style: CliStyle, title: &str, items: &[(String, Option)]) -> String { +pub(crate) fn listing( + (style, limit): (CliStyle, Option), + title: &str, + items: &[(String, Option)], +) -> String { if items.is_empty() { return String::new(); } let mut text: String = format!("\n{}\n", style.success(title)); - for (name, detail) in items { + let (shown, hidden) = visible(items, limit); + for (name, detail) in shown { match detail { Some(detail) => text.push_str(&format!( " {} {}\n", @@ -282,5 +328,54 @@ pub(crate) fn listing(style: CliStyle, title: &str, items: &[(String, Option text.push_str(&format!(" {}\n", style.object_name(name))), } } + text.push_str(&overflow(style, hidden, " ")); + text +} + +/// Skipped objects: each with its reason when `limit` is none, otherwise counted by reason. +pub(crate) fn skipped_objects( + plan: &JanitorPlanText, + limit: Option, + style: CliStyle, +) -> String { + let Some(limit) = limit else { + let skipped: Vec<(String, Option)> = plan + .skipped_relations + .iter() + .map(|(name, reason)| (name.clone(), Some(reason.clone()))) + .collect(); + return listing((style, None), "Skipped objects", &skipped); + }; + if plan.skipped_relations.is_empty() { + return String::new(); + } + let mut groups: Vec<(usize, &str)> = Vec::new(); + for (_, reason) in &plan.skipped_relations { + match groups + .iter_mut() + .find(|(_, known)| *known == reason.as_str()) + { + Some((count, _)) => *count += 1, + None => groups.push((1, reason.as_str())), + } + } + groups.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(right.1))); + let width: usize = groups + .first() + .map_or(1, |(count, _)| count.to_string().len()); + let mut text: String = format!( + "\n{} {}\n", + style.success("Skipped objects"), + style.muted(&format!("({})", plan.skipped_relations.len())) + ); + let (shown, hidden) = visible(&groups, Some(limit)); + for (count, reason) in shown { + text.push_str(&format!( + " {} {}\n", + style.accent(&format!("{count:>width$}")), + style.muted(reason) + )); + } + text.push_str(&overflow(style, hidden, " ")); text } diff --git a/crates/sqlbuild-core/src/janitor_text/constants.rs b/crates/sqlbuild-core/src/janitor_text/constants.rs index d9ebfac56c..13f8108f08 100644 --- a/crates/sqlbuild-core/src/janitor_text/constants.rs +++ b/crates/sqlbuild-core/src/janitor_text/constants.rs @@ -11,3 +11,14 @@ pub(crate) const DROP_EXPIRED: &str = "expired"; pub(crate) const DROP_MISSING: &str = "missing"; pub(crate) const DISABLED_DETAIL: &str = "Janitor is opt-in. It previews stale warehouse objects \ and asks before archiving or deleting anything."; +/// How many entries each preview section shows without `--verbose`. +pub(crate) const PREVIEW_LIMIT: usize = 10; +pub(crate) const VERBOSE_FLAG: &str = "--verbose"; +/// The width of the longest janitor action value. +pub(crate) const ACTION_WIDTH: usize = "archived".len(); +pub(crate) const RECORDED_ACTION: &str = "recorded"; +pub(crate) const DROPPED_ACTION: &str = "dropped"; +pub(crate) const SECONDS_PER_MINUTE: u64 = 60; +/// Durations below this many seconds show seconds with two decimals. +pub(crate) const MINUTE_SECONDS: f64 = 60.0; +pub(crate) const SECONDS_PER_HOUR: u64 = 3600; diff --git a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs new file mode 100644 index 0000000000..8f644e58ab --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs @@ -0,0 +1,44 @@ +//! `sqb janitor --json`. + +use crate::janitor_text::_helpers::plan_json::{plan_payload, run_payload}; +use crate::janitor_text::models::{JanitorOutcome, JanitorPlanText, JanitorRunText}; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// `json.dumps(payload, indent=2)` of the outcome, the plan (none when disabled), run and error. +#[must_use] +pub fn format_janitor_json( + plan: Option<&JanitorPlanText>, + outcome: JanitorOutcome, + run: Option<&JanitorRunText>, + error: Option<&str>, +) -> String { + let payload: JsonValue = JsonValue::Object(vec![ + ( + "target".to_owned(), + plan.and_then(|plan| plan.target_name.as_deref()) + .map_or(JsonValue::Null, |name| JsonValue::String(name.to_owned())), + ), + ( + "status".to_owned(), + JsonValue::String(outcome.value().to_owned()), + ), + ( + "plan".to_owned(), + plan.map_or(JsonValue::Null, plan_payload), + ), + ( + "result".to_owned(), + run.map_or(JsonValue::Null, run_payload), + ), + ( + "error".to_owned(), + error.map_or(JsonValue::Null, |error| JsonValue::String(error.to_owned())), + ), + ]); + dumps( + &payload, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + ) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs deleted file mode 100644 index c6d4ef48b9..0000000000 --- a/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! The summary of a completed janitor run. - -use crate::cli_text::main::format_count_noun::format_count_noun; -use crate::cli_text::models::CliStyle; -use crate::janitor_text::models::JanitorResultCounts; - -fn noun(count: usize, singular: &str) -> String { - format_count_noun(i64::try_from(count).unwrap_or(i64::MAX), singular, None) -} - -/// `[Archived N relations. ]Deleted M objects[ and pruned K direct state tables].` -#[must_use] -pub fn janitor_completion_text(counts: &JanitorResultCounts, style: CliStyle) -> String { - let archived: String = if counts.archived == 0 { - String::new() - } else { - format!("Archived {}. ", noun(counts.archived, "relation")) - }; - let objects: String = noun( - counts.deleted - + counts.deleted_archives - + counts.deleted_query_artifacts - + counts.dropped_old_name_views, - "object", - ); - let deleted: String = if counts.pruned_direct_state == 0 { - format!("Deleted {objects}.") - } else { - format!( - "Deleted {objects} and pruned {}.", - noun(counts.pruned_direct_state, "direct state table") - ) - }; - format!("{}\n", style.success(&format!("{archived}{deleted}"))) -} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_events_unwritten_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_events_unwritten_text.rs new file mode 100644 index 0000000000..6a6e33d139 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_events_unwritten_text.rs @@ -0,0 +1,27 @@ +//! The warning that completed janitor actions have no audit events. + +use crate::cli_text::main::format_phase_line::format_phase_line; +use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::phases::noun; + +/// `✗ Audit events not written for 5 completed actions in analytics: `. +#[must_use] +pub fn janitor_events_unwritten_text( + (count, schemas): (usize, &[String]), + error: &str, + style: CliStyle, +) -> String { + format_phase_line( + style, + false, + ( + "Audit events not written", + &format!( + "for {} in {}: {}", + noun(count, "completed action"), + schemas.join(", "), + error.lines().next().unwrap_or_default() + ), + ), + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_item_counts.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_item_counts.rs new file mode 100644 index 0000000000..4a754c360f --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_item_counts.rs @@ -0,0 +1,24 @@ +//! What the finished actions of a janitor run changed. + +use crate::janitor_text::constants::DROPPED_ACTION; +use crate::janitor_text::models::{JanitorItemText, JanitorPhase, JanitorResultCounts}; + +/// Counts of the successful `items`, as the execution result would count them. +#[must_use] +pub fn janitor_item_counts(items: &[JanitorItemText]) -> JanitorResultCounts { + let mut counts: JanitorResultCounts = JanitorResultCounts::default(); + for item in items.iter().filter(|item| item.error.is_none()) { + match item.phase { + JanitorPhase::DeleteRelations => counts.deleted += 1, + JanitorPhase::Archive => counts.archived += 1, + JanitorPhase::DeleteArchives => counts.deleted_archives += 1, + JanitorPhase::DropOldNameViews if item.action == DROPPED_ACTION => { + counts.dropped_old_name_views += 1; + } + JanitorPhase::DropOldNameViews => {} + JanitorPhase::DropQueryArtifacts => counts.deleted_query_artifacts += 1, + JanitorPhase::PruneDirectState => counts.pruned_direct_state += 1, + } + } + counts +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_nothing_to_do_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_nothing_to_do_text.rs new file mode 100644 index 0000000000..253f9d8ef1 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_nothing_to_do_text.rs @@ -0,0 +1,18 @@ +//! The close of a janitor run whose plan changes nothing. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::models::{CliStyle, CompletionState}; + +/// `✓ Janitor completed nothing to clean up`. +#[must_use] +pub fn janitor_nothing_to_do_text(style: CliStyle) -> String { + format!( + "{}\n", + format_completion_line( + style, + CompletionState::Ok, + "Janitor completed", + Some(&style.muted("nothing to clean up")) + ) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_complete_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_complete_text.rs new file mode 100644 index 0000000000..5920bcc517 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_complete_text.rs @@ -0,0 +1,33 @@ +//! The close of one janitor apply phase. + +use crate::janitor_text::_helpers::phases::{duration, noun, phase_words}; +use crate::janitor_text::constants::RECORDED_ACTION; +use crate::janitor_text::models::{JanitorItemText, JanitorPhase}; + +/// `Archived 608 relations. (4m 51s)`, also counting old-name views recorded as already gone. +#[must_use] +pub fn janitor_phase_complete_text( + phase: JanitorPhase, + items: &[JanitorItemText], + elapsed_seconds: f64, +) -> String { + let (_, done, singular) = phase_words(phase); + let finished: Vec<&JanitorItemText> = items + .iter() + .filter(|item| item.phase == phase && item.error.is_none()) + .collect(); + let recorded: usize = finished + .iter() + .filter(|item| item.action == RECORDED_ACTION) + .count(); + let recorded_text: String = if recorded == 0 { + String::new() + } else { + format!(" and recorded {recorded} already gone") + }; + format!( + "{done} {}{recorded_text}. ({})", + noun(finished.len() - recorded, singular), + duration(elapsed_seconds) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_failed_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_failed_text.rs new file mode 100644 index 0000000000..9a2016a824 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_failed_text.rs @@ -0,0 +1,29 @@ +//! The close of a janitor apply phase that an action stopped. + +use crate::cli_text::main::format_phase_line::format_phase_line; +use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::phases::{duration, noun, phase_words}; +use crate::janitor_text::models::JanitorPhase; + +/// `✗ Archiving failed after 11 of 608 relations (1m 02s)`. +#[must_use] +pub fn janitor_phase_failed_text( + phase: JanitorPhase, + (completed, total): (usize, usize), + elapsed_seconds: f64, + style: CliStyle, +) -> String { + let (doing, _, singular) = phase_words(phase); + format_phase_line( + style, + false, + ( + &format!("{doing} failed"), + &format!( + "after {completed} of {} ({})", + noun(total, singular), + duration(elapsed_seconds) + ), + ), + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_flow_width.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_flow_width.rs new file mode 100644 index 0000000000..2920f2b22c --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_flow_width.rs @@ -0,0 +1,39 @@ +//! The width of a janitor phase's action column. + +use crate::janitor_text::_helpers::phases::flow; +use crate::janitor_text::models::{JanitorPhase, JanitorPlanText}; + +/// The widest plain `name` or `name -> destination` the plan's `phase` reports. +#[must_use] +pub fn janitor_phase_flow_width(plan: &JanitorPlanText, phase: JanitorPhase) -> usize { + let flows: Vec = match phase { + JanitorPhase::DeleteRelations => plan.candidates.clone(), + JanitorPhase::Archive => plan + .archive_candidates + .iter() + .map(|candidate| flow(&candidate.name, Some(&candidate.archive_name))) + .collect(), + JanitorPhase::DeleteArchives => plan + .archive_deletions + .iter() + .map(|archive| archive.name.clone()) + .collect(), + JanitorPhase::DropOldNameViews => plan + .old_name_drops + .iter() + .chain(&plan.old_name_missing) + .map(|view| view.name.clone()) + .collect(), + JanitorPhase::DropQueryArtifacts => plan.query_artifacts.clone(), + JanitorPhase::PruneDirectState => plan + .state_prunes + .iter() + .map(|(name, _)| name.clone()) + .collect(), + }; + flows + .iter() + .map(|flow| flow.chars().count()) + .max() + .unwrap_or(0) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_start_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_start_text.rs new file mode 100644 index 0000000000..dd4e42d46e --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_start_text.rs @@ -0,0 +1,11 @@ +//! The announcement of one janitor apply phase. + +use crate::janitor_text::_helpers::phases::{noun, phase_words}; +use crate::janitor_text::models::JanitorPhase; + +/// `Archiving 608 relations...`. +#[must_use] +pub fn janitor_phase_start_text(phase: JanitorPhase, total: usize) -> String { + let (doing, _, singular) = phase_words(phase); + format!("{doing} {}...", noun(total, singular)) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/mod.rs b/crates/sqlbuild-core/src/janitor_text/main/mod.rs index d73e81e797..eb83932969 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/mod.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/mod.rs @@ -1,9 +1,18 @@ -pub mod janitor_completion_text; +pub mod format_janitor_json; pub mod janitor_confirmation_phrase; pub mod janitor_confirmation_prompt; pub mod janitor_disabled_text; +pub mod janitor_events_unwritten_text; +pub mod janitor_item_counts; +pub mod janitor_nothing_to_do_text; +pub mod janitor_phase_complete_text; +pub mod janitor_phase_failed_text; +pub mod janitor_phase_flow_width; +pub mod janitor_phase_start_text; pub mod janitor_plan_has_work; pub mod janitor_preview_complete_text; pub mod janitor_preview_start_text; +pub mod render_janitor_completion; +pub mod render_janitor_item_line; pub mod render_janitor_plan; pub mod validate_janitor_settings; diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs new file mode 100644 index 0000000000..8523335723 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs @@ -0,0 +1,44 @@ +//! The final line of a janitor run. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::models::{CliStyle, CompletionState}; +use crate::janitor_text::_helpers::phases::duration; +use crate::janitor_text::models::JanitorResultCounts; + +/// `✓ Janitor completed archived N, deleted M, pruned K (5m 02s)`, or `✗ Janitor failed ...`. +#[must_use] +pub fn render_janitor_completion( + counts: &JanitorResultCounts, + elapsed_seconds: Option, + failed: bool, + style: CliStyle, +) -> String { + let deleted: usize = counts.deleted + + counts.deleted_archives + + counts.deleted_query_artifacts + + counts.dropped_old_name_views; + let mut summary: String = [ + ("archived", counts.archived), + ("deleted", deleted), + ("pruned", counts.pruned_direct_state), + ] + .iter() + .map(|(label, count)| format!("{label} {}", style.value(&count.to_string()))) + .collect::>() + .join(", "); + if let Some(seconds) = elapsed_seconds { + summary.push_str(&format!( + " {}", + style.muted(&format!("({})", duration(seconds))) + )); + } + let (state, label) = if failed { + (CompletionState::Fail, "Janitor failed") + } else { + (CompletionState::Ok, "Janitor completed") + }; + format!( + "{}\n", + format_completion_line(style, state, label, Some(&summary)) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs new file mode 100644 index 0000000000..0dec945eeb --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs @@ -0,0 +1,46 @@ +//! One streamed janitor action line. + +use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::phases::{duration, flow}; +use crate::janitor_text::constants::ACTION_WIDTH; +use crate::janitor_text::models::JanitorItemText; + +/// Position, action, padded `name -> destination`, status, duration and any error of one action. +#[must_use] +pub fn render_janitor_item_line( + (index, total): (usize, usize), + item: &JanitorItemText, + flow_width: usize, + style: CliStyle, +) -> String { + let position: String = format!("{index:>width$}/{total}", width = total.to_string().len()); + let plain_flow: String = flow(&item.name, item.destination.as_deref()); + let styled_flow: String = match &item.destination { + Some(destination) => format!( + "{} {} {}", + style.object_name(&item.name), + style.muted("->"), + style.object_name(destination) + ), + None => style.object_name(&item.name), + }; + let plain_width: usize = plain_flow.chars().count(); + let flow_padding: String = " ".repeat(flow_width.max(plain_width) - plain_width); + let action_padding: String = + " ".repeat(ACTION_WIDTH.saturating_sub(item.action.chars().count())); + let status: &str = if item.error.is_some() { "FAIL" } else { "OK" }; + let elapsed: String = item.duration_seconds.map_or_else(String::new, |seconds| { + format!(" {}", style.muted(&duration(seconds))) + }); + let error: String = item + .error + .as_deref() + .and_then(|error| error.lines().next()) + .map_or_else(String::new, |error| format!(" {}", style.error(error))); + format!( + " {} {}{action_padding} {styled_flow}{flow_padding} {}{elapsed}{error}", + style.muted(&position), + style.accent(&item.action), + style.status(status) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs index f9191b40ed..4c3747f3be 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs @@ -3,20 +3,22 @@ use crate::cli_text::models::CliStyle; use crate::janitor_text::_helpers::counts::environment_label; use crate::janitor_text::_helpers::plan_sections::{ - archives, blocked, listing, old_name_views, summary, + archives, blocked, listing, old_name_views, skipped_objects, summary, }; +use crate::janitor_text::constants::PREVIEW_LIMIT; use crate::janitor_text::models::JanitorPlanText; -/// Every section of the preview, ending with a blank line. +/// Every preview section, capped and grouped unless `verbose`, ending with a blank line. #[must_use] -pub fn render_janitor_plan(plan: &JanitorPlanText, style: CliStyle) -> String { +pub fn render_janitor_plan(plan: &JanitorPlanText, verbose: bool, style: CliStyle) -> String { + let limit: Option = (!verbose).then_some(PREVIEW_LIMIT); let mut text: String = format!( "{} {}\n\n", style.title("Janitor preview"), style.object_name(environment_label(plan)) ); text.push_str(&summary(plan, style)); - text.push_str(&blocked(plan, style)); + text.push_str(&blocked(plan, limit, style)); let skipped_schemas: Vec<(String, Option)> = plan .skipped_schemas .iter() @@ -30,24 +32,28 @@ pub fn render_janitor_plan(plan: &JanitorPlanText, style: CliStyle) -> String { ) }) .collect(); - text.push_str(&listing(style, "Skipped schemas", &skipped_schemas)); + text.push_str(&listing( + (style, limit), + "Skipped schemas", + &skipped_schemas, + )); if !plan.direct_mode { let candidates: Vec<(String, Option)> = plan .candidates .iter() .map(|name| (name.clone(), None)) .collect(); - text.push_str(&listing(style, "Eligible objects", &candidates)); + text.push_str(&listing((style, limit), "Eligible objects", &candidates)); } - text.push_str(&archives(plan, style)); - text.push_str(&old_name_views(plan, style)); + text.push_str(&archives(plan, limit, style)); + text.push_str(&old_name_views(plan, limit, style)); let artifacts: Vec<(String, Option)> = plan .query_artifacts .iter() .map(|name| (name.clone(), None)) .collect(); text.push_str(&listing( - style, + (style, limit), "Eligible expired query artifacts", &artifacts, )); @@ -56,13 +62,12 @@ pub fn render_janitor_plan(plan: &JanitorPlanText, style: CliStyle) -> String { .iter() .map(|(name, versions)| (name.clone(), Some(format!("keep latest {versions}")))) .collect(); - text.push_str(&listing(style, "Eligible direct state pruning", &prunes)); - let skipped: Vec<(String, Option)> = plan - .skipped_relations - .iter() - .map(|(name, reason)| (name.clone(), Some(reason.clone()))) - .collect(); - text.push_str(&listing(style, "Skipped objects", &skipped)); + text.push_str(&listing( + (style, limit), + "Eligible direct state pruning", + &prunes, + )); + text.push_str(&skipped_objects(plan, limit, style)); text.push('\n'); text } diff --git a/crates/sqlbuild-core/src/janitor_text/models.rs b/crates/sqlbuild-core/src/janitor_text/models.rs index b979149679..cbd49ccd0b 100644 --- a/crates/sqlbuild-core/src/janitor_text/models.rs +++ b/crates/sqlbuild-core/src/janitor_text/models.rs @@ -81,3 +81,98 @@ pub struct JanitorResultCounts { pub dropped_old_name_views: usize, pub pruned_direct_state: usize, } + +/// One janitor apply phase, in execution order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JanitorPhase { + DeleteRelations, + Archive, + DeleteArchives, + DropOldNameViews, + DropQueryArtifacts, + PruneDirectState, +} + +impl JanitorPhase { + /// The phase from the executor's `JanitorPhase` value. + #[must_use] + pub fn from_value(value: &str) -> Option { + match value { + "delete_relations" => Some(Self::DeleteRelations), + "archive" => Some(Self::Archive), + "delete_archives" => Some(Self::DeleteArchives), + "drop_old_name_views" => Some(Self::DropOldNameViews), + "drop_query_artifacts" => Some(Self::DropQueryArtifacts), + "prune_direct_state" => Some(Self::PruneDirectState), + _ => None, + } + } + + /// The executor's `JanitorPhase` value. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::DeleteRelations => "delete_relations", + Self::Archive => "archive", + Self::DeleteArchives => "delete_archives", + Self::DropOldNameViews => "drop_old_name_views", + Self::DropQueryArtifacts => "drop_query_artifacts", + Self::PruneDirectState => "prune_direct_state", + } + } +} + +/// One finished janitor action. +#[derive(Clone, Debug, PartialEq)] +pub struct JanitorItemText { + pub phase: JanitorPhase, + /// The `JanitorAction` value, such as `archived`. + pub action: String, + pub name: String, + /// Where the action moved the relation, such as its archive name. + pub destination: Option, + /// The error that stopped the action, or none when it succeeded. + pub error: Option, + pub duration_seconds: Option, +} + +/// The actions a janitor run finished, what it changed, and how long it took. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct JanitorRunText { + pub items: Vec, + pub counts: JanitorResultCounts, + pub elapsed_seconds: f64, +} + +/// How a janitor invocation ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JanitorOutcome { + /// Janitor is not enabled for the project. + Disabled, + /// An inspection-only `--as` preview. + Previewed, + /// Active sources block every action. + Blocked, + /// The plan changes nothing. + NothingToDo, + Cancelled, + Completed, + /// An action failed after the cleanup started. + Failed, +} + +impl JanitorOutcome { + /// The `status` of `sqb janitor --json`. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::Disabled => "disabled", + Self::Previewed => "previewed", + Self::Blocked => "blocked", + Self::NothingToDo => "nothing_to_do", + Self::Cancelled => "cancelled", + Self::Completed => "completed", + Self::Failed => "failed", + } + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests.rs b/crates/sqlbuild-core/src/janitor_text/tests.rs index 67b4ac8a4c..3735ed1a78 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_janitor_progress_text.rs"] +mod janitor_progress_text; #[path = "tests/test_janitor_text.rs"] mod janitor_text; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index f304628d0a..48c075193d 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -1,4 +1,7 @@ -use crate::janitor_text::models::{ArchiveCandidateText, ArchivedRelationText, JanitorPlanText}; +use crate::janitor_text::models::{ + ArchiveCandidateText, ArchivedRelationText, JanitorItemText, JanitorPhase, JanitorPlanText, + OldNameViewText, +}; /// A direct-mode plan for `dev` that archives one relation and deletes one archive. pub(super) fn direct_archive_plan() -> JanitorPlanText { @@ -18,3 +21,60 @@ pub(super) fn direct_archive_plan() -> JanitorPlanText { ..JanitorPlanText::default() } } + +/// A finished action of `phase`, failed when `error` is given. +pub(super) fn item( + phase: JanitorPhase, + (action, name, destination): (&str, &str, Option<&str>), + error: Option<&str>, +) -> JanitorItemText { + JanitorItemText { + phase, + action: action.to_owned(), + name: name.to_owned(), + destination: destination.map(str::to_owned), + error: error.map(str::to_owned), + duration_seconds: Some(0.5), + } +} + +/// A direct-mode plan archiving `count` relations, with `skipped` objects for two reasons. +pub(super) fn large_plan(count: usize, skipped: usize) -> JanitorPlanText { + JanitorPlanText { + target_name: Some("dev".to_owned()), + retention_days: 14, + direct_mode: true, + archive_retention_days: 14, + archive_candidates: (0..count) + .map(|index| ArchiveCandidateText { + name: format!("dev.old_orders_{index:03}"), + archive_name: format!("dev._sqb_archive__20260924t101500z__old_orders_{index:03}"), + ..ArchiveCandidateText::default() + }) + .collect(), + skipped_relations: (0..skipped) + .map(|index| { + let reasons: [&str; 2] = + ["relation is newer than 14 days", "not tracked by SQLBuild"]; + let reason: &str = reasons[usize::from(index % 3 != 0)]; + (format!("dev.recent_{index:03}"), reason.to_owned()) + }) + .collect(), + ..JanitorPlanText::default() + } +} + +/// A plan with `count` live old-name views expiring on one day. +pub(super) fn old_name_plan(count: usize) -> JanitorPlanText { + JanitorPlanText { + old_name_live: (0..count) + .map(|index| OldNameViewText { + name: format!("dev.orders_v{index:02}"), + destination_model: format!("orders_{index:02}"), + expires_on: Some("2026-11-05".to_owned()), + ..OldNameViewText::default() + }) + .collect(), + ..JanitorPlanText::default() + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs new file mode 100644 index 0000000000..ae0fa83a86 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs @@ -0,0 +1,421 @@ +use crate::cli_text::models::CliStyle; +use crate::janitor_text::main::format_janitor_json::format_janitor_json; +use crate::janitor_text::main::janitor_events_unwritten_text::janitor_events_unwritten_text; +use crate::janitor_text::main::janitor_item_counts::janitor_item_counts; +use crate::janitor_text::main::janitor_nothing_to_do_text::janitor_nothing_to_do_text; +use crate::janitor_text::main::janitor_phase_complete_text::janitor_phase_complete_text; +use crate::janitor_text::main::janitor_phase_failed_text::janitor_phase_failed_text; +use crate::janitor_text::main::janitor_phase_flow_width::janitor_phase_flow_width; +use crate::janitor_text::main::janitor_phase_start_text::janitor_phase_start_text; +use crate::janitor_text::main::render_janitor_completion::render_janitor_completion; +use crate::janitor_text::main::render_janitor_item_line::render_janitor_item_line; +use crate::janitor_text::main::render_janitor_plan::render_janitor_plan; +use crate::janitor_text::models::{ + JanitorOutcome, JanitorPhase, JanitorResultCounts, JanitorRunText, +}; +use crate::janitor_text::tests::helpers::{direct_archive_plan, item, large_plan, old_name_plan}; +use crate::janitor_text::tests::test_types::{ + CompletionTestCase, EventsUnwrittenTestCase, ItemCountsTestCase, ItemLineTestCase, + JsonTestCase, PhaseTextTestCase, PreviewTestCase, +}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_finished_action_when_rendering_then_line_aligns_position_flow_and_status() { + let test_cases = [ + ItemLineTestCase { + description: "a rename pads its flow to the phase width", + position: (1, 12), + item: item( + JanitorPhase::Archive, + ("archived", "dev.orders", Some("dev._a")), + None, + ), + flow_width: 24, + expected_line: " 1/12 archived dev.orders -> dev._a OK 0.50s", + }, + ItemLineTestCase { + description: "a failure shows the first line of its error", + position: (3, 3), + item: item( + JanitorPhase::DeleteArchives, + ("deleted", "dev.old", None), + Some("permission denied\nquery id 1"), + ), + flow_width: 0, + expected_line: " 3/3 deleted dev.old FAIL 0.50s permission denied", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_janitor_item_line( + test_case.position, + &test_case.item, + test_case.flow_width, + PLAIN + ), + test_case.expected_line, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_phase_when_rendering_progress_then_start_complete_and_failure_count_its_actions() { + let test_cases = [ + PhaseTextTestCase { + description: "one archive uses singular nouns and seconds", + phase: JanitorPhase::Archive, + items: vec![item( + JanitorPhase::Archive, + ("archived", "dev.a", Some("dev.b")), + None, + )], + total: 1, + elapsed_seconds: 0.25, + expected_start: "Archiving 1 relation...", + expected_complete: "Archived 1 relation. (0.25s)", + expected_failed: "\u{2717} Archiving failed after 1 of 1 relation (0.25s)", + }, + PhaseTextTestCase { + description: "old-name views count drops apart from views already gone", + phase: JanitorPhase::DropOldNameViews, + items: vec![ + item( + JanitorPhase::DropOldNameViews, + ("dropped", "dev.a", None), + None, + ), + item( + JanitorPhase::DropOldNameViews, + ("recorded", "dev.b", None), + None, + ), + item( + JanitorPhase::DropOldNameViews, + ("dropped", "dev.c", None), + Some("denied"), + ), + ], + total: 3, + elapsed_seconds: 75.4, + expected_start: "Dropping 3 old-name views...", + expected_complete: "Dropped 1 old-name view and recorded 1 already gone. (1m 15s)", + expected_failed: "\u{2717} Dropping failed after 2 of 3 old-name views (1m 15s)", + }, + PhaseTextTestCase { + description: "hour-long phases show hours, minutes and seconds", + phase: JanitorPhase::PruneDirectState, + items: Vec::new(), + total: 2, + elapsed_seconds: 3725.0, + expected_start: "Pruning 2 direct state tables...", + expected_complete: "Pruned 0 direct state tables. (1h 02m 05s)", + expected_failed: "\u{2717} Pruning failed after 0 of 2 direct state tables (1h 02m 05s)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + janitor_phase_start_text(test_case.phase, test_case.total), + test_case.expected_start, + "{}", + test_case.description + ); + assert_eq!( + janitor_phase_complete_text( + test_case.phase, + &test_case.items, + test_case.elapsed_seconds + ), + test_case.expected_complete, + "{}", + test_case.description + ); + let completed: usize = test_case + .items + .iter() + .filter(|item| item.error.is_none()) + .count(); + assert_eq!( + janitor_phase_failed_text( + test_case.phase, + (completed, test_case.total), + test_case.elapsed_seconds, + PLAIN + ), + test_case.expected_failed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_run_counts_when_rendering_completion_then_final_line_is_unambiguous() { + let test_cases = [ + CompletionTestCase { + description: "a completed run totals archives, deletions and prunes", + counts: JanitorResultCounts { + archived: 608, + deleted_archives: 6, + pruned_direct_state: 6, + ..JanitorResultCounts::default() + }, + elapsed_seconds: Some(302.0), + failed: false, + expected_text: "\u{2713} Janitor completed archived 608, deleted 6, pruned 6 (5m 02s)\n", + }, + CompletionTestCase { + description: "a failed run reports what finished before the failure", + counts: JanitorResultCounts { + archived: 11, + ..JanitorResultCounts::default() + }, + elapsed_seconds: Some(65.0), + failed: true, + expected_text: "\u{2717} Janitor failed archived 11, deleted 0, pruned 0 (1m 05s)\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_janitor_completion( + &test_case.counts, + test_case.elapsed_seconds, + test_case.failed, + PLAIN + ), + test_case.expected_text, + "{}", + test_case.description + ); + } + assert_eq!( + janitor_nothing_to_do_text(PLAIN), + "\u{2713} Janitor completed nothing to clean up\n" + ); +} + +#[test] +fn given_finished_actions_when_counting_then_only_successes_count_by_kind() { + let test_cases = [ItemCountsTestCase { + description: "failed actions and views already gone are not counted", + items: vec![ + item(JanitorPhase::Archive, ("archived", "a", Some("b")), None), + item( + JanitorPhase::Archive, + ("archived", "c", Some("d")), + Some("denied"), + ), + item(JanitorPhase::DeleteArchives, ("deleted", "e", None), None), + item(JanitorPhase::DropOldNameViews, ("dropped", "f", None), None), + item( + JanitorPhase::DropOldNameViews, + ("recorded", "g", None), + None, + ), + item( + JanitorPhase::DropQueryArtifacts, + ("dropped", "h", None), + None, + ), + item(JanitorPhase::PruneDirectState, ("pruned", "i", None), None), + ], + plan: large_plan(3, 0), + expected_counts: JanitorResultCounts { + archived: 1, + deleted: 0, + deleted_archives: 1, + deleted_query_artifacts: 1, + dropped_old_name_views: 1, + pruned_direct_state: 1, + }, + expected_archive_flow_width: 72, + }]; + + for test_case in test_cases { + assert_eq!( + janitor_item_counts(&test_case.items), + test_case.expected_counts, + "{}", + test_case.description + ); + assert_eq!( + janitor_phase_flow_width(&test_case.plan, JanitorPhase::Archive), + test_case.expected_archive_flow_width, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_long_plan_when_previewing_then_lists_are_capped_unless_verbose() { + let test_cases = [ + PreviewTestCase { + description: "long lists show ten entries and skipped objects group by reason", + plan: large_plan(25, 7), + verbose: false, + expected_fragments: &[ + " dev.old_orders_009 -> ", + " ... and 15 more (use --verbose to show all)\n", + "Skipped objects (7)\n 4 not tracked by SQLBuild\n 3 relation is newer than 14 days\n", + " relations to archive 25\n", + ], + unexpected_fragments: &["dev.old_orders_010 ->", "dev.recent_000"], + }, + PreviewTestCase { + description: "verbose lists every entry and every skipped object", + plan: large_plan(25, 7), + verbose: true, + expected_fragments: &[ + " dev.old_orders_024 -> ", + "Skipped objects\n dev.recent_000 relation is newer than 14 days\n", + ], + unexpected_fragments: &["... and", "Skipped objects (7)"], + }, + PreviewTestCase { + description: "a capped old-name view tree closes with its overflow line", + plan: old_name_plan(12), + verbose: false, + expected_fragments: &[ + "\u{251c}\u{2500}\u{2500} dev.orders_v09 -> model:orders_09\n\u{2502} \u{2514}\u{2500}\u{2500} expires", + "\u{2514}\u{2500}\u{2500} ... and 2 more (use --verbose to show all)\n", + ], + unexpected_fragments: &["\u{2514}\u{2500}\u{2500} dev.orders_v09", "dev.orders_v10"], + }, + PreviewTestCase { + description: "an uncapped old-name view tree closes with its last view", + plan: old_name_plan(3), + verbose: false, + expected_fragments: &[ + "\u{2514}\u{2500}\u{2500} dev.orders_v02 -> model:orders_02\n \u{2514}", + ], + unexpected_fragments: &["... and"], + }, + ]; + + for test_case in test_cases { + let text: String = render_janitor_plan(&test_case.plan, test_case.verbose, PLAIN); + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: missing fragments in {text}", + test_case.description + ); + assert!( + !test_case + .unexpected_fragments + .iter() + .any(|fragment| text.contains(fragment)), + "{}: unexpected fragments in {text}", + test_case.description + ); + } +} + +#[test] +fn given_outcome_when_formatting_json_then_document_carries_plan_result_and_error() { + let run: JanitorRunText = JanitorRunText { + items: vec![item( + JanitorPhase::Archive, + ("archived", "dev.old_orders", Some("dev._a")), + None, + )], + counts: JanitorResultCounts { + archived: 1, + ..JanitorResultCounts::default() + }, + elapsed_seconds: 1.5, + }; + let test_cases = [ + JsonTestCase { + description: "a completed run lists its actions", + plan: Some(direct_archive_plan()), + outcome: JanitorOutcome::Completed, + run: Some(run.clone()), + error: None, + expected_fragments: &[ + "\"target\": \"dev\"", + "\"status\": \"completed\"", + "\"relations_to_archive\": [", + "\"name\": \"dev.old_orders\"", + "\"phase\": \"archive\"", + "\"status\": \"ok\"", + "\"archived\": 1", + "\"error\": null", + ], + }, + JsonTestCase { + description: "a failed run carries its error", + plan: Some(direct_archive_plan()), + outcome: JanitorOutcome::Failed, + run: Some(run), + error: Some("rename denied"), + expected_fragments: &["\"status\": \"failed\"", "\"error\": \"rename denied\""], + }, + JsonTestCase { + description: "a preview has no result", + plan: Some(direct_archive_plan()), + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &["\"status\": \"previewed\"", "\"result\": null"], + }, + JsonTestCase { + description: "a disabled janitor has no plan", + plan: None, + outcome: JanitorOutcome::Disabled, + run: None, + error: None, + expected_fragments: &["\"status\": \"disabled\"", "\"plan\": null"], + }, + ]; + + for test_case in test_cases { + let text: String = format_janitor_json( + test_case.plan.as_ref(), + test_case.outcome, + test_case.run.as_ref(), + test_case.error, + ); + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: missing fragments in {text}", + test_case.description + ); + } +} + +#[test] +fn given_failed_event_write_when_warning_then_line_counts_actions_per_schema() { + let test_cases = [EventsUnwrittenTestCase { + description: "the warning names the schemas and the first line of the error", + count: 5, + schemas: vec!["analytics".to_owned(), "staging".to_owned()], + error: "events table is locked\nquery id 7", + expected_text: "\u{2717} Audit events not written for 5 completed actions in analytics, staging: events table is locked", + }]; + + for test_case in test_cases { + assert_eq!( + janitor_events_unwritten_text( + (test_case.count, &test_case.schemas), + test_case.error, + PLAIN + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs index 6a91f57366..ae6072dc7c 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs @@ -1,4 +1,7 @@ -use crate::janitor_text::models::JanitorPlanText; +use crate::janitor_text::models::{ + JanitorItemText, JanitorOutcome, JanitorPhase, JanitorPlanText, JanitorResultCounts, + JanitorRunText, +}; pub(super) struct ConfirmationTestCase { pub(super) description: &'static str, @@ -18,3 +21,63 @@ pub(super) struct DisabledTestCase { pub(super) use_color: bool, pub(super) expected_text: &'static str, } + +pub(super) struct ItemLineTestCase { + pub(super) description: &'static str, + pub(super) position: (usize, usize), + pub(super) item: JanitorItemText, + pub(super) flow_width: usize, + pub(super) expected_line: &'static str, +} + +pub(super) struct PhaseTextTestCase { + pub(super) description: &'static str, + pub(super) phase: JanitorPhase, + pub(super) items: Vec, + pub(super) total: usize, + pub(super) elapsed_seconds: f64, + pub(super) expected_start: &'static str, + pub(super) expected_complete: &'static str, + pub(super) expected_failed: &'static str, +} + +pub(super) struct CompletionTestCase { + pub(super) description: &'static str, + pub(super) counts: JanitorResultCounts, + pub(super) elapsed_seconds: Option, + pub(super) failed: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct ItemCountsTestCase { + pub(super) description: &'static str, + pub(super) items: Vec, + pub(super) plan: JanitorPlanText, + pub(super) expected_counts: JanitorResultCounts, + pub(super) expected_archive_flow_width: usize, +} + +pub(super) struct PreviewTestCase { + pub(super) description: &'static str, + pub(super) plan: JanitorPlanText, + pub(super) verbose: bool, + pub(super) expected_fragments: &'static [&'static str], + pub(super) unexpected_fragments: &'static [&'static str], +} + +pub(super) struct JsonTestCase { + pub(super) description: &'static str, + pub(super) plan: Option, + pub(super) outcome: JanitorOutcome, + pub(super) run: Option, + pub(super) error: Option<&'static str>, + pub(super) expected_fragments: &'static [&'static str], +} + +pub(super) struct EventsUnwrittenTestCase { + pub(super) description: &'static str, + pub(super) count: usize, + pub(super) schemas: Vec, + pub(super) error: &'static str, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index 33c6f16149..abc2f86bdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -51,7 +51,7 @@ impl TransientStatusReporter { within(self.lock.bind(py), |_| body()) } - fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { + pub(crate) fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { let options: Bound<'_, PyDict> = PyDict::new(py); options.set_item("stream", self.stream.bind(py))?; options.set_item("text", text)?; @@ -161,7 +161,7 @@ impl TransientStatusReporter { }) } - fn start(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { + pub(crate) fn start(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { let py: Python<'_> = slf.py(); let reporter: &Self = slf.get(); if reporter.enabled { @@ -182,7 +182,7 @@ impl TransientStatusReporter { reporter.write_message(py, &message, true) } - fn update(&self, py: Python<'_>, message: String) -> PyResult<()> { + pub(crate) fn update(&self, py: Python<'_>, message: String) -> PyResult<()> { if self.enabled { let updated: bool = self.locked(py, || { let drawn: bool = { @@ -206,7 +206,11 @@ impl TransientStatusReporter { } #[pyo3(signature = (*, message, blank_line_after = false))] - fn complete(slf: &Bound<'_, Self>, message: &str, blank_line_after: bool) -> PyResult<()> { + pub(crate) fn complete( + slf: &Bound<'_, Self>, + message: &str, + blank_line_after: bool, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); Self::close(slf)?; slf.get().write_message(py, message, true)?; @@ -218,7 +222,7 @@ impl TransientStatusReporter { /// Complete with a pre-styled message written verbatim. #[pyo3(signature = (*, message, blank_line_after = false))] - fn complete_styled( + pub(crate) fn complete_styled( slf: &Bound<'_, Self>, message: &str, blank_line_after: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 61132a215e..73f7aced98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; use crate::bindings::_helpers::cli::launch_state::offer_launch_record; +use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, }; @@ -207,7 +208,7 @@ fn write_compile_machine_error( print_object(&document, None, None) } -/// With `sqb recall … --json` or `sqb plan --json`, the failure as `{"error": {"code", "message", "help"}}` on stdout, so callers parsing the output read the error instead of failing to parse it. +/// With `sqb recall … --json`, `sqb plan --json` or `sqb janitor --json` (before it printed its own document), the failure as `{"error": {"code", "message", "help"}}` on stdout, so callers parsing the output read the error instead of failing to parse it. fn write_machine_error( invocation: &Invocation<'_>, error: &Bound<'_, PyAny>, @@ -220,6 +221,8 @@ fn write_machine_error( .map(|format| format.extract()) .transpose()?; format.as_deref() == Some(JSON_FORMAT) + } else if invocation.command_is("janitor") { + invocation.flag("json")? && !take_machine_output_written() } else { invocation.command_is("plan") && invocation.flag("json")? }; @@ -599,6 +602,14 @@ fn observed_dispatch<'py>( /// Dispatch once and project every expected failure to its coded error and exit code. fn handled_dispatch<'py>(invocation: &Invocation<'py>) -> PyResult> { + let _ = take_machine_output_written(); + let handled: PyResult> = report_dispatch(invocation); + let _ = take_machine_output_written(); + handled +} + +/// Dispatch the command and report its failure, writing any machine-readable error document. +fn report_dispatch<'py>(invocation: &Invocation<'py>) -> PyResult> { let py: Python<'py> = invocation.py(); let policy: InvocationPolicy = invocation_policy(&invocation.facts); report_skill_freshness(invocation, policy)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/machine_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/machine_output.rs new file mode 100644 index 0000000000..2b33330d34 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/machine_output.rs @@ -0,0 +1,18 @@ +//! Whether the running command already printed its machine-readable document, so a failure adds no second one. + +use std::cell::Cell; + +thread_local! { + /// Per thread, since an invocation dispatches and reports on the thread that runs it. + static MACHINE_OUTPUT_WRITTEN: Cell = const { Cell::new(false) }; +} + +/// Record that the running command printed its machine-readable document on stdout. +pub(crate) fn report_machine_output_written() { + MACHINE_OUTPUT_WRITTEN.with(|written| written.set(true)); +} + +/// Whether the running command printed its document, cleared for the next invocation. +pub(crate) fn take_machine_output_written() -> bool { + MACHINE_OUTPUT_WRITTEN.with(|written| written.replace(false)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index ee447df5e8..17fbdc632b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod dispatch; pub(crate) mod entry; pub(crate) mod failure_cause; pub(crate) mod launch_state; +pub(crate) mod machine_output; pub(crate) mod namespace; pub(crate) mod observability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 48e4e1ded7..5dba134a6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -11,26 +11,34 @@ use sqlbuild_analysis::janitor_planning::main::resolve_retention_days::resolve_r use sqlbuild_cli::diff_report::errors::DiffUserError; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::janitor_text::errors::JanitorSettingsError; -use sqlbuild_core::janitor_text::main::janitor_completion_text::janitor_completion_text; +use sqlbuild_core::janitor_text::main::format_janitor_json::format_janitor_json; use sqlbuild_core::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation_phrase; use sqlbuild_core::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation_prompt; use sqlbuild_core::janitor_text::main::janitor_disabled_text::janitor_disabled_text; +use sqlbuild_core::janitor_text::main::janitor_item_counts::janitor_item_counts; +use sqlbuild_core::janitor_text::main::janitor_nothing_to_do_text::janitor_nothing_to_do_text; use sqlbuild_core::janitor_text::main::janitor_plan_has_work::janitor_plan_has_work; use sqlbuild_core::janitor_text::main::janitor_preview_complete_text::janitor_preview_complete_text; use sqlbuild_core::janitor_text::main::janitor_preview_start_text::janitor_preview_start_text; +use sqlbuild_core::janitor_text::main::render_janitor_completion::render_janitor_completion; use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::main::validate_janitor_settings::validate_janitor_settings; -use sqlbuild_core::janitor_text::models::JanitorPlanText; +use sqlbuild_core::janitor_text::models::{JanitorOutcome, JanitorPlanText, JanitorRunText}; +use std::time::Instant; +use crate::bindings::_helpers::cli::machine_output::{ + report_machine_output_written, take_machine_output_written, +}; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::diff_report::support::cli_error; use crate::bindings::_helpers::janitor::janitor_phases::{ JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, - compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, + compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, output_stream, }; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; +use crate::bindings::_helpers::janitor::janitor_progress::JanitorProgressPrinter; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::error_class; @@ -74,15 +82,28 @@ fn settings_error(py: Python<'_>, error: JanitorSettingsError) -> PyErr { } } -fn write(py: Python<'_>, text: &str) -> PyResult<()> { - py.import("sys")? - .getattr("stdout")? - .call_method1("write", (text,))?; +fn write(stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; Ok(()) } -fn flush(py: Python<'_>) -> PyResult<()> { - py.import("sys")?.getattr("stdout")?.call_method0("flush")?; +/// Print the `--json` document on stdout when JSON output was requested. +fn write_json( + (request, plan): (&Bound<'_, PyAny>, Option<&JanitorPlanText>), + outcome: JanitorOutcome, + run: Option<&JanitorRunText>, + error: Option<&str>, +) -> PyResult<()> { + if !native_field(request, "json_output")?.extract::()? { + return Ok(()); + } + let stdout: Bound<'_, PyAny> = imported(request.py(), "sys", "stdout")?; + write( + &stdout, + &format!("{}\n", format_janitor_json(plan, outcome, run, error)), + )?; + report_machine_output_written(); Ok(()) } @@ -111,8 +132,10 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult()? && stdout_supports_color(py)?; + let machine_output: bool = native_field(request, "json_output")?.extract()?; + let use_color: bool = !machine_output + && !native_field(request, "no_color")?.extract::()? + && stdout_supports_color(py)?; let options: Bound<'py, PyDict> = keywords( py, &[ @@ -124,6 +147,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( } /// Ask for the exact confirmation phrase; an interrupt cancels, and a non-interactive stdin refuses. -fn confirmed(py: Python<'_>, plan: &JanitorPlanText) -> PyResult { +fn confirmed(stream: &Bound<'_, PyAny>, plan: &JanitorPlanText) -> PyResult { + let py: Python<'_> = stream.py(); let stdin: Bound<'_, PyAny> = imported(py, "sys", "stdin")?; if stdin.is_none() || !stream_isatty(&stdin)? { return Err(cli_error( @@ -180,22 +205,87 @@ fn confirmed(py: Python<'_>, plan: &JanitorPlanText) -> PyResult { }, )); } - write(py, &janitor_confirmation_prompt(plan))?; - flush(py)?; + write(stream, &janitor_confirmation_prompt(plan))?; match stdin.call_method0("readline") { Ok(response) => Ok(response.extract::()?.trim_end_matches(LINE_ENDINGS) == janitor_confirmation_phrase(plan)), Err(error) if error.is_instance_of::(py) => { - write(py, "\n")?; + write(stream, "\n")?; Ok(false) } Err(error) => Err(error), } } +/// Run the confirmed plan, streaming each action, then print the final line and `--json`. +fn apply( + (request, invocation): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (stream, style): (&Bound<'_, PyAny>, CliStyle), + (compiled, connection): (&JanitorCompiled<'_>, &Bound<'_, PyAny>), + (planning, plan): (&Bound<'_, PyAny>, &JanitorPlanText), +) -> PyResult { + let py: Python<'_> = stream.py(); + let progress: Bound<'_, JanitorProgressPrinter> = Bound::new( + py, + JanitorProgressPrinter::new(stream, style, plan.clone())?, + )?; + let started: Instant = Instant::now(); + let executed: PyResult> = execute_janitor_cleanup( + invocation, + (compiled, connection), + (planning, progress.as_any()), + ); + let elapsed_seconds: f64 = started.elapsed().as_secs_f64(); + let items = progress.get().items(); + match executed { + Ok(result) => { + let run: JanitorRunText = JanitorRunText { + items, + counts: janitor_result_counts(&result)?, + elapsed_seconds, + }; + write( + stream, + &format!( + "\n{}", + render_janitor_completion(&run.counts, Some(elapsed_seconds), false, style) + ), + )?; + write_json( + (request, Some(plan)), + JanitorOutcome::Completed, + Some(&run), + None, + )?; + Ok(0) + } + Err(error) => { + let run: JanitorRunText = JanitorRunText { + counts: janitor_item_counts(&items), + items, + elapsed_seconds, + }; + write( + stream, + &format!( + "\n{}", + render_janitor_completion(&run.counts, Some(elapsed_seconds), true, style) + ), + )?; + let message: String = error.value(py).str()?.extract()?; + write_json( + (request, Some(plan)), + JanitorOutcome::Failed, + Some(&run), + Some(&message), + )?; + Err(error) + } + } +} + /// Plan, preview, confirm and clean up with an open warehouse connection. fn clean( - py: Python<'_>, request: &Bound<'_, PyAny>, (invocation, settings): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), (compiled, connection): (&JanitorCompiled<'_>, &Bound<'_, PyAny>), @@ -203,42 +293,65 @@ fn clean( let style: CliStyle = CliStyle { use_color: native_field(invocation, "use_color")?.extract()?, }; + let stream: Bound<'_, PyAny> = output_stream(invocation)?; let planning: Bound<'_, PyAny> = build_janitor_execution_plan((invocation, settings), compiled, connection)?; let plan: JanitorPlanText = janitor_plan_text(&planning)?; - write(py, &render_janitor_plan(&plan, style))?; + let verbose: bool = native_field(request, "verbose")?.extract()?; + write(&stream, &render_janitor_plan(&plan, verbose, style))?; let blocked: i32 = i32::from(!plan.blocked_schemas.is_empty()); let as_target: Option = native_field(invocation, "as_target")?.extract()?; if let Some(as_target) = as_target { let active: Option = native_field(invocation, "active_target_name")?.extract()?; write( - py, + &stream, &janitor_preview_complete_text(&as_target, active.as_deref(), style), )?; + write_json( + (request, Some(&plan)), + JanitorOutcome::Previewed, + None, + None, + )?; return Ok(blocked); } if blocked != 0 { + write_json((request, Some(&plan)), JanitorOutcome::Blocked, None, None)?; return Ok(1); } if !janitor_plan_has_work(&plan) { + write(&stream, &janitor_nothing_to_do_text(style))?; + write_json( + (request, Some(&plan)), + JanitorOutcome::NothingToDo, + None, + None, + )?; return Ok(0); } - if !native_field(request, "auto_approve")?.extract::()? && !confirmed(py, &plan)? { - write(py, "Janitor cancelled.\n")?; + if !native_field(request, "auto_approve")?.extract::()? && !confirmed(&stream, &plan)? { + write(&stream, "Janitor cancelled.\n")?; + write_json( + (request, Some(&plan)), + JanitorOutcome::Cancelled, + None, + None, + )?; return Ok(1); } - let result: Bound<'_, PyAny> = execute_janitor_cleanup(compiled, connection, &planning)?; - write( - py, - &janitor_completion_text(&janitor_result_counts(&result)?, style), - )?; - Ok(0) + apply( + (request, invocation), + (&stream, style), + (compiled, connection), + (&planning, &plan), + ) } /// Run one `sqb janitor` request. #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let _ = take_machine_output_written(); let invocation: Bound<'_, PyAny> = invocation(py, request)?; let style: CliStyle = CliStyle { use_color: native_field(&invocation, "use_color")?.extract()?, @@ -246,8 +359,10 @@ pub(crate) fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> let janitor: Bound<'_, PyAny> = native_field(&invocation, "discovered_inputs")? .getattr("project_config")? .getattr("janitor")?; + let stream: Bound<'_, PyAny> = output_stream(&invocation)?; if !native_field(&janitor, "enabled")?.is_truthy()? { - write(py, &janitor_disabled_text(style))?; + write(&stream, &janitor_disabled_text(style))?; + write_json((request, None), JanitorOutcome::Disabled, None, None)?; return Ok(0); } let settings: Bound<'_, PyAny> = settings(py, request, &janitor)?; @@ -255,19 +370,13 @@ pub(crate) fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> if let Some(as_target) = &as_target { let active: Option = native_field(&invocation, "active_target_name")?.extract()?; write( - py, + &stream, &janitor_preview_start_text(as_target, active.as_deref()), )?; - flush(py)?; } let compiled: JanitorCompiled<'_> = compile_janitor_project(&invocation)?; let connection: Bound<'_, PyAny> = connect_janitor_warehouse(&invocation, &compiled)?; - let outcome: PyResult = clean( - py, - request, - (&invocation, &settings), - (&compiled, &connection), - ); + let outcome: PyResult = clean(request, (&invocation, &settings), (&compiled, &connection)); let closed: PyResult<()> = close_janitor_warehouse(&compiled, &connection); let code: i32 = outcome?; closed?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 9ff0f2b19e..052592d9e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -18,7 +18,8 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ use crate::bindings::_helpers::records::fields::replaced; const QUERY_ARTIFACT_LIFECYCLE: ExecutorHost = ExecutorHost::QueryDiffArtifactLifecycle; -const RELATION_LOOKUP: AdapterHost = AdapterHost::BuildRelationLookup; +const INSPECTION_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; +const ACTIVE_CATALOG: AdapterHost = AdapterHost::ActiveInspectionCatalog; const LOG_DEBUG_EVENT: ExecutorHost = ExecutorHost::LogDebugEvent; const EXECUTION_LOGGER: &str = "sqlbuild.execution"; use crate::bindings::_helpers::native_host::host; @@ -49,6 +50,93 @@ impl<'py> Warehouse<'py> { call_with(&native_field(&self.adapter, method)?, &arguments) } + /// `list_relations`, served from the open inspection catalog so each schema is read once. + pub(crate) fn list_relations( + &self, + database: Option<&str>, + (schemas, names): (Option<&[String]>, Option<&[String]>), + ) -> PyResult> { + let py: Python<'py> = self.py(); + let texts = |values: Option<&[String]>| -> PyResult> { + match values { + Some(values) => tuple(py, values.iter().map(|value| text(py, value))), + None => Ok(py.None().into_bound(py)), + } + }; + call_with( + &host::entry(py, INSPECTION_LIST)?, + &[ + ("adapter", self.adapter.clone()), + ("connection", self.connection.clone()), + ("database", optional(py, database)?), + ("schemas", texts(schemas)?), + ("names", texts(names)?), + ], + ) + } + + /// The inspection catalog open on this connection, or none. + fn catalog(&self) -> PyResult>> { + let catalog: Bound<'py, PyAny> = call_with( + &host::entry(self.py(), ACTIVE_CATALOG)?, + &[ + ("adapter", self.adapter.clone()), + ("connection", self.connection.clone()), + ], + )?; + Ok((!catalog.is_none()).then_some(catalog)) + } + + /// Read every not-yet-listed schema concurrently when the open catalog lists whole schemas. + pub(crate) fn prefetch_schema_listings( + &self, + scopes: &[(Option, String)], + ) -> PyResult<()> { + let py: Python<'py> = self.py(); + let Some(catalog) = self.catalog()?.filter(|_| !scopes.is_empty()) else { + return Ok(()); + }; + let mut items: Vec> = Vec::new(); + for (database, schema) in scopes { + items.push(tuple( + py, + [optional(py, database.as_deref())?, text(py, schema)], + )?); + } + call_with( + &native_field(&catalog, "prefetch_schema_listings")?, + &[("scopes", tuple(py, items)?)], + )?; + Ok(()) + } + + /// Read these views' definitions with one read per schema, when a catalog is open. + pub(crate) fn prefetch_view_definitions( + &self, + requests: &[(Option, String, Vec)], + ) -> PyResult<()> { + let py: Python<'py> = self.py(); + let Some(catalog) = self.catalog()?.filter(|_| !requests.is_empty()) else { + return Ok(()); + }; + let mut items: Vec> = Vec::new(); + for (database, schema, names) in requests { + items.push(tuple( + py, + [ + optional(py, database.as_deref())?, + text(py, schema), + tuple(py, names.iter().map(|name| text(py, name)))?, + ], + )?); + } + call_with( + &native_field(&catalog, "prefetch_view_definitions")?, + &[("requests", tuple(py, items)?)], + )?; + Ok(()) + } + pub(crate) fn age_metadata_support(&self) -> PyResult { self.adapter .call_method0("supports_relation_age_metadata")? @@ -156,7 +244,7 @@ fn source_schemas( Ok(schemas) } -/// Relations matched case-insensitively and respelled as their target schema. +/// Each target schema's relations, listed once through the catalog and respelled as targeted. fn list_target_schema_relations<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -168,6 +256,7 @@ fn list_target_schema_relations<'py>( databases.push(database); } } + warehouse.prefetch_schema_listings(&concrete_schemas(target_schemas))?; let mut result: SchemaRelations<'py> = Vec::new(); for database in databases { let mut concrete: Vec = target_schemas @@ -177,87 +266,82 @@ fn list_target_schema_relations<'py>( .collect(); concrete.sort(); concrete.dedup(); - let schemas: Bound<'py, PyAny> = if concrete.is_empty() { - py.None().into_bound(py) + let requests: Vec>> = if concrete.is_empty() { + vec![None] } else { - tuple(py, concrete.iter().map(|schema| text(py, schema)))? + concrete + .into_iter() + .map(|schema| Some(vec![schema])) + .collect() }; - let listed: Bound<'py, PyAny> = warehouse.call( - "list_relations", - &[ - ("database", optional(py, database.as_deref())?), - ("schemas", schemas), - ], - )?; - for relation in listed.try_iter()? { - let relation: Bound<'py, PyAny> = relation?; - let folded: SchemaKey = folded_schema(&( - native_field(&relation, "database")?.extract()?, - native_field(&relation, "schema")?.extract()?, - )); - let Some(target) = target_schemas - .iter() - .find(|target| folded_schema(target) == folded) - else { - continue; - }; - let respelled: Bound<'py, PyAny> = replaced( - &relation, - &[ - ("database", optional(py, target.0.as_deref())?), - ("schema", optional(py, target.1.as_deref())?), - ], - )?; - match result.iter_mut().find(|(listed, _)| listed == target) { - Some((_, relations)) => relations.push(respelled), - None => result.push((target.clone(), vec![respelled])), + for schemas in requests { + let listed: Bound<'py, PyAny> = + warehouse.list_relations(database.as_deref(), (schemas.as_deref(), None))?; + for relation in listed.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + let folded: SchemaKey = folded_schema(&( + native_field(&relation, "database")?.extract()?, + native_field(&relation, "schema")?.extract()?, + )); + let Some(target) = target_schemas + .iter() + .find(|target| folded_schema(target) == folded) + else { + continue; + }; + let respelled: Bound<'py, PyAny> = replaced( + &relation, + &[ + ("database", optional(py, target.0.as_deref())?), + ("schema", optional(py, target.1.as_deref())?), + ], + )?; + match result.iter_mut().find(|(listed, _)| listed == target) { + Some((_, relations)) => relations.push(respelled), + None => result.push((target.clone(), vec![respelled])), + } } } } Ok(result) } -/// `build_relation_lookup` over the given locations. -fn relation_lookup<'py>( - warehouse: &Warehouse<'py>, +/// Folded keys of relations in the locations' schemas, listed through the inspection catalog. +fn relation_lookup( + warehouse: &Warehouse<'_>, locations: &[(Option, String, &str)], -) -> PyResult> { - let py: Python<'py> = warehouse.py(); - let mut items: Vec> = Vec::new(); - for (database, schema, name) in locations { - items.push(tuple( - py, - [ - optional(py, database.as_deref())?, - text(py, schema), - text(py, name), - ], - )?); +) -> PyResult> { + let mut scopes: Vec<(Option, String)> = Vec::new(); + for (database, schema, _) in locations { + if !scopes.contains(&(database.clone(), schema.clone())) { + scopes.push((database.clone(), schema.clone())); + } } - call_with( - &host::entry(py, RELATION_LOOKUP)?, - &[ - ("adapter", warehouse.adapter.clone()), - ("connection", warehouse.connection.clone()), - ("locations", tuple(py, items)?), - ], - ) + let mut present: HashSet = HashSet::new(); + for (database, schema) in scopes { + let schemas: [String; 1] = [schema]; + for relation in warehouse + .list_relations(database.as_deref(), (Some(&schemas), None))? + .try_iter()? + { + present.insert(Key::of(&relation?)?.folded()); + } + } + Ok(present) } fn lookup_exists( - lookup: &Bound<'_, PyAny>, + lookup: &HashSet, (database, schema, name): (Option<&str>, &str, &str), -) -> PyResult { - let py: Python<'_> = lookup.py(); - call_with( - &native_field(lookup, "exists")?, - &[ - ("database", optional(py, database)?), - ("schema", text(py, schema)), - ("name", text(py, name)), - ], - )? - .is_truthy() +) -> bool { + lookup.contains( + &Key { + database: database.map(str::to_owned), + schema: Some(schema.to_owned()), + name: name.to_owned(), + } + .folded(), + ) } fn concrete_schemas(target_schemas: &[SchemaKey]) -> Vec<(Option, String)> { @@ -282,13 +366,13 @@ fn tracked_relation_keys( .iter() .map(|(database, schema)| (database.clone(), schema.clone(), FINGERPRINT_TABLE_NAME)) .collect(); - let lookup: Bound<'_, PyAny> = relation_lookup(warehouse, &locations)?; + let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut tracked: HashSet = HashSet::new(); for (database, schema) in &schemas { let exists: bool = lookup_exists( &lookup, (database.as_deref(), schema, FINGERPRINT_TABLE_NAME), - )?; + ); let read: PyResult> = read_latest_fingerprints(Some(&keywords( py, &[ @@ -411,7 +495,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( SOURCE_FRESHNESS_TABLE_NAME, )); } - let lookup: Bound<'py, PyAny> = relation_lookup(warehouse, &locations)?; + let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); for (database, schema) in &schemas { for (table_name, render) in [ @@ -424,7 +508,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( "render_prune_source_freshness_history_sql", ), ] { - if !lookup_exists(&lookup, (database.as_deref(), schema, table_name))? { + if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { continue; } let prune_sql: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index e6a3ba599b..161a1b5c0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -115,7 +115,6 @@ fn physical_old_names<'py>( histories: &[Bound<'py, PyAny>], relations: &SchemaRelations<'py>, ) -> PyResult>> { - let py: Python<'py> = warehouse.py(); let mut known: Vec<(String, &Vec>)> = Vec::new(); for ((_, schema), listed) in relations { let folded: String = schema.as_deref().unwrap_or("").to_lowercase(); @@ -143,17 +142,8 @@ fn physical_old_names<'py>( let mut names: Vec = unscanned.iter().map(|key| key.name.clone()).collect(); names.sort(); names.dedup(); - let listed: Bound<'py, PyAny> = warehouse.call( - "list_relations", - &[ - ("database", optional(py, first.database.as_deref())?), - ( - "schemas", - tuple(py, schemas.iter().map(|schema| text(py, schema)))?, - ), - ("names", tuple(py, names.iter().map(|name| text(py, name)))?), - ], - )?; + let listed: Bound<'py, PyAny> = + warehouse.list_relations(first.database.as_deref(), (Some(&schemas), Some(&names)))?; for relation in listed.try_iter()? { scanned.push(relation?); } @@ -287,6 +277,23 @@ pub(crate) fn plan_old_name_views<'py>( requested.push((name.clone(), false)); } } + let mut definitions: Vec<(Option, String, Vec)> = Vec::new(); + for history in &histories { + let key: Key = old_key(history)?; + let (Some(schema), true) = (&key.schema, physical.contains_key(&key.folded())) else { + continue; + }; + match definitions + .iter_mut() + .find(|(database, listed, _)| *database == key.database && listed == schema) + { + Some((_, _, names)) => names.push(key.name.clone()), + None => { + definitions.push((key.database.clone(), schema.clone(), vec![key.name.clone()])) + } + } + } + warehouse.prefetch_view_definitions(&definitions)?; let is_compatibility_view: Bound<'py, PyAny> = host::entry(py, IS_COMPATIBILITY_VIEW)?; let (mut live, mut drops, mut missing) = (Vec::new(), Vec::new(), Vec::new()); let mut keys: HashSet = HashSet::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index 9141d10afc..ec56d16473 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -1,12 +1,13 @@ //! `sqb janitor` phases: compile, connect, inspect and plan, clean up and close. -use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::native_host::connections::connection_failure; use std::time::Instant; +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; @@ -15,6 +16,7 @@ use crate::bindings::_helpers::build_progress::progress_reporters::{ ConnectionProgressReporter, PlanningProgressReporter, }; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; use crate::bindings::_helpers::janitor::janitor_planning::{ @@ -43,8 +45,13 @@ pub(crate) struct JanitorCompiled<'py> { pub(crate) connection_config: Bound<'py, PyAny>, } -fn stdout(py: Python<'_>) -> PyResult> { - py.import("sys")?.getattr("stdout") +/// Where human output goes: stderr when stdout carries `--json`, otherwise stdout. +pub(crate) fn output_stream<'py>(invocation: &Bound<'py, PyAny>) -> PyResult> { + let machine_output: bool = native_field(invocation, "machine_output")?.is_truthy()?; + invocation + .py() + .import("sys")? + .getattr(if machine_output { "stderr" } else { "stdout" }) } fn use_color(invocation: &Bound<'_, PyAny>) -> PyResult { @@ -76,7 +83,7 @@ pub(crate) fn compile_janitor_project<'py>( let started: Instant = Instant::now(); let status: Bound<'py, PyAny> = Bound::new( py, - PlanningProgressReporter::new(py, &stdout(py)?, use_color(invocation)?)?, + PlanningProgressReporter::new(py, &output_stream(invocation)?, use_color(invocation)?)?, )? .into_any(); let as_target: Bound<'py, PyAny> = native_field(invocation, "as_target")?; @@ -145,7 +152,7 @@ pub(crate) fn connect_janitor_warehouse<'py>( py, &[ ("adapter_name", compiled.adapter_name.clone()), - ("stream", stdout(py)?), + ("stream", output_stream(invocation)?), ( "use_color", PyBool::new(py, use_color(invocation)?) @@ -221,7 +228,8 @@ pub(crate) fn build_janitor_execution_plan<'py>( ) -> PyResult> { let py: Python<'py> = invocation.py(); let started: Instant = Instant::now(); - let status: Bound<'py, PyAny> = status_reporter(&stdout(py)?, use_color(invocation)?)?; + let status: Bound<'py, PyAny> = + status_reporter(&output_stream(invocation)?, use_color(invocation)?)?; status.call_method1("start", ("Inspecting warehouse state...",))?; let janitor: Bound<'py, PyAny> = native_field(invocation, "discovered_inputs")? .getattr("project_config")? @@ -229,36 +237,45 @@ pub(crate) fn build_janitor_execution_plan<'py>( let plan: Bound<'py, PyAny> = within( &lifecycle(py, ("janitor", "janitor_candidate_planning"))?, |lifecycle| { - let plan: Bound<'py, PyAny> = build_plan( - ( - &compiled.project, - &Warehouse { - adapter: compiled.adapter.clone(), - connection: connection.clone(), - }, - ), - &JanitorSettings { - retention_days: native_field(settings, "retention_days")?.extract()?, - delete_tracked_only: native_field(&janitor, "delete_tracked_only")? - .is_truthy()?, - exclude_patterns: native_field(&janitor, "exclude_patterns")?.extract()?, - direct_mode: true, - state_history_versions: native_field( - settings, - "direct_state_history_versions", - )? - .extract()?, - archive_retention_days: native_field(settings, "archive_retention_days")? - .extract()?, - early_old_name_view_drops: native_field(settings, "drop_old_name_views")? - .extract()?, - }, - &RelationScope { - scan_keys: Vec::new(), - protected_keys: Vec::new(), - reasons: Vec::new(), - }, + let catalog: Bound<'py, PyAny> = call_with( + &host::entry(py, AdapterHost::OpenInspectionCatalog)?, + &[ + ("adapter", compiled.adapter.clone()), + ("connection", connection.clone()), + ], )?; + let plan: Bound<'py, PyAny> = within(&catalog, |_| { + build_plan( + ( + &compiled.project, + &Warehouse { + adapter: compiled.adapter.clone(), + connection: connection.clone(), + }, + ), + &JanitorSettings { + retention_days: native_field(settings, "retention_days")?.extract()?, + delete_tracked_only: native_field(&janitor, "delete_tracked_only")? + .is_truthy()?, + exclude_patterns: native_field(&janitor, "exclude_patterns")?.extract()?, + direct_mode: true, + state_history_versions: native_field( + settings, + "direct_state_history_versions", + )? + .extract()?, + archive_retention_days: native_field(settings, "archive_retention_days")? + .extract()?, + early_old_name_view_drops: native_field(settings, "drop_old_name_views")? + .extract()?, + }, + &RelationScope { + scan_keys: Vec::new(), + protected_keys: Vec::new(), + reasons: Vec::new(), + }, + ) + })?; let metadata: Bound<'py, PyDict> = PyDict::new(py); metadata.set_item("item_count", candidate_count(&plan)?)?; let options: Bound<'py, PyDict> = PyDict::new(py); @@ -285,18 +302,49 @@ pub(crate) fn build_janitor_execution_plan<'py>( Ok(plan) } -/// Execute the warehouse cleanup plan. +/// Execute the cleanup plan on up to the command's concurrency, reporting to `progress`. pub(crate) fn execute_janitor_cleanup<'py>( - compiled: &JanitorCompiled<'py>, - connection: &Bound<'py, PyAny>, - plan: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + (compiled, connection): (&JanitorCompiled<'py>, &Bound<'py, PyAny>), + (plan, progress): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { + let py: Python<'py> = invocation.py(); + let concurrency: i64 = resolve_concurrency( + &py.None().into_bound(py), + &native_field(invocation, "discovered_inputs")?, + &native_field(invocation, "selected_target")?, + )? + .limit; + let connection_progress: Bound<'py, PyAny> = Bound::new( + py, + ConnectionProgressReporter::new( + py, + Some(keywords( + py, + &[ + ("adapter_name", compiled.adapter_name.clone()), + ("stream", output_stream(invocation)?), + ( + "use_color", + PyBool::new(py, use_color(invocation)?) + .to_owned() + .into_any(), + ), + ], + )?), + )?, + )? + .into_any(); call_with( - &host::entry(compiled.adapter.py(), ExecutorHost::ExecuteJanitorPlan)?, + &host::entry(py, ExecutorHost::ExecuteJanitorPlan)?, &[ ("plan", plan.clone()), ("adapter", compiled.adapter.clone()), ("connection", connection.clone()), + ("progress", progress.clone()), + ("connection_config", compiled.connection_config.clone()), + ("concurrency", concurrency.into_pyobject(py)?.into_any()), + ("connection_progress", connection_progress), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs index 994ac0d36c..38a3a16c4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs @@ -4,7 +4,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::janitor_text::main::janitor_completion_text::janitor_completion_text; +use sqlbuild_core::janitor_text::main::render_janitor_completion::render_janitor_completion; use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::models::{ ArchiveCandidateText, ArchivedRelationText, JanitorPlanText, JanitorResultCounts, @@ -193,20 +193,34 @@ pub(crate) fn janitor_result_counts(result: &Bound<'_, PyAny>) -> PyResult, use_color: bool) -> PyResult { +#[pyo3(name = "render_janitor_plan", signature = (*, plan, use_color = false, verbose = false))] +fn render_janitor_plan_text( + plan: &Bound<'_, PyAny>, + use_color: bool, + verbose: bool, +) -> PyResult { Ok(render_janitor_plan( &janitor_plan_text(plan)?, + verbose, CliStyle { use_color }, )) } /// Render one executor janitor result as the `sqb janitor` completion line. #[pyfunction] -#[pyo3(name = "render_janitor_completion", signature = (*, result, use_color = false))] -fn render_janitor_completion_text(result: &Bound<'_, PyAny>, use_color: bool) -> PyResult { - Ok(janitor_completion_text( +#[pyo3( + name = "render_janitor_completion", + signature = (*, result, elapsed_seconds = None, use_color = false) +)] +fn render_janitor_completion_text( + result: &Bound<'_, PyAny>, + elapsed_seconds: Option, + use_color: bool, +) -> PyResult { + Ok(render_janitor_completion( &janitor_result_counts(result)?, + elapsed_seconds, + false, CliStyle { use_color }, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs new file mode 100644 index 0000000000..6a717a5964 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs @@ -0,0 +1,175 @@ +//! Streamed janitor phase and action lines, as the executor's progress callbacks. + +use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::Instant; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::janitor_text::main::janitor_events_unwritten_text::janitor_events_unwritten_text; +use sqlbuild_core::janitor_text::main::janitor_phase_complete_text::janitor_phase_complete_text; +use sqlbuild_core::janitor_text::main::janitor_phase_failed_text::janitor_phase_failed_text; +use sqlbuild_core::janitor_text::main::janitor_phase_flow_width::janitor_phase_flow_width; +use sqlbuild_core::janitor_text::main::janitor_phase_start_text::janitor_phase_start_text; +use sqlbuild_core::janitor_text::main::render_janitor_item_line::render_janitor_item_line; +use sqlbuild_core::janitor_text::models::{JanitorItemText, JanitorPhase, JanitorPlanText}; + +use crate::bindings::_helpers::build_progress::terminal::tty_flag; +use crate::bindings::_helpers::build_progress::transient_status::{ + TransientStatusReporter, progress_spinners_disabled, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// The running phase's start text, size, finished count, column width and start time. +struct ActivePhase { + message: String, + total: usize, + finished: usize, + flow_width: usize, + started: Instant, +} + +#[derive(Default)] +struct ProgressState { + items: Vec, + active: Option, +} + +/// Print each janitor phase and action as it finishes, and keep the finished actions. +#[pyclass(frozen)] +pub(crate) struct JanitorProgressPrinter { + status: Py, + style: CliStyle, + live: bool, + plan: JanitorPlanText, + state: Mutex, +} + +impl JanitorProgressPrinter { + pub(crate) fn new( + stream: &Bound<'_, PyAny>, + style: CliStyle, + plan: JanitorPlanText, + ) -> PyResult { + let py: Python<'_> = stream.py(); + Ok(Self { + status: Py::new( + py, + TransientStatusReporter::new(py, stream, style.use_color, true)?, + )?, + style, + live: tty_flag(stream)? && !progress_spinners_disabled(), + plan, + state: Mutex::new(ProgressState::default()), + }) + } + + fn state(&self) -> MutexGuard<'_, ProgressState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Every action finished so far, in order. + pub(crate) fn items(&self) -> Vec { + self.state().items.clone() + } +} + +fn phase(value: &str) -> PyResult { + JanitorPhase::from_value(value) + .ok_or_else(|| PyValueError::new_err(format!("unknown janitor phase: {value}"))) +} + +#[pymethods] +impl JanitorProgressPrinter { + #[pyo3(signature = (*, phase, total))] + fn phase_started(&self, py: Python<'_>, phase: &str, total: usize) -> PyResult<()> { + let phase: JanitorPhase = self::phase(phase)?; + let message: String = janitor_phase_start_text(phase, total); + self.state().active = Some(ActivePhase { + message: message.clone(), + total, + finished: 0, + flow_width: janitor_phase_flow_width(&self.plan, phase), + started: Instant::now(), + }); + TransientStatusReporter::start(self.status.bind(py), message) + } + + /// Record and print one finished action; `item` is its finished `JanitorProgressItem`. + #[pyo3(signature = (*, phase, item))] + fn item_finished(&self, py: Python<'_>, phase: &str, item: &Bound<'_, PyAny>) -> PyResult<()> { + let item: JanitorItemText = JanitorItemText { + phase: self::phase(phase)?, + action: native_field(item, "action")?.str()?.extract()?, + name: native_field(item, "name")?.extract()?, + destination: native_field(item, "destination")?.extract()?, + error: native_field(item, "error")?.extract()?, + duration_seconds: Some(native_field(item, "duration_seconds")?.extract()?), + }; + let (line, progress) = { + let mut state = self.state(); + let (index, total, width, message) = match state.active.as_mut() { + Some(active) => { + active.finished += 1; + ( + active.finished, + active.total, + active.flow_width, + active.message.clone(), + ) + } + None => (1, 1, 0, String::new()), + }; + let line: String = render_janitor_item_line((index, total), &item, width, self.style); + state.items.push(item); + (line, format!("{message} {index}/{total}")) + }; + let status: &TransientStatusReporter = self.status.get(); + status.write_persistent(py, &format!("{line}\n"))?; + if self.live { + status.update(py, progress)?; + } + Ok(()) + } + + /// Warn that the audit events of `count` completed actions in `schemas` were not written. + #[pyo3(signature = (*, count, schemas, error))] + fn events_unwritten( + &self, + py: Python<'_>, + count: usize, + schemas: Vec, + error: &str, + ) -> PyResult<()> { + let line: String = janitor_events_unwritten_text((count, &schemas), error, self.style); + self.status.get().write_persistent(py, &format!("{line}\n")) + } + + #[pyo3(signature = (*, phase, failed))] + fn phase_finished(&self, py: Python<'_>, phase: &str, failed: bool) -> PyResult<()> { + let phase: JanitorPhase = self::phase(phase)?; + let (active, items) = { + let mut state = self.state(); + (state.active.take(), state.items.clone()) + }; + let (total, elapsed) = active.as_ref().map_or((0, 0.0), |active| { + (active.total, active.started.elapsed().as_secs_f64()) + }); + let status: &Bound<'_, TransientStatusReporter> = self.status.bind(py); + if failed { + let completed: usize = items + .iter() + .filter(|item| item.phase == phase && item.error.is_none()) + .count(); + let message: String = + janitor_phase_failed_text(phase, (completed, total), elapsed, self.style); + return TransientStatusReporter::complete_styled(status, &message, false); + } + TransientStatusReporter::complete( + status, + &janitor_phase_complete_text(phase, &items, elapsed), + false, + ) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs index 9f9557101e..d2231d7bc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs @@ -6,4 +6,5 @@ pub(crate) mod janitor_old_names; pub(crate) mod janitor_phases; pub(crate) mod janitor_plan; pub(crate) mod janitor_planning; +pub(crate) mod janitor_progress; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index f29979c521..5ff12ba1d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -463,6 +463,8 @@ const JANITOR_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("drop_old_name_views", FieldDefault::EmptyTuple), FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("as_target", FieldDefault::None), + FieldSpec::new("verbose", FieldDefault::False), + FieldSpec::new("json_output", FieldDefault::False), ], }; const JANITOR_INVOCATION: RecordSpec = RecordSpec { @@ -475,6 +477,7 @@ const JANITOR_INVOCATION: RecordSpec = RecordSpec { FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("active_target_name", FieldDefault::None), FieldSpec::new("as_target", FieldDefault::None), + FieldSpec::new("machine_output", FieldDefault::False), ], }; const JANITOR_SETTINGS: RecordSpec = RecordSpec { diff --git a/fensu.toml b/fensu.toml index 7acee3cbaf..ec0575896d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1611,12 +1611,6 @@ path = "src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.p symbols = ["invoke_lifecycle_event_sink"] reason = "The delivery thread claims each call in the dict it shares with the native dispatcher; the claim is the mutation, atomic under the GIL, so a cancelled call is never both run and counted dropped." -[[targets.python.rule_exceptions]] -rule = "XSB051" -path = "src/sqlbuild/executor/old_name_views/_helpers/janitor.py" -symbols = ["apply_old_name_view_drops"] -reason = "Janitor reads each compatibility view's stored definition right before deciding to drop it, so a relation that replaced the view is never dropped; a project has few live old names." - [targets.rust.rule_options.XSB072] edge_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/native_host", @@ -1910,10 +1904,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs=2", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 7d23b51609..25074a0d76 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -10,6 +10,8 @@ Online: https://sqlbuild.com/docs/cli/janitor/ - Usage - Flags +- Output +- Speed - Targets - Lifecycle - Compatibility views @@ -39,14 +41,49 @@ sqb --project-dir janitor [flags] | `--auto-approve` | Skip the confirmation prompt and run the planned actions immediately | | `--retention-days` | Override the configured retention period (days) before a stale relation is archived | | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | +| `--verbose`, `-v` | List every entry of each preview section and every skipped object with its reason. Without it, each section shows its first 10 entries and skipped objects are counted by reason | +| `--json` | Print the plan and the result as one JSON document on stdout; the preview, prompt and progress go to stderr | | `--drop-old-name-view ` | Drop the [compatibility view](../concepts/models/migrations.md#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | +## Output + +After the preview and confirmation, each phase with work announces how many actions it runs, prints one line per action as it finishes, and closes with its duration. The run ends with one line of totals: + +``` +Archiving 3 relations... + 1/3 archived analytics.old_orders -> analytics._sqb_archive__20261022t101500z__old_orders OK 0.21s + 2/3 archived analytics.old_customers -> analytics._sqb_archive__20261022t101500z__old_customers OK 0.19s + 3/3 archived analytics.old_products -> analytics._sqb_archive__20261022t101500z__old_products OK 0.24s +Archived 3 relations. (0.64s) +Deleting 1 expired archive... + 1/1 deleted analytics._sqb_archive__20261001t080000z__old_inventory OK 0.12s +Deleted 1 expired archive. (0.12s) + +✓ Janitor completed archived 3, deleted 1, pruned 0 (0.81s) +``` + +The phases are archiving stale relations, deleting expired archives, dropping old-name views, dropping expired query artifacts and pruning direct state, in that order. If an action fails, its line shows `FAIL` and the error, no further action starts, and the run ends with `✗ Janitor failed` and the totals completed before the failure, exiting with `1`. + +Long preview sections show their first 10 entries and then `... and N more (use --verbose to show all)`. Skipped objects are counted by reason: + +``` +Skipped objects (385) + 210 relation is newer than 14 days + 175 relation is not tracked by SQLBuild +``` + +With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. + +## Speed + +The janitor reads each schema's relations once per run, and the definitions of compatibility views with one read per schema. On warehouses that run DDL concurrently (Snowflake, BigQuery and Databricks), the independent actions of each phase run on up to the target's [`concurrency`](../concepts/project-configuration.md#target-concurrency) connections at once; phases still run one after another. DuckDB, SQLite and the other adapters apply one action at a time. + ## Targets By default the janitor cleans up the active target. `--target ` selects another configured target, exactly like `sqb build --target`: the janitor connects with that target's connection and inspects and cleans that target's schemas and state tables. Other targets are never touched. -`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](plan.md). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists every relation it would archive, every archive it would delete, every compatibility view it would drop and every state table it would prune, then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. +`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](plan.md). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists the relations it would archive, the archives it would delete, the compatibility views it would drop and the state tables it would prune (all of them with `--verbose`), then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. ``` Previewing janitor as target 'prod' through the connection of target 'dev' (inspection only). @@ -123,6 +160,8 @@ Ages are only looked up for relations that are otherwise candidates, and not at Every archive and every delete is recorded in an append-only `_sqlbuild_janitor_events` table in the affected schema. It records the original name, the archive name, the time and the invocation. The janitor never reads this table to decide what to do. It exists for auditing, and for restoring archives whose names were shortened. +Events are written in batches of up to 100 per schema, after each batch of actions and after a failure. An archive whose name was shortened gets its event right after its rename. If the process is killed, at most one batch of actions in a schema can lack events; those archives still expire by the timestamp in their names. + ## Restoring an archive Rename the archive back to its original name before it expires, using your warehouse's rename statement. For example, on Snowflake: @@ -173,4 +212,4 @@ In addition to archiving stale relations, the janitor prunes old rows from `_sql ## Safety -The janitor shows every planned archive and delete, with exact names, ages and expiry, then asks you to type an exact confirmation phrase. Relations whose names differ only by case from another relation in the same schema are reported and never dropped, and exclude patterns match regardless of case. Use `--auto-approve` only in automation, or when you're certain. +The janitor shows the planned archives and deletes, with exact names, ages and expiry (every one of them with `--verbose`, or in the `--json` plan), then asks you to type an exact confirmation phrase. Relations whose names differ only by case from another relation in the same schema are reported and never dropped, and exclude patterns match regardless of case. Use `--auto-approve` only in automation, or when you're certain. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index c64f5cdca5..25e52f4ea7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -689,7 +689,7 @@ delta_compile = true | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](models/configuration.md#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](models/contracts.md). | -| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`, and for `sqb janitor` actions on warehouses that run DDL concurrently; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](python-nodes/loaders.md). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 81c6d8ccd7..aeee4b0ddc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5050,7 +5050,9 @@ def observe_source_freshness_json( previous_records: Mapping[SourceFreshnessIdentity, SourceFreshnessRecord] | None = None, previous_records_by_source_name: Mapping[str, SourceFreshnessRecord] | None = None, ) -> str: ... -def render_janitor_plan(*, plan: JanitorPlan, use_color: bool = False) -> str: ... +def render_janitor_plan( + *, plan: JanitorPlan, use_color: bool = False, verbose: bool = False +) -> str: ... def discover_python_node_functions( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] | None = None ) -> DiscoveredPythonNodeFunctions: ... @@ -5147,7 +5149,10 @@ def build_janitor_archive_name( def build_query_diff_artifact_name(*, run_id: str, side: str) -> str: ... def parse_query_diff_artifact_name(name: str, /) -> tuple[str, str] | None: ... def render_janitor_completion( - *, result: JanitorExecutionResult, use_color: bool = False + *, + result: JanitorExecutionResult, + elapsed_seconds: float | None = None, + use_color: bool = False, ) -> str: ... # Native analysis: SQL test planning glue. @@ -6663,6 +6668,8 @@ class JanitorCommandRequest(NativeRecord): drop_old_name_views: tuple[str, ...] selected_target: str | None as_target: str | None + verbose: bool + json_output: bool def __init__( self, project_dir: Path | None, @@ -6673,6 +6680,8 @@ class JanitorCommandRequest(NativeRecord): drop_old_name_views: tuple[str, ...] = (), selected_target: str | None = None, as_target: str | None = None, + verbose: bool = False, + json_output: bool = False, ) -> None: ... class LoadCommandRequest(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 8bf89cfc53..db9ef689a7 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -114,6 +114,7 @@ class BaseAdapter(RetentionAdapterMixin, StrictAdapter): statement_cancellation_is_asynchronous: ClassVar[bool] = False statement_limit_excludes_queueing: ClassVar[bool] = False metadata_inspection_concurrency: ClassVar[int] = 1 + concurrent_ddl_limit: ClassVar[int] = 1 supports_session_warehouse: ClassVar[bool] = False state_timestamp_type: ClassVar[str | None] = None _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index cfc6da6ee4..3c1beddbe3 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -85,6 +85,7 @@ StatementSizeLimit, TablePromotionMode, ) +from sqlbuild.adapter.relations.main.cached_view_definition import cached_view_definition from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) @@ -1684,6 +1685,11 @@ def render_replace_view_keeping_grants( def read_view_definition( self, *, connection: Any, database: str | None, schema: str, name: str ) -> str | None: + prefetched: tuple[bool, str | None] = cached_view_definition( + adapter=self, connection=connection, database=database, schema=schema, name=name + ) + if prefetched[0]: + return prefetched[1] database_filter: str = ( "" if database is None else f" AND database_name = {_duckdb_string_literal(database)}" ) @@ -1697,6 +1703,32 @@ def read_view_definition( ).fetchall() return None if not rows else str(rows[0][0]) + def read_view_definitions( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> dict[str, str]: + """Read the stored definitions of several views in one schema with one query.""" + + if not names: + return {} + database_filter: str = ( + "" if database is None else f" AND database_name = {_duckdb_string_literal(database)}" + ) + rows: list[tuple[Any, ...]] = self.execute( + connection=connection, + sql=( + "SELECT view_name, sql FROM duckdb_views() WHERE schema_name = " + f"{_duckdb_string_literal(schema)} AND view_name IN (" + + ", ".join(_duckdb_string_literal(name) for name in names) + + ")" + + database_filter + ), + ).fetchall() + definitions: dict[str, str] = {} + row: tuple[Any, ...] + for row in rows: + definitions.setdefault(str(row[0]), str(row[1])) + return definitions + def view_definition_matches( self, *, connection: Any, database: str | None, schema: str, name: str, sql: str ) -> bool: diff --git a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py index 8ef8562216..078f055e3b 100644 --- a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py +++ b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py @@ -19,7 +19,7 @@ SchemaColumnListing, SchemaRelationListing, ) -from sqlbuild.adapter.relations.types import SchemaScopedMetadataReader +from sqlbuild.adapter.relations.types import SchemaScopedMetadataReader, ViewDefinitionBatchReader type _RelationIdentity = tuple[str | None, str | None, str] type _SchemaKey = tuple[str | None, str] @@ -43,6 +43,7 @@ def __init__(self, *, adapter: Any, connection: Any) -> None: self._remember_lock: threading.Lock = threading.Lock() self._relation_requests: dict[_RelationRequestKey, tuple[RelationInfo, ...]] = {} self._relation_columns: dict[_RelationIdentity, tuple[ColumnInfo, ...] | None] = {} + self._view_definitions: dict[_RelationIdentity, str | None] = {} def remember[ValueT](self, *, key: tuple[str, ...], compute: Callable[[], ValueT]) -> ValueT: """Compute one adapter-owned metadata fact at most once per planning invocation.""" @@ -131,6 +132,77 @@ def prefetch_schema_listings( if listing is not None: self._relation_listings[missing_key] = listing + def prefetch_view_definitions( + self, *, requests: tuple[tuple[str | None, str, tuple[str, ...]], ...] + ) -> None: + """Read each (database, schema, names) request's view definitions in one batched read.""" + + read_batch: ViewDefinitionBatchReader | None = getattr( + self.adapter, "read_view_definitions", None + ) + if read_batch is None: + return + pending: list[tuple[str | None, str, tuple[str, ...]]] = [] + database: str | None + schema: str + names: tuple[str, ...] + for database, schema, names in requests: + missing: tuple[str, ...] = tuple( + name + for name in dict.fromkeys(names) + if (database, schema, name) not in self._view_definitions + ) + if missing: + pending.append((database, schema, missing)) + results: list[dict[str, str]] = run_bounded_inspections( + tasks=tuple( + self._bind_view_definitions( + read_batch=read_batch, database=database, schema=schema, names=names + ) + for database, schema, names in pending + ), + concurrency=self._concurrency, + ) + definitions: dict[str, str] + for (database, schema, names), definitions in zip(pending, results, strict=True): + name: str + for name in names: + self._view_definitions[(database, schema, name)] = definitions.get(name) + + def cached_view_definition( + self, *, database: str | None, schema: str, name: str + ) -> tuple[bool, str | None]: + """Return whether a prefetched definition is known for this view, and the definition.""" + + key: _RelationIdentity = (database, schema, name) + if key not in self._view_definitions: + return False, None + return True, self._view_definitions[key] + + def _bind_view_definitions( + self, + *, + read_batch: ViewDefinitionBatchReader, + database: str | None, + schema: str, + names: tuple[str, ...], + ) -> Callable[[], dict[str, str]]: + def read_definitions() -> dict[str, str]: + definitions: dict[str, str] = {} + start: int + for start in range(0, len(names), INSPECTION_IN_LIST_LIMIT): + definitions.update( + read_batch( + connection=self.connection, + database=database, + schema=schema, + names=names[start : start + INSPECTION_IN_LIST_LIMIT], + ) + ) + return definitions + + return read_definitions + def get_columns_for_relations( self, *, relations: tuple[RelationInfo, ...] ) -> dict[_RelationIdentity, tuple[ColumnInfo, ...]]: diff --git a/src/sqlbuild/adapter/relations/main/cached_view_definition.py b/src/sqlbuild/adapter/relations/main/cached_view_definition.py new file mode 100644 index 0000000000..7466779ab2 --- /dev/null +++ b/src/sqlbuild/adapter/relations/main/cached_view_definition.py @@ -0,0 +1,19 @@ +"""Serve a view definition the open inspection catalog already read.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.relations._helpers.inspection_context import current_catalog +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog + + +def cached_view_definition( + *, adapter: Any, connection: Any, database: str | None, schema: str, name: str +) -> tuple[bool, str | None]: + """Return whether the open catalog prefetched this view's definition, and the definition.""" + + catalog: InspectionCatalog | None = current_catalog(adapter=adapter, connection=connection) + if catalog is None: + return False, None + return catalog.cached_view_definition(database=database, schema=schema, name=name) diff --git a/src/sqlbuild/adapter/relations/types.py b/src/sqlbuild/adapter/relations/types.py index acdad6b3b2..30d9bf395c 100644 --- a/src/sqlbuild/adapter/relations/types.py +++ b/src/sqlbuild/adapter/relations/types.py @@ -57,6 +57,16 @@ def metadata_name_key(self, name: str) -> str: ... +class ViewDefinitionBatchReader(Protocol): + """An adapter's ``read_view_definitions``: stored definitions of named views in one schema.""" + + def __call__( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> dict[str, str]: + """Return each found view's definition keyed by the requested name.""" + ... + + class InspectionCompletion[ResultT](Protocol): """Callback receiving one finished inspection read on the calling thread.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index e4b1b6b8d0..8d90520f08 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -170,6 +170,7 @@ def get_columns_for_relations( ) views_read_with_reader_access: ClassVar[bool] = True max_identifier_length: ClassVar[int] = 1024 + concurrent_ddl_limit: ClassVar[int] = 16 _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", distinct_condition=render_is_distinct_from, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index ad6baf39ba..03ff3cbd5c 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -160,6 +160,7 @@ def get_columns_for_relations( nested_block_comments=True, ) max_identifier_length: ClassVar[int] = 255 + concurrent_ddl_limit: ClassVar[int] = 16 _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", distinct_condition=render_is_distinct_from, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 55e7ddfdc9..1e17650c96 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -101,6 +101,7 @@ from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog from sqlbuild.adapter.relations.constants import INSPECTION_IN_LIST_LIMIT from sqlbuild.adapter.relations.main.active_inspection_catalog import active_inspection_catalog +from sqlbuild.adapter.relations.main.cached_view_definition import cached_view_definition from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.relations.main.render_inspection_sql import render_inspection_sql from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections @@ -191,6 +192,7 @@ class SnowflakeAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): execution_duration_limit_seconds: ClassVar[int | None] = MAX_STATEMENT_TIMEOUT_SECONDS statement_limit_excludes_queueing: ClassVar[bool] = True metadata_inspection_concurrency: ClassVar[int] = _METADATA_INSPECTION_CONCURRENCY + concurrent_ddl_limit: ClassVar[int] = 16 exact_column_inspection_limit: ClassVar[int] = _EXACT_COLUMN_INSPECTION_LIMIT max_identifier_length: ClassVar[int] = 255 @@ -3011,6 +3013,11 @@ def render_replace_view_keeping_grants( def read_view_definition( self, *, connection: Any, database: str | None, schema: str, name: str ) -> str | None: + prefetched: tuple[bool, str | None] = cached_view_definition( + adapter=self, connection=connection, database=database, schema=schema, name=name + ) + if prefetched[0]: + return prefetched[1] views: str = ( "INFORMATION_SCHEMA.VIEWS" if database is None @@ -3032,6 +3039,33 @@ def read_view_definition( ).fetchall() return None if not rows else str(rows[0][0]) + def read_view_definitions( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> dict[str, str]: + """Read several views' definitions in one schema, matching names case-insensitively.""" + + if not names: + return {} + views: str = ( + "INFORMATION_SCHEMA.VIEWS" + if database is None + else (f"{database}.INFORMATION_SCHEMA.VIEWS") + ) + literals: str = ", ".join("UPPER('" + name.replace("'", "''") + "')" for name in names) + rows: list[tuple[Any, ...]] = self.execute( + connection=connection, + sql=( + f"SELECT TABLE_NAME, VIEW_DEFINITION FROM {views} WHERE UPPER(TABLE_SCHEMA) = " + "UPPER('" + schema.replace("'", "''") + "') " + f"AND UPPER(TABLE_NAME) IN ({literals})" + ), + ).fetchall() + found: dict[str, str] = {} + row: tuple[Any, ...] + for row in rows: + found.setdefault(str(row[0]).upper(), str(row[1])) + return {name: found[name.upper()] for name in names if name.upper() in found} + def view_definition_matches( self, *, connection: Any, database: str | None, schema: str, name: str, sql: str ) -> bool: diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index c22d58f774..eea80a0fa1 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -20,6 +20,8 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog +from sqlbuild.adapter.relations.main.active_inspection_catalog import active_inspection_catalog from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_QUERY_DIFF_ARTIFACT from sqlbuild.errors.contracts.exceptions import ExecutorInputError @@ -184,10 +186,13 @@ def inspect_query_diff_artifacts( ) -> QueryDiffArtifactInspection: """Classify strict-name query artifacts using immutable fingerprint ownership.""" - relations: tuple[RelationInfo, ...] = adapter.list_relations( - connection=connection, - database=database, - schemas=(schema,), + catalog: InspectionCatalog | None = active_inspection_catalog( + adapter=adapter, connection=connection + ) + relations: tuple[RelationInfo, ...] = ( + adapter.list_relations(connection=connection, database=database, schemas=(schema,)) + if catalog is None + else catalog.list_relations(database=database, schemas=(schema,)) ) candidates: tuple[RelationInfo, ...] = tuple( relation diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py index 14b48201e4..c7b2dd52ee 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py @@ -14,15 +14,24 @@ ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.constants import ARCHIVE_NAME_SEPARATOR +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( + JanitorActionDone, JanitorArchiveCandidate, JanitorArchivedRelation, JanitorPlan, + JanitorProgressItem, JanitorRelationKey, ) -from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_SCHEMA_VERSION +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress +from sqlbuild.executor.janitor_events.classes.janitor_event_writer import JanitorEventWriter +from sqlbuild.executor.janitor_events.constants import ( + JANITOR_EVENT_CHUNK_SIZE, + JANITOR_EVENT_SCHEMA_VERSION, +) from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id -from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record from sqlbuild.executor.janitor_events.models import JanitorEventRecord from sqlbuild.executor.janitor_events.types import JanitorEventType from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -43,59 +52,125 @@ def apply_direct_archives( *, plan: JanitorPlan, adapter: BaseAdapter, - connection: Any, + workers: JanitorWorkers, recorder: StatementRecorder, run_id: str, + progress: JanitorProgress | None = None, ) -> tuple[tuple[JanitorArchiveCandidate, ...], tuple[JanitorArchivedRelation, ...]]: - """Rename stale relations to archives, then drop expired archives by name.""" + """Rename stale relations to archives, then drop expired archives, auditing each chunk.""" - _create_event_tables(adapter=adapter, connection=connection, plan=plan) - archived: list[JanitorArchiveCandidate] = [] - candidate: JanitorArchiveCandidate - for candidate in plan.archive_candidates: + schemas: tuple[tuple[str | None, str], ...] = _create_event_tables( + adapter=adapter, connection=workers.connection, plan=plan + ) + events: JanitorEventWriter = JanitorEventWriter( + execute=adapter.execute, + render_qualified_name=adapter.render_qualified_name, + run_id=run_id, + ) + if schemas: + events.read_existing(connection=workers.connection, schemas=schemas) + + def archive_relation(*, item: JanitorArchiveCandidate, connection: Any) -> JanitorActionDone: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): _rename_relation( adapter=adapter, connection=connection, - candidate=candidate, + candidate=item, recorder=recorder, ) - archived.append(candidate) - _write_event( + record: JanitorEventRecord = _event_record( adapter=adapter, - connection=connection, - record=_event_record( - adapter=adapter, - event_type=JanitorEventType.ARCHIVE, - run_id=run_id, - archive=JanitorArchivedRelation( - key=candidate.archive_key, - relation_type=candidate.relation.relation_type, - archived_at=candidate.archived_at, - expires_at=candidate.expires_at, - original_key=candidate.key, - ), + event_type=JanitorEventType.ARCHIVE, + run_id=run_id, + archive=JanitorArchivedRelation( + key=item.archive_key, + relation_type=item.relation.relation_type, + archived_at=item.archived_at, + expires_at=item.expires_at, + original_key=item.key, ), ) - deleted: list[JanitorArchivedRelation] = [] - archive: JanitorArchivedRelation - for archive in plan.archive_deletion_candidates: + if not _shortened(item): + return JanitorActionDone(event=record) + events.write(connection=connection, records=(record,)) + return JanitorActionDone() + + archived: tuple[tuple[JanitorArchiveCandidate, JanitorActionDone], ...] = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=plan.archive_candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.ARCHIVED, + name=candidate.key.display_name(), + destination=candidate.archive_key.display_name(), + ), + apply=archive_relation, + workers=workers, + chunks=_event_chunks(tuple(candidate.archive_key for candidate in plan.archive_candidates)), + after_chunk=lambda finished: events.write( + connection=workers.connection, records=_chunk_events(finished) + ), + ) + + def delete_archive(*, item: JanitorArchivedRelation, connection: Any) -> JanitorActionDone: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): - _drop_archive( - adapter=adapter, connection=connection, archive=archive, recorder=recorder - ) - deleted.append(archive) - _write_event( - adapter=adapter, - connection=connection, - record=_event_record( + _drop_archive(adapter=adapter, connection=connection, archive=item, recorder=recorder) + return JanitorActionDone( + event=_event_record( adapter=adapter, event_type=JanitorEventType.DELETE, run_id=run_id, - archive=archive, - ), + archive=item, + ) ) - return tuple(archived), tuple(deleted) + + deleted: tuple[tuple[JanitorArchivedRelation, JanitorActionDone], ...] = run_janitor_phase( + progress=progress, + phase=JanitorPhase.DELETE_ARCHIVES, + items=plan.archive_deletion_candidates, + describe=lambda archive: JanitorProgressItem( + action=JanitorAction.DELETED, name=archive.key.display_name() + ), + apply=delete_archive, + workers=workers, + chunks=_event_chunks(tuple(archive.key for archive in plan.archive_deletion_candidates)), + after_chunk=lambda finished: events.write( + connection=workers.connection, records=_chunk_events(finished) + ), + ) + return ( + tuple(candidate for candidate, _ in archived), + tuple(archive for archive, _ in deleted), + ) + + +def _chunk_events(finished: tuple[JanitorActionDone, ...]) -> tuple[JanitorEventRecord, ...]: + return tuple(done.event for done in finished if done.event is not None) + + +def _shortened(candidate: JanitorArchiveCandidate) -> bool: + """Whether the archive name no longer ends with the relation's full original name.""" + + return not candidate.archive_key.name.lower().endswith( + ARCHIVE_NAME_SEPARATOR + candidate.key.name.lower() + ) + + +def _event_chunks(keys: tuple[JanitorRelationKey, ...]) -> tuple[tuple[int, ...], ...]: + """Item positions grouped by schema, in order, and split into event-sized chunks.""" + + by_schema: dict[tuple[str | None, str | None], list[int]] = {} + index: int + key: JanitorRelationKey + for index, key in enumerate(keys): + by_schema.setdefault((key.database, key.schema), []).append(index) + chunks: list[tuple[int, ...]] = [] + positions: list[int] + for positions in by_schema.values(): + start: int + for start in range(0, len(positions), JANITOR_EVENT_CHUNK_SIZE): + chunks.append(tuple(positions[start : start + JANITOR_EVENT_CHUNK_SIZE])) + return tuple(chunks) def _required_schema(key: JanitorRelationKey) -> str: @@ -162,7 +237,9 @@ def _drop_archive( ) -def _create_event_tables(*, adapter: BaseAdapter, connection: Any, plan: JanitorPlan) -> None: +def _create_event_tables( + *, adapter: BaseAdapter, connection: Any, plan: JanitorPlan +) -> tuple[tuple[str | None, str], ...]: keys: tuple[JanitorRelationKey, ...] = ( *(candidate.key for candidate in plan.archive_candidates), *(archive.key for archive in plan.archive_deletion_candidates), @@ -177,6 +254,7 @@ def _create_event_tables(*, adapter: BaseAdapter, connection: Any, plan: Janitor connection=connection, sql=adapter.render_create_janitor_event_table_sql(database=database, schema=schema), ) + return tuple(schemas) def _event_record( @@ -211,12 +289,3 @@ def _event_record( archive_qualified_name=_qualified(adapter=adapter, key=archive.key), archived_at=archive.archived_at, ) - - -def _write_event(*, adapter: BaseAdapter, connection: Any, record: JanitorEventRecord) -> None: - _ = write_janitor_event_record( - connection=connection, - execute=adapter.execute, - record=record, - render_qualified_name=adapter.render_qualified_name, - ) diff --git a/src/sqlbuild/executor/janitor/_helpers/deletion.py b/src/sqlbuild/executor/janitor/_helpers/deletion.py index bc83ff3363..c68b1905b9 100644 --- a/src/sqlbuild/executor/janitor/_helpers/deletion.py +++ b/src/sqlbuild/executor/janitor/_helpers/deletion.py @@ -1,23 +1,47 @@ -"""Generic janitor candidate deletion application.""" +"""Direct state history pruning application.""" from __future__ import annotations from collections.abc import Callable +from typing import Any +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase +from sqlbuild.executor.janitor.models import ( + JanitorActionDone, + JanitorDirectStatePruneCandidate, + JanitorProgressItem, +) +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -def apply_janitor_deletions[T]( +def apply_janitor_deletions( *, - candidates: tuple[T, ...], - delete: Callable[[T], object] | None, -) -> tuple[T, ...]: - """Run the delete callback for each candidate, returning the deleted ones.""" + candidates: tuple[JanitorDirectStatePruneCandidate, ...], + delete: Callable[..., object] | None, + workers: JanitorWorkers, + progress: JanitorProgress | None = None, +) -> tuple[JanitorDirectStatePruneCandidate, ...]: + """Run `delete(candidate=, connection=)` for each candidate, returning the deleted ones.""" if delete is None: return () - candidate: T - for candidate in candidates: + + def prune(*, item: JanitorDirectStatePruneCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): - _ = delete(candidate) - return candidates + _ = delete(candidate=item, connection=connection) + + pruned: tuple[tuple[JanitorDirectStatePruneCandidate, JanitorActionDone], ...] = ( + run_janitor_phase( + progress=progress, + phase=JanitorPhase.PRUNE_DIRECT_STATE, + items=candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.PRUNED, name=candidate.display_name() + ), + apply=prune, + workers=workers, + ) + ) + return tuple(candidate for candidate, _ in pruned) diff --git a/src/sqlbuild/executor/janitor/_helpers/phase_steps.py b/src/sqlbuild/executor/janitor/_helpers/phase_steps.py new file mode 100644 index 0000000000..b25801dda2 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/phase_steps.py @@ -0,0 +1,124 @@ +"""Steps of one janitor apply phase: announcements, chunk positions and finished actions.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from sqlbuild.executor.janitor.models import ( + JanitorActionDone, + JanitorActionOutcome, + JanitorChunkOutcome, +) +from sqlbuild.executor.janitor.types import JanitorPhase, JanitorProgress +from sqlbuild.executor.janitor_events.models import JanitorEventRecord + + +def announce_phase(*, progress: JanitorProgress | None, phase: JanitorPhase, total: int) -> None: + """Report that a phase starts `total` actions.""" + + if progress is not None: + progress.phase_started(phase=phase.value, total=total) + + +def close_phase(*, progress: JanitorProgress | None, phase: JanitorPhase, failed: bool) -> None: + """Report that a phase finished or failed.""" + + if progress is not None: + progress.phase_finished(phase=phase.value, failed=failed) + + +def chunk_positions( + *, total: int, chunks: Sequence[Sequence[int]] | None +) -> tuple[tuple[int, ...], ...]: + """The item positions of each chunk, or every position as one chunk.""" + + if chunks is None: + return (tuple(range(total)),) + return tuple(tuple(chunk) for chunk in chunks) + + +def finished_actions( + *, + positions: tuple[int, ...], + chunk: JanitorChunkOutcome[JanitorActionDone | None], +) -> tuple[tuple[int, JanitorActionDone], ...]: + """Each succeeded item's phase position and what it did.""" + + pairs: list[tuple[int, JanitorActionDone]] = [] + position: int + outcome: JanitorActionOutcome[JanitorActionDone | None] | None + for position, outcome in zip(positions, chunk.outcomes, strict=True): + if outcome is not None and outcome.error is None: + pairs.append((position, outcome.result or JanitorActionDone())) + return tuple(pairs) + + +def chunk_failure(chunk: JanitorChunkOutcome[JanitorActionDone | None]) -> BaseException | None: + """The chunk's interrupt, else its earliest item failure, else its report error.""" + + if chunk.interruption is not None: + return chunk.interruption + return next( + (outcome.error for outcome in chunk.outcomes if outcome is not None and outcome.error), + chunk.report_error, + ) + + +def report_unwritten_events( + *, + progress: JanitorProgress | None, + finished: tuple[JanitorActionDone, ...], + error: BaseException, +) -> None: + """Warn which completed actions' audit events a failed write left out, by schema.""" + + events: tuple[JanitorEventRecord, ...] = tuple( + done.event for done in finished if done.event is not None + ) + if progress is None or not events: + return + schemas: tuple[str, ...] = tuple(sorted({_schema_name(event) for event in events})) + progress.events_unwritten( + count=len(events), schemas=schemas, error=str(error) or type(error).__name__ + ) + + +def raise_phase_failure( + *, + progress: JanitorProgress | None, + phase: JanitorPhase, + failure: BaseException | None, + flush_error: BaseException | None, +) -> None: + """Close the phase as failed and raise its failure; an interrupt wins, the other is chained.""" + + primary: BaseException | None + secondary: BaseException | None + primary, secondary = _ranked(failure=failure, flush_error=flush_error) + if primary is None: + return + if secondary is not None: + primary.__context__ = secondary + if flush_error is not None and primary is not flush_error: + primary.add_note(f"Audit events of completed actions were not written: {flush_error}") + try: + close_phase(progress=progress, phase=phase, failed=True) + finally: + raise primary + + +def _ranked( + *, failure: BaseException | None, flush_error: BaseException | None +) -> tuple[BaseException | None, BaseException | None]: + """The error to raise and the one to chain: an interrupt first, else the original failure.""" + + flush_interrupted: bool = flush_error is not None and not isinstance(flush_error, Exception) + if failure is None or (flush_interrupted and isinstance(failure, Exception)): + return flush_error, failure + return failure, flush_error + + +def _schema_name(event: JanitorEventRecord) -> str: + if event.relation_database is None: + return event.relation_schema + return f"{event.relation_database}.{event.relation_schema}" diff --git a/src/sqlbuild/executor/janitor/_helpers/worker_runs.py b/src/sqlbuild/executor/janitor/_helpers/worker_runs.py new file mode 100644 index 0000000000..db7d70b3ca --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/worker_runs.py @@ -0,0 +1,36 @@ +"""Attempt janitor actions and submit them to a worker pool.""" + +from __future__ import annotations + +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from contextvars import copy_context +from time import perf_counter +from typing import Any, cast + +from sqlbuild.executor.janitor.models import JanitorActionOutcome + + +def attempt_janitor_action[ResultT]( + *, action: Callable[..., ResultT], item: object, connection: Any +) -> JanitorActionOutcome[ResultT]: + """Run `action(item=, connection=)`, capturing its result or error and its duration.""" + + started: float = perf_counter() + try: + result: ResultT = action(item=item, connection=connection) + except BaseException as error: + return JanitorActionOutcome(error=error, duration_seconds=perf_counter() - started) + return JanitorActionOutcome(result=result, duration_seconds=perf_counter() - started) + + +def submit_janitor_action[ItemT, ResultT]( + *, + pool: ThreadPoolExecutor, + item: ItemT, + attempt: Callable[[ItemT], JanitorActionOutcome[ResultT]], +) -> Future[JanitorActionOutcome[ResultT]]: + """The submitted attempt of one item, run in a copy of the caller's context.""" + + submitted: Future[Any] = pool.submit(copy_context().run, attempt, item) + return cast(Future[JanitorActionOutcome[ResultT]], submitted) diff --git a/src/sqlbuild/executor/janitor/_helpers/workers.py b/src/sqlbuild/executor/janitor/_helpers/workers.py new file mode 100644 index 0000000000..b4c316a625 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/workers.py @@ -0,0 +1,61 @@ +"""Size and open the extra warehouse connections of concurrent janitor actions.""" + +from __future__ import annotations + +import time +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.janitor.models import JanitorPlan +from sqlbuild.executor.janitor.types import JanitorConnectionProgress +from sqlbuild.executor.pipeline.main._open_worker_connection_pool import ( + open_worker_connection_pool, +) +from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( + raise_warehouse_connection_failure, +) + + +def janitor_worker_count(*, plan: JanitorPlan, adapter: BaseAdapter, concurrency: int) -> int: + """Connections the largest phase can use: the concurrency, capped by the adapter's DDL limit.""" + + largest_phase: int = max( + 0 if plan.direct_mode else len(plan.candidates), + len(plan.archive_candidates) if plan.direct_mode else 0, + len(plan.archive_deletion_candidates) if plan.direct_mode else 0, + len(plan.old_name_views.drops) + len(plan.old_name_views.missing), + len(plan.query_diff_artifact_candidates), + len(plan.direct_state_prune_candidates), + ) + return max(1, min(concurrency, int(adapter.concurrent_ddl_limit), largest_phase)) + + +def open_janitor_worker_connections( + *, + adapter: BaseAdapter, + connection_config: dict[str, object] | None, + count: int, + progress: JanitorConnectionProgress | None, +) -> tuple[Any, ...]: + """Open `count` extra connections, reporting the connect phase to `progress`.""" + + if count <= 0 or connection_config is None: + return () + if progress is not None: + progress.on_connection_start(count) + started: float = time.monotonic() + try: + connections: tuple[Any, ...] = open_worker_connection_pool( + adapter=adapter, connection_config=connection_config, connection_count=count + ) + except BaseException as error: + if progress is not None: + progress.on_connection_error( + connection_count=count, elapsed_seconds=time.monotonic() - started + ) + raise_warehouse_connection_failure(adapter=adapter, error=error) + if progress is not None: + progress.on_connection_complete( + connection_count=count, elapsed_seconds=time.monotonic() - started + ) + return connections diff --git a/src/sqlbuild/executor/janitor/classes/__init__.py b/src/sqlbuild/executor/janitor/classes/__init__.py new file mode 100644 index 0000000000..8f8b3afa2b --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/__init__.py @@ -0,0 +1 @@ +"""Janitor executor classes.""" diff --git a/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py b/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py new file mode 100644 index 0000000000..ea0d8bb598 --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py @@ -0,0 +1,98 @@ +"""Keep every finished janitor action of one chunk, even when reporting it or waiting fails.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait + +from sqlbuild.executor.janitor._helpers.worker_runs import submit_janitor_action +from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorChunkOutcome + + +class JanitorChunkCollector[ResultT]: + """Record each outcome before reporting it; a report error or interrupt stops new work only.""" + + def __init__(self, *, size: int, on_done: Callable[..., None]) -> None: + self._size: int = size + self._on_done: Callable[..., None] = on_done + self._outcomes: dict[int, JanitorActionOutcome[ResultT]] = {} + self._interruption: BaseException | None = None + self._report_error: BaseException | None = None + + @property + def accepting(self) -> bool: + """Whether more actions may start: nothing failed, raised or interrupted yet.""" + + return ( + self._interruption is None + and self._report_error is None + and all(outcome.error is None for outcome in self._outcomes.values()) + ) + + def record(self, *, index: int, outcome: JanitorActionOutcome[ResultT]) -> None: + """Keep `outcome`, then report it unless reporting already failed.""" + + if index in self._outcomes: + return + self._outcomes[index] = outcome + if self._report_error is not None: + return + try: + self._on_done(index=index, outcome=outcome) + except BaseException as error: + self._report_error = error + + def interrupt(self, error: BaseException) -> None: + """Remember the first interrupt; running actions still finish and are recorded.""" + + if self._interruption is None: + self._interruption = error + + def collect[ItemT]( + self, + *, + pool: ThreadPoolExecutor, + items: Sequence[ItemT], + attempt: Callable[[ItemT], JanitorActionOutcome[ResultT]], + ) -> None: + """Submit items until stopped, then record every submitted action as it finishes.""" + + futures: dict[Future[JanitorActionOutcome[ResultT]], int] = {} + index: int + item: ItemT + for index, item in enumerate(items): + if not self.accepting: + break + try: + futures[submit_janitor_action(pool=pool, item=item, attempt=attempt)] = index + except BaseException as error: + self.interrupt(error) + self.drain(futures=futures) + + def drain(self, *, futures: dict[Future[JanitorActionOutcome[ResultT]], int]) -> None: + """Record every submitted action as it finishes, cancelling unstarted ones once stopped.""" + + pending: set[Future[JanitorActionOutcome[ResultT]]] = set(futures) + while pending: + try: + done: set[Future[JanitorActionOutcome[ResultT]]] = wait( + pending, return_when=FIRST_COMPLETED + ).done + future: Future[JanitorActionOutcome[ResultT]] + for future in sorted(done, key=futures.__getitem__): + if not future.cancelled(): + self.record(index=futures[future], outcome=future.result()) + pending.discard(future) + except BaseException as error: + self.interrupt(error) + if not self.accepting: + pending = {future for future in pending if not future.cancel()} + + def outcome(self) -> JanitorChunkOutcome[ResultT]: + """Every recorded outcome by item position, with the interrupt and report error.""" + + return JanitorChunkOutcome( + outcomes=tuple(self._outcomes.get(index) for index in range(self._size)), + interruption=self._interruption, + report_error=self._report_error, + ) diff --git a/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py b/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py new file mode 100644 index 0000000000..a566904cd2 --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py @@ -0,0 +1,48 @@ +"""Report each finished janitor action of one chunk to the progress output.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.executor.janitor.models import ( + JanitorActionDone, + JanitorActionOutcome, + JanitorProgressItem, +) +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress + + +class JanitorItemReporter: + """Report one chunk's finished actions by their position in the chunk.""" + + def __init__( + self, + *, + progress: JanitorProgress | None, + phase: JanitorPhase, + labels: tuple[JanitorProgressItem, ...], + ) -> None: + self._progress: JanitorProgress | None = progress + self._phase: JanitorPhase = phase + self._labels: tuple[JanitorProgressItem, ...] = labels + + def __call__( + self, *, index: int, outcome: JanitorActionOutcome[JanitorActionDone | None] + ) -> None: + if self._progress is None: + return + done: JanitorActionDone | None = outcome.result + taken: JanitorAction | None = None if done is None else done.taken + label: JanitorProgressItem = self._labels[index] + error: str | None = ( + None if outcome.error is None else str(outcome.error) or type(outcome.error).__name__ + ) + self._progress.item_finished( + phase=self._phase.value, + item=replace( + label, + action=label.action if taken is None else taken, + error=error, + duration_seconds=outcome.duration_seconds, + ), + ) diff --git a/src/sqlbuild/executor/janitor/classes/janitor_workers.py b/src/sqlbuild/executor/janitor/classes/janitor_workers.py new file mode 100644 index 0000000000..1116f4754d --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/janitor_workers.py @@ -0,0 +1,75 @@ +"""Bounded concurrent janitor actions, one warehouse connection per worker.""" + +from __future__ import annotations + +import queue +from collections.abc import Callable, Sequence +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from sqlbuild.executor.janitor._helpers.worker_runs import attempt_janitor_action +from sqlbuild.executor.janitor.classes.janitor_chunk_collector import JanitorChunkCollector +from sqlbuild.executor.janitor.classes.pooled_janitor_attempt import PooledJanitorAttempt + + +class JanitorWorkers: + """Run independent janitor actions, each on a connection no other action is using.""" + + def __init__(self, *, connections: tuple[Any, ...]) -> None: + self.connections: tuple[Any, ...] = connections + + @property + def connection(self) -> Any: + """The invocation's own connection, used for everything outside a phase's actions.""" + + return self.connections[0] + + def run[ItemT, ResultT]( + self, + *, + items: Sequence[ItemT], + action: Callable[..., ResultT], + collector: JanitorChunkCollector[ResultT], + ) -> None: + """Run `action(item=, connection=)` per item into `collector` until it stops accepting.""" + + if len(self.connections) == 1 or len(items) <= 1: + self._run_in_order(items=items, action=action, collector=collector) + return + connections: queue.Queue[Any] = queue.Queue() + connection: Any + for connection in self.connections: + connections.put(connection) + pool: ThreadPoolExecutor = ThreadPoolExecutor( + max_workers=min(len(self.connections), len(items)) + ) + try: + collector.collect( + pool=pool, + items=items, + attempt=PooledJanitorAttempt(action=action, connections=connections), + ) + finally: + pool.shutdown(wait=True, cancel_futures=True) + + def _run_in_order[ItemT, ResultT]( + self, + *, + items: Sequence[ItemT], + action: Callable[..., ResultT], + collector: JanitorChunkCollector[ResultT], + ) -> None: + index: int + item: ItemT + for index, item in enumerate(items): + if not collector.accepting: + return + try: + collector.record( + index=index, + outcome=attempt_janitor_action( + action=action, item=item, connection=self.connection + ), + ) + except BaseException as error: + collector.interrupt(error) diff --git a/src/sqlbuild/executor/janitor/classes/pooled_janitor_attempt.py b/src/sqlbuild/executor/janitor/classes/pooled_janitor_attempt.py new file mode 100644 index 0000000000..74fba835a3 --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/pooled_janitor_attempt.py @@ -0,0 +1,25 @@ +"""One janitor action attempt on a connection borrowed from a worker pool.""" + +from __future__ import annotations + +import queue +from collections.abc import Callable +from typing import Any + +from sqlbuild.executor.janitor._helpers.worker_runs import attempt_janitor_action +from sqlbuild.executor.janitor.models import JanitorActionOutcome + + +class PooledJanitorAttempt[ResultT]: + """Run an action on a borrowed connection and return the connection afterwards.""" + + def __init__(self, *, action: Callable[..., ResultT], connections: queue.Queue[Any]) -> None: + self._action: Callable[..., ResultT] = action + self._connections: queue.Queue[Any] = connections + + def __call__(self, item: object) -> JanitorActionOutcome[ResultT]: + borrowed: Any = self._connections.get() + try: + return attempt_janitor_action(action=self._action, item=item, connection=borrowed) + finally: + self._connections.put(borrowed) diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py new file mode 100644 index 0000000000..4b6e6f2a97 --- /dev/null +++ b/src/sqlbuild/executor/janitor/constants.py @@ -0,0 +1,3 @@ +"""Janitor executor constants.""" + +ARCHIVE_NAME_SEPARATOR: str = "__" diff --git a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py new file mode 100644 index 0000000000..dab78408dc --- /dev/null +++ b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py @@ -0,0 +1,86 @@ +"""Run one janitor apply phase, reporting each action as it finishes.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence + +from sqlbuild.executor.janitor._helpers.phase_steps import ( + announce_phase, + chunk_failure, + chunk_positions, + close_phase, + finished_actions, + raise_phase_failure, + report_unwritten_events, +) +from sqlbuild.executor.janitor.classes.janitor_chunk_collector import JanitorChunkCollector +from sqlbuild.executor.janitor.classes.janitor_item_reporter import JanitorItemReporter +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.models import ( + JanitorActionDone, + JanitorChunkOutcome, + JanitorProgressItem, +) +from sqlbuild.executor.janitor.types import JanitorPhase, JanitorProgress + + +def run_janitor_phase[T]( + *, + progress: JanitorProgress | None, + phase: JanitorPhase, + items: Sequence[T], + describe: Callable[[T], JanitorProgressItem], + apply: Callable[..., JanitorActionDone | None], + workers: JanitorWorkers, + chunks: Sequence[Sequence[int]] | None = None, + after_chunk: Callable[[tuple[JanitorActionDone, ...]], None] | None = None, +) -> tuple[tuple[T, JanitorActionDone], ...]: + """Apply `apply(item=, connection=)` chunk by chunk; return succeeded items in plan order.""" + + if not items: + return () + _ = announce_phase(progress=progress, phase=phase, total=len(items)) + finished: list[tuple[int, JanitorActionDone]] = [] + positions: tuple[int, ...] + for positions in chunk_positions(total=len(items), chunks=chunks): + collector: JanitorChunkCollector[JanitorActionDone | None] = JanitorChunkCollector( + size=len(positions), + on_done=JanitorItemReporter( + progress=progress, + phase=phase, + labels=tuple(describe(items[position]) for position in positions), + ), + ) + try: + _ = workers.run( + items=tuple(items[position] for position in positions), + action=apply, + collector=collector, + ) + except BaseException as error: + _ = collector.interrupt(error) + chunk: JanitorChunkOutcome[JanitorActionDone | None] = collector.outcome() + succeeded: tuple[tuple[int, JanitorActionDone], ...] = finished_actions( + positions=positions, chunk=chunk + ) + finished.extend(succeeded) + completed: tuple[JanitorActionDone, ...] = tuple(done for _, done in succeeded) + flush_error: BaseException | None = None + try: + if after_chunk is not None: + _ = after_chunk(completed) + except BaseException as error: + flush_error = error + _ = report_unwritten_events(progress=progress, finished=completed, error=error) + _ = raise_phase_failure( + progress=progress, + phase=phase, + failure=chunk_failure(chunk), + flush_error=flush_error, + ) + _ = close_phase(progress=progress, phase=phase, failed=False) + return tuple((items[position], done) for position, done in sorted(finished, key=_position)) + + +def _position(pair: tuple[int, JanitorActionDone]) -> int: + return pair[0] diff --git a/src/sqlbuild/executor/janitor/main/execute.py b/src/sqlbuild/executor/janitor/main/execute.py index bcda614982..95faf450c5 100644 --- a/src/sqlbuild/executor/janitor/main/execute.py +++ b/src/sqlbuild/executor/janitor/main/execute.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sys from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -11,6 +12,12 @@ janitor_run_id, ) from sqlbuild.executor.janitor._helpers.deletion import apply_janitor_deletions +from sqlbuild.executor.janitor._helpers.workers import ( + janitor_worker_count, + open_janitor_worker_connections, +) +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( JanitorArchiveCandidate, JanitorArchivedRelation, @@ -19,9 +26,19 @@ JanitorExecutionResult, JanitorOldNameView, JanitorPlan, + JanitorProgressItem, JanitorQueryDiffArtifactCandidate, ) +from sqlbuild.executor.janitor.types import ( + JanitorAction, + JanitorConnectionProgress, + JanitorPhase, + JanitorProgress, +) from sqlbuild.executor.old_name_views.main._drop_old_name_views import drop_old_name_views +from sqlbuild.executor.pipeline.main._close_worker_connection_pool import ( + close_worker_connection_pool, +) from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -30,46 +47,75 @@ def execute_janitor_plan( plan: JanitorPlan, adapter: BaseAdapter, connection: Any, + progress: JanitorProgress | None = None, + connection_config: dict[str, object] | None = None, + concurrency: int = 1, + connection_progress: JanitorConnectionProgress | None = None, ) -> JanitorExecutionResult: - """Delete all candidates in a janitor plan.""" + """Apply a janitor plan phase by phase, running each phase's actions on up to `concurrency`.""" with OperationLifecycle( operation_kind="janitor", operation_name="janitor_execution", metadata={"item_count": _janitor_action_count(plan)}, ): - return _execute_janitor_plan( - plan=plan, + extra: tuple[Any, ...] = open_janitor_worker_connections( adapter=adapter, - connection=connection, + connection_config=connection_config, + count=janitor_worker_count(plan=plan, adapter=adapter, concurrency=concurrency) - 1, + progress=connection_progress, ) + try: + return _execute_janitor_plan( + plan=plan, + adapter=adapter, + workers=JanitorWorkers(connections=(connection, *extra)), + progress=progress, + ) + finally: + _ = close_worker_connection_pool( + adapter=adapter, connections=extra, active_error=sys.exc_info()[1] + ) def _execute_janitor_plan( *, plan: JanitorPlan, adapter: BaseAdapter, - connection: Any, + workers: JanitorWorkers, + progress: JanitorProgress | None, ) -> JanitorExecutionResult: recorder: StatementRecorder = StatementRecorder() - candidate: JanitorDeleteCandidate - for candidate in () if plan.direct_mode else plan.candidates: + + def drop_candidate(*, item: JanitorDeleteCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): adapter.drop( connection=connection, - destination=candidate.key.display_name(), + destination=item.key.display_name(), if_exists=True, statement_recorder=recorder, ) + + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.DELETE_RELATIONS, + items=() if plan.direct_mode else plan.candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.DELETED, name=candidate.key.display_name() + ), + apply=drop_candidate, + workers=workers, + ) archived: tuple[JanitorArchiveCandidate, ...] deleted_archives: tuple[JanitorArchivedRelation, ...] archived, deleted_archives = ( apply_direct_archives( plan=plan, adapter=adapter, - connection=connection, + workers=workers, recorder=recorder, run_id=janitor_run_id(), + progress=progress, ) if plan.direct_mode else ((), ()) @@ -77,33 +123,47 @@ def _execute_janitor_plan( dropped_old_name_views: tuple[JanitorOldNameView, ...] = drop_old_name_views( plan=plan.old_name_views, adapter=adapter, - connection=connection, + workers=workers, recorder=recorder, run_id=janitor_run_id(), + progress=progress, ) - query_artifact_candidate: JanitorQueryDiffArtifactCandidate - for query_artifact_candidate in plan.query_diff_artifact_candidates: + + def drop_query_artifact(*, item: JanitorQueryDiffArtifactCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): query_artifact_destination: str | None = adapter.render_qualified_name( - database=query_artifact_candidate.key.database, - schema=query_artifact_candidate.key.schema, - name=query_artifact_candidate.key.name, + database=item.key.database, + schema=item.key.schema, + name=item.key.name, ) adapter.drop( connection=connection, destination=( query_artifact_destination if query_artifact_destination is not None - else query_artifact_candidate.key.display_name() + else item.key.display_name() ), if_exists=True, statement_recorder=recorder, ) + + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.DROP_QUERY_ARTIFACTS, + items=plan.query_diff_artifact_candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.DROPPED, name=candidate.key.display_name() + ), + apply=drop_query_artifact, + workers=workers, + ) pruned_direct_state: tuple[JanitorDirectStatePruneCandidate, ...] = apply_janitor_deletions( candidates=plan.direct_state_prune_candidates, - delete=lambda direct_state_candidate: adapter.execute( - connection=connection, sql=direct_state_candidate.prune_sql + delete=lambda *, candidate, connection: adapter.execute( + connection=connection, sql=candidate.prune_sql ), + workers=workers, + progress=progress, ) return JanitorExecutionResult( deleted=() if plan.direct_mode else plan.candidates, diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 1eb83014a3..5313227c0a 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -7,6 +7,8 @@ from sqlbuild._native import OldNameViewDropReason, OldNameViewHistory from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup +from sqlbuild.executor.janitor.types import JanitorAction +from sqlbuild.executor.janitor_events.models import JanitorEventRecord @dataclass(frozen=True, eq=False) @@ -226,3 +228,40 @@ class JanitorExecutionResult: ) pruned_direct_state: tuple[JanitorDirectStatePruneCandidate, ...] = field(default_factory=tuple) dropped_old_name_views: tuple[JanitorOldNameView, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorProgressItem: + """How the progress output names one janitor action, and once finished its error and time.""" + + action: JanitorAction + name: str + destination: str | None = None + error: str | None = None + duration_seconds: float = 0.0 + + +@dataclass(frozen=True) +class JanitorActionOutcome[ResultT]: + """What one janitor action returned or raised, and how long it ran.""" + + result: ResultT | None = None + error: BaseException | None = None + duration_seconds: float = 0.0 + + +@dataclass(frozen=True) +class JanitorChunkOutcome[ResultT]: + """Each item's outcome in one chunk (none if never started), interrupt and report error.""" + + outcomes: tuple[JanitorActionOutcome[ResultT] | None, ...] + interruption: BaseException | None = None + report_error: BaseException | None = None + + +@dataclass(frozen=True) +class JanitorActionDone: + """A finished action: the action taken when it differs from the described one, and its event.""" + + taken: JanitorAction | None = None + event: JanitorEventRecord | None = None diff --git a/src/sqlbuild/executor/janitor/types.py b/src/sqlbuild/executor/janitor/types.py new file mode 100644 index 0000000000..5549abfb21 --- /dev/null +++ b/src/sqlbuild/executor/janitor/types.py @@ -0,0 +1,44 @@ +"""Janitor execution types.""" + +from __future__ import annotations + +from enum import StrEnum +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from sqlbuild.executor.janitor.models import JanitorProgressItem + + +class JanitorPhase(StrEnum): + DELETE_RELATIONS = "delete_relations" + ARCHIVE = "archive" + DELETE_ARCHIVES = "delete_archives" + DROP_OLD_NAME_VIEWS = "drop_old_name_views" + DROP_QUERY_ARTIFACTS = "drop_query_artifacts" + PRUNE_DIRECT_STATE = "prune_direct_state" + + +class JanitorAction(StrEnum): + ARCHIVED = "archived" + DELETED = "deleted" + DROPPED = "dropped" + RECORDED = "recorded" + PRUNED = "pruned" + + +class JanitorProgress(Protocol): + def phase_started(self, *, phase: str, total: int) -> None: ... + + def item_finished(self, *, phase: str, item: JanitorProgressItem) -> None: ... + + def phase_finished(self, *, phase: str, failed: bool) -> None: ... + + def events_unwritten(self, *, count: int, schemas: tuple[str, ...], error: str) -> None: ... + + +class JanitorConnectionProgress(Protocol): + def on_connection_start(self, connection_count: int) -> None: ... + + def on_connection_complete(self, *, connection_count: int, elapsed_seconds: float) -> None: ... + + def on_connection_error(self, *, connection_count: int, elapsed_seconds: float) -> None: ... diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py index 1de4e10a49..b56c2653e2 100644 --- a/src/sqlbuild/executor/janitor_events/_helpers/sql.py +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -58,30 +58,40 @@ def build_create_table_sql( ) -def build_existing_event_sql( +def build_existing_run_events_sql( *, - record: JanitorEventRecord, + database: str | None, + schema: str, + run_id: str, render_qualified_name: Callable[..., str | None], ) -> str: + """Select the IDs of every event one run already wrote to one schema's table.""" + qualified_name: str = build_qualified_table_name( - database=record.relation_database, - schema=record.relation_schema, - render_qualified_name=render_qualified_name, + database=database, schema=schema, render_qualified_name=render_qualified_name ) - event_id: str = _column_literal(column=COLUMN_EVENT_ID, value=record.event_id) - return f"SELECT {COLUMN_EVENT_ID} FROM {qualified_name} WHERE {COLUMN_EVENT_ID} = {event_id}" + run: str = _column_literal(column=COLUMN_RUN_ID, value=run_id) + return f"SELECT {COLUMN_EVENT_ID} FROM {qualified_name} WHERE {COLUMN_RUN_ID} = {run}" -def build_insert_sql( +def build_insert_many_sql( *, - record: JanitorEventRecord, + records: tuple[JanitorEventRecord, ...], render_qualified_name: Callable[..., str | None], ) -> str: + """Append several events to the table of the first record's schema in one statement.""" + + first: JanitorEventRecord = records[0] qualified_name: str = build_qualified_table_name( - database=record.relation_database, - schema=record.relation_schema, + database=first.relation_database, + schema=first.relation_schema, render_qualified_name=render_qualified_name, ) + rows: str = ", ".join(f"({_row_literals(record=record)})" for record in records) + return f"INSERT INTO {qualified_name} ({', '.join(JANITOR_EVENT_COLUMNS)}) VALUES {rows}" + + +def _row_literals(*, record: JanitorEventRecord) -> str: values: dict[str, object | None] = { COLUMN_EVENT_ID: record.event_id, COLUMN_SCHEMA_VERSION: record.schema_version, @@ -97,10 +107,9 @@ def build_insert_sql( COLUMN_ARCHIVE_QUALIFIED_NAME: record.archive_qualified_name, COLUMN_ARCHIVED_AT: record.archived_at, } - literals: str = ", ".join( + return ", ".join( _column_literal(column=column, value=values[column]) for column in JANITOR_EVENT_COLUMNS ) - return f"INSERT INTO {qualified_name} ({', '.join(JANITOR_EVENT_COLUMNS)}) VALUES ({literals})" def _column_literal(*, column: str, value: object | None) -> str: diff --git a/src/sqlbuild/executor/janitor_events/classes/__init__.py b/src/sqlbuild/executor/janitor_events/classes/__init__.py new file mode 100644 index 0000000000..2eeccb4088 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/classes/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event classes.""" diff --git a/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py b/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py new file mode 100644 index 0000000000..8554afb30c --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py @@ -0,0 +1,84 @@ +"""Append-only janitor audit events, written in multi-row chunks per schema event table.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.executor.janitor_events._helpers.sql import ( + build_existing_run_events_sql, + build_insert_many_sql, +) +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_CHUNK_SIZE +from sqlbuild.executor.janitor_events.models import JanitorEventRecord + +type _SchemaKey = tuple[str | None, str] + + +class JanitorEventWriter: + """Write each deterministic event ID at most once, even across retries of one invocation.""" + + def __init__( + self, + *, + execute: AdapterExecute[Any, Any], + render_qualified_name: Callable[..., str | None], + run_id: str, + ) -> None: + self._execute: AdapterExecute[Any, Any] = execute + self._render_qualified_name: Callable[..., str | None] = render_qualified_name + self._run_id: str = run_id + self._written: set[str] = set() + self._lock: threading.Lock = threading.Lock() + + def read_existing(self, *, connection: Any, schemas: tuple[_SchemaKey, ...]) -> None: + """Remember the IDs of events this run already wrote to each schema's table.""" + + database: str | None + schema: str + for database, schema in schemas: + cursor: Any = self._execute( + connection=connection, + sql=build_existing_run_events_sql( + database=database, + schema=schema, + run_id=self._run_id, + render_qualified_name=self._render_qualified_name, + ), + ) + ids: set[str] = {str(row[0]) for row in cursor.fetchall()} + with self._lock: + self._written |= ids + + def write(self, *, connection: Any, records: tuple[JanitorEventRecord, ...]) -> None: + """Append the unwritten events, one INSERT per schema and chunk of rows.""" + + with self._lock: + unwritten: dict[str, JanitorEventRecord] = { + record.event_id: record + for record in records + if record.event_id not in self._written + } + by_schema: dict[_SchemaKey, list[JanitorEventRecord]] = {} + record: JanitorEventRecord + for record in unwritten.values(): + by_schema.setdefault((record.relation_database, record.relation_schema), []).append( + record + ) + schema_records: list[JanitorEventRecord] + for schema_records in by_schema.values(): + start: int + for start in range(0, len(schema_records), JANITOR_EVENT_CHUNK_SIZE): + chunk: tuple[JanitorEventRecord, ...] = tuple( + schema_records[start : start + JANITOR_EVENT_CHUNK_SIZE] + ) + _ = self._execute( + connection=connection, + sql=build_insert_many_sql( + records=chunk, render_qualified_name=self._render_qualified_name + ), + ) + with self._lock: + self._written |= {written.event_id for written in chunk} diff --git a/src/sqlbuild/executor/janitor_events/constants.py b/src/sqlbuild/executor/janitor_events/constants.py index 9e30530b97..f120e39903 100644 --- a/src/sqlbuild/executor/janitor_events/constants.py +++ b/src/sqlbuild/executor/janitor_events/constants.py @@ -55,3 +55,5 @@ COLUMN_OCCURRED_AT: StateSqlValueType.TEXT_TIMESTAMP, COLUMN_ARCHIVED_AT: StateSqlValueType.TEXT_TIMESTAMP, } + +JANITOR_EVENT_CHUNK_SIZE: int = 100 diff --git a/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py b/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py deleted file mode 100644 index 3dc22a22e4..0000000000 --- a/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Janitor audit event warehouse write operation.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.executor.janitor_events._helpers.sql import ( - build_existing_event_sql, - build_insert_sql, -) -from sqlbuild.executor.janitor_events.models import JanitorEventRecord - - -def write_janitor_event_record( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - record: JanitorEventRecord, - render_qualified_name: Callable[..., str | None], -) -> bool: - """Append one janitor event unless its deterministic ID was already written.""" - - cursor: Any = execute( - connection=connection, - sql=build_existing_event_sql(record=record, render_qualified_name=render_qualified_name), - ) - if cursor.fetchall(): - return False - _ = execute( - connection=connection, - sql=build_insert_sql(record=record, render_qualified_name=render_qualified_name), - ) - return True diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index e2b7e4e095..3e1758eb6a 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -20,11 +20,16 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( + JanitorActionDone, JanitorOldNameView, JanitorOldNameViewPlanning, + JanitorProgressItem, JanitorRelationKey, ) +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress from sqlbuild.executor.migrations.main._compatibility_view_sql import compatibility_view_sql @@ -32,15 +37,20 @@ def apply_old_name_view_drops( *, plan: JanitorOldNameViewPlanning, adapter: BaseAdapter, - connection: Any, + workers: JanitorWorkers, recorder: StatementRecorder, run_id: str, + progress: JanitorProgress | None = None, ) -> tuple[JanitorOldNameView, ...]: """Drop expired or requested views, then record every drop and every vanished view.""" - dropped: list[JanitorOldNameView] = [] - view: JanitorOldNameView - for view in plan.drops: + def apply(*, item: tuple[JanitorOldNameView, bool], connection: Any) -> JanitorActionDone: + view: JanitorOldNameView + planned_missing: bool + view, planned_missing = item + if planned_missing: + _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) + return JanitorActionDone() if not is_compatibility_view(adapter=adapter, connection=connection, view=view): _record_drop( adapter=adapter, @@ -48,7 +58,7 @@ def apply_old_name_view_drops( view=_with_reason(view=view, reason=OldNameViewDropReason.MISSING), run_id=run_id, ) - continue + return JanitorActionDone(taken=JanitorAction.RECORDED) adapter.drop_view( connection=connection, destination=resolve_qualified_name_parts( @@ -61,10 +71,29 @@ def apply_old_name_view_drops( statement_recorder=recorder, ) _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) - dropped.append(view) - for view in plan.missing: - _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) - return tuple(dropped) + return JanitorActionDone() + + finished: tuple[tuple[tuple[JanitorOldNameView, bool], JanitorActionDone], ...] = ( + run_janitor_phase( + progress=progress, + phase=JanitorPhase.DROP_OLD_NAME_VIEWS, + items=( + *((view, False) for view in plan.drops), + *((view, True) for view in plan.missing), + ), + describe=lambda entry: JanitorProgressItem( + action=JanitorAction.RECORDED if entry[1] else JanitorAction.DROPPED, + name=entry[0].key.display_name(), + ), + apply=apply, + workers=workers, + ) + ) + return tuple( + view + for (view, planned_missing), done in finished + if not planned_missing and done.taken is None + ) def _record_drop( diff --git a/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py b/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py index b66fa14d88..1c3a0273d2 100644 --- a/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py +++ b/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py @@ -2,11 +2,11 @@ from __future__ import annotations -from typing import Any - from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.models import JanitorOldNameView, JanitorOldNameViewPlanning +from sqlbuild.executor.janitor.types import JanitorProgress from sqlbuild.executor.old_name_views._helpers.janitor import apply_old_name_view_drops @@ -14,12 +14,18 @@ def drop_old_name_views( *, plan: JanitorOldNameViewPlanning, adapter: BaseAdapter, - connection: Any, + workers: JanitorWorkers, recorder: StatementRecorder, run_id: str, + progress: JanitorProgress | None = None, ) -> tuple[JanitorOldNameView, ...]: """Drop planned views, then append a drop fact for each dropped or vanished view.""" return apply_old_name_view_drops( - plan=plan, adapter=adapter, connection=connection, recorder=recorder, run_id=run_id + plan=plan, + adapter=adapter, + workers=workers, + recorder=recorder, + run_id=run_id, + progress=progress, ) diff --git a/src/sqlbuild/executor/pipeline/main/_close_worker_connection_pool.py b/src/sqlbuild/executor/pipeline/main/_close_worker_connection_pool.py new file mode 100644 index 0000000000..3423594249 --- /dev/null +++ b/src/sqlbuild/executor/pipeline/main/_close_worker_connection_pool.py @@ -0,0 +1,16 @@ +"""Public entry for closing the extra connections of concurrent workers.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.pipeline._helpers.connections import close_connections + + +def close_worker_connection_pool( + *, adapter: BaseAdapter, connections: tuple[Any, ...], active_error: BaseException | None +) -> None: + """Close every connection, raising a close failure only when no error is already active.""" + + _ = close_connections(adapter=adapter, connections=connections, active_error=active_error) diff --git a/src/sqlbuild/executor/pipeline/main/_open_worker_connection_pool.py b/src/sqlbuild/executor/pipeline/main/_open_worker_connection_pool.py new file mode 100644 index 0000000000..5b04daa14e --- /dev/null +++ b/src/sqlbuild/executor/pipeline/main/_open_worker_connection_pool.py @@ -0,0 +1,18 @@ +"""Public entry for opening the extra connections of concurrent workers.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.pipeline._helpers.connections import open_worker_connections + + +def open_worker_connection_pool( + *, adapter: BaseAdapter, connection_config: dict[str, object], connection_count: int +) -> tuple[Any, ...]: + """Open `connection_count` connections concurrently, closing any opened when one fails.""" + + return open_worker_connections( + adapter=adapter, connection_config=connection_config, connection_count=connection_count + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index dfe682a71f..7c9005104e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -186,3 +186,105 @@ class JanitorNonInteractiveE2ETestCase: expected_exit_code: int expected_stderr_fragments: tuple[str, ...] expected_existing_tables: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorProgressE2ETestCase: + """Test case for streamed janitor phase and action lines.""" + + description: str + janitor_command: tuple[str, ...] + stale_tables: int + expected_exit_code: int + expected_ordered_lines: tuple[str, ...] + expected_archive_events: int + unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) + fail_rename: str | None = None + expected_remaining_tables: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorJsonE2ETestCase: + """Test case for `sqb janitor --json` output streams.""" + + description: str + janitor_command: tuple[str, ...] + stale_tables: int + expected_status: str + expected_archived: int + expected_item_statuses: tuple[str, ...] + expected_stderr_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorInspectionCountE2ETestCase: + """Test case for warehouse reads and audit writes per schema.""" + + description: str + stale_tables: dict[str, int] + renamed_models: tuple[str, ...] + expected_listings_per_schema: int + expected_multi_schema_listings: int + expected_existence_queries_per_schema: int + expected_view_definition_reads: int + expected_batched_view_definition_reads: int + expected_event_inserts: dict[str, int] + + +@dataclass(frozen=True) +class JanitorConcurrencyE2ETestCase: + """Test case comparing concurrent janitor actions with sequential ones.""" + + description: str + stale_tables: dict[str, int] + concurrency: int + expected_exit_code: int + expected_connection_line: str + + +@dataclass(frozen=True) +class JanitorConcurrentFailureE2ETestCase: + """Test case for one failing action among concurrent janitor actions.""" + + description: str + stale_tables: dict[str, int] + concurrency: int + fail_rename: str + expected_exit_code: int + expected_failed_lines: int + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorShortenedArchiveE2ETestCase: + """Test case for the audit event of an archive whose name was shortened.""" + + description: str + long_relation_name: str + identifier_limit: int + expected_rows_in_event_insert: int + expected_event_count: int + + +@dataclass(frozen=True) +class JanitorJsonErrorE2ETestCase: + """Test case for the `--json` error document of a janitor run that fails before cleanup.""" + + description: str + janitor_command: tuple[str, ...] + expected_exit_code: int + expected_error_code: str + expected_stderr_fragment: str + + +@dataclass(frozen=True) +class JanitorJsonFailedRunE2ETestCase: + """Test case for the single `--json` document of a janitor run that fails mid-phase.""" + + description: str + janitor_command: tuple[str, ...] + fail_rename: str + expected_exit_code: int + expected_status: str + expected_error_fragment: str + expected_failed_items: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index dd575d49d8..ef611e854e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -2,9 +2,12 @@ from __future__ import annotations +import json +import re import subprocess from collections.abc import Mapping from datetime import UTC, datetime +from itertools import product from pathlib import Path from textwrap import dedent from types import MappingProxyType @@ -233,7 +236,7 @@ def supports_relation_age_metadata(self) -> bool: return True def maximum_identifier_length(self) -> int: - return int(_settings().get("identifier_limit", super().maximum_identifier_length())) + return int(_settings().get("identifier_limit") or super().maximum_identifier_length()) def list_relations( self, @@ -460,3 +463,238 @@ def probe_fingerprint_count(*, db_path: Path, schema: str) -> int: ), ) return int(str(rows[0][0])) + + +PROBE_JANITOR_ADAPTER_NAME: str = "probe_janitor_duckdb" +PROBE_JANITOR_SETTINGS_FILENAME: str = "janitor_probe_settings.json" +PROBE_STATEMENT_LOG_FILENAME: str = "janitor_statements.log" +PROBE_JANITOR_ADAPTER_SOURCE: str = """ +import json +import threading +from pathlib import Path +from typing import Any + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter + +_ROOT = Path(__file__).resolve().parent.parent +_LOG_LOCK = threading.Lock() + + +def _settings() -> dict[str, Any]: + path = _ROOT / "janitor_probe_settings.json" + if not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +class ProbeJanitorDuckDbAdapter(DuckDbAdapter): + adapter_name = "probe_janitor_duckdb" + + @property + def concurrent_ddl_limit(self) -> int: + return int(_settings().get("concurrent_ddl_limit", 1)) + + def maximum_identifier_length(self) -> int: + return int(_settings().get("identifier_limit") or super().maximum_identifier_length()) + + def _execute(self, *, connection: Any, sql: str) -> Any: + settings = _settings() + failing = settings.get("fail_rename") + if failing and sql.startswith("ALTER TABLE") and f".{failing} RENAME" in sql: + raise AdapterUserError(f"simulated rename failure for {failing}") + if settings.get("log"): + with _LOG_LOCK, (_ROOT / "janitor_statements.log").open("a") as handle: + handle.write(" ".join(sql.split()) + "\\n") + return super()._execute(connection=connection, sql=sql) +""" + + +ARCHIVE_ITEM_LINE: re.Pattern[str] = re.compile( + r"^\s+(?P\d+)/(?P\d+) archived ", re.MULTILINE +) + + +def _probe_model_sql(*, schema: str, migrate_from: str) -> str: + migration: str = {"": ""}.get(migrate_from, f', migrate_from "{migrate_from}"') + return f"MODEL (description 'Test model.', schema {schema}{migration});\n\nSELECT 1 AS id\n" + + +def prepare_probe_janitor_project( + *, + tmp_path: Path, + project_name: str, + stale_tables: Mapping[str, int], + renamed_models: tuple[str, ...] = (), + concurrency: int = 1, +) -> Path: + """Build a DuckDB project with stale tables and an expired archive in each schema.""" + + models: dict[str, str] = { + f"models/{schema}_{name}.sql": _probe_model_sql(schema=schema, migrate_from="") + for schema, name in product(stale_tables, ("orders", *renamed_models)) + } + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name=project_name, + repo_files={ + "sqlbuild_project.toml": ( + f'name = "{project_name}"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + f'database = "{(tmp_path / project_name / "janitor.duckdb").as_posix()}"\n\n' + f"[settings]\nconcurrency = {concurrency}\n\n" + "[janitor]\n" + "enabled = true\n" + "retention_days = 0\n" + "archive_retention_days = 14\n" + "delete_tracked_only = false\n" + "direct_state_history_versions = 2\n\n" + "[defaults]\n" + 'materialized = "table"\n' + ), + f"adapters/{PROBE_JANITOR_ADAPTER_NAME}.py": PROBE_JANITOR_ADAPTER_SOURCE, + **models, + }, + ) + _build(project_dir) + db_path: Path = project_dir / "janitor.duckdb" + schema: str + count: int + for schema, count in stale_tables.items(): + tables: str = "; ".join( + f"CREATE TABLE {schema}.old_orders_{index:03d} AS SELECT {index} AS id" + for index in range(count) + ) + execute_duckdb( + db_path=db_path, + sql=f'{tables}; CREATE TABLE {schema}."{EXPIRED_ARCHIVE_NAME}" AS SELECT 1 AS id', + ) + return project_dir + + +def rename_probe_models( + *, project_dir: Path, schemas: tuple[str, ...], renamed_models: tuple[str, ...] +) -> None: + """Rename each model with `migrate_from` and rebuild, leaving compatibility views behind.""" + + schema: str + name: str + for schema, name in product(schemas, renamed_models): + (project_dir / "models" / f"{schema}_{name}.sql").unlink() + (project_dir / "models" / f"{schema}_{name}_v2.sql").write_text( + _probe_model_sql(schema=schema, migrate_from=f"{schema}.{schema}_{name}"), + encoding="utf-8", + ) + _build(project_dir) + + +def _build(project_dir: Path) -> None: + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def write_probe_settings( + *, + project_dir: Path, + concurrent_ddl_limit: int = 1, + fail_rename: str | None = None, + identifier_limit: int | None = None, +) -> None: + """Run janitor on the probe adapter with statement logging and these fault switches.""" + + (project_dir / "sqlbuild_local.toml").write_text( + f'adapter = "{PROBE_JANITOR_ADAPTER_NAME}"\n', encoding="utf-8" + ) + (project_dir / PROBE_STATEMENT_LOG_FILENAME).write_text("", encoding="utf-8") + (project_dir / PROBE_JANITOR_SETTINGS_FILENAME).write_text( + json.dumps( + { + "log": True, + "concurrent_ddl_limit": concurrent_ddl_limit, + "fail_rename": fail_rename, + "identifier_limit": identifier_limit, + } + ), + encoding="utf-8", + ) + + +def read_probe_statements(*, project_dir: Path) -> tuple[str, ...]: + """Return every statement the probe adapter logged, in order.""" + + return tuple( + (project_dir / PROBE_STATEMENT_LOG_FILENAME).read_text(encoding="utf-8").splitlines() + ) + + +def count_statements( + *, statements: tuple[str, ...], fragments: tuple[str, ...], excluded: tuple[str, ...] = () +) -> int: + """Count statements containing every fragment and none of the excluded fragments.""" + + return len( + tuple( + filter( + lambda statement: ( + all(fragment in statement for fragment in fragments) + and not any(fragment in statement for fragment in excluded) + ), + statements, + ) + ) + ) + + +def statement_index(*, statements: tuple[str, ...], prefix: str) -> int: + """The position of the first statement starting with `prefix`.""" + + return [statement.startswith(prefix) for statement in statements].index(True) + + +def archive_line_positions(output: str) -> tuple[tuple[int, int], ...]: + """The `n/N` counter of every archived action line, in printed order.""" + + return tuple( + (int(match["index"]), int(match["total"])) for match in ARCHIVE_ITEM_LINE.finditer(output) + ) + + +def assert_in_order(*, output: str, lines: tuple[str, ...]) -> None: + """Assert each line appears in `output` after the previous one.""" + + position: int = 0 + line: str + for line in lines: + found: int = output.find(line, position) + assert found >= 0, f"{line!r} missing after position {position} in:\n{output}" + position = found + len(line) + + +def janitor_end_state(*, db_path: Path, schemas: tuple[str, ...]) -> tuple[tuple[object, ...], ...]: + """Every relation and every audit event in `schemas`, with archive timestamps masked.""" + + relations: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT table_schema, regexp_replace(table_name, '[0-9]{8}t[0-9]{6}z', 'ts') " + "FROM information_schema.tables WHERE table_schema NOT IN ('information_schema') " + "ORDER BY 1, 2" + ), + ) + events: list[tuple[object, ...]] = [] + schema: str + for schema in schemas: + events.extend( + query_duckdb( + db_path=db_path, + sql=( + "SELECT relation_schema, event_type, original_name, " + "regexp_replace(archive_name, '[0-9]{8}t[0-9]{6}z', 'ts') " + f"FROM {schema}._sqlbuild_janitor_events ORDER BY 1, 2, 3, 4" + ), + ) + ) + return (*relations, *events) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 35f6ba4e68..f4896cd156 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -72,7 +72,7 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( JanitorCleanupE2ETestCase( description="tracked-only direct janitor archives only tracked stale relations", build_command=("--no-color", "build", "--full-refresh"), - janitor_command=("janitor", "--auto-approve"), + janitor_command=("janitor", "--auto-approve", "--verbose"), expected_exit_code=0, expected_stdout_fragments=( "relations to archive 1", @@ -311,7 +311,7 @@ def test_given_source_in_managed_schema_when_running_direct_janitor_then_blocks_ JanitorCleanupE2ETestCase( description="tracked-only direct janitor archives strict scenario artifacts", build_command=("--no-color", "build", "--full-refresh"), - janitor_command=("janitor", "--auto-approve"), + janitor_command=("janitor", "--auto-approve", "--verbose"), expected_exit_code=0, expected_stdout_fragments=( "relations to archive 2", @@ -395,7 +395,7 @@ def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_t "Eligible direct state pruning", "main._sqlbuild_fingerprints keep latest 2", "main._sqlbuild_source_freshness keep latest 2", - "pruned 2 direct state tables", + "Pruned 2 direct state tables.", ), expected_fingerprint_count_before=5, expected_fingerprint_count_after=3, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py index dbd110d627..1b830691c3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py @@ -47,7 +47,7 @@ [ JanitorArchiveRetentionE2ETestCase( description="retired tracked relation is archived after retention and then retained", - janitor_command=("--no-color", "janitor", "--auto-approve"), + janitor_command=("--no-color", "janitor", "--auto-approve", "--verbose"), retired_model_ages_days={"customers": 20, "products": 3}, expected_archived_names=("customers",), expected_retained_names=("products",), @@ -136,7 +136,7 @@ def test_given_retired_tracked_relation_past_retention_when_running_janitor_then [ JanitorArchiveExpiryE2ETestCase( description="only expired strict archives in managed schemas are deleted", - janitor_command=("--no-color", "janitor", "--auto-approve"), + janitor_command=("--no-color", "janitor", "--auto-approve", "--verbose"), expected_deleted_names=( "_sqb_archive__20200101t000000z__old_customers_view", "_sqb_archive__20200101t000000z__old_products", @@ -163,7 +163,7 @@ def test_given_retired_tracked_relation_past_retention_when_running_janitor_then "main._sqb_archive_notes name resembles a janitor archive", "main._SQB_ARCHIVE__20200101T000000Z__legacy_orders relation name is not a plain " "lowercase identifier and may require quoting; janitor does not act on it", - "Deleted 2 objects", + "\u2713 Janitor completed archived 0, deleted 2", ), expected_delete_event_count=2, expected_total_event_count=5, @@ -262,7 +262,7 @@ def test_given_archive_named_relations_when_running_janitor_then_deletes_only_ex "archived in this run", "Eligible direct state pruning", "main._sqlbuild_fingerprints keep latest 2", - "Archived 2 relations. Deleted 2 objects", + "\u2713 Janitor completed archived 2, deleted 2", ), expected_event_types=( ("archive", "stale_customers_view"), @@ -405,7 +405,7 @@ def test_given_small_identifier_limit_when_archiving_long_name_then_fits_name_an expected_second_stdout_fragments=( "archives to delete 1", "Archives to delete", - "Deleted 1 object", + "\u2713 Janitor completed archived 0, deleted 1", ), ) ], @@ -478,7 +478,7 @@ def test_given_audit_write_interrupted_after_rename_when_janitor_reruns_then_arc [ JanitorUnaddressableRelationE2ETestCase( description="quoted mixed-case relation is reported and left while plain one archives", - janitor_command=("--no-color", "janitor", "--auto-approve"), + janitor_command=("--no-color", "janitor", "--auto-approve", "--verbose"), plain_relation="old_orders", quoted_relation="LegacyCustomers", expected_stdout_fragments=( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py new file mode 100644 index 0000000000..22d535943f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py @@ -0,0 +1,477 @@ +"""E2E tests for janitor progress output, inspection reads, concurrency and audit batching.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorConcurrencyE2ETestCase, + JanitorConcurrentFailureE2ETestCase, + JanitorInspectionCountE2ETestCase, + JanitorJsonE2ETestCase, + JanitorJsonErrorE2ETestCase, + JanitorJsonFailedRunE2ETestCase, + JanitorProgressE2ETestCase, + JanitorShortenedArchiveE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( + archive_line_positions, + assert_in_order, + count_statements, + janitor_end_state, + prepare_probe_janitor_project, + read_probe_statements, + rename_probe_models, + statement_index, + write_probe_settings, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + query_duckdb, + run_sqb, + table_exists, +) + +APPROVED_JANITOR: tuple[str, ...] = ("--no-color", "janitor", "--auto-approve") + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorProgressE2ETestCase( + description="each phase announces its work, streams each action and closes", + janitor_command=APPROVED_JANITOR, + stale_tables=12, + expected_exit_code=0, + expected_ordered_lines=( + "Janitor preview default", + "Relations to archive", + " main.old_orders_009 -> main._sqb_archive__", + " ... and 2 more (use --verbose to show all)", + "Skipped objects (", + "Archiving 12 relations...", + " 1/12 archived main.old_orders_000 -> main._sqb_archive__", + " OK ", + " 12/12 archived main.old_orders_011 -> main._sqb_archive__", + "Archived 12 relations. (", + "Deleting 1 expired archive...", + " 1/1 deleted main._sqb_archive__20200101t000000z__old_products OK", + "Deleted 1 expired archive. (", + "Pruning 1 direct state table...", + " 1/1 pruned main._sqlbuild_fingerprints OK", + "Pruned 1 direct state table. (", + "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", + ), + expected_archive_events=12, + unexpected_fragments=( + " main.old_orders_010 -> ", + "main._sqlbuild_fingerprints relation matches exclude pattern", + ), + ), + JanitorProgressE2ETestCase( + description="verbose preview lists every relation and skipped object", + janitor_command=(*APPROVED_JANITOR, "--verbose"), + stale_tables=12, + expected_exit_code=0, + expected_ordered_lines=( + " main.old_orders_011 -> main._sqb_archive__", + "Skipped objects\n main._sqlbuild_fingerprints relation matches exclude pattern", + "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", + ), + expected_archive_events=12, + unexpected_fragments=("... and", "Skipped objects ("), + ), + JanitorProgressE2ETestCase( + description="a failing rename keeps earlier lines, marks the item and fails the run", + janitor_command=APPROVED_JANITOR, + stale_tables=12, + expected_exit_code=1, + fail_rename="old_orders_005", + expected_ordered_lines=( + "Archiving 12 relations...", + " 5/12 archived main.old_orders_004 -> main._sqb_archive__", + " 6/12 archived main.old_orders_005 -> main._sqb_archive__", + " FAIL ", + "simulated rename failure for old_orders_005", + "\u2717 Archiving failed after 5 of 12 relations (", + "\u2717 Janitor failed archived 5, deleted 0, pruned 0 (", + ), + expected_archive_events=5, + unexpected_fragments=("7/12", "Deleting 1 expired archive", "Janitor completed"), + expected_remaining_tables=("old_orders_005", "old_orders_011"), + ), + ], + ids=lambda case: case.description, +) +def test_given_stale_relations_when_running_janitor_then_progress_streams_in_order( + test_case: JanitorProgressE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, + project_name="janitor_progress", + stale_tables={"main": test_case.stale_tables}, + ) + write_probe_settings(project_dir=project_dir, fail_rename=test_case.fail_rename) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + db_path: Path = project_dir / "janitor.duckdb" + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + assert_in_order(output=result.stdout, lines=test_case.expected_ordered_lines) + fragment: str + for fragment in test_case.unexpected_fragments: + assert fragment not in result.stdout + assert query_duckdb( + db_path=db_path, + sql="SELECT count(*) FROM main._sqlbuild_janitor_events WHERE event_type = 'archive'", + ) == [(test_case.expected_archive_events,)] + table: str + for table in test_case.expected_remaining_tables: + assert table_exists(db_path=db_path, table_name=table) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorJsonE2ETestCase( + description="stdout is one JSON document and progress goes to stderr", + janitor_command=("janitor", "--auto-approve", "--json"), + stale_tables=3, + expected_status="completed", + expected_archived=3, + expected_item_statuses=("ok",) * 5, + expected_stderr_fragments=( + "Janitor preview default", + "Archiving 3 relations...", + " 1/3 archived main.old_orders_000 -> main._sqb_archive__", + "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_json_flag_when_running_janitor_then_stdout_holds_plan_and_result( + test_case: JanitorJsonE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, + project_name="janitor_json", + stale_tables={"main": test_case.stale_tables}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["status"] == test_case.expected_status + assert len(payload["plan"]["relations_to_archive"]) == test_case.stale_tables + assert payload["result"]["counts"]["archived"] == test_case.expected_archived + assert tuple(item["status"] for item in payload["result"]["items"]) == ( + test_case.expected_item_statuses + ) + assert payload["error"] is None + assert_in_order(output=result.stderr, lines=test_case.expected_stderr_fragments) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorJsonErrorE2ETestCase( + description="a refused non-interactive confirmation prints a JSON error document", + janitor_command=("janitor", "--json"), + expected_exit_code=1, + expected_error_code="C503", + expected_stderr_fragment="Janitor preview default", + ), + JanitorJsonErrorE2ETestCase( + description="an invalid setting prints a JSON error document before any inspection", + janitor_command=("janitor", "--json", "--retention-days", "-1"), + expected_exit_code=1, + expected_error_code="C501", + expected_stderr_fragment="error[C501]", + ), + ], + ids=lambda case: case.description, +) +def test_given_json_flag_when_janitor_fails_before_cleanup_then_stdout_holds_error_document( + test_case: JanitorJsonErrorE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_json_error", stale_tables={"main": 2} + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["error"]["code"] == test_case.expected_error_code + assert test_case.expected_stderr_fragment in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorJsonFailedRunE2ETestCase( + description="a run failing mid-phase prints exactly one JSON document", + janitor_command=("janitor", "--auto-approve", "--json"), + fail_rename="old_orders_001", + expected_exit_code=1, + expected_status="failed", + expected_error_fragment="simulated rename failure for old_orders_001", + expected_failed_items=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_json_flag_when_janitor_fails_mid_phase_then_stdout_holds_one_run_document( + test_case: JanitorJsonFailedRunE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_json_failed", stale_tables={"main": 3} + ) + write_probe_settings(project_dir=project_dir, fail_rename=test_case.fail_rename) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["status"] == test_case.expected_status + assert test_case.expected_error_fragment in payload["error"] + assert [item["status"] for item in payload["result"]["items"]].count("failed") == ( + test_case.expected_failed_items + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorInspectionCountE2ETestCase( + description="each schema is listed once, its views read once, and events chunked", + stale_tables={"main": 150, "mart": 3, "staging": 3}, + renamed_models=("revenue", "customers"), + expected_listings_per_schema=1, + expected_multi_schema_listings=0, + expected_existence_queries_per_schema=1, + expected_view_definition_reads=3, + expected_batched_view_definition_reads=3, + expected_event_inserts={"main": 3, "mart": 2, "staging": 2}, + ), + ], + ids=lambda case: case.description, +) +def test_given_several_schemas_when_running_janitor_then_reads_and_writes_scale_per_schema( + test_case: JanitorInspectionCountE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, + project_name="janitor_counts", + stale_tables=test_case.stale_tables, + renamed_models=test_case.renamed_models, + ) + rename_probe_models( + project_dir=project_dir, + schemas=tuple(test_case.stale_tables), + renamed_models=test_case.renamed_models, + ) + write_probe_settings(project_dir=project_dir) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=APPROVED_JANITOR, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + statements: tuple[str, ...] = read_probe_statements(project_dir=project_dir) + schema: str + inserts: int + for schema, inserts in test_case.expected_event_inserts.items(): + assert count_statements( + statements=statements, + fragments=("FROM information_schema.tables", f"table_schema IN ('{schema}')"), + excluded=("table_catalog",), + ) == (test_case.expected_listings_per_schema) + assert count_statements( + statements=statements, + fragments=(f"SELECT event_id FROM {schema}._sqlbuild_janitor_events WHERE",), + ) == (test_case.expected_existence_queries_per_schema) + assert count_statements( + statements=statements, + fragments=(f"INSERT INTO {schema}._sqlbuild_janitor_events",), + ) == (inserts) + assert count_statements( + statements=statements, fragments=("FROM information_schema.tables", "', '") + ) == (test_case.expected_multi_schema_listings) + assert count_statements(statements=statements, fragments=("FROM duckdb_views()",)) == ( + test_case.expected_view_definition_reads + ) + assert count_statements( + statements=statements, fragments=("FROM duckdb_views()", "view_name IN (") + ) == (test_case.expected_batched_view_definition_reads) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorConcurrencyE2ETestCase( + description="concurrent actions leave the same end state as sequential ones", + stale_tables={"main": 30, "staging": 5}, + concurrency=4, + expected_exit_code=0, + expected_connection_line="Connecting to probe_janitor_duckdb (3 connections)", + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_adapter_when_running_janitor_then_matches_sequential_end_state( + test_case: JanitorConcurrencyE2ETestCase, tmp_path: Path +) -> None: + sequential_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path / "sequential", + project_name="janitor_sequential", + stale_tables=test_case.stale_tables, + ) + concurrent_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path / "concurrent", + project_name="janitor_concurrent", + stale_tables=test_case.stale_tables, + concurrency=test_case.concurrency, + ) + write_probe_settings(project_dir=sequential_dir) + write_probe_settings(project_dir=concurrent_dir, concurrent_ddl_limit=test_case.concurrency) + + sequential: subprocess.CompletedProcess[str] = run_sqb( + command=APPROVED_JANITOR, project_dir=sequential_dir + ) + concurrent: subprocess.CompletedProcess[str] = run_sqb( + command=APPROVED_JANITOR, project_dir=concurrent_dir + ) + + total: int = sum(test_case.stale_tables.values()) + schemas: tuple[str, ...] = tuple(test_case.stale_tables) + assert sequential.returncode == test_case.expected_exit_code, sequential.stdout + assert concurrent.returncode == test_case.expected_exit_code, concurrent.stdout + assert test_case.expected_connection_line in concurrent.stdout + assert test_case.expected_connection_line not in sequential.stdout + assert archive_line_positions(concurrent.stdout) == tuple( + (index, total) for index in range(1, total + 1) + ) + assert janitor_end_state(db_path=concurrent_dir / "janitor.duckdb", schemas=schemas) == ( + janitor_end_state(db_path=sequential_dir / "janitor.duckdb", schemas=schemas) + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorConcurrentFailureE2ETestCase( + description="a failing concurrent rename is reported alone and events stay complete", + stale_tables={"main": 30}, + concurrency=4, + fail_rename="old_orders_012", + expected_exit_code=1, + expected_failed_lines=1, + expected_fragments=( + "simulated rename failure for old_orders_012", + "\u2717 Archiving failed after ", + "\u2717 Janitor failed archived ", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_concurrent_rename_when_running_janitor_then_events_match_renames( + test_case: JanitorConcurrentFailureE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, + project_name="janitor_concurrent_failure", + stale_tables=test_case.stale_tables, + concurrency=test_case.concurrency, + ) + write_probe_settings( + project_dir=project_dir, + concurrent_ddl_limit=test_case.concurrency, + fail_rename=test_case.fail_rename, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=APPROVED_JANITOR, project_dir=project_dir + ) + + db_path: Path = project_dir / "janitor.duckdb" + positions: tuple[tuple[int, int], ...] = archive_line_positions(result.stdout) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + assert result.stdout.count(" FAIL ") == test_case.expected_failed_lines + assert_in_order(output=result.stdout, lines=test_case.expected_fragments) + assert tuple(index for index, _ in positions) == tuple(range(1, len(positions) + 1)) + assert table_exists(db_path=db_path, table_name=test_case.fail_rename) + assert query_duckdb( + db_path=db_path, + sql="SELECT count(*) FROM main._sqlbuild_janitor_events WHERE event_type = 'archive'", + ) == query_duckdb( + db_path=db_path, + sql=( + "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'main' " + "AND starts_with(table_name, '_sqb_archive__') " + "AND contains(table_name, '__old_orders_')" + ), + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorShortenedArchiveE2ETestCase( + description="a shortened archive name gets its event right after its rename", + long_relation_name="legacy_orders_with_a_very_long_descriptive_name_kept_for_audit", + identifier_limit=48, + expected_rows_in_event_insert=1, + expected_event_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_shortened_archive_name_when_archiving_then_event_is_written_immediately( + test_case: JanitorShortenedArchiveE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_shortened", stale_tables={"main": 3} + ) + execute_duckdb( + db_path=project_dir / "janitor.duckdb", + sql=f"CREATE TABLE main.{test_case.long_relation_name} AS SELECT 1 AS id", + ) + write_probe_settings(project_dir=project_dir, identifier_limit=test_case.identifier_limit) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=APPROVED_JANITOR, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + statements: tuple[str, ...] = read_probe_statements(project_dir=project_dir) + rename: int = statement_index( + statements=statements, prefix=f"ALTER TABLE main.{test_case.long_relation_name} RENAME" + ) + event_insert: str = statements[rename + 1] + assert event_insert.startswith("INSERT INTO main._sqlbuild_janitor_events") + assert f"'{test_case.long_relation_name}'" in event_insert + assert event_insert.count("), (") + 1 == test_case.expected_rows_in_event_insert + assert query_duckdb( + db_path=project_dir / "janitor.duckdb", + sql=( + "SELECT count(*) FROM main._sqlbuild_janitor_events " + f"WHERE original_name = '{test_case.long_relation_name}'" + ), + ) == [(test_case.expected_event_count,)] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py index dc0169530d..75a481aa83 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -162,7 +162,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview prod", "prod.customers -> prod._sqb_archive__", - "Archived 1 relation. Deleted 1 object and pruned 2 direct state tables.", + "\u2713 Janitor completed archived 1, deleted 1, pruned 2", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), @@ -176,7 +176,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview dev", "dev.customers -> dev._sqb_archive__", - "Archived 1 relation. Deleted 1 object and pruned 2 direct state tables.", + "\u2713 Janitor completed archived 1, deleted 1, pruned 2", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py index f890b505b4..ce18fc8673 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py @@ -27,7 +27,7 @@ [ PendingOriginJanitorE2ETestCase( description="janitor keeps the origin until the rename is migrated", - janitor_args=("janitor", "--retention-days", "0", "--auto-approve"), + janitor_args=("janitor", "--retention-days", "0", "--auto-approve", "--verbose"), expected_order_ids=(1, 2, 3, 4), ), ], diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index 5be5472a2a..4df4329ad5 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -124,3 +124,14 @@ class BrokenPipeTestCase: prepare: Callable[[Path], None] expected_exit_code: int expected_stderr_end: bytes + + +@dataclass(frozen=True) +class MachineOutputResetTestCase: + """A successful machine-output command followed in-process by an early failure.""" + + description: str + first_args: tuple[str, ...] + failing_args: tuple[str, ...] + failing_extra_toml: str + expected_error_code: str diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index c479d5d40f..33c4425cfd 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -338,3 +338,19 @@ def run_without_reader(program: Path, *, args: tuple[str, ...], cwd: Path) -> La finally: os.close(writer) return completed.returncode, b"", completed.stderr + + +def write_janitor_project(project_dir: Path, *, extra_toml: str) -> Path: + """Write a DuckDB project with one model, janitor left disabled, and extra settings.""" + + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "models").mkdir(exist_ok=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders_shop"\nadapter = "duckdb"\n\n[connection]\n' + f'database = "orders.duckdb"\n{extra_toml}', + encoding="utf-8", + ) + (project_dir / "models" / "orders.sql").write_text( + "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", encoding="utf-8" + ) + return project_dir diff --git a/tests/integration/src/sqlbuild/cli/entry/test_machine_output_reset.py b/tests/integration/src/sqlbuild/cli/entry/test_machine_output_reset.py new file mode 100644 index 0000000000..a609edd14d --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_machine_output_reset.py @@ -0,0 +1,50 @@ +"""A machine-output document printed by one in-process command never hides the next one's error.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import run_cli +from tests.integration.src.sqlbuild.cli.entry._test_types import MachineOutputResetTestCase +from tests.integration.src.sqlbuild.cli.entry.helpers import write_janitor_project + + +@pytest.mark.parametrize( + "test_case", + [ + MachineOutputResetTestCase( + description="a recall settings error after a janitor json run prints its document", + first_args=("janitor", "--json"), + failing_args=("janitor", "--json"), + failing_extra_toml=( + '\n[connections.history]\nadapter = "sqlite"\npath = "history.sqlite3"\n\n' + '[recall]\nwrite_connection = "history"\n' + ), + expected_error_code="C975", + ), + ], + ids=lambda case: case.description, +) +def test_given_janitor_json_success_when_next_command_fails_early_then_error_document_prints( + test_case: MachineOutputResetTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HOME", str(tmp_path / "home")) + succeeding: Path = write_janitor_project(tmp_path / "succeeding", extra_toml="") + failing: Path = write_janitor_project( + tmp_path / "failing", extra_toml=test_case.failing_extra_toml + ) + + _ = run_cli(["--project-dir", str(succeeding), *test_case.first_args]) + first: dict[str, Any] = json.loads(capsys.readouterr().out) + _ = run_cli(["--project-dir", str(failing), *test_case.failing_args]) + second: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert first["status"] == "disabled" + assert second["error"]["code"] == test_case.expected_error_code diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/__init__.py similarity index 100% rename from tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py rename to tests/integration/src/sqlbuild/executor/janitor_events/classes/__init__.py diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py new file mode 100644 index 0000000000..3e4bf29302 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class JanitorEventWriterTestCase: + description: str + record_count: int + written_before: int + write_passes: int + expected_row_count: int + expected_insert_count: int + expected_existence_query_count: int diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/classes/helpers.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/helpers.py new file mode 100644 index 0000000000..6e2c5f368e --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/classes/helpers.py @@ -0,0 +1,65 @@ +"""Synthetic janitor events and a statement-recording executor for event writer tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.janitor_events.classes.janitor_event_writer import JanitorEventWriter +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_SCHEMA_VERSION +from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType + +RUN_ID: str = "run_orders_cleanup" +ARCHIVED_AT: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) + + +def archive_event(index: int) -> JanitorEventRecord: + """The archive event of the synthetic relation `orders_` in schema `main`.""" + + archive_name: str = f"_sqb_archive__20260924t101500z__orders_{index:04d}" + return JanitorEventRecord( + event_id=build_janitor_event_id( + event_type=JanitorEventType.ARCHIVE, + run_id=RUN_ID, + relation_database=None, + relation_schema="main", + archive_name=archive_name, + ), + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=JanitorEventType.ARCHIVE, + occurred_at=ARCHIVED_AT, + run_id=RUN_ID, + relation_database=None, + relation_schema="main", + relation_type="BASE TABLE", + original_name=f"orders_{index:04d}", + original_qualified_name=f"main.orders_{index:04d}", + archive_name=archive_name, + archive_qualified_name=f"main.{archive_name}", + archived_at=ARCHIVED_AT, + ) + + +class RecordingExecute: + """An adapter's `execute` that remembers every statement it ran.""" + + def __init__(self, adapter: DuckDbAdapter) -> None: + self.adapter: DuckDbAdapter = adapter + self.statements: list[str] = [] + + def __call__(self, *, connection: Any, sql: str) -> Any: + self.statements.append(sql) + return self.adapter.execute(connection=connection, sql=sql) + + +def event_writer(execute: RecordingExecute) -> JanitorEventWriter: + """A writer for `RUN_ID` on `execute`.""" + + return JanitorEventWriter( + execute=execute, + render_qualified_name=execute.adapter.render_qualified_name, + run_id=RUN_ID, + ) diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py new file mode 100644 index 0000000000..8c0b72ad57 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py @@ -0,0 +1,123 @@ +"""DuckDB integration tests for chunked, append-only janitor audit events.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.janitor_events.classes.janitor_event_writer import JanitorEventWriter +from sqlbuild.executor.janitor_events.constants import ( + JANITOR_EVENT_COLUMNS, + JANITOR_EVENT_SCHEMA_VERSION, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from tests.integration.src.sqlbuild.executor.janitor_events.classes._test_types import ( + JanitorEventWriterTestCase, +) +from tests.integration.src.sqlbuild.executor.janitor_events.classes.helpers import ( + ARCHIVED_AT, + RUN_ID, + RecordingExecute, + archive_event, + event_writer, +) + + +@pytest.mark.parametrize( + "test_case", + ( + JanitorEventWriterTestCase( + description="events past one chunk are appended with one insert per chunk", + record_count=250, + written_before=0, + write_passes=1, + expected_row_count=250, + expected_insert_count=3, + expected_existence_query_count=1, + ), + JanitorEventWriterTestCase( + description="a retried run skips every event it already wrote", + record_count=5, + written_before=5, + write_passes=1, + expected_row_count=5, + expected_insert_count=0, + expected_existence_query_count=1, + ), + JanitorEventWriterTestCase( + description="a retried run appends only the events it had not written", + record_count=5, + written_before=3, + write_passes=1, + expected_row_count=5, + expected_insert_count=1, + expected_existence_query_count=1, + ), + JanitorEventWriterTestCase( + description="writing the same events again in one run appends nothing more", + record_count=4, + written_before=0, + write_passes=3, + expected_row_count=4, + expected_insert_count=1, + expected_existence_query_count=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_janitor_events_when_writing_in_chunks_then_each_event_is_appended_once( + test_case: JanitorEventWriterTestCase, + tmp_path: Path, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": str(tmp_path / "janitor_events.duckdb")}) + records: tuple[JanitorEventRecord, ...] = tuple( + archive_event(index) for index in range(test_case.record_count) + ) + try: + adapter.execute( + connection=connection, + sql=adapter.render_create_janitor_event_table_sql(database=None, schema="main"), + ) + event_writer(RecordingExecute(adapter)).write( + connection=connection, records=records[: test_case.written_before] + ) + execute: RecordingExecute = RecordingExecute(adapter) + writer: JanitorEventWriter = event_writer(execute) + writer.read_existing(connection=connection, schemas=((None, "main"),)) + _: int + for _ in range(test_case.write_passes): + writer.write(connection=connection, records=records) + rows: list[tuple[object, ...]] = connection.execute( + f"SELECT {', '.join(JANITOR_EVENT_COLUMNS)} FROM main._sqlbuild_janitor_events " + "ORDER BY archive_name" + ).fetchall() + finally: + adapter.close(connection) + + assert len(rows) == test_case.expected_row_count + assert len({row[0] for row in rows}) == test_case.expected_row_count + assert rows[0] == ( + records[0].event_id, + JANITOR_EVENT_SCHEMA_VERSION, + "archive", + ARCHIVED_AT.replace(tzinfo=None), + RUN_ID, + None, + "main", + "BASE TABLE", + "orders_0000", + "main.orders_0000", + records[0].archive_name, + f"main.{records[0].archive_name}", + ARCHIVED_AT.replace(tzinfo=None), + ) + assert sum(sql.startswith("INSERT INTO") for sql in execute.statements) == ( + test_case.expected_insert_count + ) + assert sum(sql.startswith("SELECT") for sql in execute.statements) == ( + test_case.expected_existence_query_count + ) diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py deleted file mode 100644 index bf162f5406..0000000000 --- a/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class JanitorEventWriteTestCase: - description: str - write_attempts: int - expected_row_count: int - expected_write_results: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py deleted file mode 100644 index c1ff5d333d..0000000000 --- a/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py +++ /dev/null @@ -1,103 +0,0 @@ -"""DuckDB integration tests for append-only janitor audit events.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import pytest - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.executor.janitor_events.constants import ( - JANITOR_EVENT_COLUMNS, - JANITOR_EVENT_SCHEMA_VERSION, -) -from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id -from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record -from sqlbuild.executor.janitor_events.models import JanitorEventRecord -from sqlbuild.executor.janitor_events.types import JanitorEventType -from tests.integration.src.sqlbuild.executor.janitor_events.main._test_types import ( - JanitorEventWriteTestCase, -) - -ARCHIVE_NAME: str = "_SQB_ARCHIVE__20260924T101500Z__orders" - - -@pytest.mark.parametrize( - "test_case", - ( - JanitorEventWriteTestCase( - description="repeated write of one deterministic event appends a single row", - write_attempts=2, - expected_row_count=1, - expected_write_results=(True, False), - ), - ), - ids=lambda case: case.description, -) -def test_given_repeated_janitor_event_when_writing_then_insert_is_idempotent( - test_case: JanitorEventWriteTestCase, - tmp_path: Path, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - connection: Any = adapter.connect({"database": str(tmp_path / "janitor_events.duckdb")}) - archived_at: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) - record: JanitorEventRecord = JanitorEventRecord( - event_id=build_janitor_event_id( - event_type=JanitorEventType.ARCHIVE, - run_id="run_orders_cleanup", - relation_database=None, - relation_schema="main", - archive_name=ARCHIVE_NAME, - ), - schema_version=JANITOR_EVENT_SCHEMA_VERSION, - event_type=JanitorEventType.ARCHIVE, - occurred_at=archived_at, - run_id="run_orders_cleanup", - relation_database=None, - relation_schema="main", - relation_type="BASE TABLE", - original_name="orders", - original_qualified_name="main.orders", - archive_name=ARCHIVE_NAME, - archive_qualified_name=f"main.{ARCHIVE_NAME}", - archived_at=archived_at, - ) - try: - adapter.execute( - connection=connection, - sql=adapter.render_create_janitor_event_table_sql(database=None, schema="main"), - ) - write_results: tuple[bool, ...] = tuple( - write_janitor_event_record( - connection=connection, - execute=adapter.execute, - record=record, - render_qualified_name=adapter.render_qualified_name, - ) - for _ in range(test_case.write_attempts) - ) - rows: list[tuple[object, ...]] = connection.execute( - f"SELECT {', '.join(JANITOR_EVENT_COLUMNS)} FROM main._sqlbuild_janitor_events" - ).fetchall() - finally: - adapter.close(connection) - - assert write_results == test_case.expected_write_results - assert len(rows) == test_case.expected_row_count - assert rows[0] == ( - record.event_id, - JANITOR_EVENT_SCHEMA_VERSION, - "archive", - archived_at.replace(tzinfo=None), - "run_orders_cleanup", - None, - "main", - "BASE TABLE", - "orders", - "main.orders", - ARCHIVE_NAME, - f"main.{ARCHIVE_NAME}", - archived_at.replace(tzinfo=None), - ) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 4637b11380..2ff4b3fc2f 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -284,3 +284,13 @@ class SnowflakeStateTableSchemaFallbackTestCase: rejecting_schema: str accepting_schema: str expected_executed: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class SnowflakeViewDefinitionsTestCase: + description: str + stored_rows: tuple[tuple[str, str], ...] + prefetched_names: tuple[str, ...] + read_names: tuple[str, ...] + expected_definitions: tuple[str | None, ...] + expected_query_count: int diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py new file mode 100644 index 0000000000..5939d278fd --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py @@ -0,0 +1,89 @@ +"""Snowflake view definitions read one schema at a time through the inspection catalog.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog +from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeViewDefinitionsTestCase, +) + + +class _Rows: + def __init__(self, rows: list[tuple[str, ...]]) -> None: + self._rows: list[tuple[str, ...]] = rows + + def fetchall(self) -> list[tuple[str, ...]]: + return self._rows + + +class _ViewDefinitionAdapter(SnowflakeAdapter): + """Answers INFORMATION_SCHEMA.VIEWS reads from stored rows, matching names like Snowflake.""" + + def __init__(self, *, stored: tuple[tuple[str, str], ...]) -> None: + super().__init__() + self.stored: tuple[tuple[str, str], ...] = stored + self.queries: list[str] = [] + + def execute(self, *, connection: Any, sql: str) -> Any: + del connection + self.queries.append(sql) + if sql.startswith("SELECT TABLE_NAME, VIEW_DEFINITION"): + return _Rows([row for row in self.stored if f"UPPER('{row[0]}')" in sql.upper()]) + return _Rows( + [ + (definition,) + for name, definition in self.stored + if f"'{name.upper()}'" in sql.upper() + ] + ) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeViewDefinitionsTestCase( + description="prefetched views are served without further queries, absent ones as none", + stored_rows=(("ORDERS_VIEW", "SELECT 1"), ("CUSTOMERS_VIEW", "SELECT 2")), + prefetched_names=("orders_view", "customers_view", "missing_view"), + read_names=("orders_view", "customers_view", "missing_view"), + expected_definitions=("SELECT 1", "SELECT 2", None), + expected_query_count=1, + ), + SnowflakeViewDefinitionsTestCase( + description="a view outside the prefetch is still read on its own", + stored_rows=(("ORDERS_VIEW", "SELECT 1"), ("PRODUCTS_VIEW", "SELECT 3")), + prefetched_names=("orders_view",), + read_names=("orders_view", "products_view"), + expected_definitions=("SELECT 1", "SELECT 3"), + expected_query_count=2, + ), + ), + ids=lambda case: case.description, +) +def test_given_prefetched_schema_when_reading_view_definitions_then_one_query_serves_them( + test_case: SnowflakeViewDefinitionsTestCase, +) -> None: + adapter: _ViewDefinitionAdapter = _ViewDefinitionAdapter(stored=test_case.stored_rows) + connection: object = object() + + catalog: InspectionCatalog + with open_inspection_catalog(adapter=adapter, connection=connection) as catalog: + catalog.prefetch_view_definitions( + requests=(("analytics_db", "analytics", test_case.prefetched_names),) + ) + definitions: tuple[str | None, ...] = tuple( + adapter.read_view_definition( + connection=connection, database="analytics_db", schema="analytics", name=name + ) + for name in test_case.read_names + ) + + assert definitions == test_case.expected_definitions + assert len(adapter.queries) == test_case.expected_query_count + assert "UPPER(TABLE_NAME) IN (" in adapter.queries[0] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py index 6b08277a95..3931ef67b2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py @@ -8,3 +8,4 @@ class JanitorCompletionTestCase: deleted: int pruned_direct_state: int expected_output: str + elapsed_seconds: float | None = None diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py index 16012e165a..88f88d54ea 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py @@ -1,4 +1,4 @@ -"""Janitor completion summary wording.""" +"""Janitor completion line wording.""" from __future__ import annotations @@ -18,32 +18,31 @@ "test_case", ( JanitorCompletionTestCase( - description="single archive and object use singular nouns", + description="counts archived, deleted and pruned items", archived=1, deleted=1, pruned_direct_state=0, - expected_output="Archived 1 relation. Deleted 1 object.\n", + expected_output="\u2713 Janitor completed archived 1, deleted 1, pruned 0\n", ), JanitorCompletionTestCase( - description="single pruned state table uses a singular noun", + description="zero counts stay visible", archived=0, deleted=0, pruned_direct_state=1, - expected_output="Deleted 0 objects and pruned 1 direct state table.\n", + expected_output="\u2713 Janitor completed archived 0, deleted 0, pruned 1\n", ), JanitorCompletionTestCase( - description="plural counts keep plural nouns", + description="elapsed time follows the counts", archived=2, deleted=3, pruned_direct_state=2, - expected_output=( - "Archived 2 relations. Deleted 3 objects and pruned 2 direct state tables.\n" - ), + elapsed_seconds=302.4, + expected_output="\u2713 Janitor completed archived 2, deleted 3, pruned 2 (5m 02s)\n", ), ), ids=lambda case: case.description, ) -def test_given_janitor_counts_when_writing_completion_then_nouns_match_counts( +def test_given_janitor_counts_when_writing_completion_then_line_reports_totals( test_case: JanitorCompletionTestCase, ) -> None: result: JanitorExecutionResult = JanitorExecutionResult( @@ -52,7 +51,9 @@ def test_given_janitor_counts_when_writing_completion_then_nouns_match_counts( pruned_direct_state=placeholder_candidates(test_case.pruned_direct_state), ) - output: str = render_janitor_completion(result=result, use_color=False) + output: str = render_janitor_completion( + result=result, elapsed_seconds=test_case.elapsed_seconds, use_color=False + ) assert output == test_case.expected_output diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index c3cc3ae1e4..14bd03402a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -30,7 +30,7 @@ "suppressed archive: blocked.old_model", "No janitor actions will be performed.", "Skipped schemas\n dev contains active source raw.orders", - "Skipped objects\n dev.source_table source relation", + "Skipped objects (1)\n 1 source relation", ), unexpected_output_fragments=( "\033[", @@ -113,3 +113,27 @@ def test_given_direct_archive_plan_when_writing_preview_then_lists_archive_actio assert fragment in output for fragment in test_case.unexpected_output_fragments: assert fragment not in output + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPlanOutputTestCase( + description="verbose preview lists each skipped object with its reason", + use_color=False, + expected_output_fragments=("Skipped objects\n dev.source_table source relation",), + unexpected_output_fragments=("Skipped objects (1)",), + ) + ], + ids=lambda case: case.description, +) +def test_given_janitor_plan_when_writing_verbose_preview_then_lists_every_skipped_object( + test_case: JanitorPlanOutputTestCase, +) -> None: + output: str = render_janitor_plan( + plan=build_janitor_plan(), use_color=test_case.use_color, verbose=True + ) + for fragment in test_case.expected_output_fragments: + assert fragment in output + for fragment in test_case.unexpected_output_fragments: + assert fragment not in output diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py index 4a3bc59c44..e18bc6d880 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py @@ -14,3 +14,11 @@ class ArchiveNameBuildTestCase: expected_length: int expected_logical_name_is_original: bool kind: str | None = None + + +@dataclass(frozen=True) +class JanitorWorkerConnectionFailureTestCase: + description: str + connection_count: int + expected_code: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_worker_connections.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_worker_connections.py new file mode 100644 index 0000000000..aa27f98861 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_worker_connections.py @@ -0,0 +1,39 @@ +"""Worker connection failures surface as the coded warehouse connection error.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.janitor._helpers.workers import open_janitor_worker_connections +from sqlbuild.executor.pipeline.exceptions import WarehouseConnectionError +from tests.unit.src.sqlbuild.executor.janitor._helpers._test_types import ( + JanitorWorkerConnectionFailureTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorWorkerConnectionFailureTestCase( + description="an unopenable worker connection raises the coded connection error", + connection_count=2, + expected_code="C115", + expected_message="could not connect to the duckdb warehouse", + ), + ], + ids=lambda case: case.description, +) +def test_given_unreachable_warehouse_when_opening_worker_connections_then_error_is_coded( + test_case: JanitorWorkerConnectionFailureTestCase, +) -> None: + with pytest.raises(WarehouseConnectionError) as raised: + _ = open_janitor_worker_connections( + adapter=DuckDbAdapter(), + connection_config={"database": "/nonexistent-directory/orders/janitor.duckdb"}, + count=test_case.connection_count, + progress=None, + ) + + assert raised.value.code == test_case.expected_code + assert test_case.expected_message in raised.value.message diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 528edca1ef..b891ddf619 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -87,6 +87,7 @@ class JanitorArchiveExecutionTestCase: expected_dropped_targets: tuple[str, ...] expected_dropped_view_targets: tuple[str, ...] expected_event_insert_count: int + expected_event_row_count: int expected_event_table_create_count: int @@ -158,3 +159,45 @@ class JanitorCaseSafetyTestCase: exclude_patterns: tuple[str, ...] expected_skipped_relations: tuple[tuple[str, str], ...] expected_dropped_targets: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorPhaseEventSafetyTestCase: + description: str + connection_count: int + item_count: int + failing_report: int + report_error: type[BaseException] + expected_error: type[BaseException] + expected_minimum_completed: int + + +@dataclass(frozen=True) +class JanitorPhaseInterruptTestCase: + description: str + connection_count: int + item_count: int + expected_error: type[BaseException] + expected_minimum_completed: int + + +@dataclass(frozen=True) +class JanitorPhaseSubmitInterruptTestCase: + description: str + connection_count: int + item_count: int + failing_submit: int + expected_error: type[BaseException] + expected_maximum_completed: int + + +@dataclass(frozen=True) +class JanitorPhaseFlushFailureTestCase: + description: str + item_count: int + failing_item: int + action_error: type[BaseException] + flush_error: type[BaseException] + expected_error: type[BaseException] + expected_chained_error: type[BaseException] + expected_unwritten: tuple[tuple[int, tuple[str, ...], str], ...] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 402c648ef6..d4bc3910a3 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -5,7 +5,7 @@ import base64 from collections.abc import Callable from dataclasses import replace -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from typing import Any, ClassVar, cast @@ -39,7 +39,14 @@ CompiledProjectView, CompiledSourceView, ) -from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME +from sqlbuild.executor.janitor.models import JanitorActionDone, JanitorProgressItem +from sqlbuild.executor.janitor.types import JanitorAction +from sqlbuild.executor.janitor_events.constants import ( + JANITOR_EVENT_SCHEMA_VERSION, + JANITOR_EVENTS_TABLE_NAME, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS UNQUOTED_ADDRESSING_REASON: str = ( @@ -610,3 +617,162 @@ def _ignore_relation(selected: list[RelationInfo], relation: RelationInfo) -> No True: _append_relation, False: _ignore_relation, } + + +PHASE_EVENT_TIME: datetime = datetime(2026, 9, 24, 10, 15, tzinfo=UTC) + + +def phase_event(index: int) -> JanitorEventRecord: + """The synthetic audit event of the archive action at `index`.""" + + archive_name: str = f"_sqb_archive__20260924t101500z__orders_{index:04d}" + return JanitorEventRecord( + event_id=f"event_{index:04d}", + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=JanitorEventType.ARCHIVE, + occurred_at=PHASE_EVENT_TIME, + run_id="run_orders_cleanup", + relation_database=None, + relation_schema="analytics", + relation_type="BASE TABLE", + original_name=f"orders_{index:04d}", + original_qualified_name=f"analytics.orders_{index:04d}", + archive_name=archive_name, + archive_qualified_name=f"analytics.{archive_name}", + archived_at=PHASE_EVENT_TIME, + ) + + +class PhaseActions: + """Archive actions that each return their event and remember that they completed.""" + + def __init__(self) -> None: + self.completed: list[int] = [] + self.flushed: list[str] = [] + + def apply(self, *, item: int, connection: Any) -> JanitorActionDone: + del connection + self.completed.append(item) + return JanitorActionDone(event=phase_event(item)) + + def flush(self, finished: tuple[JanitorActionDone, ...]) -> None: + self.flushed.extend(cast(JanitorEventRecord, done.event).event_id for done in finished) + + def completed_event_ids(self) -> list[str]: + return sorted(phase_event(item).event_id for item in self.completed) + + +class FailingReportProgress: + """Progress whose `failing_report`th finished-action report raises `error`.""" + + def __init__(self, *, failing_report: int, error: type[BaseException], size: int) -> None: + def quiet() -> None: + return None + + def fail() -> None: + raise error("progress stream closed") + + self._reactions: list[Callable[[], None]] = [ + *([quiet] * (failing_report - 1)), + fail, + *([quiet] * size), + ] + self._calls: int = 0 + self.closed_failed: list[bool] = [] + self.unwritten: list[tuple[int, tuple[str, ...], str]] = [] + + def phase_started(self, *, phase: str, total: int) -> None: + del phase, total + + def item_finished(self, *, phase: str, item: object) -> None: + del phase, item + self._calls += 1 + self._reactions[self._calls - 1]() + + def phase_finished(self, *, phase: str, failed: bool) -> None: + del phase + self.closed_failed.append(failed) + + def events_unwritten(self, *, count: int, schemas: tuple[str, ...], error: str) -> None: + self.unwritten.append((count, schemas, error)) + + +def interrupting_wait(real_wait: Callable[..., Any]) -> Callable[..., Any]: + """`concurrent.futures.wait` that raises KeyboardInterrupt on its first call only.""" + + def interrupt(*args: object, **kwargs: object) -> Any: + del args, kwargs + raise KeyboardInterrupt + + calls: list[Callable[..., Any]] = [interrupt] + + def wait(*args: Any, **kwargs: Any) -> Any: + reaction: Callable[..., Any] = [*calls, real_wait][0] + calls.clear() + return reaction(*args, **kwargs) + + return wait + + +def phase_label(item: int) -> JanitorProgressItem: + """The progress label of the archive action at `item`.""" + + return JanitorProgressItem(action=JanitorAction.ARCHIVED, name=f"analytics.orders_{item:04d}") + + +def _raiser(error: type[BaseException], message: str) -> Callable[[], None]: + def fail() -> None: + raise error(message) + + return fail + + +def _quiet() -> None: + return None + + +class FailingPhaseActions(PhaseActions): + """Archive actions where the `failing_item`th raises `error`, and whose event flush raises.""" + + def __init__( + self, + *, + failing_item: int, + error: type[BaseException], + flush_error: type[BaseException], + size: int, + ) -> None: + super().__init__() + self._reactions: list[Callable[[], None]] = [ + *([_quiet] * failing_item), + _raiser(error, f"rename failed for analytics.orders_{failing_item:04d}"), + *([_quiet] * size), + ] + self._flush_error: type[BaseException] = flush_error + + def apply(self, *, item: int, connection: Any) -> JanitorActionDone: + self._reactions[item]() + return super().apply(item=item, connection=connection) + + def flush(self, finished: tuple[JanitorActionDone, ...]) -> None: + del finished + raise self._flush_error("events table is locked") + + +def interrupting_submit( + real_submit: Callable[..., Any], *, failing_call: int +) -> Callable[..., Any]: + """`submit_janitor_action` that raises KeyboardInterrupt on its `failing_call`th call.""" + + reactions: list[Callable[[], None]] = [ + *([_quiet] * (failing_call - 1)), + _raiser(KeyboardInterrupt, "interrupted while submitting"), + ] + calls: list[int] = [] + + def submit(**kwargs: Any) -> Any: + calls.append(len(calls)) + [*reactions, *([_quiet] * len(calls))][len(calls) - 1]() + return real_submit(**kwargs) + + return submit diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py index 5ad86f4cfa..d0aa8eaa5a 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py @@ -31,7 +31,7 @@ "test_case", [ JanitorArchiveExecutionTestCase( - description="renames stale table and view, drops expired archive, and audits each", + description="renames stale table and view, drops expired archive, and audits each per phase", relation_infos=( relation_info("old_orders", created_at=OLD_TIME), relation_info("old_customers_view", relation_type="VIEW", created_at=OLD_TIME), @@ -42,7 +42,8 @@ expected_view_rename_origins=("analytics.old_customers_view",), expected_dropped_targets=("analytics._sqb_archive__20200101t000000z__old_products",), expected_dropped_view_targets=(), - expected_event_insert_count=3, + expected_event_insert_count=2, + expected_event_row_count=3, expected_event_table_create_count=1, ), JanitorArchiveExecutionTestCase( @@ -56,7 +57,8 @@ expected_view_rename_origins=("analytics.old_customers_view",), expected_dropped_targets=("analytics._sqb_archive__",), expected_dropped_view_targets=("analytics._sqb_archive__",), - expected_event_insert_count=4, + expected_event_insert_count=2, + expected_event_row_count=4, expected_event_table_create_count=1, ), ], @@ -110,6 +112,10 @@ def test_given_direct_archive_plan_when_executing_then_renames_drops_and_writes_ filter(lambda sql: sql.startswith("CREATE TABLE"), event_statements) ) assert len(event_inserts) == test_case.expected_event_insert_count + assert sum(sql.count("), (") + 1 for sql in event_inserts) == ( + test_case.expected_event_row_count + ) + assert len(tuple(filter(lambda sql: sql.startswith("SELECT"), event_statements))) == 1 assert len(event_creates) == test_case.expected_event_table_create_count assert tuple(candidate.key.name for candidate in result.archived) == tuple( origin.rsplit(".", maxsplit=1)[-1] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py new file mode 100644 index 0000000000..be732bd6a8 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py @@ -0,0 +1,248 @@ +"""Every completed janitor action keeps its audit event when reporting or waiting fails.""" + +from __future__ import annotations + +from concurrent.futures import wait + +import pytest + +from sqlbuild.executor.janitor.classes import janitor_chunk_collector +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase +from sqlbuild.executor.janitor.types import JanitorPhase +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorPhaseEventSafetyTestCase, + JanitorPhaseFlushFailureTestCase, + JanitorPhaseInterruptTestCase, + JanitorPhaseSubmitInterruptTestCase, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FailingPhaseActions, + FailingReportProgress, + PhaseActions, + interrupting_submit, + interrupting_wait, + phase_label, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPhaseEventSafetyTestCase( + description="sequential actions keep their events when the progress stream breaks", + connection_count=1, + item_count=10, + failing_report=3, + report_error=BrokenPipeError, + expected_error=BrokenPipeError, + expected_minimum_completed=3, + ), + JanitorPhaseEventSafetyTestCase( + description="concurrent actions keep their events when the progress stream breaks", + connection_count=3, + item_count=10, + failing_report=3, + report_error=BrokenPipeError, + expected_error=BrokenPipeError, + expected_minimum_completed=3, + ), + JanitorPhaseEventSafetyTestCase( + description="an interrupt while reporting still writes every completed event", + connection_count=3, + item_count=10, + failing_report=2, + report_error=KeyboardInterrupt, + expected_error=KeyboardInterrupt, + expected_minimum_completed=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_progress_when_running_phase_then_completed_actions_keep_events( + test_case: JanitorPhaseEventSafetyTestCase, +) -> None: + actions: PhaseActions = PhaseActions() + progress: FailingReportProgress = FailingReportProgress( + failing_report=test_case.failing_report, + error=test_case.report_error, + size=test_case.item_count, + ) + + with pytest.raises(test_case.expected_error): + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=tuple(range(test_case.item_count)), + describe=phase_label, + apply=actions.apply, + workers=JanitorWorkers( + connections=tuple(object() for _ in range(test_case.connection_count)) + ), + after_chunk=actions.flush, + ) + + assert len(actions.completed) >= test_case.expected_minimum_completed + assert len(actions.completed) < test_case.item_count + assert sorted(actions.flushed) == actions.completed_event_ids() + assert progress.closed_failed == [True] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPhaseInterruptTestCase( + description="an interrupt while waiting for workers still writes completed events", + connection_count=3, + item_count=10, + expected_error=KeyboardInterrupt, + expected_minimum_completed=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupt_while_waiting_when_running_phase_then_completed_actions_keep_events( + test_case: JanitorPhaseInterruptTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + actions: PhaseActions = PhaseActions() + progress: FailingReportProgress = FailingReportProgress( + failing_report=test_case.item_count + 1, + error=BrokenPipeError, + size=test_case.item_count, + ) + monkeypatch.setattr(janitor_chunk_collector, "wait", interrupting_wait(wait)) + + with pytest.raises(test_case.expected_error): + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=tuple(range(test_case.item_count)), + describe=phase_label, + apply=actions.apply, + workers=JanitorWorkers( + connections=tuple(object() for _ in range(test_case.connection_count)) + ), + after_chunk=actions.flush, + ) + + assert len(actions.completed) >= test_case.expected_minimum_completed + assert sorted(actions.flushed) == actions.completed_event_ids() + assert progress.closed_failed == [True] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPhaseSubmitInterruptTestCase( + description="an interrupt while submitting still writes every completed event", + connection_count=3, + item_count=10, + failing_submit=3, + expected_error=KeyboardInterrupt, + expected_maximum_completed=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupt_while_submitting_when_running_phase_then_completed_actions_keep_events( + test_case: JanitorPhaseSubmitInterruptTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + actions: PhaseActions = PhaseActions() + progress: FailingReportProgress = FailingReportProgress( + failing_report=test_case.item_count + 1, + error=BrokenPipeError, + size=test_case.item_count, + ) + monkeypatch.setattr( + janitor_chunk_collector, + "submit_janitor_action", + interrupting_submit( + janitor_chunk_collector.submit_janitor_action, failing_call=test_case.failing_submit + ), + ) + + with pytest.raises(test_case.expected_error): + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=tuple(range(test_case.item_count)), + describe=phase_label, + apply=actions.apply, + workers=JanitorWorkers( + connections=tuple(object() for _ in range(test_case.connection_count)) + ), + after_chunk=actions.flush, + ) + + assert len(actions.completed) <= test_case.expected_maximum_completed + assert sorted(actions.flushed) == actions.completed_event_ids() + assert progress.closed_failed == [True] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPhaseFlushFailureTestCase( + description="a failed event write keeps the action failure and warns per schema", + item_count=6, + failing_item=3, + action_error=RuntimeError, + flush_error=OSError, + expected_error=RuntimeError, + expected_chained_error=OSError, + expected_unwritten=((3, ("analytics",), "events table is locked"),), + ), + JanitorPhaseFlushFailureTestCase( + description="an interrupted action still wins over a failed event write", + item_count=6, + failing_item=2, + action_error=KeyboardInterrupt, + flush_error=OSError, + expected_error=KeyboardInterrupt, + expected_chained_error=OSError, + expected_unwritten=((2, ("analytics",), "events table is locked"),), + ), + JanitorPhaseFlushFailureTestCase( + description="an interrupted event write wins over the action failure", + item_count=6, + failing_item=4, + action_error=RuntimeError, + flush_error=KeyboardInterrupt, + expected_error=KeyboardInterrupt, + expected_chained_error=RuntimeError, + expected_unwritten=((4, ("analytics",), "events table is locked"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_event_write_when_phase_fails_then_original_error_is_raised_and_chained( + test_case: JanitorPhaseFlushFailureTestCase, +) -> None: + actions: FailingPhaseActions = FailingPhaseActions( + failing_item=test_case.failing_item, + error=test_case.action_error, + flush_error=test_case.flush_error, + size=test_case.item_count, + ) + progress: FailingReportProgress = FailingReportProgress( + failing_report=test_case.item_count + 1, + error=BrokenPipeError, + size=test_case.item_count, + ) + + with pytest.raises(test_case.expected_error) as raised: + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=tuple(range(test_case.item_count)), + describe=phase_label, + apply=actions.apply, + workers=JanitorWorkers(connections=(object(),)), + after_chunk=actions.flush, + ) + + assert isinstance(raised.value.__context__, test_case.expected_chained_error) + assert progress.unwritten == list(test_case.expected_unwritten) + assert progress.closed_failed == [True] diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 91089bee3f..6ac27db06b 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -22,14 +22,49 @@ sqb --project-dir janitor [flags] | `--auto-approve` | Skip the confirmation prompt and run the planned actions immediately | | `--retention-days` | Override the configured retention period (days) before a stale relation is archived | | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | +| `--verbose`, `-v` | List every entry of each preview section and every skipped object with its reason. Without it, each section shows its first 10 entries and skipped objects are counted by reason | +| `--json` | Print the plan and the result as one JSON document on stdout; the preview, prompt and progress go to stderr | | `--drop-old-name-view ` | Drop the [compatibility view](/docs/concepts/models/migrations#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | +## Output + +After the preview and confirmation, each phase with work announces how many actions it runs, prints one line per action as it finishes, and closes with its duration. The run ends with one line of totals: + +``` +Archiving 3 relations... + 1/3 archived analytics.old_orders -> analytics._sqb_archive__20261022t101500z__old_orders OK 0.21s + 2/3 archived analytics.old_customers -> analytics._sqb_archive__20261022t101500z__old_customers OK 0.19s + 3/3 archived analytics.old_products -> analytics._sqb_archive__20261022t101500z__old_products OK 0.24s +Archived 3 relations. (0.64s) +Deleting 1 expired archive... + 1/1 deleted analytics._sqb_archive__20261001t080000z__old_inventory OK 0.12s +Deleted 1 expired archive. (0.12s) + +✓ Janitor completed archived 3, deleted 1, pruned 0 (0.81s) +``` + +The phases are archiving stale relations, deleting expired archives, dropping old-name views, dropping expired query artifacts and pruning direct state, in that order. If an action fails, its line shows `FAIL` and the error, no further action starts, and the run ends with `✗ Janitor failed` and the totals completed before the failure, exiting with `1`. + +Long preview sections show their first 10 entries and then `... and N more (use --verbose to show all)`. Skipped objects are counted by reason: + +``` +Skipped objects (385) + 210 relation is newer than 14 days + 175 relation is not tracked by SQLBuild +``` + +With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. + +## Speed + +The janitor reads each schema's relations once per run, and the definitions of compatibility views with one read per schema. On warehouses that run DDL concurrently (Snowflake, BigQuery and Databricks), the independent actions of each phase run on up to the target's [`concurrency`](/docs/concepts/project-configuration#target-concurrency) connections at once; phases still run one after another. DuckDB, SQLite and the other adapters apply one action at a time. + ## Targets By default the janitor cleans up the active target. `--target ` selects another configured target, exactly like `sqb build --target`: the janitor connects with that target's connection and inspects and cleans that target's schemas and state tables. Other targets are never touched. -`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](/docs/cli/plan). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists every relation it would archive, every archive it would delete, every compatibility view it would drop and every state table it would prune, then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. +`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](/docs/cli/plan). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists the relations it would archive, the archives it would delete, the compatibility views it would drop and the state tables it would prune (all of them with `--verbose`), then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. ``` Previewing janitor as target 'prod' through the connection of target 'dev' (inspection only). @@ -106,6 +141,8 @@ Ages are only looked up for relations that are otherwise candidates, and not at Every archive and every delete is recorded in an append-only `_sqlbuild_janitor_events` table in the affected schema. It records the original name, the archive name, the time and the invocation. The janitor never reads this table to decide what to do. It exists for auditing, and for restoring archives whose names were shortened. +Events are written in batches of up to 100 per schema, after each batch of actions and after a failure. An archive whose name was shortened gets its event right after its rename. If the process is killed, at most one batch of actions in a schema can lack events; those archives still expire by the timestamp in their names. + ## Restoring an archive Rename the archive back to its original name before it expires, using your warehouse's rename statement. For example, on Snowflake: @@ -156,4 +193,4 @@ In addition to archiving stale relations, the janitor prunes old rows from `_sql ## Safety -The janitor shows every planned archive and delete, with exact names, ages and expiry, then asks you to type an exact confirmation phrase. Relations whose names differ only by case from another relation in the same schema are reported and never dropped, and exclude patterns match regardless of case. Use `--auto-approve` only in automation, or when you're certain. +The janitor shows the planned archives and deletes, with exact names, ages and expiry (every one of them with `--verbose`, or in the `--json` plan), then asks you to type an exact confirmation phrase. Relations whose names differ only by case from another relation in the same schema are reported and never dropped, and exclude patterns match regardless of case. Use `--auto-approve` only in automation, or when you're certain. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 22e328e7ab..ef2438826b 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -666,7 +666,7 @@ delta_compile = true | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](/docs/concepts/models/configuration#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](/docs/concepts/models/contracts). | -| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`, and for `sqb janitor` actions on warehouses that run DDL concurrently; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](/docs/concepts/python-nodes/loaders). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) |