From d2a44e213d45b21d09a90058a899d13d5da84ad1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:10:43 +0100 Subject: [PATCH 01/10] feat: stream janitor phases and actions, shorten the preview, add --verbose and --json --- .../sqlbuild-cli/src/argv/command_table.json | 30 +++ crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- .../src/janitor_text/_helpers/mod.rs | 2 + .../src/janitor_text/_helpers/phases.rs | 44 ++++ .../src/janitor_text/_helpers/plan_json.rs | 245 ++++++++++++++++++ .../janitor_text/_helpers/plan_sections.rs | 110 +++++++- .../src/janitor_text/constants.rs | 9 + .../janitor_text/main/format_janitor_json.rs | 43 +++ .../main/janitor_completion_text.rs | 35 --- .../janitor_text/main/janitor_item_counts.rs | 24 ++ .../main/janitor_nothing_to_do_text.rs | 18 ++ .../main/janitor_phase_complete_text.rs | 33 +++ .../main/janitor_phase_failed_text.rs | 29 +++ .../main/janitor_phase_flow_width.rs | 39 +++ .../main/janitor_phase_start_text.rs | 11 + .../src/janitor_text/main/mod.rs | 10 +- .../main/render_janitor_completion.rs | 42 +++ .../main/render_janitor_item_line.rs | 47 ++++ .../janitor_text/main/render_janitor_plan.rs | 33 +-- .../sqlbuild-core/src/janitor_text/models.rs | 95 +++++++ .../_helpers/janitor/janitor_command.rs | 144 +++++++--- .../_helpers/janitor/janitor_phases.rs | 20 +- .../bindings/_helpers/janitor/janitor_plan.rs | 26 +- .../_helpers/janitor/janitor_progress.rs | 183 +++++++++++++ .../src/bindings/_helpers/janitor/mod.rs | 1 + .../bindings/_helpers/records/constants.rs | 3 + src/sqlbuild/_native.pyi | 13 +- .../janitor/_helpers/archive_execution.py | 34 ++- .../executor/janitor/_helpers/deletion.py | 28 +- .../janitor/main/_run_janitor_phase.py | 65 +++++ src/sqlbuild/executor/janitor/main/execute.py | 47 +++- src/sqlbuild/executor/janitor/models.py | 10 + src/sqlbuild/executor/janitor/types.py | 41 +++ .../old_name_views/_helpers/janitor.py | 30 ++- .../main/_drop_old_name_views.py | 9 +- 35 files changed, 1421 insertions(+), 136 deletions(-) create mode 100644 crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs delete mode 100644 crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/janitor_item_counts.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/janitor_nothing_to_do_text.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/janitor_phase_complete_text.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/janitor_phase_failed_text.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/janitor_phase_flow_width.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/janitor_phase_start_text.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs create mode 100644 crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs create mode 100644 src/sqlbuild/executor/janitor/main/_run_janitor_phase.py create mode 100644 src/sqlbuild/executor/janitor/types.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 95a778e52a..6ba7934b09 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -5198,6 +5198,7 @@ "nargs": 0, "const": true, "default": false, + "help": "skip the confirmation prompt and run the planned archives, deletes and prunes", "group": 1 }, { @@ -5207,6 +5208,7 @@ ], "dest": "retention_days", "type": "int", + "help": "days a stale relation must be unchanged before it is archived (overrides janitor.retention_days)", "group": 1 }, { @@ -5216,6 +5218,7 @@ ], "dest": "direct_state_history_versions", "type": "int", + "help": "state history rows to keep per identity when pruning direct state tables (overrides janitor.direct_state_history_versions)", "group": 1 }, { @@ -5226,6 +5229,32 @@ "dest": "drop_old_name_views", "default": [], "metavar": "NAME", + "help": "drop the compatibility view at this old model name before it expires; repeat for several views", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--verbose", + "-v" + ], + "dest": "verbose", + "nargs": 0, + "const": true, + "default": false, + "help": "list every entry of each preview section and every skipped object", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--json" + ], + "dest": "json", + "nargs": 0, + "const": true, + "default": false, + "help": "print the plan and result as JSON on stdout; progress goes to stderr", "group": 1 }, { @@ -5234,6 +5263,7 @@ "--target" ], "dest": "target", + "help": "clean up this configured target instead of the active target", "group": 1 }, { diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index da56abafec..9070e10680 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -484,7 +484,7 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); -const JANITOR_FIELDS: [Field; 8] = [ +const JANITOR_FIELDS: [Field; 10] = [ PROJECT_DIR, NO_COLOR, same("auto_approve"), @@ -496,6 +496,8 @@ const JANITOR_FIELDS: [Field; 8] = [ ), SELECTED_TARGET, same("as_target"), + same("verbose"), + JSON_OUTPUT, ]; const JANITOR_ARGUMENTS: [Field; 1] = [request(class("JanitorCommandRequest"), &JANITOR_FIELDS)]; pub(crate) const JANITOR_ROUTE: Route = routed( diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs index c6213b2bdb..fa60de261a 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod counts; +pub(crate) mod phases; +pub(crate) mod plan_json; pub(crate) mod plan_sections; diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs new file mode 100644 index 0000000000..c6a49b3d54 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs @@ -0,0 +1,44 @@ +//! Wording and durations of janitor apply phases. + +use crate::cli_text::main::format_count_noun::format_count_noun; +use crate::janitor_text::constants::{SECONDS_PER_HOUR, SECONDS_PER_MINUTE}; +use crate::janitor_text::models::JanitorPhase; + +/// The phase's present participle, past tense and singular noun. +pub(crate) fn phase_words(phase: JanitorPhase) -> (&'static str, &'static str, &'static str) { + match phase { + JanitorPhase::DeleteRelations => ("Deleting", "Deleted", "relation"), + JanitorPhase::Archive => ("Archiving", "Archived", "relation"), + JanitorPhase::DeleteArchives => ("Deleting", "Deleted", "expired archive"), + JanitorPhase::DropOldNameViews => ("Dropping", "Dropped", "old-name view"), + JanitorPhase::DropQueryArtifacts => ("Dropping", "Dropped", "query artifact"), + JanitorPhase::PruneDirectState => ("Pruning", "Pruned", "direct state table"), + } +} + +/// `count` with the singular noun for one and its plural otherwise. +pub(crate) fn noun(count: usize, singular: &str) -> String { + format_count_noun(i64::try_from(count).unwrap_or(i64::MAX), singular, None) +} + +/// Seconds with two decimals under a minute, then whole minutes and seconds. +pub(crate) fn duration(seconds: f64) -> String { + if seconds < 60.0 { + return format!("{:.2}s", seconds.max(0.0)); + } + let whole: u64 = seconds.round() as u64; + let (hours, minutes, rest) = ( + whole / SECONDS_PER_HOUR, + whole % SECONDS_PER_HOUR / SECONDS_PER_MINUTE, + whole % SECONDS_PER_MINUTE, + ); + if hours > 0 { + return format!("{hours}h {minutes:02}m {rest:02}s"); + } + format!("{minutes}m {rest:02}s") +} + +/// `name -> destination`, or the name alone. +pub(crate) fn flow(name: &str, destination: Option<&str>) -> String { + destination.map_or_else(|| name.to_owned(), |destination| format!("{name} -> {destination}")) +} diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs new file mode 100644 index 0000000000..b07123f4ff --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -0,0 +1,245 @@ +//! The JSON payload of `sqb janitor --json`. + +use crate::janitor_text::constants::{DROP_EARLY, DROP_EXPIRED}; +use crate::janitor_text::models::{ + JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText, OldNameViewText, + SourceSchemaText, +}; +use crate::json::models::{JsonInteger, JsonValue}; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn optional_text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, text) +} + +fn texts(values: &[String]) -> JsonValue { + JsonValue::Array(values.iter().map(|value| text(value)).collect()) +} + +fn integer(value: impl ToString) -> JsonValue { + JsonInteger::parse(&value.to_string()).map_or(JsonValue::Null, JsonValue::Integer) +} + +fn optional_integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, integer) +} + +fn object(fields: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + fields + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +/// A preview `YYYY-MM-DD HH:MM:SS` UTC time as ISO 8601. +fn utc_timestamp(value: &str) -> JsonValue { + if value.is_empty() { + return JsonValue::Null; + } + text(&format!("{}Z", value.replacen(' ', "T", 1))) +} + +fn old_name_view(view: &OldNameViewText, planned: &str) -> JsonValue { + let action: &str = match (planned, view.drop_reason.as_deref()) { + ("drop", Some(DROP_EARLY | DROP_EXPIRED)) => "drop", + ("live", _) => "keep", + _ => "record_dropped", + }; + object(vec![ + ("name", text(&view.name)), + ("destination_model", text(&view.destination_model)), + ("expires_on", optional_text(view.expires_on.as_deref())), + ("action", text(action)), + ("drop_reason", optional_text(view.drop_reason.as_deref())), + ("occupied", JsonValue::Bool(view.occupied)), + ("claimed_by", optional_text(view.claimed_by.as_deref())), + ]) +} + +fn source_schema(schema: &SourceSchemaText, blocked: bool) -> JsonValue { + let mut fields: Vec<(&str, JsonValue)> = vec![ + ("name", text(&schema.name)), + ("source_names", texts(&schema.source_names)), + ]; + if blocked { + fields.push(( + "suppressed_candidates", + texts(&schema.suppressed_candidates), + )); + fields.push(( + "suppressed_archive_deletions", + texts(&schema.suppressed_archive_deletions), + )); + } + object(fields) +} + +/// Every planned action, retained archive and skipped object of the plan. +pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { + let views = plan + .old_name_drops + .iter() + .map(|view| old_name_view(view, "drop")) + .chain( + plan.old_name_live + .iter() + .map(|view| old_name_view(view, "live")), + ) + .chain( + plan.old_name_missing + .iter() + .map(|view| old_name_view(view, "missing")), + ) + .collect(); + object(vec![ + ("direct_mode", JsonValue::Bool(plan.direct_mode)), + ("retention_days", integer(plan.retention_days)), + ("archive_retention_days", integer(plan.archive_retention_days)), + ( + "age_metadata_supported", + JsonValue::Bool(plan.age_metadata_supported), + ), + ("schemas_scanned", integer(plan.scanned_schema_count)), + ("eligible_for_deletion", texts(&plan.candidates)), + ( + "relations_to_archive", + JsonValue::Array( + plan.archive_candidates + .iter() + .map(|candidate| { + object(vec![ + ("name", text(&candidate.name)), + ("archive_name", text(&candidate.archive_name)), + ("age_days", optional_integer(candidate.age_days)), + ("expires_at", utc_timestamp(&candidate.expires_at)), + ("deleted_in_run", JsonValue::Bool(candidate.expires_now)), + ]) + }) + .collect(), + ), + ), + ( + "archives_to_delete", + JsonValue::Array( + plan.archive_deletions + .iter() + .map(|archive| { + object(vec![ + ("name", text(&archive.name)), + ("archived_in_run", JsonValue::Bool(archive.archived_in_run)), + ("archived_at", utc_timestamp(&archive.archived_at)), + ("age_days", optional_integer(archive.age_days)), + ("expires_at", utc_timestamp(&archive.expires_at)), + ]) + }) + .collect(), + ), + ), + ( + "retained_archives", + JsonValue::Array( + plan.retained_archives + .iter() + .map(|archive| { + object(vec![ + ("name", text(&archive.name)), + ("archived_at", utc_timestamp(&archive.archived_at)), + ("expires_at", utc_timestamp(&archive.expires_at)), + ]) + }) + .collect(), + ), + ), + ("old_name_views", JsonValue::Array(views)), + ("query_artifacts", texts(&plan.query_artifacts)), + ( + "direct_state_prunes", + JsonValue::Array( + plan.state_prunes + .iter() + .map(|(name, versions)| { + object(vec![ + ("name", text(name)), + ("retain_versions", integer(*versions)), + ]) + }) + .collect(), + ), + ), + ( + "skipped_objects", + JsonValue::Array( + plan.skipped_relations + .iter() + .map(|(name, reason)| { + object(vec![("name", text(name)), ("reason", text(reason))]) + }) + .collect(), + ), + ), + ( + "skipped_schemas", + JsonValue::Array( + plan.skipped_schemas + .iter() + .map(|schema| source_schema(schema, false)) + .collect(), + ), + ), + ( + "blocked_schemas", + JsonValue::Array( + plan.blocked_schemas + .iter() + .map(|schema| source_schema(schema, true)) + .collect(), + ), + ), + ]) +} + +fn counts_payload(counts: &JanitorResultCounts) -> JsonValue { + object(vec![ + ("archived", integer(counts.archived)), + ("deleted", integer(counts.deleted)), + ("deleted_archives", integer(counts.deleted_archives)), + ("deleted_query_artifacts", integer(counts.deleted_query_artifacts)), + ("dropped_old_name_views", integer(counts.dropped_old_name_views)), + ("pruned_direct_state", integer(counts.pruned_direct_state)), + ]) +} + +fn item_payload(item: &JanitorItemText) -> JsonValue { + object(vec![ + ("phase", text(item.phase.value())), + ("action", text(&item.action)), + ("name", text(&item.name)), + ("destination", optional_text(item.destination.as_deref())), + ( + "status", + text(if item.error.is_some() { "failed" } else { "ok" }), + ), + ("error", optional_text(item.error.as_deref())), + ( + "duration_seconds", + item.duration_seconds.map_or(JsonValue::Null, JsonValue::Float), + ), + ]) +} + +/// The counts, actions and duration of a run. +pub(crate) fn run_payload(run: &JanitorRunText) -> JsonValue { + object(vec![ + ("counts", counts_payload(&run.counts)), + ("elapsed_seconds", JsonValue::Float(run.elapsed_seconds)), + ( + "items", + JsonValue::Array(run.items.iter().map(item_payload).collect()), + ), + ]) +} diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs index 87d0aeb42f..1c0235c6d2 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs @@ -3,10 +3,29 @@ use crate::cli_text::models::CliStyle; use crate::janitor_text::constants::{ DROP_EARLY, DROP_EXPIRED, DROP_MISSING, LABEL_WIDTH, TREE_BLANK, TREE_BRANCH_GLYPH, - TREE_CONTINUATION, TREE_LAST_GLYPH, + TREE_CONTINUATION, TREE_LAST_GLYPH, VERBOSE_FLAG, }; use crate::janitor_text::models::{JanitorPlanText, OldNameViewText}; +/// The first `limit` items, or all of them, and how many are hidden. +fn visible(items: &[T], limit: Option) -> (&[T], usize) { + let shown: usize = limit.map_or(items.len(), |limit| items.len().min(limit)); + (&items[..shown], items.len() - shown) +} + +/// The dim `... and N more` line of a section that hides entries. +fn overflow(style: CliStyle, hidden: usize, indent: &str) -> String { + if hidden == 0 { + return String::new(); + } + format!( + "{}\n", + style.muted(&format!( + "{indent}... and {hidden} more (use {VERBOSE_FLAG} to show all)" + )) + ) +} + fn row(label: &str, rendered: &str) -> String { format!(" {label: String { } /// Schemas whose active sources block every janitor action. -pub(crate) fn blocked(plan: &JanitorPlanText, style: CliStyle) -> String { +pub(crate) fn blocked(plan: &JanitorPlanText, limit: Option, style: CliStyle) -> String { if plan.blocked_schemas.is_empty() { return String::new(); } @@ -126,18 +145,22 @@ pub(crate) fn blocked(plan: &JanitorPlanText, style: CliStyle) -> String { schema.source_names.join(", ") )) )); - for name in &schema.suppressed_candidates { + let (candidates, hidden) = visible(&schema.suppressed_candidates, limit); + for name in candidates { text.push_str(&format!( " {}\n", style.error_muted(&format!("suppressed {action}: {name}")) )); } - for name in &schema.suppressed_archive_deletions { + text.push_str(&overflow(style, hidden, " ")); + let (deletions, hidden) = visible(&schema.suppressed_archive_deletions, limit); + for name in deletions { text.push_str(&format!( " {}\n", style.error_muted(&format!("suppressed archive deletion: {name}")) )); } + text.push_str(&overflow(style, hidden, " ")); } text.push_str(&format!( " {}\n", @@ -147,11 +170,12 @@ pub(crate) fn blocked(plan: &JanitorPlanText, style: CliStyle) -> String { } /// Relations to archive, archives to delete, and retained archives. -pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { +pub(crate) fn archives(plan: &JanitorPlanText, limit: Option, style: CliStyle) -> String { let mut text: String = String::new(); if !plan.archive_candidates.is_empty() { text.push_str(&format!("\n{}\n", style.success("Relations to archive"))); - for candidate in &plan.archive_candidates { + let (candidates, hidden) = visible(&plan.archive_candidates, limit); + for candidate in candidates { let expiry: String = if candidate.expires_now { "deleted in this run".to_owned() } else { @@ -164,10 +188,12 @@ pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { style.muted(&format!("{}, {expiry}", age(candidate.age_days))) )); } + text.push_str(&overflow(style, hidden, " ")); } if !plan.archive_deletions.is_empty() { text.push_str(&format!("\n{}\n", style.success("Archives to delete"))); - for deletion in &plan.archive_deletions { + let (deletions, hidden) = visible(&plan.archive_deletions, limit); + for deletion in deletions { let detail: String = if deletion.archived_in_run { "archived in this run".to_owned() } else { @@ -184,10 +210,12 @@ pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { style.muted(&detail) )); } + text.push_str(&overflow(style, hidden, " ")); } if !plan.retained_archives.is_empty() { text.push_str(&format!("\n{}\n", style.success("Retained archives"))); - for retained in &plan.retained_archives { + let (retained_archives, hidden) = visible(&plan.retained_archives, limit); + for retained in retained_archives { text.push_str(&format!( " {} {}\n", style.object_name(&retained.name), @@ -197,6 +225,7 @@ pub(crate) fn archives(plan: &JanitorPlanText, style: CliStyle) -> String { )) )); } + text.push_str(&overflow(style, hidden, " ")); } text } @@ -228,7 +257,11 @@ fn old_name_detail(view: &OldNameViewText) -> (&'static str, String) { } /// Compatibility views at old model names and what happens to each. -pub(crate) fn old_name_views(plan: &JanitorPlanText, style: CliStyle) -> String { +pub(crate) fn old_name_views( + plan: &JanitorPlanText, + limit: Option, + style: CliStyle, +) -> String { let listed: Vec<&OldNameViewText> = plan .old_name_drops .iter() @@ -243,7 +276,8 @@ pub(crate) fn old_name_views(plan: &JanitorPlanText, style: CliStyle) -> String style.success("Old name views"), style.muted(&format!("({})", listed.len())) ); - for (index, view) in listed.iter().enumerate() { + let (shown, hidden) = visible(&listed, limit); + for (index, view) in shown.iter().enumerate() { let last: bool = index + 1 == listed.len(); text.push_str(&format!( "{} {} {}\n", @@ -263,16 +297,22 @@ pub(crate) fn old_name_views(plan: &JanitorPlanText, style: CliStyle) -> String style.muted(label) )); } + text.push_str(&overflow(style, hidden, " ")); text } /// A titled list of ` name` or ` name detail` lines, or nothing when empty. -pub(crate) fn listing(style: CliStyle, title: &str, items: &[(String, Option)]) -> String { +pub(crate) fn listing( + (style, limit): (CliStyle, Option), + title: &str, + items: &[(String, Option)], +) -> String { if items.is_empty() { return String::new(); } let mut text: String = format!("\n{}\n", style.success(title)); - for (name, detail) in items { + let (shown, hidden) = visible(items, limit); + for (name, detail) in shown { match detail { Some(detail) => text.push_str(&format!( " {} {}\n", @@ -282,5 +322,51 @@ pub(crate) fn listing(style: CliStyle, title: &str, items: &[(String, Option text.push_str(&format!(" {}\n", style.object_name(name))), } } + text.push_str(&overflow(style, hidden, " ")); + text +} + +/// Skipped objects: each with its reason when `limit` is none, otherwise counted by reason. +pub(crate) fn skipped_objects( + plan: &JanitorPlanText, + limit: Option, + style: CliStyle, +) -> String { + let Some(limit) = limit else { + let skipped: Vec<(String, Option)> = plan + .skipped_relations + .iter() + .map(|(name, reason)| (name.clone(), Some(reason.clone()))) + .collect(); + return listing((style, None), "Skipped objects", &skipped); + }; + if plan.skipped_relations.is_empty() { + return String::new(); + } + let mut groups: Vec<(usize, &str)> = Vec::new(); + for (_, reason) in &plan.skipped_relations { + match groups.iter_mut().find(|(_, known)| *known == reason.as_str()) { + Some((count, _)) => *count += 1, + None => groups.push((1, reason.as_str())), + } + } + groups.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(right.1))); + let width: usize = groups + .first() + .map_or(1, |(count, _)| count.to_string().len()); + let mut text: String = format!( + "\n{} {}\n", + style.success("Skipped objects"), + style.muted(&format!("({})", plan.skipped_relations.len())) + ); + let (shown, hidden) = visible(&groups, Some(limit)); + for (count, reason) in shown { + text.push_str(&format!( + " {} {}\n", + style.accent(&format!("{count:>width$}")), + style.muted(reason) + )); + } + text.push_str(&overflow(style, hidden, " ")); text } diff --git a/crates/sqlbuild-core/src/janitor_text/constants.rs b/crates/sqlbuild-core/src/janitor_text/constants.rs index d9ebfac56c..e8bbff3d49 100644 --- a/crates/sqlbuild-core/src/janitor_text/constants.rs +++ b/crates/sqlbuild-core/src/janitor_text/constants.rs @@ -11,3 +11,12 @@ pub(crate) const DROP_EXPIRED: &str = "expired"; pub(crate) const DROP_MISSING: &str = "missing"; pub(crate) const DISABLED_DETAIL: &str = "Janitor is opt-in. It previews stale warehouse objects \ and asks before archiving or deleting anything."; +/// How many entries each preview section shows without `--verbose`. +pub(crate) const PREVIEW_LIMIT: usize = 10; +pub(crate) const VERBOSE_FLAG: &str = "--verbose"; +/// The width of the longest janitor action value. +pub(crate) const ACTION_WIDTH: usize = "archived".len(); +pub(crate) const RECORDED_ACTION: &str = "recorded"; +pub(crate) const DROPPED_ACTION: &str = "dropped"; +pub(crate) const SECONDS_PER_MINUTE: u64 = 60; +pub(crate) const SECONDS_PER_HOUR: u64 = 3600; diff --git a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs new file mode 100644 index 0000000000..da23689da9 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs @@ -0,0 +1,43 @@ +//! `sqb janitor --json`. + +use crate::janitor_text::_helpers::plan_json::{plan_payload, run_payload}; +use crate::janitor_text::models::{JanitorOutcome, JanitorPlanText, JanitorRunText}; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// `json.dumps(payload, indent=2)` of the outcome, the plan, what the run did and its error. +/// +/// The plan is none when janitor is disabled for the project. +#[must_use] +pub fn format_janitor_json( + plan: Option<&JanitorPlanText>, + outcome: JanitorOutcome, + run: Option<&JanitorRunText>, + error: Option<&str>, +) -> String { + let payload: JsonValue = JsonValue::Object(vec![ + ( + "target".to_owned(), + plan.and_then(|plan| plan.target_name.as_deref()) + .map_or(JsonValue::Null, |name| JsonValue::String(name.to_owned())), + ), + ( + "status".to_owned(), + JsonValue::String(outcome.value().to_owned()), + ), + ("plan".to_owned(), plan.map_or(JsonValue::Null, plan_payload)), + ( + "result".to_owned(), + run.map_or(JsonValue::Null, run_payload), + ), + ( + "error".to_owned(), + error.map_or(JsonValue::Null, |error| JsonValue::String(error.to_owned())), + ), + ]); + dumps( + &payload, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + ) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs deleted file mode 100644 index c6d4ef48b9..0000000000 --- a/crates/sqlbuild-core/src/janitor_text/main/janitor_completion_text.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! The summary of a completed janitor run. - -use crate::cli_text::main::format_count_noun::format_count_noun; -use crate::cli_text::models::CliStyle; -use crate::janitor_text::models::JanitorResultCounts; - -fn noun(count: usize, singular: &str) -> String { - format_count_noun(i64::try_from(count).unwrap_or(i64::MAX), singular, None) -} - -/// `[Archived N relations. ]Deleted M objects[ and pruned K direct state tables].` -#[must_use] -pub fn janitor_completion_text(counts: &JanitorResultCounts, style: CliStyle) -> String { - let archived: String = if counts.archived == 0 { - String::new() - } else { - format!("Archived {}. ", noun(counts.archived, "relation")) - }; - let objects: String = noun( - counts.deleted - + counts.deleted_archives - + counts.deleted_query_artifacts - + counts.dropped_old_name_views, - "object", - ); - let deleted: String = if counts.pruned_direct_state == 0 { - format!("Deleted {objects}.") - } else { - format!( - "Deleted {objects} and pruned {}.", - noun(counts.pruned_direct_state, "direct state table") - ) - }; - format!("{}\n", style.success(&format!("{archived}{deleted}"))) -} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_item_counts.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_item_counts.rs new file mode 100644 index 0000000000..4a754c360f --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_item_counts.rs @@ -0,0 +1,24 @@ +//! What the finished actions of a janitor run changed. + +use crate::janitor_text::constants::DROPPED_ACTION; +use crate::janitor_text::models::{JanitorItemText, JanitorPhase, JanitorResultCounts}; + +/// Counts of the successful `items`, as the execution result would count them. +#[must_use] +pub fn janitor_item_counts(items: &[JanitorItemText]) -> JanitorResultCounts { + let mut counts: JanitorResultCounts = JanitorResultCounts::default(); + for item in items.iter().filter(|item| item.error.is_none()) { + match item.phase { + JanitorPhase::DeleteRelations => counts.deleted += 1, + JanitorPhase::Archive => counts.archived += 1, + JanitorPhase::DeleteArchives => counts.deleted_archives += 1, + JanitorPhase::DropOldNameViews if item.action == DROPPED_ACTION => { + counts.dropped_old_name_views += 1; + } + JanitorPhase::DropOldNameViews => {} + JanitorPhase::DropQueryArtifacts => counts.deleted_query_artifacts += 1, + JanitorPhase::PruneDirectState => counts.pruned_direct_state += 1, + } + } + counts +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_nothing_to_do_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_nothing_to_do_text.rs new file mode 100644 index 0000000000..253f9d8ef1 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_nothing_to_do_text.rs @@ -0,0 +1,18 @@ +//! The close of a janitor run whose plan changes nothing. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::models::{CliStyle, CompletionState}; + +/// `✓ Janitor completed nothing to clean up`. +#[must_use] +pub fn janitor_nothing_to_do_text(style: CliStyle) -> String { + format!( + "{}\n", + format_completion_line( + style, + CompletionState::Ok, + "Janitor completed", + Some(&style.muted("nothing to clean up")) + ) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_complete_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_complete_text.rs new file mode 100644 index 0000000000..5920bcc517 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_complete_text.rs @@ -0,0 +1,33 @@ +//! The close of one janitor apply phase. + +use crate::janitor_text::_helpers::phases::{duration, noun, phase_words}; +use crate::janitor_text::constants::RECORDED_ACTION; +use crate::janitor_text::models::{JanitorItemText, JanitorPhase}; + +/// `Archived 608 relations. (4m 51s)`, also counting old-name views recorded as already gone. +#[must_use] +pub fn janitor_phase_complete_text( + phase: JanitorPhase, + items: &[JanitorItemText], + elapsed_seconds: f64, +) -> String { + let (_, done, singular) = phase_words(phase); + let finished: Vec<&JanitorItemText> = items + .iter() + .filter(|item| item.phase == phase && item.error.is_none()) + .collect(); + let recorded: usize = finished + .iter() + .filter(|item| item.action == RECORDED_ACTION) + .count(); + let recorded_text: String = if recorded == 0 { + String::new() + } else { + format!(" and recorded {recorded} already gone") + }; + format!( + "{done} {}{recorded_text}. ({})", + noun(finished.len() - recorded, singular), + duration(elapsed_seconds) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_failed_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_failed_text.rs new file mode 100644 index 0000000000..9a2016a824 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_failed_text.rs @@ -0,0 +1,29 @@ +//! The close of a janitor apply phase that an action stopped. + +use crate::cli_text::main::format_phase_line::format_phase_line; +use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::phases::{duration, noun, phase_words}; +use crate::janitor_text::models::JanitorPhase; + +/// `✗ Archiving failed after 11 of 608 relations (1m 02s)`. +#[must_use] +pub fn janitor_phase_failed_text( + phase: JanitorPhase, + (completed, total): (usize, usize), + elapsed_seconds: f64, + style: CliStyle, +) -> String { + let (doing, _, singular) = phase_words(phase); + format_phase_line( + style, + false, + ( + &format!("{doing} failed"), + &format!( + "after {completed} of {} ({})", + noun(total, singular), + duration(elapsed_seconds) + ), + ), + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_flow_width.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_flow_width.rs new file mode 100644 index 0000000000..2920f2b22c --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_flow_width.rs @@ -0,0 +1,39 @@ +//! The width of a janitor phase's action column. + +use crate::janitor_text::_helpers::phases::flow; +use crate::janitor_text::models::{JanitorPhase, JanitorPlanText}; + +/// The widest plain `name` or `name -> destination` the plan's `phase` reports. +#[must_use] +pub fn janitor_phase_flow_width(plan: &JanitorPlanText, phase: JanitorPhase) -> usize { + let flows: Vec = match phase { + JanitorPhase::DeleteRelations => plan.candidates.clone(), + JanitorPhase::Archive => plan + .archive_candidates + .iter() + .map(|candidate| flow(&candidate.name, Some(&candidate.archive_name))) + .collect(), + JanitorPhase::DeleteArchives => plan + .archive_deletions + .iter() + .map(|archive| archive.name.clone()) + .collect(), + JanitorPhase::DropOldNameViews => plan + .old_name_drops + .iter() + .chain(&plan.old_name_missing) + .map(|view| view.name.clone()) + .collect(), + JanitorPhase::DropQueryArtifacts => plan.query_artifacts.clone(), + JanitorPhase::PruneDirectState => plan + .state_prunes + .iter() + .map(|(name, _)| name.clone()) + .collect(), + }; + flows + .iter() + .map(|flow| flow.chars().count()) + .max() + .unwrap_or(0) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_start_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_start_text.rs new file mode 100644 index 0000000000..dd4e42d46e --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_phase_start_text.rs @@ -0,0 +1,11 @@ +//! The announcement of one janitor apply phase. + +use crate::janitor_text::_helpers::phases::{noun, phase_words}; +use crate::janitor_text::models::JanitorPhase; + +/// `Archiving 608 relations...`. +#[must_use] +pub fn janitor_phase_start_text(phase: JanitorPhase, total: usize) -> String { + let (doing, _, singular) = phase_words(phase); + format!("{doing} {}...", noun(total, singular)) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/mod.rs b/crates/sqlbuild-core/src/janitor_text/main/mod.rs index d73e81e797..bb3822d571 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/mod.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/mod.rs @@ -1,9 +1,17 @@ -pub mod janitor_completion_text; +pub mod format_janitor_json; pub mod janitor_confirmation_phrase; pub mod janitor_confirmation_prompt; pub mod janitor_disabled_text; +pub mod janitor_item_counts; +pub mod janitor_nothing_to_do_text; +pub mod janitor_phase_complete_text; +pub mod janitor_phase_failed_text; +pub mod janitor_phase_flow_width; +pub mod janitor_phase_start_text; pub mod janitor_plan_has_work; pub mod janitor_preview_complete_text; pub mod janitor_preview_start_text; +pub mod render_janitor_completion; +pub mod render_janitor_item_line; pub mod render_janitor_plan; pub mod validate_janitor_settings; diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs new file mode 100644 index 0000000000..410107822c --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs @@ -0,0 +1,42 @@ +//! The final line of a janitor run. + +use crate::cli_text::main::format_completion_line::format_completion_line; +use crate::cli_text::models::{CliStyle, CompletionState}; +use crate::janitor_text::_helpers::phases::duration; +use crate::janitor_text::models::JanitorResultCounts; + +/// `✓ Janitor completed archived N, deleted M, pruned K (5m 02s)`, or `✗ Janitor failed ...` +/// with what was done before the failure. +#[must_use] +pub fn render_janitor_completion( + counts: &JanitorResultCounts, + elapsed_seconds: Option, + failed: bool, + style: CliStyle, +) -> String { + let deleted: usize = counts.deleted + + counts.deleted_archives + + counts.deleted_query_artifacts + + counts.dropped_old_name_views; + let mut summary: String = [ + ("archived", counts.archived), + ("deleted", deleted), + ("pruned", counts.pruned_direct_state), + ] + .iter() + .map(|(label, count)| format!("{label} {}", style.value(&count.to_string()))) + .collect::>() + .join(", "); + if let Some(seconds) = elapsed_seconds { + summary.push_str(&format!(" {}", style.muted(&format!("({})", duration(seconds))))); + } + let (state, label) = if failed { + (CompletionState::Fail, "Janitor failed") + } else { + (CompletionState::Ok, "Janitor completed") + }; + format!( + "{}\n", + format_completion_line(style, state, label, Some(&summary)) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs new file mode 100644 index 0000000000..cea767a76a --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs @@ -0,0 +1,47 @@ +//! One streamed janitor action line. + +use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::phases::{duration, flow}; +use crate::janitor_text::constants::ACTION_WIDTH; +use crate::janitor_text::models::JanitorItemText; + +/// Position, action, name or `name -> destination` padded to `flow_width`, status and duration, +/// then the error of a failed action. +#[must_use] +pub fn render_janitor_item_line( + (index, total): (usize, usize), + item: &JanitorItemText, + flow_width: usize, + style: CliStyle, +) -> String { + let position: String = format!("{index:>width$}/{total}", width = total.to_string().len()); + let plain_flow: String = flow(&item.name, item.destination.as_deref()); + let styled_flow: String = match &item.destination { + Some(destination) => format!( + "{} {} {}", + style.object_name(&item.name), + style.muted("->"), + style.object_name(destination) + ), + None => style.object_name(&item.name), + }; + let plain_width: usize = plain_flow.chars().count(); + let flow_padding: String = " ".repeat(flow_width.max(plain_width) - plain_width); + let action_padding: String = + " ".repeat(ACTION_WIDTH.saturating_sub(item.action.chars().count())); + let status: &str = if item.error.is_some() { "FAIL" } else { "OK" }; + let elapsed: String = item.duration_seconds.map_or_else(String::new, |seconds| { + format!(" {}", style.muted(&duration(seconds))) + }); + let error: String = item + .error + .as_deref() + .and_then(|error| error.lines().next()) + .map_or_else(String::new, |error| format!(" {}", style.error(error))); + format!( + " {} {}{action_padding} {styled_flow}{flow_padding} {}{elapsed}{error}", + style.muted(&position), + style.accent(&item.action), + style.status(status) + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs index f9191b40ed..19163ae34d 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs @@ -3,20 +3,24 @@ use crate::cli_text::models::CliStyle; use crate::janitor_text::_helpers::counts::environment_label; use crate::janitor_text::_helpers::plan_sections::{ - archives, blocked, listing, old_name_views, summary, + archives, blocked, listing, old_name_views, skipped_objects, summary, }; +use crate::janitor_text::constants::PREVIEW_LIMIT; use crate::janitor_text::models::JanitorPlanText; /// Every section of the preview, ending with a blank line. +/// +/// Without `verbose`, each list shows its first entries and skipped objects are counted by reason. #[must_use] -pub fn render_janitor_plan(plan: &JanitorPlanText, style: CliStyle) -> String { +pub fn render_janitor_plan(plan: &JanitorPlanText, verbose: bool, style: CliStyle) -> String { + let limit: Option = (!verbose).then_some(PREVIEW_LIMIT); let mut text: String = format!( "{} {}\n\n", style.title("Janitor preview"), style.object_name(environment_label(plan)) ); text.push_str(&summary(plan, style)); - text.push_str(&blocked(plan, style)); + text.push_str(&blocked(plan, limit, style)); let skipped_schemas: Vec<(String, Option)> = plan .skipped_schemas .iter() @@ -30,24 +34,24 @@ pub fn render_janitor_plan(plan: &JanitorPlanText, style: CliStyle) -> String { ) }) .collect(); - text.push_str(&listing(style, "Skipped schemas", &skipped_schemas)); + text.push_str(&listing((style, limit), "Skipped schemas", &skipped_schemas)); if !plan.direct_mode { let candidates: Vec<(String, Option)> = plan .candidates .iter() .map(|name| (name.clone(), None)) .collect(); - text.push_str(&listing(style, "Eligible objects", &candidates)); + text.push_str(&listing((style, limit), "Eligible objects", &candidates)); } - text.push_str(&archives(plan, style)); - text.push_str(&old_name_views(plan, style)); + text.push_str(&archives(plan, limit, style)); + text.push_str(&old_name_views(plan, limit, style)); let artifacts: Vec<(String, Option)> = plan .query_artifacts .iter() .map(|name| (name.clone(), None)) .collect(); text.push_str(&listing( - style, + (style, limit), "Eligible expired query artifacts", &artifacts, )); @@ -56,13 +60,12 @@ pub fn render_janitor_plan(plan: &JanitorPlanText, style: CliStyle) -> String { .iter() .map(|(name, versions)| (name.clone(), Some(format!("keep latest {versions}")))) .collect(); - text.push_str(&listing(style, "Eligible direct state pruning", &prunes)); - let skipped: Vec<(String, Option)> = plan - .skipped_relations - .iter() - .map(|(name, reason)| (name.clone(), Some(reason.clone()))) - .collect(); - text.push_str(&listing(style, "Skipped objects", &skipped)); + text.push_str(&listing( + (style, limit), + "Eligible direct state pruning", + &prunes, + )); + text.push_str(&skipped_objects(plan, limit, style)); text.push('\n'); text } diff --git a/crates/sqlbuild-core/src/janitor_text/models.rs b/crates/sqlbuild-core/src/janitor_text/models.rs index b979149679..cbd49ccd0b 100644 --- a/crates/sqlbuild-core/src/janitor_text/models.rs +++ b/crates/sqlbuild-core/src/janitor_text/models.rs @@ -81,3 +81,98 @@ pub struct JanitorResultCounts { pub dropped_old_name_views: usize, pub pruned_direct_state: usize, } + +/// One janitor apply phase, in execution order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JanitorPhase { + DeleteRelations, + Archive, + DeleteArchives, + DropOldNameViews, + DropQueryArtifacts, + PruneDirectState, +} + +impl JanitorPhase { + /// The phase from the executor's `JanitorPhase` value. + #[must_use] + pub fn from_value(value: &str) -> Option { + match value { + "delete_relations" => Some(Self::DeleteRelations), + "archive" => Some(Self::Archive), + "delete_archives" => Some(Self::DeleteArchives), + "drop_old_name_views" => Some(Self::DropOldNameViews), + "drop_query_artifacts" => Some(Self::DropQueryArtifacts), + "prune_direct_state" => Some(Self::PruneDirectState), + _ => None, + } + } + + /// The executor's `JanitorPhase` value. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::DeleteRelations => "delete_relations", + Self::Archive => "archive", + Self::DeleteArchives => "delete_archives", + Self::DropOldNameViews => "drop_old_name_views", + Self::DropQueryArtifacts => "drop_query_artifacts", + Self::PruneDirectState => "prune_direct_state", + } + } +} + +/// One finished janitor action. +#[derive(Clone, Debug, PartialEq)] +pub struct JanitorItemText { + pub phase: JanitorPhase, + /// The `JanitorAction` value, such as `archived`. + pub action: String, + pub name: String, + /// Where the action moved the relation, such as its archive name. + pub destination: Option, + /// The error that stopped the action, or none when it succeeded. + pub error: Option, + pub duration_seconds: Option, +} + +/// The actions a janitor run finished, what it changed, and how long it took. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct JanitorRunText { + pub items: Vec, + pub counts: JanitorResultCounts, + pub elapsed_seconds: f64, +} + +/// How a janitor invocation ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JanitorOutcome { + /// Janitor is not enabled for the project. + Disabled, + /// An inspection-only `--as` preview. + Previewed, + /// Active sources block every action. + Blocked, + /// The plan changes nothing. + NothingToDo, + Cancelled, + Completed, + /// An action failed after the cleanup started. + Failed, +} + +impl JanitorOutcome { + /// The `status` of `sqb janitor --json`. + #[must_use] + pub fn value(self) -> &'static str { + match self { + Self::Disabled => "disabled", + Self::Previewed => "previewed", + Self::Blocked => "blocked", + Self::NothingToDo => "nothing_to_do", + Self::Cancelled => "cancelled", + Self::Completed => "completed", + Self::Failed => "failed", + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 48e4e1ded7..8c05d690ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -11,16 +11,20 @@ use sqlbuild_analysis::janitor_planning::main::resolve_retention_days::resolve_r use sqlbuild_cli::diff_report::errors::DiffUserError; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::janitor_text::errors::JanitorSettingsError; -use sqlbuild_core::janitor_text::main::janitor_completion_text::janitor_completion_text; +use sqlbuild_core::janitor_text::main::format_janitor_json::format_janitor_json; use sqlbuild_core::janitor_text::main::janitor_confirmation_phrase::janitor_confirmation_phrase; use sqlbuild_core::janitor_text::main::janitor_confirmation_prompt::janitor_confirmation_prompt; use sqlbuild_core::janitor_text::main::janitor_disabled_text::janitor_disabled_text; +use sqlbuild_core::janitor_text::main::janitor_item_counts::janitor_item_counts; +use sqlbuild_core::janitor_text::main::janitor_nothing_to_do_text::janitor_nothing_to_do_text; use sqlbuild_core::janitor_text::main::janitor_plan_has_work::janitor_plan_has_work; use sqlbuild_core::janitor_text::main::janitor_preview_complete_text::janitor_preview_complete_text; use sqlbuild_core::janitor_text::main::janitor_preview_start_text::janitor_preview_start_text; +use sqlbuild_core::janitor_text::main::render_janitor_completion::render_janitor_completion; use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::main::validate_janitor_settings::validate_janitor_settings; -use sqlbuild_core::janitor_text::models::JanitorPlanText; +use sqlbuild_core::janitor_text::models::{JanitorOutcome, JanitorPlanText, JanitorRunText}; +use std::time::Instant; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; @@ -28,8 +32,9 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::diff_report::support::cli_error; use crate::bindings::_helpers::janitor::janitor_phases::{ JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, - compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, + compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, output_stream, }; +use crate::bindings::_helpers::janitor::janitor_progress::JanitorProgressPrinter; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; @@ -74,16 +79,27 @@ fn settings_error(py: Python<'_>, error: JanitorSettingsError) -> PyErr { } } -fn write(py: Python<'_>, text: &str) -> PyResult<()> { - py.import("sys")? - .getattr("stdout")? - .call_method1("write", (text,))?; +fn write(stream: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; Ok(()) } -fn flush(py: Python<'_>) -> PyResult<()> { - py.import("sys")?.getattr("stdout")?.call_method0("flush")?; - Ok(()) +/// Print the `--json` document on stdout when JSON output was requested. +fn write_json( + (request, plan): (&Bound<'_, PyAny>, Option<&JanitorPlanText>), + outcome: JanitorOutcome, + run: Option<&JanitorRunText>, + error: Option<&str>, +) -> PyResult<()> { + if !native_field(request, "json_output")?.extract::()? { + return Ok(()); + } + let stdout: Bound<'_, PyAny> = imported(request.py(), "sys", "stdout")?; + write( + &stdout, + &format!("{}\n", format_janitor_json(plan, outcome, run, error)), + ) } /// Discover the project and resolve the target and output context. @@ -111,8 +127,10 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult()? && stdout_supports_color(py)?; + let machine_output: bool = native_field(request, "json_output")?.extract()?; + let use_color: bool = !machine_output + && !native_field(request, "no_color")?.extract::()? + && stdout_supports_color(py)?; let options: Bound<'py, PyDict> = keywords( py, &[ @@ -124,6 +142,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( } /// Ask for the exact confirmation phrase; an interrupt cancels, and a non-interactive stdin refuses. -fn confirmed(py: Python<'_>, plan: &JanitorPlanText) -> PyResult { +fn confirmed(stream: &Bound<'_, PyAny>, plan: &JanitorPlanText) -> PyResult { + let py: Python<'_> = stream.py(); let stdin: Bound<'_, PyAny> = imported(py, "sys", "stdin")?; if stdin.is_none() || !stream_isatty(&stdin)? { return Err(cli_error( @@ -180,22 +200,78 @@ fn confirmed(py: Python<'_>, plan: &JanitorPlanText) -> PyResult { }, )); } - write(py, &janitor_confirmation_prompt(plan))?; - flush(py)?; + write(stream, &janitor_confirmation_prompt(plan))?; match stdin.call_method0("readline") { Ok(response) => Ok(response.extract::()?.trim_end_matches(LINE_ENDINGS) == janitor_confirmation_phrase(plan)), Err(error) if error.is_instance_of::(py) => { - write(py, "\n")?; + write(stream, "\n")?; Ok(false) } Err(error) => Err(error), } } +/// Run the confirmed plan, streaming each action, then print the final line and `--json`. +fn apply( + (request, stream, style): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, CliStyle), + (compiled, connection): (&JanitorCompiled<'_>, &Bound<'_, PyAny>), + (planning, plan): (&Bound<'_, PyAny>, &JanitorPlanText), +) -> PyResult { + let py: Python<'_> = stream.py(); + let progress: Bound<'_, JanitorProgressPrinter> = Bound::new( + py, + JanitorProgressPrinter::new(stream, style, plan.clone())?, + )?; + let started: Instant = Instant::now(); + let executed: PyResult> = + execute_janitor_cleanup(compiled, connection, (planning, progress.as_any())); + let elapsed_seconds: f64 = started.elapsed().as_secs_f64(); + let items = progress.get().items(); + match executed { + Ok(result) => { + let run: JanitorRunText = JanitorRunText { + items, + counts: janitor_result_counts(&result)?, + elapsed_seconds, + }; + write( + stream, + &format!( + "\n{}", + render_janitor_completion(&run.counts, Some(elapsed_seconds), false, style) + ), + )?; + write_json((request, Some(plan)), JanitorOutcome::Completed, Some(&run), None)?; + Ok(0) + } + Err(error) => { + let run: JanitorRunText = JanitorRunText { + counts: janitor_item_counts(&items), + items, + elapsed_seconds, + }; + write( + stream, + &format!( + "\n{}", + render_janitor_completion(&run.counts, Some(elapsed_seconds), true, style) + ), + )?; + let message: String = error.value(py).str()?.extract()?; + write_json( + (request, Some(plan)), + JanitorOutcome::Failed, + Some(&run), + Some(&message), + )?; + Err(error) + } + } +} + /// Plan, preview, confirm and clean up with an open warehouse connection. fn clean( - py: Python<'_>, request: &Bound<'_, PyAny>, (invocation, settings): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), (compiled, connection): (&JanitorCompiled<'_>, &Bound<'_, PyAny>), @@ -203,36 +279,42 @@ fn clean( let style: CliStyle = CliStyle { use_color: native_field(invocation, "use_color")?.extract()?, }; + let stream: Bound<'_, PyAny> = output_stream(invocation)?; let planning: Bound<'_, PyAny> = build_janitor_execution_plan((invocation, settings), compiled, connection)?; let plan: JanitorPlanText = janitor_plan_text(&planning)?; - write(py, &render_janitor_plan(&plan, style))?; + let verbose: bool = native_field(request, "verbose")?.extract()?; + write(&stream, &render_janitor_plan(&plan, verbose, style))?; let blocked: i32 = i32::from(!plan.blocked_schemas.is_empty()); let as_target: Option = native_field(invocation, "as_target")?.extract()?; if let Some(as_target) = as_target { let active: Option = native_field(invocation, "active_target_name")?.extract()?; write( - py, + &stream, &janitor_preview_complete_text(&as_target, active.as_deref(), style), )?; + write_json((request, Some(&plan)), JanitorOutcome::Previewed, None, None)?; return Ok(blocked); } if blocked != 0 { + write_json((request, Some(&plan)), JanitorOutcome::Blocked, None, None)?; return Ok(1); } if !janitor_plan_has_work(&plan) { + write(&stream, &janitor_nothing_to_do_text(style))?; + write_json((request, Some(&plan)), JanitorOutcome::NothingToDo, None, None)?; return Ok(0); } - if !native_field(request, "auto_approve")?.extract::()? && !confirmed(py, &plan)? { - write(py, "Janitor cancelled.\n")?; + if !native_field(request, "auto_approve")?.extract::()? && !confirmed(&stream, &plan)? { + write(&stream, "Janitor cancelled.\n")?; + write_json((request, Some(&plan)), JanitorOutcome::Cancelled, None, None)?; return Ok(1); } - let result: Bound<'_, PyAny> = execute_janitor_cleanup(compiled, connection, &planning)?; - write( - py, - &janitor_completion_text(&janitor_result_counts(&result)?, style), - )?; - Ok(0) + apply( + (request, &stream, style), + (compiled, connection), + (&planning, &plan), + ) } /// Run one `sqb janitor` request. @@ -246,8 +328,10 @@ pub(crate) fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> let janitor: Bound<'_, PyAny> = native_field(&invocation, "discovered_inputs")? .getattr("project_config")? .getattr("janitor")?; + let stream: Bound<'_, PyAny> = output_stream(&invocation)?; if !native_field(&janitor, "enabled")?.is_truthy()? { - write(py, &janitor_disabled_text(style))?; + write(&stream, &janitor_disabled_text(style))?; + write_json((request, None), JanitorOutcome::Disabled, None, None)?; return Ok(0); } let settings: Bound<'_, PyAny> = settings(py, request, &janitor)?; @@ -255,15 +339,13 @@ pub(crate) fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> if let Some(as_target) = &as_target { let active: Option = native_field(&invocation, "active_target_name")?.extract()?; write( - py, + &stream, &janitor_preview_start_text(as_target, active.as_deref()), )?; - flush(py)?; } let compiled: JanitorCompiled<'_> = compile_janitor_project(&invocation)?; let connection: Bound<'_, PyAny> = connect_janitor_warehouse(&invocation, &compiled)?; let outcome: PyResult = clean( - py, request, (&invocation, &settings), (&compiled, &connection), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index 9141d10afc..7129456cea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -43,8 +43,13 @@ pub(crate) struct JanitorCompiled<'py> { pub(crate) connection_config: Bound<'py, PyAny>, } -fn stdout(py: Python<'_>) -> PyResult> { - py.import("sys")?.getattr("stdout") +/// Where human output goes: stderr when stdout carries `--json`, otherwise stdout. +pub(crate) fn output_stream<'py>(invocation: &Bound<'py, PyAny>) -> PyResult> { + let machine_output: bool = native_field(invocation, "machine_output")?.is_truthy()?; + invocation + .py() + .import("sys")? + .getattr(if machine_output { "stderr" } else { "stdout" }) } fn use_color(invocation: &Bound<'_, PyAny>) -> PyResult { @@ -76,7 +81,7 @@ pub(crate) fn compile_janitor_project<'py>( let started: Instant = Instant::now(); let status: Bound<'py, PyAny> = Bound::new( py, - PlanningProgressReporter::new(py, &stdout(py)?, use_color(invocation)?)?, + PlanningProgressReporter::new(py, &output_stream(invocation)?, use_color(invocation)?)?, )? .into_any(); let as_target: Bound<'py, PyAny> = native_field(invocation, "as_target")?; @@ -145,7 +150,7 @@ pub(crate) fn connect_janitor_warehouse<'py>( py, &[ ("adapter_name", compiled.adapter_name.clone()), - ("stream", stdout(py)?), + ("stream", output_stream(invocation)?), ( "use_color", PyBool::new(py, use_color(invocation)?) @@ -221,7 +226,7 @@ pub(crate) fn build_janitor_execution_plan<'py>( ) -> PyResult> { let py: Python<'py> = invocation.py(); let started: Instant = Instant::now(); - let status: Bound<'py, PyAny> = status_reporter(&stdout(py)?, use_color(invocation)?)?; + let status: Bound<'py, PyAny> = status_reporter(&output_stream(invocation)?, use_color(invocation)?)?; status.call_method1("start", ("Inspecting warehouse state...",))?; let janitor: Bound<'py, PyAny> = native_field(invocation, "discovered_inputs")? .getattr("project_config")? @@ -285,11 +290,11 @@ pub(crate) fn build_janitor_execution_plan<'py>( Ok(plan) } -/// Execute the warehouse cleanup plan. +/// Execute the warehouse cleanup plan, reporting each phase and action to `progress`. pub(crate) fn execute_janitor_cleanup<'py>( compiled: &JanitorCompiled<'py>, connection: &Bound<'py, PyAny>, - plan: &Bound<'py, PyAny>, + (plan, progress): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { call_with( &host::entry(compiled.adapter.py(), ExecutorHost::ExecuteJanitorPlan)?, @@ -297,6 +302,7 @@ pub(crate) fn execute_janitor_cleanup<'py>( ("plan", plan.clone()), ("adapter", compiled.adapter.clone()), ("connection", connection.clone()), + ("progress", progress.clone()), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs index 994ac0d36c..38a3a16c4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs @@ -4,7 +4,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::janitor_text::main::janitor_completion_text::janitor_completion_text; +use sqlbuild_core::janitor_text::main::render_janitor_completion::render_janitor_completion; use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::models::{ ArchiveCandidateText, ArchivedRelationText, JanitorPlanText, JanitorResultCounts, @@ -193,20 +193,34 @@ pub(crate) fn janitor_result_counts(result: &Bound<'_, PyAny>) -> PyResult, use_color: bool) -> PyResult { +#[pyo3(name = "render_janitor_plan", signature = (*, plan, use_color = false, verbose = false))] +fn render_janitor_plan_text( + plan: &Bound<'_, PyAny>, + use_color: bool, + verbose: bool, +) -> PyResult { Ok(render_janitor_plan( &janitor_plan_text(plan)?, + verbose, CliStyle { use_color }, )) } /// Render one executor janitor result as the `sqb janitor` completion line. #[pyfunction] -#[pyo3(name = "render_janitor_completion", signature = (*, result, use_color = false))] -fn render_janitor_completion_text(result: &Bound<'_, PyAny>, use_color: bool) -> PyResult { - Ok(janitor_completion_text( +#[pyo3( + name = "render_janitor_completion", + signature = (*, result, elapsed_seconds = None, use_color = false) +)] +fn render_janitor_completion_text( + result: &Bound<'_, PyAny>, + elapsed_seconds: Option, + use_color: bool, +) -> PyResult { + Ok(render_janitor_completion( &janitor_result_counts(result)?, + elapsed_seconds, + false, CliStyle { use_color }, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs new file mode 100644 index 0000000000..7ba1d3e3cd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs @@ -0,0 +1,183 @@ +//! Streamed janitor phase and action lines, as the executor's progress callbacks. + +use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::Instant; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::janitor_text::main::janitor_phase_complete_text::janitor_phase_complete_text; +use sqlbuild_core::janitor_text::main::janitor_phase_failed_text::janitor_phase_failed_text; +use sqlbuild_core::janitor_text::main::janitor_phase_flow_width::janitor_phase_flow_width; +use sqlbuild_core::janitor_text::main::janitor_phase_start_text::janitor_phase_start_text; +use sqlbuild_core::janitor_text::main::render_janitor_item_line::render_janitor_item_line; +use sqlbuild_core::janitor_text::models::{JanitorItemText, JanitorPhase, JanitorPlanText}; + +use crate::bindings::_helpers::build_progress::terminal::{shared_lines, tty_flag}; +use crate::bindings::_helpers::build_progress::transient_status::{ + progress_spinners_disabled, status_reporter, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// The running phase's start text, size, finished count, column width and start time. +struct ActivePhase { + message: String, + total: usize, + finished: usize, + flow_width: usize, + started: Instant, +} + +#[derive(Default)] +struct ProgressState { + items: Vec, + active: Option, +} + +/// Print each janitor phase and action as it finishes, and keep the finished actions. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct JanitorProgressPrinter { + stream: Py, + status: Py, + style: CliStyle, + live: bool, + plan: JanitorPlanText, + state: Mutex, +} + +impl JanitorProgressPrinter { + pub(crate) fn new( + stream: &Bound<'_, PyAny>, + style: CliStyle, + plan: JanitorPlanText, + ) -> PyResult { + Ok(Self { + stream: stream.clone().unbind(), + status: status_reporter(stream, style.use_color)?.unbind(), + style, + live: tty_flag(stream)? && !progress_spinners_disabled(), + plan, + state: Mutex::new(ProgressState::default()), + }) + } + + fn state(&self) -> MutexGuard<'_, ProgressState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Every action finished so far, in order. + pub(crate) fn items(&self) -> Vec { + self.state().items.clone() + } + + fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", self.stream.bind(py))?; + options.set_item("text", text)?; + shared_lines(py)?.call_method("write_persistent", (), Some(&options))?; + Ok(()) + } +} + +fn phase(value: &str) -> PyResult { + JanitorPhase::from_value(value) + .ok_or_else(|| PyValueError::new_err(format!("unknown janitor phase: {value}"))) +} + +#[pymethods] +impl JanitorProgressPrinter { + #[pyo3(signature = (*, phase, total))] + fn phase_started(&self, py: Python<'_>, phase: &str, total: usize) -> PyResult<()> { + let phase: JanitorPhase = self::phase(phase)?; + let message: String = janitor_phase_start_text(phase, total); + self.state().active = Some(ActivePhase { + message: message.clone(), + total, + finished: 0, + flow_width: janitor_phase_flow_width(&self.plan, phase), + started: Instant::now(), + }); + self.status.bind(py).call_method1("start", (message,))?; + Ok(()) + } + + /// Record and print one finished action; `item` is its `JanitorProgressItem`. + #[pyo3(signature = (*, phase, item, error, duration_seconds))] + fn item_finished( + &self, + py: Python<'_>, + phase: &str, + item: &Bound<'_, PyAny>, + error: Option, + duration_seconds: f64, + ) -> PyResult<()> { + let item: JanitorItemText = JanitorItemText { + phase: self::phase(phase)?, + action: native_field(item, "action")?.str()?.extract()?, + name: native_field(item, "name")?.extract()?, + destination: native_field(item, "destination")?.extract()?, + error, + duration_seconds: Some(duration_seconds), + }; + let (line, progress) = { + let mut state = self.state(); + let (index, total, width, message) = match state.active.as_mut() { + Some(active) => { + active.finished += 1; + ( + active.finished, + active.total, + active.flow_width, + active.message.clone(), + ) + } + None => (1, 1, 0, String::new()), + }; + let line: String = render_janitor_item_line((index, total), &item, width, self.style); + state.items.push(item); + (line, format!("{message} {index}/{total}")) + }; + self.write_persistent(py, &format!("{line}\n"))?; + if self.live { + self.status.bind(py).call_method1("update", (progress,))?; + } + Ok(()) + } + + #[pyo3(signature = (*, phase, failed))] + fn phase_finished(&self, py: Python<'_>, phase: &str, failed: bool) -> PyResult<()> { + let phase: JanitorPhase = self::phase(phase)?; + let (active, items) = { + let mut state = self.state(); + (state.active.take(), state.items.clone()) + }; + let (total, elapsed) = active.as_ref().map_or((0, 0.0), |active| { + (active.total, active.started.elapsed().as_secs_f64()) + }); + let options: Bound<'_, PyDict> = PyDict::new(py); + if failed { + let completed: usize = items + .iter() + .filter(|item| item.phase == phase && item.error.is_none()) + .count(); + options.set_item( + "message", + janitor_phase_failed_text(phase, (completed, total), elapsed, self.style), + )?; + self.status + .bind(py) + .call_method("complete_styled", (), Some(&options))?; + return Ok(()); + } + options.set_item( + "message", + janitor_phase_complete_text(phase, &items, elapsed), + )?; + self.status + .bind(py) + .call_method("complete", (), Some(&options))?; + Ok(()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs index 9f9557101e..d2231d7bc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs @@ -6,4 +6,5 @@ pub(crate) mod janitor_old_names; pub(crate) mod janitor_phases; pub(crate) mod janitor_plan; pub(crate) mod janitor_planning; +pub(crate) mod janitor_progress; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index f29979c521..5ff12ba1d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -463,6 +463,8 @@ const JANITOR_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("drop_old_name_views", FieldDefault::EmptyTuple), FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("as_target", FieldDefault::None), + FieldSpec::new("verbose", FieldDefault::False), + FieldSpec::new("json_output", FieldDefault::False), ], }; const JANITOR_INVOCATION: RecordSpec = RecordSpec { @@ -475,6 +477,7 @@ const JANITOR_INVOCATION: RecordSpec = RecordSpec { FieldSpec::new("selected_target", FieldDefault::None), FieldSpec::new("active_target_name", FieldDefault::None), FieldSpec::new("as_target", FieldDefault::None), + FieldSpec::new("machine_output", FieldDefault::False), ], }; const JANITOR_SETTINGS: RecordSpec = RecordSpec { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 81c6d8ccd7..aeee4b0ddc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5050,7 +5050,9 @@ def observe_source_freshness_json( previous_records: Mapping[SourceFreshnessIdentity, SourceFreshnessRecord] | None = None, previous_records_by_source_name: Mapping[str, SourceFreshnessRecord] | None = None, ) -> str: ... -def render_janitor_plan(*, plan: JanitorPlan, use_color: bool = False) -> str: ... +def render_janitor_plan( + *, plan: JanitorPlan, use_color: bool = False, verbose: bool = False +) -> str: ... def discover_python_node_functions( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] | None = None ) -> DiscoveredPythonNodeFunctions: ... @@ -5147,7 +5149,10 @@ def build_janitor_archive_name( def build_query_diff_artifact_name(*, run_id: str, side: str) -> str: ... def parse_query_diff_artifact_name(name: str, /) -> tuple[str, str] | None: ... def render_janitor_completion( - *, result: JanitorExecutionResult, use_color: bool = False + *, + result: JanitorExecutionResult, + elapsed_seconds: float | None = None, + use_color: bool = False, ) -> str: ... # Native analysis: SQL test planning glue. @@ -6663,6 +6668,8 @@ class JanitorCommandRequest(NativeRecord): drop_old_name_views: tuple[str, ...] selected_target: str | None as_target: str | None + verbose: bool + json_output: bool def __init__( self, project_dir: Path | None, @@ -6673,6 +6680,8 @@ class JanitorCommandRequest(NativeRecord): drop_old_name_views: tuple[str, ...] = (), selected_target: str | None = None, as_target: str | None = None, + verbose: bool = False, + json_output: bool = False, ) -> None: ... class LoadCommandRequest(NativeRecord): diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py index 14b48201e4..7702bef2e9 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py @@ -14,12 +14,15 @@ ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( JanitorArchiveCandidate, JanitorArchivedRelation, JanitorPlan, + JanitorProgressItem, JanitorRelationKey, ) +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_SCHEMA_VERSION from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record @@ -46,13 +49,14 @@ def apply_direct_archives( connection: Any, recorder: StatementRecorder, run_id: str, + progress: JanitorProgress | None = None, ) -> tuple[tuple[JanitorArchiveCandidate, ...], tuple[JanitorArchivedRelation, ...]]: """Rename stale relations to archives, then drop expired archives by name.""" _create_event_tables(adapter=adapter, connection=connection, plan=plan) archived: list[JanitorArchiveCandidate] = [] - candidate: JanitorArchiveCandidate - for candidate in plan.archive_candidates: + + def archive_relation(candidate: JanitorArchiveCandidate) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): _rename_relation( adapter=adapter, @@ -77,9 +81,21 @@ def apply_direct_archives( ), ), ) + + run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=plan.archive_candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.ARCHIVED, + name=candidate.key.display_name(), + destination=candidate.archive_key.display_name(), + ), + apply=archive_relation, + ) deleted: list[JanitorArchivedRelation] = [] - archive: JanitorArchivedRelation - for archive in plan.archive_deletion_candidates: + + def delete_archive(archive: JanitorArchivedRelation) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): _drop_archive( adapter=adapter, connection=connection, archive=archive, recorder=recorder @@ -95,6 +111,16 @@ def apply_direct_archives( archive=archive, ), ) + + run_janitor_phase( + progress=progress, + phase=JanitorPhase.DELETE_ARCHIVES, + items=plan.archive_deletion_candidates, + describe=lambda archive: JanitorProgressItem( + action=JanitorAction.DELETED, name=archive.key.display_name() + ), + apply=delete_archive, + ) return tuple(archived), tuple(deleted) diff --git a/src/sqlbuild/executor/janitor/_helpers/deletion.py b/src/sqlbuild/executor/janitor/_helpers/deletion.py index bc83ff3363..ebaaf63a6f 100644 --- a/src/sqlbuild/executor/janitor/_helpers/deletion.py +++ b/src/sqlbuild/executor/janitor/_helpers/deletion.py @@ -1,23 +1,37 @@ -"""Generic janitor candidate deletion application.""" +"""Direct state history pruning application.""" from __future__ import annotations from collections.abc import Callable +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase +from sqlbuild.executor.janitor.models import JanitorDirectStatePruneCandidate, JanitorProgressItem +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -def apply_janitor_deletions[T]( +def apply_janitor_deletions( *, - candidates: tuple[T, ...], - delete: Callable[[T], object] | None, -) -> tuple[T, ...]: + candidates: tuple[JanitorDirectStatePruneCandidate, ...], + delete: Callable[[JanitorDirectStatePruneCandidate], object] | None, + progress: JanitorProgress | None = None, +) -> tuple[JanitorDirectStatePruneCandidate, ...]: """Run the delete callback for each candidate, returning the deleted ones.""" if delete is None: return () - candidate: T - for candidate in candidates: + + def prune(candidate: JanitorDirectStatePruneCandidate) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): _ = delete(candidate) + + run_janitor_phase( + progress=progress, + phase=JanitorPhase.PRUNE_DIRECT_STATE, + items=candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.PRUNED, name=candidate.display_name() + ), + apply=prune, + ) return candidates diff --git a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py new file mode 100644 index 0000000000..ce402ae3b6 --- /dev/null +++ b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py @@ -0,0 +1,65 @@ +"""Run one janitor apply phase, reporting each action as it finishes.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import replace +from time import perf_counter + +from sqlbuild.executor.janitor.models import JanitorProgressItem +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress + + +def run_janitor_phase[T]( + *, + progress: JanitorProgress | None, + phase: JanitorPhase, + items: Sequence[T], + describe: Callable[[T], JanitorProgressItem], + apply: Callable[[T], JanitorAction | None], +) -> None: + """Apply `apply` to each item in order; a failing item is reported, then re-raised. + + `apply` may return the action actually taken when it differs from the described one. + """ + + if not items: + return + if progress is not None: + progress.phase_started(phase=phase.value, total=len(items)) + item: T + for item in items: + started: float = perf_counter() + try: + taken: JanitorAction | None = apply(item) + except BaseException as error: + message: str = str(error) or type(error).__name__ + _report(progress, phase, (describe(item), None), (message, started)) + if progress is not None: + progress.phase_finished(phase=phase.value, failed=True) + raise + _report(progress, phase, (describe(item), taken), (None, started)) + if progress is not None: + progress.phase_finished(phase=phase.value, failed=False) + + +def _report( + progress: JanitorProgress | None, + phase: JanitorPhase, + described: tuple[JanitorProgressItem, JanitorAction | None], + outcome: tuple[str | None, float], +) -> None: + if progress is None: + return + label: JanitorProgressItem + taken: JanitorAction | None + label, taken = described + error: str | None + started: float + error, started = outcome + progress.item_finished( + phase=phase.value, + item=label if taken is None else replace(label, action=taken), + error=error, + duration_seconds=perf_counter() - started, + ) diff --git a/src/sqlbuild/executor/janitor/main/execute.py b/src/sqlbuild/executor/janitor/main/execute.py index bcda614982..a341f1589a 100644 --- a/src/sqlbuild/executor/janitor/main/execute.py +++ b/src/sqlbuild/executor/janitor/main/execute.py @@ -11,6 +11,7 @@ janitor_run_id, ) from sqlbuild.executor.janitor._helpers.deletion import apply_janitor_deletions +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( JanitorArchiveCandidate, JanitorArchivedRelation, @@ -19,8 +20,10 @@ JanitorExecutionResult, JanitorOldNameView, JanitorPlan, + JanitorProgressItem, JanitorQueryDiffArtifactCandidate, ) +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress from sqlbuild.executor.old_name_views.main._drop_old_name_views import drop_old_name_views from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -30,8 +33,9 @@ def execute_janitor_plan( plan: JanitorPlan, adapter: BaseAdapter, connection: Any, + progress: JanitorProgress | None = None, ) -> JanitorExecutionResult: - """Delete all candidates in a janitor plan.""" + """Delete all candidates in a janitor plan, reporting each phase and action to `progress`.""" with OperationLifecycle( operation_kind="janitor", @@ -42,6 +46,7 @@ def execute_janitor_plan( plan=plan, adapter=adapter, connection=connection, + progress=progress, ) @@ -50,10 +55,11 @@ def _execute_janitor_plan( plan: JanitorPlan, adapter: BaseAdapter, connection: Any, + progress: JanitorProgress | None, ) -> JanitorExecutionResult: recorder: StatementRecorder = StatementRecorder() - candidate: JanitorDeleteCandidate - for candidate in () if plan.direct_mode else plan.candidates: + + def drop_candidate(candidate: JanitorDeleteCandidate) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): adapter.drop( connection=connection, @@ -61,6 +67,16 @@ def _execute_janitor_plan( if_exists=True, statement_recorder=recorder, ) + + run_janitor_phase( + progress=progress, + phase=JanitorPhase.DELETE_RELATIONS, + items=() if plan.direct_mode else plan.candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.DELETED, name=candidate.key.display_name() + ), + apply=drop_candidate, + ) archived: tuple[JanitorArchiveCandidate, ...] deleted_archives: tuple[JanitorArchivedRelation, ...] archived, deleted_archives = ( @@ -70,6 +86,7 @@ def _execute_janitor_plan( connection=connection, recorder=recorder, run_id=janitor_run_id(), + progress=progress, ) if plan.direct_mode else ((), ()) @@ -80,30 +97,42 @@ def _execute_janitor_plan( connection=connection, recorder=recorder, run_id=janitor_run_id(), + progress=progress, ) - query_artifact_candidate: JanitorQueryDiffArtifactCandidate - for query_artifact_candidate in plan.query_diff_artifact_candidates: + + def drop_query_artifact(candidate: JanitorQueryDiffArtifactCandidate) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): query_artifact_destination: str | None = adapter.render_qualified_name( - database=query_artifact_candidate.key.database, - schema=query_artifact_candidate.key.schema, - name=query_artifact_candidate.key.name, + database=candidate.key.database, + schema=candidate.key.schema, + name=candidate.key.name, ) adapter.drop( connection=connection, destination=( query_artifact_destination if query_artifact_destination is not None - else query_artifact_candidate.key.display_name() + else candidate.key.display_name() ), if_exists=True, statement_recorder=recorder, ) + + run_janitor_phase( + progress=progress, + phase=JanitorPhase.DROP_QUERY_ARTIFACTS, + items=plan.query_diff_artifact_candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.DROPPED, name=candidate.key.display_name() + ), + apply=drop_query_artifact, + ) pruned_direct_state: tuple[JanitorDirectStatePruneCandidate, ...] = apply_janitor_deletions( candidates=plan.direct_state_prune_candidates, delete=lambda direct_state_candidate: adapter.execute( connection=connection, sql=direct_state_candidate.prune_sql ), + progress=progress, ) return JanitorExecutionResult( deleted=() if plan.direct_mode else plan.candidates, diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 1eb83014a3..1d49a19393 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -7,6 +7,7 @@ from sqlbuild._native import OldNameViewDropReason, OldNameViewHistory from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup +from sqlbuild.executor.janitor.types import JanitorAction @dataclass(frozen=True, eq=False) @@ -226,3 +227,12 @@ class JanitorExecutionResult: ) pruned_direct_state: tuple[JanitorDirectStatePruneCandidate, ...] = field(default_factory=tuple) dropped_old_name_views: tuple[JanitorOldNameView, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorProgressItem: + """How the progress output names one janitor action before it runs.""" + + action: JanitorAction + name: str + destination: str | None = None diff --git a/src/sqlbuild/executor/janitor/types.py b/src/sqlbuild/executor/janitor/types.py new file mode 100644 index 0000000000..8821306d8c --- /dev/null +++ b/src/sqlbuild/executor/janitor/types.py @@ -0,0 +1,41 @@ +"""Janitor execution types.""" + +from __future__ import annotations + +from enum import StrEnum +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from sqlbuild.executor.janitor.models import JanitorProgressItem + + +class JanitorPhase(StrEnum): + DELETE_RELATIONS = "delete_relations" + ARCHIVE = "archive" + DELETE_ARCHIVES = "delete_archives" + DROP_OLD_NAME_VIEWS = "drop_old_name_views" + DROP_QUERY_ARTIFACTS = "drop_query_artifacts" + PRUNE_DIRECT_STATE = "prune_direct_state" + + +class JanitorAction(StrEnum): + ARCHIVED = "archived" + DELETED = "deleted" + DROPPED = "dropped" + RECORDED = "recorded" + PRUNED = "pruned" + + +class JanitorProgress(Protocol): + def phase_started(self, *, phase: str, total: int) -> None: ... + + def item_finished( + self, + *, + phase: str, + item: JanitorProgressItem, + error: str | None, + duration_seconds: float, + ) -> None: ... + + def phase_finished(self, *, phase: str, failed: bool) -> None: ... diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index e2b7e4e095..7dbe7bda9b 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -20,11 +20,14 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( JanitorOldNameView, JanitorOldNameViewPlanning, + JanitorProgressItem, JanitorRelationKey, ) +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress from sqlbuild.executor.migrations.main._compatibility_view_sql import compatibility_view_sql @@ -35,12 +38,19 @@ def apply_old_name_view_drops( connection: Any, recorder: StatementRecorder, run_id: str, + progress: JanitorProgress | None = None, ) -> tuple[JanitorOldNameView, ...]: """Drop expired or requested views, then record every drop and every vanished view.""" dropped: list[JanitorOldNameView] = [] - view: JanitorOldNameView - for view in plan.drops: + + def apply(entry: tuple[JanitorOldNameView, bool]) -> JanitorAction | None: + view: JanitorOldNameView + planned_missing: bool + view, planned_missing = entry + if planned_missing: + _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) + return None if not is_compatibility_view(adapter=adapter, connection=connection, view=view): _record_drop( adapter=adapter, @@ -48,7 +58,7 @@ def apply_old_name_view_drops( view=_with_reason(view=view, reason=OldNameViewDropReason.MISSING), run_id=run_id, ) - continue + return JanitorAction.RECORDED adapter.drop_view( connection=connection, destination=resolve_qualified_name_parts( @@ -62,8 +72,18 @@ def apply_old_name_view_drops( ) _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) dropped.append(view) - for view in plan.missing: - _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) + return None + + run_janitor_phase( + progress=progress, + phase=JanitorPhase.DROP_OLD_NAME_VIEWS, + items=(*((view, False) for view in plan.drops), *((view, True) for view in plan.missing)), + describe=lambda entry: JanitorProgressItem( + action=JanitorAction.RECORDED if entry[1] else JanitorAction.DROPPED, + name=entry[0].key.display_name(), + ), + apply=apply, + ) return tuple(dropped) diff --git a/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py b/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py index b66fa14d88..7a656b8a48 100644 --- a/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py +++ b/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py @@ -7,6 +7,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.janitor.models import JanitorOldNameView, JanitorOldNameViewPlanning +from sqlbuild.executor.janitor.types import JanitorProgress from sqlbuild.executor.old_name_views._helpers.janitor import apply_old_name_view_drops @@ -17,9 +18,15 @@ def drop_old_name_views( connection: Any, recorder: StatementRecorder, run_id: str, + progress: JanitorProgress | None = None, ) -> tuple[JanitorOldNameView, ...]: """Drop planned views, then append a drop fact for each dropped or vanished view.""" return apply_old_name_view_drops( - plan=plan, adapter=adapter, connection=connection, recorder=recorder, run_id=run_id + plan=plan, + adapter=adapter, + connection=connection, + recorder=recorder, + run_id=run_id, + progress=progress, ) From abeecbf9b5f9d75868d3ead306267e11431378e8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:50:38 +0100 Subject: [PATCH 02/10] perf: list each janitor schema once and batch view definition reads --- .../_helpers/janitor/janitor_facts.rs | 229 ++++++++++++------ .../_helpers/janitor/janitor_old_names.rs | 29 ++- .../_helpers/janitor/janitor_phases.rs | 13 +- .../contract/classes/duckdb_backed_adapter.py | 32 +++ .../relations/classes/inspection_catalog.py | 78 +++++- .../relations/main/cached_view_definition.py | 19 ++ src/sqlbuild/adapter/relations/types.py | 10 + .../snowflake/classes/snowflake_adapter.py | 38 +++ .../executor/diff/_helpers/query_artifacts.py | 13 +- 9 files changed, 369 insertions(+), 92 deletions(-) create mode 100644 src/sqlbuild/adapter/relations/main/cached_view_definition.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 9ff0f2b19e..5d368b488a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -18,7 +18,8 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ use crate::bindings::_helpers::records::fields::replaced; const QUERY_ARTIFACT_LIFECYCLE: ExecutorHost = ExecutorHost::QueryDiffArtifactLifecycle; -const RELATION_LOOKUP: AdapterHost = AdapterHost::BuildRelationLookup; +const INSPECTION_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; +const ACTIVE_CATALOG: AdapterHost = AdapterHost::ActiveInspectionCatalog; const LOG_DEBUG_EVENT: ExecutorHost = ExecutorHost::LogDebugEvent; const EXECUTION_LOGGER: &str = "sqlbuild.execution"; use crate::bindings::_helpers::native_host::host; @@ -49,6 +50,93 @@ impl<'py> Warehouse<'py> { call_with(&native_field(&self.adapter, method)?, &arguments) } + /// `list_relations`, served from the open inspection catalog so each schema is read once. + pub(crate) fn list_relations( + &self, + database: Option<&str>, + (schemas, names): (Option<&[String]>, Option<&[String]>), + ) -> PyResult> { + let py: Python<'py> = self.py(); + let texts = |values: Option<&[String]>| -> PyResult> { + match values { + Some(values) => tuple(py, values.iter().map(|value| text(py, value))), + None => Ok(py.None().into_bound(py)), + } + }; + call_with( + &host::entry(py, INSPECTION_LIST)?, + &[ + ("adapter", self.adapter.clone()), + ("connection", self.connection.clone()), + ("database", optional(py, database)?), + ("schemas", texts(schemas)?), + ("names", texts(names)?), + ], + ) + } + + /// The inspection catalog open on this connection, or none. + fn catalog(&self) -> PyResult>> { + let catalog: Bound<'py, PyAny> = call_with( + &host::entry(self.py(), ACTIVE_CATALOG)?, + &[ + ("adapter", self.adapter.clone()), + ("connection", self.connection.clone()), + ], + )?; + Ok((!catalog.is_none()).then_some(catalog)) + } + + /// Read every not-yet-listed schema concurrently when the open catalog lists whole schemas. + pub(crate) fn prefetch_schema_listings( + &self, + scopes: &[(Option, String)], + ) -> PyResult<()> { + let py: Python<'py> = self.py(); + let Some(catalog) = self.catalog()?.filter(|_| !scopes.is_empty()) else { + return Ok(()); + }; + let mut items: Vec> = Vec::new(); + for (database, schema) in scopes { + items.push(tuple( + py, + [optional(py, database.as_deref())?, text(py, schema)], + )?); + } + call_with( + &native_field(&catalog, "prefetch_schema_listings")?, + &[("scopes", tuple(py, items)?)], + )?; + Ok(()) + } + + /// Read these views' definitions with one read per schema, when a catalog is open. + pub(crate) fn prefetch_view_definitions( + &self, + requests: &[(Option, String, Vec)], + ) -> PyResult<()> { + let py: Python<'py> = self.py(); + let Some(catalog) = self.catalog()?.filter(|_| !requests.is_empty()) else { + return Ok(()); + }; + let mut items: Vec> = Vec::new(); + for (database, schema, names) in requests { + items.push(tuple( + py, + [ + optional(py, database.as_deref())?, + text(py, schema), + tuple(py, names.iter().map(|name| text(py, name)))?, + ], + )?); + } + call_with( + &native_field(&catalog, "prefetch_view_definitions")?, + &[("requests", tuple(py, items)?)], + )?; + Ok(()) + } + pub(crate) fn age_metadata_support(&self) -> PyResult { self.adapter .call_method0("supports_relation_age_metadata")? @@ -157,6 +245,8 @@ fn source_schemas( } /// Relations matched case-insensitively and respelled as their target schema. +/// +/// Each schema is listed on its own, so every other planning read of it hits the catalog. fn list_target_schema_relations<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -168,6 +258,7 @@ fn list_target_schema_relations<'py>( databases.push(database); } } + warehouse.prefetch_schema_listings(&concrete_schemas(target_schemas))?; let mut result: SchemaRelations<'py> = Vec::new(); for database in databases { let mut concrete: Vec = target_schemas @@ -177,87 +268,81 @@ fn list_target_schema_relations<'py>( .collect(); concrete.sort(); concrete.dedup(); - let schemas: Bound<'py, PyAny> = if concrete.is_empty() { - py.None().into_bound(py) + let requests: Vec>> = if concrete.is_empty() { + vec![None] } else { - tuple(py, concrete.iter().map(|schema| text(py, schema)))? + concrete.into_iter().map(|schema| Some(vec![schema])).collect() }; - let listed: Bound<'py, PyAny> = warehouse.call( - "list_relations", - &[ - ("database", optional(py, database.as_deref())?), - ("schemas", schemas), - ], - )?; - for relation in listed.try_iter()? { - let relation: Bound<'py, PyAny> = relation?; - let folded: SchemaKey = folded_schema(&( - native_field(&relation, "database")?.extract()?, - native_field(&relation, "schema")?.extract()?, - )); - let Some(target) = target_schemas - .iter() - .find(|target| folded_schema(target) == folded) - else { - continue; - }; - let respelled: Bound<'py, PyAny> = replaced( - &relation, - &[ - ("database", optional(py, target.0.as_deref())?), - ("schema", optional(py, target.1.as_deref())?), - ], - )?; - match result.iter_mut().find(|(listed, _)| listed == target) { - Some((_, relations)) => relations.push(respelled), - None => result.push((target.clone(), vec![respelled])), + for schemas in requests { + let listed: Bound<'py, PyAny> = + warehouse.list_relations(database.as_deref(), (schemas.as_deref(), None))?; + for relation in listed.try_iter()? { + let relation: Bound<'py, PyAny> = relation?; + let folded: SchemaKey = folded_schema(&( + native_field(&relation, "database")?.extract()?, + native_field(&relation, "schema")?.extract()?, + )); + let Some(target) = target_schemas + .iter() + .find(|target| folded_schema(target) == folded) + else { + continue; + }; + let respelled: Bound<'py, PyAny> = replaced( + &relation, + &[ + ("database", optional(py, target.0.as_deref())?), + ("schema", optional(py, target.1.as_deref())?), + ], + )?; + match result.iter_mut().find(|(listed, _)| listed == target) { + Some((_, relations)) => relations.push(respelled), + None => result.push((target.clone(), vec![respelled])), + } } } } Ok(result) } -/// `build_relation_lookup` over the given locations. -fn relation_lookup<'py>( - warehouse: &Warehouse<'py>, +/// Relations present at the given locations, keyed as `RelationLookup.key` folds them. +/// +/// Like `build_relation_lookup`, but each schema is listed through the inspection catalog. +fn relation_lookup( + warehouse: &Warehouse<'_>, locations: &[(Option, String, &str)], -) -> PyResult> { - let py: Python<'py> = warehouse.py(); - let mut items: Vec> = Vec::new(); - for (database, schema, name) in locations { - items.push(tuple( - py, - [ - optional(py, database.as_deref())?, - text(py, schema), - text(py, name), - ], - )?); +) -> PyResult> { + let mut scopes: Vec<(Option, String)> = Vec::new(); + for (database, schema, _) in locations { + if !scopes.contains(&(database.clone(), schema.clone())) { + scopes.push((database.clone(), schema.clone())); + } } - call_with( - &host::entry(py, RELATION_LOOKUP)?, - &[ - ("adapter", warehouse.adapter.clone()), - ("connection", warehouse.connection.clone()), - ("locations", tuple(py, items)?), - ], - ) + let mut present: HashSet = HashSet::new(); + for (database, schema) in scopes { + let schemas: [String; 1] = [schema]; + for relation in warehouse + .list_relations(database.as_deref(), (Some(&schemas), None))? + .try_iter()? + { + present.insert(Key::of(&relation?)?.folded()); + } + } + Ok(present) } fn lookup_exists( - lookup: &Bound<'_, PyAny>, + lookup: &HashSet, (database, schema, name): (Option<&str>, &str, &str), -) -> PyResult { - let py: Python<'_> = lookup.py(); - call_with( - &native_field(lookup, "exists")?, - &[ - ("database", optional(py, database)?), - ("schema", text(py, schema)), - ("name", text(py, name)), - ], - )? - .is_truthy() +) -> bool { + lookup.contains( + &Key { + database: database.map(str::to_owned), + schema: Some(schema.to_owned()), + name: name.to_owned(), + } + .folded(), + ) } fn concrete_schemas(target_schemas: &[SchemaKey]) -> Vec<(Option, String)> { @@ -282,13 +367,13 @@ fn tracked_relation_keys( .iter() .map(|(database, schema)| (database.clone(), schema.clone(), FINGERPRINT_TABLE_NAME)) .collect(); - let lookup: Bound<'_, PyAny> = relation_lookup(warehouse, &locations)?; + let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut tracked: HashSet = HashSet::new(); for (database, schema) in &schemas { let exists: bool = lookup_exists( &lookup, (database.as_deref(), schema, FINGERPRINT_TABLE_NAME), - )?; + ); let read: PyResult> = read_latest_fingerprints(Some(&keywords( py, &[ @@ -411,7 +496,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( SOURCE_FRESHNESS_TABLE_NAME, )); } - let lookup: Bound<'py, PyAny> = relation_lookup(warehouse, &locations)?; + let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); for (database, schema) in &schemas { for (table_name, render) in [ @@ -424,7 +509,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( "render_prune_source_freshness_history_sql", ), ] { - if !lookup_exists(&lookup, (database.as_deref(), schema, table_name))? { + if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { continue; } let prune_sql: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index e6a3ba599b..d55d951693 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -115,7 +115,6 @@ fn physical_old_names<'py>( histories: &[Bound<'py, PyAny>], relations: &SchemaRelations<'py>, ) -> PyResult>> { - let py: Python<'py> = warehouse.py(); let mut known: Vec<(String, &Vec>)> = Vec::new(); for ((_, schema), listed) in relations { let folded: String = schema.as_deref().unwrap_or("").to_lowercase(); @@ -143,17 +142,8 @@ fn physical_old_names<'py>( let mut names: Vec = unscanned.iter().map(|key| key.name.clone()).collect(); names.sort(); names.dedup(); - let listed: Bound<'py, PyAny> = warehouse.call( - "list_relations", - &[ - ("database", optional(py, first.database.as_deref())?), - ( - "schemas", - tuple(py, schemas.iter().map(|schema| text(py, schema)))?, - ), - ("names", tuple(py, names.iter().map(|name| text(py, name)))?), - ], - )?; + let listed: Bound<'py, PyAny> = + warehouse.list_relations(first.database.as_deref(), (Some(&schemas), Some(&names)))?; for relation in listed.try_iter()? { scanned.push(relation?); } @@ -287,6 +277,21 @@ pub(crate) fn plan_old_name_views<'py>( requested.push((name.clone(), false)); } } + let mut definitions: Vec<(Option, String, Vec)> = Vec::new(); + for history in &histories { + let key: Key = old_key(history)?; + let (Some(schema), true) = (&key.schema, physical.contains_key(&key.folded())) else { + continue; + }; + match definitions + .iter_mut() + .find(|(database, listed, _)| *database == key.database && listed == schema) + { + Some((_, _, names)) => names.push(key.name.clone()), + None => definitions.push((key.database.clone(), schema.clone(), vec![key.name.clone()])), + } + } + warehouse.prefetch_view_definitions(&definitions)?; let is_compatibility_view: Bound<'py, PyAny> = host::entry(py, IS_COMPATIBILITY_VIEW)?; let (mut live, mut drops, mut missing) = (Vec::new(), Vec::new(), Vec::new()); let mut keys: HashSet = HashSet::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index 7129456cea..316d4b423b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -1,6 +1,6 @@ //! `sqb janitor` phases: compile, connect, inspect and plan, clean up and close. -use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; @@ -234,7 +234,14 @@ pub(crate) fn build_janitor_execution_plan<'py>( let plan: Bound<'py, PyAny> = within( &lifecycle(py, ("janitor", "janitor_candidate_planning"))?, |lifecycle| { - let plan: Bound<'py, PyAny> = build_plan( + let catalog: Bound<'py, PyAny> = call_with( + &host::entry(py, AdapterHost::OpenInspectionCatalog)?, + &[ + ("adapter", compiled.adapter.clone()), + ("connection", connection.clone()), + ], + )?; + let plan: Bound<'py, PyAny> = within(&catalog, |_| build_plan( ( &compiled.project, &Warehouse { @@ -263,7 +270,7 @@ pub(crate) fn build_janitor_execution_plan<'py>( protected_keys: Vec::new(), reasons: Vec::new(), }, - )?; + ))?; let metadata: Bound<'py, PyDict> = PyDict::new(py); metadata.set_item("item_count", candidate_count(&plan)?)?; let options: Bound<'py, PyDict> = PyDict::new(py); diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index cfc6da6ee4..3c1beddbe3 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -85,6 +85,7 @@ StatementSizeLimit, TablePromotionMode, ) +from sqlbuild.adapter.relations.main.cached_view_definition import cached_view_definition from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) @@ -1684,6 +1685,11 @@ def render_replace_view_keeping_grants( def read_view_definition( self, *, connection: Any, database: str | None, schema: str, name: str ) -> str | None: + prefetched: tuple[bool, str | None] = cached_view_definition( + adapter=self, connection=connection, database=database, schema=schema, name=name + ) + if prefetched[0]: + return prefetched[1] database_filter: str = ( "" if database is None else f" AND database_name = {_duckdb_string_literal(database)}" ) @@ -1697,6 +1703,32 @@ def read_view_definition( ).fetchall() return None if not rows else str(rows[0][0]) + def read_view_definitions( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> dict[str, str]: + """Read the stored definitions of several views in one schema with one query.""" + + if not names: + return {} + database_filter: str = ( + "" if database is None else f" AND database_name = {_duckdb_string_literal(database)}" + ) + rows: list[tuple[Any, ...]] = self.execute( + connection=connection, + sql=( + "SELECT view_name, sql FROM duckdb_views() WHERE schema_name = " + f"{_duckdb_string_literal(schema)} AND view_name IN (" + + ", ".join(_duckdb_string_literal(name) for name in names) + + ")" + + database_filter + ), + ).fetchall() + definitions: dict[str, str] = {} + row: tuple[Any, ...] + for row in rows: + definitions.setdefault(str(row[0]), str(row[1])) + return definitions + def view_definition_matches( self, *, connection: Any, database: str | None, schema: str, name: str, sql: str ) -> bool: diff --git a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py index 8ef8562216..b2f60283e8 100644 --- a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py +++ b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py @@ -19,7 +19,7 @@ SchemaColumnListing, SchemaRelationListing, ) -from sqlbuild.adapter.relations.types import SchemaScopedMetadataReader +from sqlbuild.adapter.relations.types import SchemaScopedMetadataReader, ViewDefinitionBatchReader type _RelationIdentity = tuple[str | None, str | None, str] type _SchemaKey = tuple[str | None, str] @@ -43,6 +43,7 @@ def __init__(self, *, adapter: Any, connection: Any) -> None: self._remember_lock: threading.Lock = threading.Lock() self._relation_requests: dict[_RelationRequestKey, tuple[RelationInfo, ...]] = {} self._relation_columns: dict[_RelationIdentity, tuple[ColumnInfo, ...] | None] = {} + self._view_definitions: dict[_RelationIdentity, str | None] = {} def remember[ValueT](self, *, key: tuple[str, ...], compute: Callable[[], ValueT]) -> ValueT: """Compute one adapter-owned metadata fact at most once per planning invocation.""" @@ -131,6 +132,81 @@ def prefetch_schema_listings( if listing is not None: self._relation_listings[missing_key] = listing + def prefetch_view_definitions( + self, *, requests: tuple[tuple[str | None, str, tuple[str, ...]], ...] + ) -> None: + """Read the definitions of these views with one batched read per schema. + + Each request is a database, schema and view names. Adapters without + ``read_view_definitions`` keep reading each definition on demand. + """ + + read_batch: ViewDefinitionBatchReader | None = getattr( + self.adapter, "read_view_definitions", None + ) + if read_batch is None: + return + pending: list[tuple[str | None, str, tuple[str, ...]]] = [] + database: str | None + schema: str + names: tuple[str, ...] + for database, schema, names in requests: + missing: tuple[str, ...] = tuple( + name + for name in dict.fromkeys(names) + if (database, schema, name) not in self._view_definitions + ) + if missing: + pending.append((database, schema, missing)) + results: list[dict[str, str]] = run_bounded_inspections( + tasks=tuple( + self._bind_view_definitions( + read_batch=read_batch, database=database, schema=schema, names=names + ) + for database, schema, names in pending + ), + concurrency=self._concurrency, + ) + definitions: dict[str, str] + for (database, schema, names), definitions in zip(pending, results, strict=True): + name: str + for name in names: + self._view_definitions[(database, schema, name)] = definitions.get(name) + + def cached_view_definition( + self, *, database: str | None, schema: str, name: str + ) -> tuple[bool, str | None]: + """Return whether a prefetched definition is known for this view, and the definition.""" + + key: _RelationIdentity = (database, schema, name) + if key not in self._view_definitions: + return False, None + return True, self._view_definitions[key] + + def _bind_view_definitions( + self, + *, + read_batch: ViewDefinitionBatchReader, + database: str | None, + schema: str, + names: tuple[str, ...], + ) -> Callable[[], dict[str, str]]: + def read_definitions() -> dict[str, str]: + definitions: dict[str, str] = {} + start: int + for start in range(0, len(names), INSPECTION_IN_LIST_LIMIT): + definitions.update( + read_batch( + connection=self.connection, + database=database, + schema=schema, + names=names[start : start + INSPECTION_IN_LIST_LIMIT], + ) + ) + return definitions + + return read_definitions + def get_columns_for_relations( self, *, relations: tuple[RelationInfo, ...] ) -> dict[_RelationIdentity, tuple[ColumnInfo, ...]]: diff --git a/src/sqlbuild/adapter/relations/main/cached_view_definition.py b/src/sqlbuild/adapter/relations/main/cached_view_definition.py new file mode 100644 index 0000000000..7466779ab2 --- /dev/null +++ b/src/sqlbuild/adapter/relations/main/cached_view_definition.py @@ -0,0 +1,19 @@ +"""Serve a view definition the open inspection catalog already read.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.relations._helpers.inspection_context import current_catalog +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog + + +def cached_view_definition( + *, adapter: Any, connection: Any, database: str | None, schema: str, name: str +) -> tuple[bool, str | None]: + """Return whether the open catalog prefetched this view's definition, and the definition.""" + + catalog: InspectionCatalog | None = current_catalog(adapter=adapter, connection=connection) + if catalog is None: + return False, None + return catalog.cached_view_definition(database=database, schema=schema, name=name) diff --git a/src/sqlbuild/adapter/relations/types.py b/src/sqlbuild/adapter/relations/types.py index acdad6b3b2..30d9bf395c 100644 --- a/src/sqlbuild/adapter/relations/types.py +++ b/src/sqlbuild/adapter/relations/types.py @@ -57,6 +57,16 @@ def metadata_name_key(self, name: str) -> str: ... +class ViewDefinitionBatchReader(Protocol): + """An adapter's ``read_view_definitions``: stored definitions of named views in one schema.""" + + def __call__( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> dict[str, str]: + """Return each found view's definition keyed by the requested name.""" + ... + + class InspectionCompletion[ResultT](Protocol): """Callback receiving one finished inspection read on the calling thread.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 55e7ddfdc9..9b8a4ad612 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -103,6 +103,7 @@ from sqlbuild.adapter.relations.main.active_inspection_catalog import active_inspection_catalog from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.relations.main.render_inspection_sql import render_inspection_sql +from sqlbuild.adapter.relations.main.cached_view_definition import cached_view_definition from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections from sqlbuild.adapter.relations.main.run_recorded_inspection_query import ( run_recorded_inspection_query, @@ -3011,6 +3012,11 @@ def render_replace_view_keeping_grants( def read_view_definition( self, *, connection: Any, database: str | None, schema: str, name: str ) -> str | None: + prefetched: tuple[bool, str | None] = cached_view_definition( + adapter=self, connection=connection, database=database, schema=schema, name=name + ) + if prefetched[0]: + return prefetched[1] views: str = ( "INFORMATION_SCHEMA.VIEWS" if database is None @@ -3032,6 +3038,38 @@ def read_view_definition( ).fetchall() return None if not rows else str(rows[0][0]) + def read_view_definitions( + self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] + ) -> dict[str, str]: + """Read the stored definitions of several views in one schema with one query. + + Names match case-insensitively, as ``read_view_definition`` matches one name. + """ + + if not names: + return {} + views: str = ( + "INFORMATION_SCHEMA.VIEWS" + if database is None + else (f"{database}.INFORMATION_SCHEMA.VIEWS") + ) + literals: str = ", ".join( + "UPPER('" + name.replace("'", "''") + "')" for name in names + ) + rows: list[tuple[Any, ...]] = self.execute( + connection=connection, + sql=( + f"SELECT TABLE_NAME, VIEW_DEFINITION FROM {views} WHERE UPPER(TABLE_SCHEMA) = " + "UPPER('" + schema.replace("'", "''") + "') " + f"AND UPPER(TABLE_NAME) IN ({literals})" + ), + ).fetchall() + found: dict[str, str] = {} + row: tuple[Any, ...] + for row in rows: + found.setdefault(str(row[0]).upper(), str(row[1])) + return {name: found[name.upper()] for name in names if name.upper() in found} + def view_definition_matches( self, *, connection: Any, database: str | None, schema: str, name: str, sql: str ) -> bool: diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index c22d58f774..eea80a0fa1 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -20,6 +20,8 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog +from sqlbuild.adapter.relations.main.active_inspection_catalog import active_inspection_catalog from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_QUERY_DIFF_ARTIFACT from sqlbuild.errors.contracts.exceptions import ExecutorInputError @@ -184,10 +186,13 @@ def inspect_query_diff_artifacts( ) -> QueryDiffArtifactInspection: """Classify strict-name query artifacts using immutable fingerprint ownership.""" - relations: tuple[RelationInfo, ...] = adapter.list_relations( - connection=connection, - database=database, - schemas=(schema,), + catalog: InspectionCatalog | None = active_inspection_catalog( + adapter=adapter, connection=connection + ) + relations: tuple[RelationInfo, ...] = ( + adapter.list_relations(connection=connection, database=database, schemas=(schema,)) + if catalog is None + else catalog.list_relations(database=database, schemas=(schema,)) ) candidates: tuple[RelationInfo, ...] = tuple( relation From fbf009c48c9cb0cc7f598145f0d7237a4472e7c8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:00:45 +0100 Subject: [PATCH 03/10] perf: run janitor actions concurrently and batch audit events --- .../_helpers/janitor/janitor_command.rs | 13 +- .../_helpers/janitor/janitor_phases.rs | 40 +++++- .../adapter/contract/classes/base_adapter.py | 3 + .../bigquery/classes/bigquery_adapter.py | 1 + .../databricks/classes/databricks_adapter.py | 1 + .../snowflake/classes/snowflake_adapter.py | 1 + .../janitor/_helpers/archive_execution.py | 123 ++++++++++++------ .../executor/janitor/_helpers/deletion.py | 12 +- .../executor/janitor/classes/__init__.py | 1 + .../janitor/classes/janitor_workers.py | 119 +++++++++++++++++ src/sqlbuild/executor/janitor/constants.py | 3 + .../janitor/main/_run_janitor_phase.py | 105 ++++++++++----- src/sqlbuild/executor/janitor/main/execute.py | 96 ++++++++++++-- src/sqlbuild/executor/janitor/models.py | 9 ++ src/sqlbuild/executor/janitor/types.py | 8 ++ .../executor/janitor_events/_helpers/sql.py | 35 +++-- .../janitor_events/classes/__init__.py | 1 + .../classes/janitor_event_writer.py | 120 +++++++++++++++++ .../executor/janitor_events/constants.py | 4 + .../main/write_janitor_event.py | 35 ----- .../old_name_views/_helpers/janitor.py | 24 ++-- .../main/_drop_old_name_views.py | 7 +- 22 files changed, 599 insertions(+), 162 deletions(-) create mode 100644 src/sqlbuild/executor/janitor/classes/__init__.py create mode 100644 src/sqlbuild/executor/janitor/classes/janitor_workers.py create mode 100644 src/sqlbuild/executor/janitor/constants.py create mode 100644 src/sqlbuild/executor/janitor_events/classes/__init__.py create mode 100644 src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py delete mode 100644 src/sqlbuild/executor/janitor_events/main/write_janitor_event.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 8c05d690ed..15a35c0352 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -214,7 +214,8 @@ fn confirmed(stream: &Bound<'_, PyAny>, plan: &JanitorPlanText) -> PyResult, &Bound<'_, PyAny>, CliStyle), + (request, invocation): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (stream, style): (&Bound<'_, PyAny>, CliStyle), (compiled, connection): (&JanitorCompiled<'_>, &Bound<'_, PyAny>), (planning, plan): (&Bound<'_, PyAny>, &JanitorPlanText), ) -> PyResult { @@ -224,8 +225,11 @@ fn apply( JanitorProgressPrinter::new(stream, style, plan.clone())?, )?; let started: Instant = Instant::now(); - let executed: PyResult> = - execute_janitor_cleanup(compiled, connection, (planning, progress.as_any())); + let executed: PyResult> = execute_janitor_cleanup( + invocation, + (compiled, connection), + (planning, progress.as_any()), + ); let elapsed_seconds: f64 = started.elapsed().as_secs_f64(); let items = progress.get().items(); match executed { @@ -311,7 +315,8 @@ fn clean( return Ok(1); } apply( - (request, &stream, style), + (request, invocation), + (&stream, style), (compiled, connection), (&planning, &plan), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index 316d4b423b..e81060cedf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::native_host::connections::connection_failure; use std::time::Instant; +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; @@ -15,6 +16,7 @@ use crate::bindings::_helpers::build_progress::progress_reporters::{ ConnectionProgressReporter, PlanningProgressReporter, }; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; use crate::bindings::_helpers::janitor::janitor_planning::{ @@ -298,18 +300,50 @@ pub(crate) fn build_janitor_execution_plan<'py>( } /// Execute the warehouse cleanup plan, reporting each phase and action to `progress`. +/// +/// Independent actions of a phase run on up to the command's concurrency of connections. pub(crate) fn execute_janitor_cleanup<'py>( - compiled: &JanitorCompiled<'py>, - connection: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + (compiled, connection): (&JanitorCompiled<'py>, &Bound<'py, PyAny>), (plan, progress): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { + let py: Python<'py> = invocation.py(); + let concurrency: i64 = resolve_concurrency( + &py.None().into_bound(py), + &native_field(invocation, "discovered_inputs")?, + &native_field(invocation, "selected_target")?, + )? + .limit; + let connection_progress: Bound<'py, PyAny> = Bound::new( + py, + ConnectionProgressReporter::new( + py, + Some(keywords( + py, + &[ + ("adapter_name", compiled.adapter_name.clone()), + ("stream", output_stream(invocation)?), + ( + "use_color", + PyBool::new(py, use_color(invocation)?) + .to_owned() + .into_any(), + ), + ], + )?), + )?, + )? + .into_any(); call_with( - &host::entry(compiled.adapter.py(), ExecutorHost::ExecuteJanitorPlan)?, + &host::entry(py, ExecutorHost::ExecuteJanitorPlan)?, &[ ("plan", plan.clone()), ("adapter", compiled.adapter.clone()), ("connection", connection.clone()), ("progress", progress.clone()), + ("connection_config", compiled.connection_config.clone()), + ("concurrency", concurrency.into_pyobject(py)?.into_any()), + ("connection_progress", connection_progress), ], ) } diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 8bf89cfc53..953d8e654d 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -114,6 +114,9 @@ class BaseAdapter(RetentionAdapterMixin, StrictAdapter): statement_cancellation_is_asynchronous: ClassVar[bool] = False statement_limit_excludes_queueing: ClassVar[bool] = False metadata_inspection_concurrency: ClassVar[int] = 1 + # Independent DDL statements (renames, drops) this warehouse runs at once on separate + # connections; single-writer warehouses keep 1. + concurrent_ddl_limit: ClassVar[int] = 1 supports_session_warehouse: ClassVar[bool] = False state_timestamp_type: ClassVar[str | None] = None _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index e4b1b6b8d0..8d90520f08 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -170,6 +170,7 @@ def get_columns_for_relations( ) views_read_with_reader_access: ClassVar[bool] = True max_identifier_length: ClassVar[int] = 1024 + concurrent_ddl_limit: ClassVar[int] = 16 _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", distinct_condition=render_is_distinct_from, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index ad6baf39ba..03ff3cbd5c 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -160,6 +160,7 @@ def get_columns_for_relations( nested_block_comments=True, ) max_identifier_length: ClassVar[int] = 255 + concurrent_ddl_limit: ClassVar[int] = 16 _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", distinct_condition=render_is_distinct_from, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 9b8a4ad612..b916ee583c 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -192,6 +192,7 @@ class SnowflakeAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): execution_duration_limit_seconds: ClassVar[int | None] = MAX_STATEMENT_TIMEOUT_SECONDS statement_limit_excludes_queueing: ClassVar[bool] = True metadata_inspection_concurrency: ClassVar[int] = _METADATA_INSPECTION_CONCURRENCY + concurrent_ddl_limit: ClassVar[int] = 16 exact_column_inspection_limit: ClassVar[int] = _EXACT_COLUMN_INSPECTION_LIMIT max_identifier_length: ClassVar[int] = 255 diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py index 7702bef2e9..4a277524c8 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py @@ -14,6 +14,8 @@ ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.constants import ARCHIVE_NAME_SEPARATOR from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( JanitorArchiveCandidate, @@ -23,9 +25,12 @@ JanitorRelationKey, ) from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress -from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_SCHEMA_VERSION +from sqlbuild.executor.janitor_events.classes.janitor_event_writer import JanitorEventWriter +from sqlbuild.executor.janitor_events.constants import ( + JANITOR_EVENT_CHUNK_SIZE, + JANITOR_EVENT_SCHEMA_VERSION, +) from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id -from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record from sqlbuild.executor.janitor_events.models import JanitorEventRecord from sqlbuild.executor.janitor_events.types import JanitorEventType from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -46,17 +51,33 @@ def apply_direct_archives( *, plan: JanitorPlan, adapter: BaseAdapter, - connection: Any, + workers: JanitorWorkers, recorder: StatementRecorder, run_id: str, progress: JanitorProgress | None = None, ) -> tuple[tuple[JanitorArchiveCandidate, ...], tuple[JanitorArchivedRelation, ...]]: - """Rename stale relations to archives, then drop expired archives by name.""" + """Rename stale relations to archives, then drop expired archives by name. + + Audit events are written after each chunk of actions in a schema, and after a failure for + every action that succeeded. An archive whose name was shortened gets its event right after + its rename, because the event is the only record of its original name. + """ + + schemas: tuple[tuple[str | None, str], ...] = _create_event_tables( + adapter=adapter, connection=workers.connection, plan=plan + ) + events: JanitorEventWriter = JanitorEventWriter( + execute=adapter.execute, + render_qualified_name=adapter.render_qualified_name, + run_id=run_id, + ) + if schemas: + events.read_existing(connection=workers.connection, schemas=schemas) - _create_event_tables(adapter=adapter, connection=connection, plan=plan) - archived: list[JanitorArchiveCandidate] = [] + def flush_events() -> None: + events.flush(connection=workers.connection) - def archive_relation(candidate: JanitorArchiveCandidate) -> None: + def archive_relation(candidate: JanitorArchiveCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): _rename_relation( adapter=adapter, @@ -64,25 +85,24 @@ def archive_relation(candidate: JanitorArchiveCandidate) -> None: candidate=candidate, recorder=recorder, ) - archived.append(candidate) - _write_event( + record: JanitorEventRecord = _event_record( adapter=adapter, - connection=connection, - record=_event_record( - adapter=adapter, - event_type=JanitorEventType.ARCHIVE, - run_id=run_id, - archive=JanitorArchivedRelation( - key=candidate.archive_key, - relation_type=candidate.relation.relation_type, - archived_at=candidate.archived_at, - expires_at=candidate.expires_at, - original_key=candidate.key, - ), + event_type=JanitorEventType.ARCHIVE, + run_id=run_id, + archive=JanitorArchivedRelation( + key=candidate.archive_key, + relation_type=candidate.relation.relation_type, + archived_at=candidate.archived_at, + expires_at=candidate.expires_at, + original_key=candidate.key, ), ) + if _shortened(candidate): + events.write_now(connection=connection, record=record) + else: + events.add(record) - run_janitor_phase( + archived: tuple[JanitorArchiveCandidate, ...] = run_janitor_phase( progress=progress, phase=JanitorPhase.ARCHIVE, items=plan.archive_candidates, @@ -92,27 +112,26 @@ def archive_relation(candidate: JanitorArchiveCandidate) -> None: destination=candidate.archive_key.display_name(), ), apply=archive_relation, + workers=workers, + chunks=_event_chunks(tuple(candidate.archive_key for candidate in plan.archive_candidates)), + after_chunk=flush_events, ) - deleted: list[JanitorArchivedRelation] = [] - def delete_archive(archive: JanitorArchivedRelation) -> None: + def delete_archive(archive: JanitorArchivedRelation, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): _drop_archive( adapter=adapter, connection=connection, archive=archive, recorder=recorder ) - deleted.append(archive) - _write_event( - adapter=adapter, - connection=connection, - record=_event_record( + events.add( + _event_record( adapter=adapter, event_type=JanitorEventType.DELETE, run_id=run_id, archive=archive, - ), + ) ) - run_janitor_phase( + deleted: tuple[JanitorArchivedRelation, ...] = run_janitor_phase( progress=progress, phase=JanitorPhase.DELETE_ARCHIVES, items=plan.archive_deletion_candidates, @@ -120,8 +139,34 @@ def delete_archive(archive: JanitorArchivedRelation) -> None: action=JanitorAction.DELETED, name=archive.key.display_name() ), apply=delete_archive, + workers=workers, + chunks=_event_chunks(tuple(archive.key for archive in plan.archive_deletion_candidates)), + after_chunk=flush_events, + ) + return archived, deleted + + +def _shortened(candidate: JanitorArchiveCandidate) -> bool: + """Whether the archive name no longer ends with the relation's full original name.""" + + return not candidate.archive_key.name.lower().endswith( + ARCHIVE_NAME_SEPARATOR + candidate.key.name.lower() + ) + + +def _event_chunks(keys: tuple[JanitorRelationKey, ...]) -> tuple[tuple[int, ...], ...]: + """Item positions grouped by schema, in order, and split into event-sized chunks.""" + + by_schema: dict[tuple[str | None, str | None], list[int]] = {} + index: int + key: JanitorRelationKey + for index, key in enumerate(keys): + by_schema.setdefault((key.database, key.schema), []).append(index) + return tuple( + tuple(positions[start : start + JANITOR_EVENT_CHUNK_SIZE]) + for positions in by_schema.values() + for start in range(0, len(positions), JANITOR_EVENT_CHUNK_SIZE) ) - return tuple(archived), tuple(deleted) def _required_schema(key: JanitorRelationKey) -> str: @@ -188,7 +233,9 @@ def _drop_archive( ) -def _create_event_tables(*, adapter: BaseAdapter, connection: Any, plan: JanitorPlan) -> None: +def _create_event_tables( + *, adapter: BaseAdapter, connection: Any, plan: JanitorPlan +) -> tuple[tuple[str | None, str], ...]: keys: tuple[JanitorRelationKey, ...] = ( *(candidate.key for candidate in plan.archive_candidates), *(archive.key for archive in plan.archive_deletion_candidates), @@ -203,6 +250,7 @@ def _create_event_tables(*, adapter: BaseAdapter, connection: Any, plan: Janitor connection=connection, sql=adapter.render_create_janitor_event_table_sql(database=database, schema=schema), ) + return tuple(schemas) def _event_record( @@ -237,12 +285,3 @@ def _event_record( archive_qualified_name=_qualified(adapter=adapter, key=archive.key), archived_at=archive.archived_at, ) - - -def _write_event(*, adapter: BaseAdapter, connection: Any, record: JanitorEventRecord) -> None: - _ = write_janitor_event_record( - connection=connection, - execute=adapter.execute, - record=record, - render_qualified_name=adapter.render_qualified_name, - ) diff --git a/src/sqlbuild/executor/janitor/_helpers/deletion.py b/src/sqlbuild/executor/janitor/_helpers/deletion.py index ebaaf63a6f..d7615e3362 100644 --- a/src/sqlbuild/executor/janitor/_helpers/deletion.py +++ b/src/sqlbuild/executor/janitor/_helpers/deletion.py @@ -3,7 +3,9 @@ from __future__ import annotations from collections.abc import Callable +from typing import Any +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import JanitorDirectStatePruneCandidate, JanitorProgressItem from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress @@ -13,17 +15,18 @@ def apply_janitor_deletions( *, candidates: tuple[JanitorDirectStatePruneCandidate, ...], - delete: Callable[[JanitorDirectStatePruneCandidate], object] | None, + delete: Callable[[JanitorDirectStatePruneCandidate, Any], object] | None, + workers: JanitorWorkers, progress: JanitorProgress | None = None, ) -> tuple[JanitorDirectStatePruneCandidate, ...]: - """Run the delete callback for each candidate, returning the deleted ones.""" + """Run `delete(candidate, connection)` for each candidate, returning the deleted ones.""" if delete is None: return () - def prune(candidate: JanitorDirectStatePruneCandidate) -> None: + def prune(candidate: JanitorDirectStatePruneCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): - _ = delete(candidate) + _ = delete(candidate, connection) run_janitor_phase( progress=progress, @@ -33,5 +36,6 @@ def prune(candidate: JanitorDirectStatePruneCandidate) -> None: action=JanitorAction.PRUNED, name=candidate.display_name() ), apply=prune, + workers=workers, ) return candidates diff --git a/src/sqlbuild/executor/janitor/classes/__init__.py b/src/sqlbuild/executor/janitor/classes/__init__.py new file mode 100644 index 0000000000..8f8b3afa2b --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/__init__.py @@ -0,0 +1 @@ +"""Janitor executor classes.""" diff --git a/src/sqlbuild/executor/janitor/classes/janitor_workers.py b/src/sqlbuild/executor/janitor/classes/janitor_workers.py new file mode 100644 index 0000000000..da128d0115 --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/janitor_workers.py @@ -0,0 +1,119 @@ +"""Bounded concurrent janitor actions, one warehouse connection per worker.""" + +from __future__ import annotations + +import queue +from collections.abc import Callable, Sequence +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from contextvars import copy_context +from time import perf_counter +from typing import Any + +from sqlbuild.executor.janitor.models import JanitorActionOutcome + + +class JanitorWorkers: + """Run independent janitor actions, each on a connection no other action is using.""" + + def __init__(self, *, connections: tuple[Any, ...]) -> None: + self.connections: tuple[Any, ...] = connections + + @property + def connection(self) -> Any: + """The invocation's own connection, used for everything outside a phase's actions.""" + + return self.connections[0] + + def run[ItemT, ResultT]( + self, + *, + items: Sequence[ItemT], + action: Callable[[ItemT, Any], ResultT], + on_done: Callable[[int, JanitorActionOutcome[ResultT]], None], + ) -> BaseException | None: + """Run `action` on every item, reporting each on the calling thread as it finishes. + + After a failure no further action starts; running ones finish and are reported. The + failure of the earliest item is returned. + """ + + if len(self.connections) == 1 or len(items) <= 1: + return self._run_in_order(items=items, action=action, on_done=on_done) + connections: queue.Queue[Any] = queue.Queue() + connection: Any + for connection in self.connections: + connections.put(connection) + + def attempt(item: ItemT) -> JanitorActionOutcome[ResultT]: + borrowed: Any = connections.get() + try: + return _attempt(action=action, item=item, connection=borrowed) + finally: + connections.put(borrowed) + + pool: ThreadPoolExecutor = ThreadPoolExecutor( + max_workers=min(len(self.connections), len(items)) + ) + try: + return _collect(pool=pool, items=items, attempt=attempt, on_done=on_done) + finally: + pool.shutdown(wait=True, cancel_futures=True) + + def _run_in_order[ItemT, ResultT]( + self, + *, + items: Sequence[ItemT], + action: Callable[[ItemT, Any], ResultT], + on_done: Callable[[int, JanitorActionOutcome[ResultT]], None], + ) -> BaseException | None: + index: int + item: ItemT + for index, item in enumerate(items): + outcome: JanitorActionOutcome[ResultT] = _attempt( + action=action, item=item, connection=self.connection + ) + on_done(index, outcome) + if outcome.error is not None: + return outcome.error + return None + + +def _attempt[ItemT, ResultT]( + *, action: Callable[[ItemT, Any], ResultT], item: ItemT, connection: Any +) -> JanitorActionOutcome[ResultT]: + started: float = perf_counter() + try: + result: ResultT = action(item, connection) + except BaseException as error: + return JanitorActionOutcome(error=error, duration_seconds=perf_counter() - started) + return JanitorActionOutcome(result=result, duration_seconds=perf_counter() - started) + + +def _collect[ItemT, ResultT]( + *, + pool: ThreadPoolExecutor, + items: Sequence[ItemT], + attempt: Callable[[ItemT], JanitorActionOutcome[ResultT]], + on_done: Callable[[int, JanitorActionOutcome[ResultT]], None], +) -> BaseException | None: + futures: dict[Future[JanitorActionOutcome[ResultT]], int] = { + pool.submit(copy_context().run, attempt, item): index for index, item in enumerate(items) + } + failures: dict[int, BaseException] = {} + pending: set[Future[JanitorActionOutcome[ResultT]]] = set(futures) + while pending: + done: set[Future[JanitorActionOutcome[ResultT]]] + done, pending = wait(pending, return_when=FIRST_COMPLETED) + future: Future[JanitorActionOutcome[ResultT]] + for future in sorted(done, key=futures.__getitem__): + if future.cancelled(): + continue + outcome: JanitorActionOutcome[ResultT] = future.result() + on_done(futures[future], outcome) + if outcome.error is None: + continue + if not failures: + for waiting in pending: + _ = waiting.cancel() + failures[futures[future]] = outcome.error + return failures[min(failures)] if failures else None diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py new file mode 100644 index 0000000000..4b6e6f2a97 --- /dev/null +++ b/src/sqlbuild/executor/janitor/constants.py @@ -0,0 +1,3 @@ +"""Janitor executor constants.""" + +ARCHIVE_NAME_SEPARATOR: str = "__" diff --git a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py index ce402ae3b6..9baf8a5b5c 100644 --- a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py +++ b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py @@ -4,9 +4,10 @@ from collections.abc import Callable, Sequence from dataclasses import replace -from time import perf_counter +from typing import Any -from sqlbuild.executor.janitor.models import JanitorProgressItem +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorProgressItem from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress @@ -16,50 +17,82 @@ def run_janitor_phase[T]( phase: JanitorPhase, items: Sequence[T], describe: Callable[[T], JanitorProgressItem], - apply: Callable[[T], JanitorAction | None], -) -> None: - """Apply `apply` to each item in order; a failing item is reported, then re-raised. + apply: Callable[[T, Any], JanitorAction | None], + workers: JanitorWorkers, + chunks: Sequence[Sequence[int]] | None = None, + after_chunk: Callable[[], None] | None = None, +) -> tuple[T, ...]: + """Apply `apply(item, connection)` to every item and return the items that succeeded. - `apply` may return the action actually taken when it differs from the described one. + Items run concurrently on `workers`, a chunk of item positions at a time (all items when + `chunks` is none); `after_chunk` runs after every chunk + and once more after a failure or interruption, before the error is raised. A failing item is + reported and no later action starts. `apply` may return the action actually taken when it + differs from the described one. """ if not items: - return + return () if progress is not None: progress.phase_started(phase=phase.value, total=len(items)) - item: T - for item in items: - started: float = perf_counter() - try: - taken: JanitorAction | None = apply(item) - except BaseException as error: - message: str = str(error) or type(error).__name__ - _report(progress, phase, (describe(item), None), (message, started)) - if progress is not None: - progress.phase_finished(phase=phase.value, failed=True) - raise - _report(progress, phase, (describe(item), taken), (None, started)) + succeeded: set[int] = set() + + def report( + chunk: Sequence[int], + ) -> Callable[[int, JanitorActionOutcome[JanitorAction | None]], None]: + def on_done(index: int, outcome: JanitorActionOutcome[JanitorAction | None]) -> None: + item: T = items[chunk[index]] + if outcome.error is None: + succeeded.add(chunk[index]) + if progress is None: + return + label: JanitorProgressItem = describe(item) + progress.item_finished( + phase=phase.value, + item=label if outcome.result is None else replace(label, action=outcome.result), + error=None if outcome.error is None else _message(outcome.error), + duration_seconds=outcome.duration_seconds, + ) + + return on_done + + failure: BaseException | None = None + try: + chunk: Sequence[int] + for chunk in (range(len(items)),) if chunks is None else chunks: + failure = workers.run( + items=tuple(items[index] for index in chunk), + action=apply, + on_done=report(chunk), + ) + if failure is not None: + break + if after_chunk is not None: + after_chunk() + except BaseException: + _finish_failed(progress=progress, phase=phase, after_chunk=after_chunk) + raise + if failure is not None: + _finish_failed(progress=progress, phase=phase, after_chunk=after_chunk) + raise failure if progress is not None: progress.phase_finished(phase=phase.value, failed=False) + return tuple(item for index, item in enumerate(items) if index in succeeded) -def _report( +def _finish_failed( + *, progress: JanitorProgress | None, phase: JanitorPhase, - described: tuple[JanitorProgressItem, JanitorAction | None], - outcome: tuple[str | None, float], + after_chunk: Callable[[], None] | None, ) -> None: - if progress is None: - return - label: JanitorProgressItem - taken: JanitorAction | None - label, taken = described - error: str | None - started: float - error, started = outcome - progress.item_finished( - phase=phase.value, - item=label if taken is None else replace(label, action=taken), - error=error, - duration_seconds=perf_counter() - started, - ) + try: + if after_chunk is not None: + after_chunk() + finally: + if progress is not None: + progress.phase_finished(phase=phase.value, failed=True) + + +def _message(error: BaseException) -> str: + return str(error) or type(error).__name__ diff --git a/src/sqlbuild/executor/janitor/main/execute.py b/src/sqlbuild/executor/janitor/main/execute.py index a341f1589a..8f3865b0ed 100644 --- a/src/sqlbuild/executor/janitor/main/execute.py +++ b/src/sqlbuild/executor/janitor/main/execute.py @@ -2,6 +2,8 @@ from __future__ import annotations +import sys +import time from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -11,6 +13,7 @@ janitor_run_id, ) from sqlbuild.executor.janitor._helpers.deletion import apply_janitor_deletions +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( JanitorArchiveCandidate, @@ -23,8 +26,17 @@ JanitorProgressItem, JanitorQueryDiffArtifactCandidate, ) -from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress +from sqlbuild.executor.janitor.types import ( + JanitorAction, + JanitorConnectionProgress, + JanitorPhase, + JanitorProgress, +) from sqlbuild.executor.old_name_views.main._drop_old_name_views import drop_old_name_views +from sqlbuild.executor.pipeline._helpers.connections import ( + close_connections, + open_worker_connections, +) from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -34,32 +46,89 @@ def execute_janitor_plan( adapter: BaseAdapter, connection: Any, progress: JanitorProgress | None = None, + connection_config: dict[str, object] | None = None, + concurrency: int = 1, + connection_progress: JanitorConnectionProgress | None = None, ) -> JanitorExecutionResult: - """Delete all candidates in a janitor plan, reporting each phase and action to `progress`.""" + """Delete all candidates in a janitor plan, reporting each phase and action to `progress`. + + Up to `concurrency` independent actions of one phase run at once, each on its own + connection, when the adapter allows concurrent DDL; phases still run one after another. + """ with OperationLifecycle( operation_kind="janitor", operation_name="janitor_execution", metadata={"item_count": _janitor_action_count(plan)}, ): - return _execute_janitor_plan( - plan=plan, + extra: tuple[Any, ...] = _open_worker_connections( adapter=adapter, - connection=connection, - progress=progress, + connection_config=connection_config, + count=_worker_count(plan=plan, adapter=adapter, concurrency=concurrency) - 1, + progress=connection_progress, ) + try: + return _execute_janitor_plan( + plan=plan, + adapter=adapter, + workers=JanitorWorkers(connections=(connection, *extra)), + progress=progress, + ) + finally: + close_connections(adapter=adapter, connections=extra, active_error=sys.exc_info()[1]) + + +def _worker_count(*, plan: JanitorPlan, adapter: BaseAdapter, concurrency: int) -> int: + largest_phase: int = max( + len(() if plan.direct_mode else plan.candidates), + len(plan.archive_candidates) if plan.direct_mode else 0, + len(plan.archive_deletion_candidates) if plan.direct_mode else 0, + len(plan.old_name_views.drops) + len(plan.old_name_views.missing), + len(plan.query_diff_artifact_candidates), + len(plan.direct_state_prune_candidates), + ) + return max(1, min(concurrency, int(adapter.concurrent_ddl_limit), largest_phase)) + + +def _open_worker_connections( + *, + adapter: BaseAdapter, + connection_config: dict[str, object] | None, + count: int, + progress: JanitorConnectionProgress | None, +) -> tuple[Any, ...]: + if count <= 0 or connection_config is None: + return () + if progress is not None: + progress.on_connection_start(count) + started: float = time.monotonic() + try: + connections: tuple[Any, ...] = open_worker_connections( + adapter=adapter, connection_config=connection_config, connection_count=count + ) + except BaseException: + if progress is not None: + progress.on_connection_error( + connection_count=count, elapsed_seconds=time.monotonic() - started + ) + raise + if progress is not None: + progress.on_connection_complete( + connection_count=count, elapsed_seconds=time.monotonic() - started + ) + return connections def _execute_janitor_plan( *, plan: JanitorPlan, adapter: BaseAdapter, - connection: Any, + workers: JanitorWorkers, progress: JanitorProgress | None, ) -> JanitorExecutionResult: recorder: StatementRecorder = StatementRecorder() - def drop_candidate(candidate: JanitorDeleteCandidate) -> None: + def drop_candidate(candidate: JanitorDeleteCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): adapter.drop( connection=connection, @@ -76,6 +145,7 @@ def drop_candidate(candidate: JanitorDeleteCandidate) -> None: action=JanitorAction.DELETED, name=candidate.key.display_name() ), apply=drop_candidate, + workers=workers, ) archived: tuple[JanitorArchiveCandidate, ...] deleted_archives: tuple[JanitorArchivedRelation, ...] @@ -83,7 +153,7 @@ def drop_candidate(candidate: JanitorDeleteCandidate) -> None: apply_direct_archives( plan=plan, adapter=adapter, - connection=connection, + workers=workers, recorder=recorder, run_id=janitor_run_id(), progress=progress, @@ -94,13 +164,13 @@ def drop_candidate(candidate: JanitorDeleteCandidate) -> None: dropped_old_name_views: tuple[JanitorOldNameView, ...] = drop_old_name_views( plan=plan.old_name_views, adapter=adapter, - connection=connection, + workers=workers, recorder=recorder, run_id=janitor_run_id(), progress=progress, ) - def drop_query_artifact(candidate: JanitorQueryDiffArtifactCandidate) -> None: + def drop_query_artifact(candidate: JanitorQueryDiffArtifactCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): query_artifact_destination: str | None = adapter.render_qualified_name( database=candidate.key.database, @@ -126,12 +196,14 @@ def drop_query_artifact(candidate: JanitorQueryDiffArtifactCandidate) -> None: action=JanitorAction.DROPPED, name=candidate.key.display_name() ), apply=drop_query_artifact, + workers=workers, ) pruned_direct_state: tuple[JanitorDirectStatePruneCandidate, ...] = apply_janitor_deletions( candidates=plan.direct_state_prune_candidates, - delete=lambda direct_state_candidate: adapter.execute( + delete=lambda direct_state_candidate, connection: adapter.execute( connection=connection, sql=direct_state_candidate.prune_sql ), + workers=workers, progress=progress, ) return JanitorExecutionResult( diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 1d49a19393..5d19256490 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -236,3 +236,12 @@ class JanitorProgressItem: action: JanitorAction name: str destination: str | None = None + + +@dataclass(frozen=True) +class JanitorActionOutcome[ResultT]: + """What one janitor action returned or raised, and how long it ran.""" + + result: ResultT | None = None + error: BaseException | None = None + duration_seconds: float = 0.0 diff --git a/src/sqlbuild/executor/janitor/types.py b/src/sqlbuild/executor/janitor/types.py index 8821306d8c..4a1ea8ff60 100644 --- a/src/sqlbuild/executor/janitor/types.py +++ b/src/sqlbuild/executor/janitor/types.py @@ -39,3 +39,11 @@ def item_finished( ) -> None: ... def phase_finished(self, *, phase: str, failed: bool) -> None: ... + + +class JanitorConnectionProgress(Protocol): + def on_connection_start(self, connection_count: int) -> None: ... + + def on_connection_complete(self, *, connection_count: int, elapsed_seconds: float) -> None: ... + + def on_connection_error(self, *, connection_count: int, elapsed_seconds: float) -> None: ... diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py index 1de4e10a49..b56c2653e2 100644 --- a/src/sqlbuild/executor/janitor_events/_helpers/sql.py +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -58,30 +58,40 @@ def build_create_table_sql( ) -def build_existing_event_sql( +def build_existing_run_events_sql( *, - record: JanitorEventRecord, + database: str | None, + schema: str, + run_id: str, render_qualified_name: Callable[..., str | None], ) -> str: + """Select the IDs of every event one run already wrote to one schema's table.""" + qualified_name: str = build_qualified_table_name( - database=record.relation_database, - schema=record.relation_schema, - render_qualified_name=render_qualified_name, + database=database, schema=schema, render_qualified_name=render_qualified_name ) - event_id: str = _column_literal(column=COLUMN_EVENT_ID, value=record.event_id) - return f"SELECT {COLUMN_EVENT_ID} FROM {qualified_name} WHERE {COLUMN_EVENT_ID} = {event_id}" + run: str = _column_literal(column=COLUMN_RUN_ID, value=run_id) + return f"SELECT {COLUMN_EVENT_ID} FROM {qualified_name} WHERE {COLUMN_RUN_ID} = {run}" -def build_insert_sql( +def build_insert_many_sql( *, - record: JanitorEventRecord, + records: tuple[JanitorEventRecord, ...], render_qualified_name: Callable[..., str | None], ) -> str: + """Append several events to the table of the first record's schema in one statement.""" + + first: JanitorEventRecord = records[0] qualified_name: str = build_qualified_table_name( - database=record.relation_database, - schema=record.relation_schema, + database=first.relation_database, + schema=first.relation_schema, render_qualified_name=render_qualified_name, ) + rows: str = ", ".join(f"({_row_literals(record=record)})" for record in records) + return f"INSERT INTO {qualified_name} ({', '.join(JANITOR_EVENT_COLUMNS)}) VALUES {rows}" + + +def _row_literals(*, record: JanitorEventRecord) -> str: values: dict[str, object | None] = { COLUMN_EVENT_ID: record.event_id, COLUMN_SCHEMA_VERSION: record.schema_version, @@ -97,10 +107,9 @@ def build_insert_sql( COLUMN_ARCHIVE_QUALIFIED_NAME: record.archive_qualified_name, COLUMN_ARCHIVED_AT: record.archived_at, } - literals: str = ", ".join( + return ", ".join( _column_literal(column=column, value=values[column]) for column in JANITOR_EVENT_COLUMNS ) - return f"INSERT INTO {qualified_name} ({', '.join(JANITOR_EVENT_COLUMNS)}) VALUES ({literals})" def _column_literal(*, column: str, value: object | None) -> str: diff --git a/src/sqlbuild/executor/janitor_events/classes/__init__.py b/src/sqlbuild/executor/janitor_events/classes/__init__.py new file mode 100644 index 0000000000..2eeccb4088 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/classes/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event classes.""" diff --git a/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py b/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py new file mode 100644 index 0000000000..d970aeb404 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py @@ -0,0 +1,120 @@ +"""Append-only janitor audit events, written in chunks per schema event table.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.executor.janitor_events._helpers.sql import ( + build_existing_run_events_sql, + build_insert_many_sql, +) +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_CHUNK_SIZE +from sqlbuild.executor.janitor_events.models import JanitorEventRecord + +type _SchemaKey = tuple[str | None, str] + + +class JanitorEventWriter: + """Write each event at most once, reading the run's existing event IDs once per schema. + + Events wait in memory until `flush`, which appends each schema's pending events with one + multi-row INSERT per chunk. `write_now` appends one event at once. Event IDs are + deterministic per run, so an ID already present is never inserted again, including when the + same invocation is retried. + """ + + def __init__( + self, + *, + execute: AdapterExecute[Any, Any], + render_qualified_name: Callable[..., str | None], + run_id: str, + ) -> None: + self._execute: AdapterExecute[Any, Any] = execute + self._render_qualified_name: Callable[..., str | None] = render_qualified_name + self._run_id: str = run_id + self._written: set[str] = set() + self._pending: dict[_SchemaKey, list[JanitorEventRecord]] = {} + self._lock: threading.Lock = threading.Lock() + + def read_existing(self, *, connection: Any, schemas: tuple[_SchemaKey, ...]) -> None: + """Remember the IDs of events this run already wrote to each schema's table.""" + + database: str | None + schema: str + for database, schema in schemas: + cursor: Any = self._execute( + connection=connection, + sql=build_existing_run_events_sql( + database=database, + schema=schema, + run_id=self._run_id, + render_qualified_name=self._render_qualified_name, + ), + ) + self._written.update(str(row[0]) for row in cursor.fetchall()) + + def add(self, record: JanitorEventRecord) -> None: + """Hold one event until the next `flush`, unless it is already written or held.""" + + with self._lock: + if record.event_id in self._written or any( + held.event_id == record.event_id + for held in self._pending.get(_schema_key(record), ()) + ): + return + self._pending.setdefault(_schema_key(record), []).append(record) + + def write_now(self, *, connection: Any, record: JanitorEventRecord) -> None: + """Append one event immediately on `connection`, unless it is already written.""" + + with self._lock: + if record.event_id in self._written: + return + self._written.add(record.event_id) + try: + self._insert(connection=connection, records=(record,)) + except BaseException: + with self._lock: + self._written.discard(record.event_id) + raise + + def flush(self, *, connection: Any) -> None: + """Append every held event, one INSERT per schema and chunk of rows.""" + + with self._lock: + pending: dict[_SchemaKey, list[JanitorEventRecord]] = self._pending + self._pending = {} + try: + key: _SchemaKey + records: list[JanitorEventRecord] + for key, records in list(pending.items()): + while records: + chunk: tuple[JanitorEventRecord, ...] = tuple( + records[:JANITOR_EVENT_CHUNK_SIZE] + ) + self._insert(connection=connection, records=chunk) + del records[: len(chunk)] + with self._lock: + self._written.update(record.event_id for record in chunk) + del pending[key] + finally: + if pending: + with self._lock: + for key, records in pending.items(): + self._pending.setdefault(key, [])[:0] = records + + def _insert(self, *, connection: Any, records: tuple[JanitorEventRecord, ...]) -> None: + _ = self._execute( + connection=connection, + sql=build_insert_many_sql( + records=records, render_qualified_name=self._render_qualified_name + ), + ) + + +def _schema_key(record: JanitorEventRecord) -> _SchemaKey: + return (record.relation_database, record.relation_schema) diff --git a/src/sqlbuild/executor/janitor_events/constants.py b/src/sqlbuild/executor/janitor_events/constants.py index 9e30530b97..dc211ed6ad 100644 --- a/src/sqlbuild/executor/janitor_events/constants.py +++ b/src/sqlbuild/executor/janitor_events/constants.py @@ -55,3 +55,7 @@ COLUMN_OCCURRED_AT: StateSqlValueType.TEXT_TIMESTAMP, COLUMN_ARCHIVED_AT: StateSqlValueType.TEXT_TIMESTAMP, } + +# Rows per multi-row INSERT: below every adapter's VALUES-list limit (SQL Server allows 1,000 and +# SQLite 500), and at most this many completed actions can lack events after a crash. +JANITOR_EVENT_CHUNK_SIZE: int = 100 diff --git a/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py b/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py deleted file mode 100644 index 3dc22a22e4..0000000000 --- a/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Janitor audit event warehouse write operation.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.executor.janitor_events._helpers.sql import ( - build_existing_event_sql, - build_insert_sql, -) -from sqlbuild.executor.janitor_events.models import JanitorEventRecord - - -def write_janitor_event_record( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - record: JanitorEventRecord, - render_qualified_name: Callable[..., str | None], -) -> bool: - """Append one janitor event unless its deterministic ID was already written.""" - - cursor: Any = execute( - connection=connection, - sql=build_existing_event_sql(record=record, render_qualified_name=render_qualified_name), - ) - if cursor.fetchall(): - return False - _ = execute( - connection=connection, - sql=build_insert_sql(record=record, render_qualified_name=render_qualified_name), - ) - return True diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index 7dbe7bda9b..b7aacbfece 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -20,6 +20,7 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( JanitorOldNameView, @@ -35,19 +36,20 @@ def apply_old_name_view_drops( *, plan: JanitorOldNameViewPlanning, adapter: BaseAdapter, - connection: Any, + workers: JanitorWorkers, recorder: StatementRecorder, run_id: str, progress: JanitorProgress | None = None, ) -> tuple[JanitorOldNameView, ...]: """Drop expired or requested views, then record every drop and every vanished view.""" - dropped: list[JanitorOldNameView] = [] + dropped: set[int] = set() - def apply(entry: tuple[JanitorOldNameView, bool]) -> JanitorAction | None: + def apply(entry: tuple[int, JanitorOldNameView, bool], connection: Any) -> JanitorAction | None: + position: int view: JanitorOldNameView planned_missing: bool - view, planned_missing = entry + position, view, planned_missing = entry if planned_missing: _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) return None @@ -71,20 +73,24 @@ def apply(entry: tuple[JanitorOldNameView, bool]) -> JanitorAction | None: statement_recorder=recorder, ) _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) - dropped.append(view) + dropped.add(position) return None run_janitor_phase( progress=progress, phase=JanitorPhase.DROP_OLD_NAME_VIEWS, - items=(*((view, False) for view in plan.drops), *((view, True) for view in plan.missing)), + items=( + *((index, view, False) for index, view in enumerate(plan.drops)), + *((len(plan.drops), view, True) for view in plan.missing), + ), describe=lambda entry: JanitorProgressItem( - action=JanitorAction.RECORDED if entry[1] else JanitorAction.DROPPED, - name=entry[0].key.display_name(), + action=JanitorAction.RECORDED if entry[2] else JanitorAction.DROPPED, + name=entry[1].key.display_name(), ), apply=apply, + workers=workers, ) - return tuple(dropped) + return tuple(view for index, view in enumerate(plan.drops) if index in dropped) def _record_drop( diff --git a/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py b/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py index 7a656b8a48..1e4663db98 100644 --- a/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py +++ b/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py @@ -2,11 +2,10 @@ from __future__ import annotations -from typing import Any - from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.janitor.models import JanitorOldNameView, JanitorOldNameViewPlanning +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.types import JanitorProgress from sqlbuild.executor.old_name_views._helpers.janitor import apply_old_name_view_drops @@ -15,7 +14,7 @@ def drop_old_name_views( *, plan: JanitorOldNameViewPlanning, adapter: BaseAdapter, - connection: Any, + workers: JanitorWorkers, recorder: StatementRecorder, run_id: str, progress: JanitorProgress | None = None, @@ -25,7 +24,7 @@ def drop_old_name_views( return apply_old_name_view_drops( plan=plan, adapter=adapter, - connection=connection, + workers=workers, recorder=recorder, run_id=run_id, progress=progress, From 37d9e0cd4ac14ff3ee896a8082f5c0800df94b10 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:19:31 +0100 Subject: [PATCH 04/10] test: cover janitor progress text, event writer and batched audit events --- .../sqlbuild-core/src/janitor_text/tests.rs | 2 + .../src/janitor_text/tests/helpers.rs | 48 ++- .../tests/test_janitor_progress_text.rs | 346 ++++++++++++++++++ .../src/janitor_text/tests/test_types.rs | 48 ++- .../{main => classes}/__init__.py | 0 .../janitor_events/classes/_test_types.py | 14 + .../classes/test_janitor_event_writer.py | 168 +++++++++ .../janitor_events/main/_test_types.py | 11 - .../main/test_write_janitor_event.py | 103 ------ .../_helpers/janitor_output/_test_types.py | 1 + .../_helpers/janitor_output/test_outputs.py | 23 +- .../cli/commands/main/janitor/test_janitor.py | 26 +- .../executor/janitor/main/_test_types.py | 1 + .../executor/janitor/main/test_execute.py | 12 +- 14 files changed, 672 insertions(+), 131 deletions(-) create mode 100644 crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs rename tests/integration/src/sqlbuild/executor/janitor_events/{main => classes}/__init__.py (100%) create mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py create mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py delete mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py delete mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py diff --git a/crates/sqlbuild-core/src/janitor_text/tests.rs b/crates/sqlbuild-core/src/janitor_text/tests.rs index 67b4ac8a4c..ef148abc3f 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests.rs @@ -4,3 +4,5 @@ mod helpers; mod janitor_text; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_janitor_progress_text.rs"] +mod janitor_progress_text; diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index f304628d0a..bebb87af7d 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -1,4 +1,6 @@ -use crate::janitor_text::models::{ArchiveCandidateText, ArchivedRelationText, JanitorPlanText}; +use crate::janitor_text::models::{ + ArchiveCandidateText, ArchivedRelationText, JanitorItemText, JanitorPhase, JanitorPlanText, +}; /// A direct-mode plan for `dev` that archives one relation and deletes one archive. pub(super) fn direct_archive_plan() -> JanitorPlanText { @@ -18,3 +20,47 @@ pub(super) fn direct_archive_plan() -> JanitorPlanText { ..JanitorPlanText::default() } } + +/// A finished action of `phase`, failed when `error` is given. +pub(super) fn item( + phase: JanitorPhase, + (action, name, destination): (&str, &str, Option<&str>), + error: Option<&str>, +) -> JanitorItemText { + JanitorItemText { + phase, + action: action.to_owned(), + name: name.to_owned(), + destination: destination.map(str::to_owned), + error: error.map(str::to_owned), + duration_seconds: Some(0.5), + } +} + +/// A direct-mode plan archiving `count` relations, with `skipped` objects for two reasons. +pub(super) fn large_plan(count: usize, skipped: usize) -> JanitorPlanText { + JanitorPlanText { + target_name: Some("dev".to_owned()), + retention_days: 14, + direct_mode: true, + archive_retention_days: 14, + archive_candidates: (0..count) + .map(|index| ArchiveCandidateText { + name: format!("dev.old_orders_{index:03}"), + archive_name: format!("dev._sqb_archive__20260924t101500z__old_orders_{index:03}"), + ..ArchiveCandidateText::default() + }) + .collect(), + skipped_relations: (0..skipped) + .map(|index| { + let reason: &str = if index % 3 == 0 { + "relation is newer than 14 days" + } else { + "not tracked by SQLBuild" + }; + (format!("dev.recent_{index:03}"), reason.to_owned()) + }) + .collect(), + ..JanitorPlanText::default() + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs new file mode 100644 index 0000000000..5bdf2e3534 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs @@ -0,0 +1,346 @@ +use crate::cli_text::models::CliStyle; +use crate::janitor_text::main::format_janitor_json::format_janitor_json; +use crate::janitor_text::main::janitor_item_counts::janitor_item_counts; +use crate::janitor_text::main::janitor_nothing_to_do_text::janitor_nothing_to_do_text; +use crate::janitor_text::main::janitor_phase_complete_text::janitor_phase_complete_text; +use crate::janitor_text::main::janitor_phase_failed_text::janitor_phase_failed_text; +use crate::janitor_text::main::janitor_phase_flow_width::janitor_phase_flow_width; +use crate::janitor_text::main::janitor_phase_start_text::janitor_phase_start_text; +use crate::janitor_text::main::render_janitor_completion::render_janitor_completion; +use crate::janitor_text::main::render_janitor_item_line::render_janitor_item_line; +use crate::janitor_text::main::render_janitor_plan::render_janitor_plan; +use crate::janitor_text::models::{ + JanitorOutcome, JanitorPhase, JanitorResultCounts, JanitorRunText, +}; +use crate::janitor_text::tests::helpers::{direct_archive_plan, item, large_plan}; +use crate::janitor_text::tests::test_types::{ + CompletionTestCase, ItemLineTestCase, JsonTestCase, PhaseTextTestCase, PreviewTestCase, +}; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_finished_action_when_rendering_then_line_aligns_position_flow_and_status() { + let test_cases = [ + ItemLineTestCase { + description: "a rename pads its flow to the phase width", + position: (1, 12), + item: item( + JanitorPhase::Archive, + ("archived", "dev.orders", Some("dev._a")), + None, + ), + flow_width: 24, + expected_line: " 1/12 archived dev.orders -> dev._a OK 0.50s", + }, + ItemLineTestCase { + description: "a failure shows the first line of its error", + position: (3, 3), + item: item( + JanitorPhase::DeleteArchives, + ("deleted", "dev.old", None), + Some("permission denied\nquery id 1"), + ), + flow_width: 0, + expected_line: " 3/3 deleted dev.old FAIL 0.50s permission denied", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_janitor_item_line( + test_case.position, + &test_case.item, + test_case.flow_width, + PLAIN + ), + test_case.expected_line, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_phase_when_rendering_progress_then_start_complete_and_failure_count_its_actions() { + let test_cases = [ + PhaseTextTestCase { + description: "one archive uses singular nouns and seconds", + phase: JanitorPhase::Archive, + items: vec![item( + JanitorPhase::Archive, + ("archived", "dev.a", Some("dev.b")), + None, + )], + total: 1, + elapsed_seconds: 0.25, + expected_start: "Archiving 1 relation...", + expected_complete: "Archived 1 relation. (0.25s)", + expected_failed: "\u{2717} Archiving failed after 1 of 1 relation (0.25s)", + }, + PhaseTextTestCase { + description: "old-name views count drops apart from views already gone", + phase: JanitorPhase::DropOldNameViews, + items: vec![ + item( + JanitorPhase::DropOldNameViews, + ("dropped", "dev.a", None), + None, + ), + item( + JanitorPhase::DropOldNameViews, + ("recorded", "dev.b", None), + None, + ), + item( + JanitorPhase::DropOldNameViews, + ("dropped", "dev.c", None), + Some("denied"), + ), + ], + total: 3, + elapsed_seconds: 75.4, + expected_start: "Dropping 3 old-name views...", + expected_complete: "Dropped 1 old-name view and recorded 1 already gone. (1m 15s)", + expected_failed: "\u{2717} Dropping failed after 2 of 3 old-name views (1m 15s)", + }, + PhaseTextTestCase { + description: "hour-long phases show hours, minutes and seconds", + phase: JanitorPhase::PruneDirectState, + items: Vec::new(), + total: 2, + elapsed_seconds: 3725.0, + expected_start: "Pruning 2 direct state tables...", + expected_complete: "Pruned 0 direct state tables. (1h 02m 05s)", + expected_failed: "\u{2717} Pruning failed after 0 of 2 direct state tables (1h 02m 05s)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + janitor_phase_start_text(test_case.phase, test_case.total), + test_case.expected_start, + "{}", + test_case.description + ); + assert_eq!( + janitor_phase_complete_text( + test_case.phase, + &test_case.items, + test_case.elapsed_seconds + ), + test_case.expected_complete, + "{}", + test_case.description + ); + let completed: usize = test_case + .items + .iter() + .filter(|item| item.error.is_none()) + .count(); + assert_eq!( + janitor_phase_failed_text( + test_case.phase, + (completed, test_case.total), + test_case.elapsed_seconds, + PLAIN + ), + test_case.expected_failed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_run_counts_when_rendering_completion_then_final_line_is_unambiguous() { + let test_cases = [ + CompletionTestCase { + description: "a completed run totals archives, deletions and prunes", + counts: JanitorResultCounts { + archived: 608, + deleted_archives: 6, + pruned_direct_state: 6, + ..JanitorResultCounts::default() + }, + elapsed_seconds: Some(302.0), + failed: false, + expected_text: "\u{2713} Janitor completed archived 608, deleted 6, pruned 6 (5m 02s)\n", + }, + CompletionTestCase { + description: "a failed run reports what finished before the failure", + counts: JanitorResultCounts { + archived: 11, + ..JanitorResultCounts::default() + }, + elapsed_seconds: Some(65.0), + failed: true, + expected_text: "\u{2717} Janitor failed archived 11, deleted 0, pruned 0 (1m 05s)\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_janitor_completion( + &test_case.counts, + test_case.elapsed_seconds, + test_case.failed, + PLAIN + ), + test_case.expected_text, + "{}", + test_case.description + ); + } + assert_eq!( + janitor_nothing_to_do_text(PLAIN), + "\u{2713} Janitor completed nothing to clean up\n" + ); +} + +#[test] +fn given_finished_actions_when_counting_then_only_successes_count_by_kind() { + let items = vec![ + item(JanitorPhase::Archive, ("archived", "a", Some("b")), None), + item( + JanitorPhase::Archive, + ("archived", "c", Some("d")), + Some("denied"), + ), + item(JanitorPhase::DeleteArchives, ("deleted", "e", None), None), + item(JanitorPhase::DropOldNameViews, ("dropped", "f", None), None), + item(JanitorPhase::DropOldNameViews, ("recorded", "g", None), None), + item(JanitorPhase::DropQueryArtifacts, ("dropped", "h", None), None), + item(JanitorPhase::PruneDirectState, ("pruned", "i", None), None), + ]; + + assert_eq!( + janitor_item_counts(&items), + JanitorResultCounts { + archived: 1, + deleted: 0, + deleted_archives: 1, + deleted_query_artifacts: 1, + dropped_old_name_views: 1, + pruned_direct_state: 1, + } + ); + assert_eq!( + janitor_phase_flow_width(&large_plan(3, 0), JanitorPhase::Archive), + 72 + ); +} + +#[test] +fn given_long_plan_when_previewing_then_lists_are_capped_unless_verbose() { + let test_cases = [ + PreviewTestCase { + description: "long lists show ten entries and skipped objects group by reason", + plan: large_plan(25, 7), + verbose: false, + expected_fragments: &[ + " dev.old_orders_009 -> ", + " ... and 15 more (use --verbose to show all)\n", + "Skipped objects (7)\n 4 not tracked by SQLBuild\n 3 relation is newer than 14 days\n", + " relations to archive 25\n", + ], + unexpected_fragments: &["dev.old_orders_010 ->", "dev.recent_000"], + }, + PreviewTestCase { + description: "verbose lists every entry and every skipped object", + plan: large_plan(25, 7), + verbose: true, + expected_fragments: &[ + " dev.old_orders_024 -> ", + "Skipped objects\n dev.recent_000 relation is newer than 14 days\n", + ], + unexpected_fragments: &["... and", "Skipped objects (7)"], + }, + ]; + + for test_case in test_cases { + let text: String = render_janitor_plan(&test_case.plan, test_case.verbose, PLAIN); + for fragment in test_case.expected_fragments { + assert!( + text.contains(fragment), + "{}: missing {fragment:?} in {text}", + test_case.description + ); + } + for fragment in test_case.unexpected_fragments { + assert!( + !text.contains(fragment), + "{}: unexpected {fragment:?}", + test_case.description + ); + } + } +} + +#[test] +fn given_outcome_when_formatting_json_then_document_carries_plan_result_and_error() { + let run: JanitorRunText = JanitorRunText { + items: vec![item( + JanitorPhase::Archive, + ("archived", "dev.old_orders", Some("dev._a")), + None, + )], + counts: JanitorResultCounts { + archived: 1, + ..JanitorResultCounts::default() + }, + elapsed_seconds: 1.5, + }; + let test_cases = [ + JsonTestCase { + description: "a completed run lists its actions", + outcome: JanitorOutcome::Completed, + run: Some(run.clone()), + error: None, + expected_fragments: &[ + "\"target\": \"dev\"", + "\"status\": \"completed\"", + "\"relations_to_archive\": [", + "\"name\": \"dev.old_orders\"", + "\"phase\": \"archive\"", + "\"status\": \"ok\"", + "\"archived\": 1", + "\"error\": null", + ], + }, + JsonTestCase { + description: "a failed run carries its error", + outcome: JanitorOutcome::Failed, + run: Some(run), + error: Some("rename denied"), + expected_fragments: &["\"status\": \"failed\"", "\"error\": \"rename denied\""], + }, + JsonTestCase { + description: "a preview has no result", + outcome: JanitorOutcome::Previewed, + run: None, + error: None, + expected_fragments: &["\"status\": \"previewed\"", "\"result\": null"], + }, + ]; + + for test_case in test_cases { + let text: String = format_janitor_json( + Some(&direct_archive_plan()), + test_case.outcome, + test_case.run.as_ref(), + test_case.error, + ); + for fragment in test_case.expected_fragments { + assert!( + text.contains(fragment), + "{}: missing {fragment:?} in {text}", + test_case.description + ); + } + } + assert!( + format_janitor_json(None, JanitorOutcome::Disabled, None, None) + .contains("\"plan\": null") + ); +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs index 6a91f57366..4201f45ccd 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs @@ -1,4 +1,7 @@ -use crate::janitor_text::models::JanitorPlanText; +use crate::janitor_text::models::{ + JanitorItemText, JanitorOutcome, JanitorPhase, JanitorPlanText, JanitorResultCounts, + JanitorRunText, +}; pub(super) struct ConfirmationTestCase { pub(super) description: &'static str, @@ -18,3 +21,46 @@ pub(super) struct DisabledTestCase { pub(super) use_color: bool, pub(super) expected_text: &'static str, } + +pub(super) struct ItemLineTestCase { + pub(super) description: &'static str, + pub(super) position: (usize, usize), + pub(super) item: JanitorItemText, + pub(super) flow_width: usize, + pub(super) expected_line: &'static str, +} + +pub(super) struct PhaseTextTestCase { + pub(super) description: &'static str, + pub(super) phase: JanitorPhase, + pub(super) items: Vec, + pub(super) total: usize, + pub(super) elapsed_seconds: f64, + pub(super) expected_start: &'static str, + pub(super) expected_complete: &'static str, + pub(super) expected_failed: &'static str, +} + +pub(super) struct CompletionTestCase { + pub(super) description: &'static str, + pub(super) counts: JanitorResultCounts, + pub(super) elapsed_seconds: Option, + pub(super) failed: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct PreviewTestCase { + pub(super) description: &'static str, + pub(super) plan: JanitorPlanText, + pub(super) verbose: bool, + pub(super) expected_fragments: &'static [&'static str], + pub(super) unexpected_fragments: &'static [&'static str], +} + +pub(super) struct JsonTestCase { + pub(super) description: &'static str, + pub(super) outcome: JanitorOutcome, + pub(super) run: Option, + pub(super) error: Option<&'static str>, + pub(super) expected_fragments: &'static [&'static str], +} diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/__init__.py similarity index 100% rename from tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py rename to tests/integration/src/sqlbuild/executor/janitor_events/classes/__init__.py diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py new file mode 100644 index 0000000000..dca13b7453 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class JanitorEventWriterTestCase: + description: str + record_count: int + written_before: int + write_now_first: bool + expected_row_count: int + expected_insert_count: int + expected_existence_query_count: int diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py new file mode 100644 index 0000000000..4574848f6d --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py @@ -0,0 +1,168 @@ +"""DuckDB integration tests for chunked, append-only janitor audit events.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.janitor_events.classes.janitor_event_writer import JanitorEventWriter +from sqlbuild.executor.janitor_events.constants import ( + JANITOR_EVENT_COLUMNS, + JANITOR_EVENT_SCHEMA_VERSION, +) +from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType +from tests.integration.src.sqlbuild.executor.janitor_events.classes._test_types import ( + JanitorEventWriterTestCase, +) + +RUN_ID: str = "run_orders_cleanup" +ARCHIVED_AT: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) + + +def _record(index: int) -> JanitorEventRecord: + archive_name: str = f"_sqb_archive__20260924t101500z__orders_{index:04d}" + return JanitorEventRecord( + event_id=build_janitor_event_id( + event_type=JanitorEventType.ARCHIVE, + run_id=RUN_ID, + relation_database=None, + relation_schema="main", + archive_name=archive_name, + ), + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=JanitorEventType.ARCHIVE, + occurred_at=ARCHIVED_AT, + run_id=RUN_ID, + relation_database=None, + relation_schema="main", + relation_type="BASE TABLE", + original_name=f"orders_{index:04d}", + original_qualified_name=f"main.orders_{index:04d}", + archive_name=archive_name, + archive_qualified_name=f"main.{archive_name}", + archived_at=ARCHIVED_AT, + ) + + +class _RecordingExecute: + def __init__(self, adapter: DuckDbAdapter) -> None: + self.adapter: DuckDbAdapter = adapter + self.statements: list[str] = [] + + def __call__(self, *, connection: Any, sql: str) -> Any: + self.statements.append(sql) + return self.adapter.execute(connection=connection, sql=sql) + + +def _writer(execute: _RecordingExecute) -> JanitorEventWriter: + return JanitorEventWriter( + execute=execute, + render_qualified_name=execute.adapter.render_qualified_name, + run_id=RUN_ID, + ) + + +@pytest.mark.parametrize( + "test_case", + ( + JanitorEventWriterTestCase( + description="events past one chunk are appended with one insert per chunk", + record_count=250, + written_before=0, + write_now_first=False, + expected_row_count=250, + expected_insert_count=3, + expected_existence_query_count=1, + ), + JanitorEventWriterTestCase( + description="a retried run skips every event it already wrote", + record_count=5, + written_before=5, + write_now_first=False, + expected_row_count=5, + expected_insert_count=0, + expected_existence_query_count=1, + ), + JanitorEventWriterTestCase( + description="a retried run appends only the events it had not written", + record_count=5, + written_before=3, + write_now_first=False, + expected_row_count=5, + expected_insert_count=1, + expected_existence_query_count=1, + ), + JanitorEventWriterTestCase( + description="an event written at once is not held for the next flush", + record_count=1, + written_before=0, + write_now_first=True, + expected_row_count=1, + expected_insert_count=1, + expected_existence_query_count=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_janitor_events_when_writing_in_chunks_then_each_event_is_appended_once( + test_case: JanitorEventWriterTestCase, + tmp_path: Path, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": str(tmp_path / "janitor_events.duckdb")}) + records: tuple[JanitorEventRecord, ...] = tuple( + _record(index) for index in range(test_case.record_count) + ) + try: + adapter.execute( + connection=connection, + sql=adapter.render_create_janitor_event_table_sql(database=None, schema="main"), + ) + earlier: JanitorEventWriter = _writer(_RecordingExecute(adapter)) + for record in records[: test_case.written_before]: + earlier.add(record) + earlier.flush(connection=connection) + execute: _RecordingExecute = _RecordingExecute(adapter) + writer: JanitorEventWriter = _writer(execute) + writer.read_existing(connection=connection, schemas=((None, "main"),)) + if test_case.write_now_first: + writer.write_now(connection=connection, record=records[0]) + for record in records: + writer.add(record) + writer.flush(connection=connection) + rows: list[tuple[object, ...]] = connection.execute( + f"SELECT {', '.join(JANITOR_EVENT_COLUMNS)} FROM main._sqlbuild_janitor_events " + "ORDER BY archive_name" + ).fetchall() + finally: + adapter.close(connection) + + assert len(rows) == test_case.expected_row_count + assert len({row[0] for row in rows}) == test_case.expected_row_count + assert rows[0] == ( + records[0].event_id, + JANITOR_EVENT_SCHEMA_VERSION, + "archive", + ARCHIVED_AT.replace(tzinfo=None), + RUN_ID, + None, + "main", + "BASE TABLE", + "orders_0000", + "main.orders_0000", + records[0].archive_name, + f"main.{records[0].archive_name}", + ARCHIVED_AT.replace(tzinfo=None), + ) + assert sum(sql.startswith("INSERT INTO") for sql in execute.statements) == ( + test_case.expected_insert_count + ) + assert sum(sql.startswith("SELECT") for sql in execute.statements) == ( + test_case.expected_existence_query_count + ) diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py deleted file mode 100644 index bf162f5406..0000000000 --- a/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class JanitorEventWriteTestCase: - description: str - write_attempts: int - expected_row_count: int - expected_write_results: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py deleted file mode 100644 index c1ff5d333d..0000000000 --- a/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py +++ /dev/null @@ -1,103 +0,0 @@ -"""DuckDB integration tests for append-only janitor audit events.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import pytest - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.executor.janitor_events.constants import ( - JANITOR_EVENT_COLUMNS, - JANITOR_EVENT_SCHEMA_VERSION, -) -from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id -from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record -from sqlbuild.executor.janitor_events.models import JanitorEventRecord -from sqlbuild.executor.janitor_events.types import JanitorEventType -from tests.integration.src.sqlbuild.executor.janitor_events.main._test_types import ( - JanitorEventWriteTestCase, -) - -ARCHIVE_NAME: str = "_SQB_ARCHIVE__20260924T101500Z__orders" - - -@pytest.mark.parametrize( - "test_case", - ( - JanitorEventWriteTestCase( - description="repeated write of one deterministic event appends a single row", - write_attempts=2, - expected_row_count=1, - expected_write_results=(True, False), - ), - ), - ids=lambda case: case.description, -) -def test_given_repeated_janitor_event_when_writing_then_insert_is_idempotent( - test_case: JanitorEventWriteTestCase, - tmp_path: Path, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - connection: Any = adapter.connect({"database": str(tmp_path / "janitor_events.duckdb")}) - archived_at: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) - record: JanitorEventRecord = JanitorEventRecord( - event_id=build_janitor_event_id( - event_type=JanitorEventType.ARCHIVE, - run_id="run_orders_cleanup", - relation_database=None, - relation_schema="main", - archive_name=ARCHIVE_NAME, - ), - schema_version=JANITOR_EVENT_SCHEMA_VERSION, - event_type=JanitorEventType.ARCHIVE, - occurred_at=archived_at, - run_id="run_orders_cleanup", - relation_database=None, - relation_schema="main", - relation_type="BASE TABLE", - original_name="orders", - original_qualified_name="main.orders", - archive_name=ARCHIVE_NAME, - archive_qualified_name=f"main.{ARCHIVE_NAME}", - archived_at=archived_at, - ) - try: - adapter.execute( - connection=connection, - sql=adapter.render_create_janitor_event_table_sql(database=None, schema="main"), - ) - write_results: tuple[bool, ...] = tuple( - write_janitor_event_record( - connection=connection, - execute=adapter.execute, - record=record, - render_qualified_name=adapter.render_qualified_name, - ) - for _ in range(test_case.write_attempts) - ) - rows: list[tuple[object, ...]] = connection.execute( - f"SELECT {', '.join(JANITOR_EVENT_COLUMNS)} FROM main._sqlbuild_janitor_events" - ).fetchall() - finally: - adapter.close(connection) - - assert write_results == test_case.expected_write_results - assert len(rows) == test_case.expected_row_count - assert rows[0] == ( - record.event_id, - JANITOR_EVENT_SCHEMA_VERSION, - "archive", - archived_at.replace(tzinfo=None), - "run_orders_cleanup", - None, - "main", - "BASE TABLE", - "orders", - "main.orders", - ARCHIVE_NAME, - f"main.{ARCHIVE_NAME}", - archived_at.replace(tzinfo=None), - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py index 6b08277a95..3931ef67b2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py @@ -8,3 +8,4 @@ class JanitorCompletionTestCase: deleted: int pruned_direct_state: int expected_output: str + elapsed_seconds: float | None = None diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py index 16012e165a..88f88d54ea 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py @@ -1,4 +1,4 @@ -"""Janitor completion summary wording.""" +"""Janitor completion line wording.""" from __future__ import annotations @@ -18,32 +18,31 @@ "test_case", ( JanitorCompletionTestCase( - description="single archive and object use singular nouns", + description="counts archived, deleted and pruned items", archived=1, deleted=1, pruned_direct_state=0, - expected_output="Archived 1 relation. Deleted 1 object.\n", + expected_output="\u2713 Janitor completed archived 1, deleted 1, pruned 0\n", ), JanitorCompletionTestCase( - description="single pruned state table uses a singular noun", + description="zero counts stay visible", archived=0, deleted=0, pruned_direct_state=1, - expected_output="Deleted 0 objects and pruned 1 direct state table.\n", + expected_output="\u2713 Janitor completed archived 0, deleted 0, pruned 1\n", ), JanitorCompletionTestCase( - description="plural counts keep plural nouns", + description="elapsed time follows the counts", archived=2, deleted=3, pruned_direct_state=2, - expected_output=( - "Archived 2 relations. Deleted 3 objects and pruned 2 direct state tables.\n" - ), + elapsed_seconds=302.4, + expected_output="\u2713 Janitor completed archived 2, deleted 3, pruned 2 (5m 02s)\n", ), ), ids=lambda case: case.description, ) -def test_given_janitor_counts_when_writing_completion_then_nouns_match_counts( +def test_given_janitor_counts_when_writing_completion_then_line_reports_totals( test_case: JanitorCompletionTestCase, ) -> None: result: JanitorExecutionResult = JanitorExecutionResult( @@ -52,7 +51,9 @@ def test_given_janitor_counts_when_writing_completion_then_nouns_match_counts( pruned_direct_state=placeholder_candidates(test_case.pruned_direct_state), ) - output: str = render_janitor_completion(result=result, use_color=False) + output: str = render_janitor_completion( + result=result, elapsed_seconds=test_case.elapsed_seconds, use_color=False + ) assert output == test_case.expected_output diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index c3cc3ae1e4..14bd03402a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -30,7 +30,7 @@ "suppressed archive: blocked.old_model", "No janitor actions will be performed.", "Skipped schemas\n dev contains active source raw.orders", - "Skipped objects\n dev.source_table source relation", + "Skipped objects (1)\n 1 source relation", ), unexpected_output_fragments=( "\033[", @@ -113,3 +113,27 @@ def test_given_direct_archive_plan_when_writing_preview_then_lists_archive_actio assert fragment in output for fragment in test_case.unexpected_output_fragments: assert fragment not in output + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPlanOutputTestCase( + description="verbose preview lists each skipped object with its reason", + use_color=False, + expected_output_fragments=("Skipped objects\n dev.source_table source relation",), + unexpected_output_fragments=("Skipped objects (1)",), + ) + ], + ids=lambda case: case.description, +) +def test_given_janitor_plan_when_writing_verbose_preview_then_lists_every_skipped_object( + test_case: JanitorPlanOutputTestCase, +) -> None: + output: str = render_janitor_plan( + plan=build_janitor_plan(), use_color=test_case.use_color, verbose=True + ) + for fragment in test_case.expected_output_fragments: + assert fragment in output + for fragment in test_case.unexpected_output_fragments: + assert fragment not in output diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 528edca1ef..e0966ea1f1 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -87,6 +87,7 @@ class JanitorArchiveExecutionTestCase: expected_dropped_targets: tuple[str, ...] expected_dropped_view_targets: tuple[str, ...] expected_event_insert_count: int + expected_event_row_count: int expected_event_table_create_count: int diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py index 5ad86f4cfa..d0aa8eaa5a 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py @@ -31,7 +31,7 @@ "test_case", [ JanitorArchiveExecutionTestCase( - description="renames stale table and view, drops expired archive, and audits each", + description="renames stale table and view, drops expired archive, and audits each per phase", relation_infos=( relation_info("old_orders", created_at=OLD_TIME), relation_info("old_customers_view", relation_type="VIEW", created_at=OLD_TIME), @@ -42,7 +42,8 @@ expected_view_rename_origins=("analytics.old_customers_view",), expected_dropped_targets=("analytics._sqb_archive__20200101t000000z__old_products",), expected_dropped_view_targets=(), - expected_event_insert_count=3, + expected_event_insert_count=2, + expected_event_row_count=3, expected_event_table_create_count=1, ), JanitorArchiveExecutionTestCase( @@ -56,7 +57,8 @@ expected_view_rename_origins=("analytics.old_customers_view",), expected_dropped_targets=("analytics._sqb_archive__",), expected_dropped_view_targets=("analytics._sqb_archive__",), - expected_event_insert_count=4, + expected_event_insert_count=2, + expected_event_row_count=4, expected_event_table_create_count=1, ), ], @@ -110,6 +112,10 @@ def test_given_direct_archive_plan_when_executing_then_renames_drops_and_writes_ filter(lambda sql: sql.startswith("CREATE TABLE"), event_statements) ) assert len(event_inserts) == test_case.expected_event_insert_count + assert sum(sql.count("), (") + 1 for sql in event_inserts) == ( + test_case.expected_event_row_count + ) + assert len(tuple(filter(lambda sql: sql.startswith("SELECT"), event_statements))) == 1 assert len(event_creates) == test_case.expected_event_table_create_count assert tuple(candidate.key.name for candidate in result.archived) == tuple( origin.rsplit(".", maxsplit=1)[-1] From b7d707c98a2ac46da8b98d49947c26e9a941df81 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:30:43 +0100 Subject: [PATCH 05/10] test: cover janitor progress, inspection reads and concurrent actions end to end --- .../cli/commands/main/janitor/_test_types.py | 59 +++ .../cli/commands/main/janitor/helpers.py | 182 +++++++++ .../cli/commands/main/janitor/test_janitor.py | 6 +- .../main/janitor/test_janitor_archive.py | 12 +- .../main/janitor/test_janitor_progress.py | 379 ++++++++++++++++++ .../main/janitor/test_janitor_targets.py | 4 +- .../refactor/test_janitor_pending_origin.py | 2 +- .../adapters/snowflake/_test_types.py | 10 + .../snowflake/test_view_definitions.py | 85 ++++ 9 files changed, 727 insertions(+), 12 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py create mode 100644 tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index dfe682a71f..6c315e0b1a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -186,3 +186,62 @@ class JanitorNonInteractiveE2ETestCase: expected_exit_code: int expected_stderr_fragments: tuple[str, ...] expected_existing_tables: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorProgressE2ETestCase: + """Test case for streamed janitor phase and action lines.""" + + description: str + janitor_command: tuple[str, ...] + stale_tables: int + expected_exit_code: int + expected_ordered_lines: tuple[str, ...] + expected_fragments: tuple[str, ...] = field(default_factory=tuple) + unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) + fail_rename: str | None = None + expected_archive_events: int | None = None + + +@dataclass(frozen=True) +class JanitorJsonE2ETestCase: + """Test case for `sqb janitor --json` output streams.""" + + description: str + janitor_command: tuple[str, ...] + stale_tables: int + expected_status: str + expected_archived: int + expected_stderr_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorInspectionCountE2ETestCase: + """Test case for warehouse reads and audit writes per schema.""" + + description: str + stale_tables: dict[str, int] + renamed_models: tuple[str, ...] + expected_listings_per_schema: int + expected_view_definition_reads: int + expected_event_inserts: dict[str, int] + + +@dataclass(frozen=True) +class JanitorConcurrencyE2ETestCase: + """Test case comparing concurrent janitor actions with sequential ones.""" + + description: str + stale_tables: dict[str, int] + concurrency: int + fail_rename: str | None + expected_exit_code: int + + +@dataclass(frozen=True) +class JanitorShortenedArchiveE2ETestCase: + """Test case for the audit event of an archive whose name was shortened.""" + + description: str + long_relation_name: str + identifier_limit: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index dd575d49d8..c986c8fcf6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -460,3 +460,185 @@ def probe_fingerprint_count(*, db_path: Path, schema: str) -> int: ), ) return int(str(rows[0][0])) + + +PROBE_JANITOR_ADAPTER_NAME: str = "probe_janitor_duckdb" +PROBE_JANITOR_SETTINGS_FILENAME: str = "janitor_probe_settings.json" +PROBE_STATEMENT_LOG_FILENAME: str = "janitor_statements.log" +PROBE_JANITOR_ADAPTER_SOURCE: str = """ +import json +import threading +from pathlib import Path +from typing import Any + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter + +_ROOT = Path(__file__).resolve().parent.parent +_LOG_LOCK = threading.Lock() + + +def _settings() -> dict[str, Any]: + path = _ROOT / "janitor_probe_settings.json" + if not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +class ProbeJanitorDuckDbAdapter(DuckDbAdapter): + adapter_name = "probe_janitor_duckdb" + + @property + def concurrent_ddl_limit(self) -> int: + return int(_settings().get("concurrent_ddl_limit", 1)) + + def maximum_identifier_length(self) -> int: + return int(_settings().get("identifier_limit", super().maximum_identifier_length())) + + def _execute(self, *, connection: Any, sql: str) -> Any: + settings = _settings() + failing = settings.get("fail_rename") + if failing and sql.startswith("ALTER TABLE") and f".{failing} RENAME" in sql: + raise AdapterUserError(f"simulated rename failure for {failing}") + if settings.get("log"): + with _LOG_LOCK, (_ROOT / "janitor_statements.log").open("a") as handle: + handle.write(" ".join(sql.split()) + "\\n") + return super()._execute(connection=connection, sql=sql) +""" + + +def prepare_probe_janitor_project( + *, + tmp_path: Path, + project_name: str, + stale_tables: Mapping[str, int], + renamed_models: tuple[str, ...] = (), + concurrency: int = 1, +) -> Path: + """Build a DuckDB project with stale tables, an expired archive and old-name views per schema. + + Every schema in `stale_tables` gets one live model, each name in `renamed_models` renamed + once (leaving a compatibility view at its old name), `stale_tables[schema]` stale tables + and one expired archive. Builds use DuckDB; janitor then runs on the probe adapter, which + logs statements once settings enable it. + """ + + def model(schema: str, migrate_from: str = "") -> str: + migration: str = f', migrate_from "{migrate_from}"' if migrate_from else "" + return f"MODEL (description 'Test model.', schema {schema}{migration});\n\nSELECT 1 AS id\n" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name=project_name, + repo_files={ + "sqlbuild_project.toml": ( + f'name = "{project_name}"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + f'database = "{(tmp_path / project_name / "janitor.duckdb").as_posix()}"\n\n' + f"[settings]\nconcurrency = {concurrency}\n\n" + "[janitor]\n" + "enabled = true\n" + "retention_days = 0\n" + "archive_retention_days = 14\n" + "delete_tracked_only = false\n" + "direct_state_history_versions = 2\n\n" + "[defaults]\n" + 'materialized = "table"\n' + ), + f"adapters/{PROBE_JANITOR_ADAPTER_NAME}.py": PROBE_JANITOR_ADAPTER_SOURCE, + **{ + f"models/{schema}_{name}.sql": model(schema) + for schema in stale_tables + for name in ("orders", *renamed_models) + }, + }, + ) + _build(project_dir) + if renamed_models: + for schema in stale_tables: + for name in renamed_models: + (project_dir / "models" / f"{schema}_{name}.sql").unlink() + (project_dir / "models" / f"{schema}_{name}_v2.sql").write_text( + model(schema, migrate_from=f"{schema}.{schema}_{name}"), encoding="utf-8" + ) + _build(project_dir) + db_path: Path = project_dir / "janitor.duckdb" + for schema, count in stale_tables.items(): + tables: str = "; ".join( + f"CREATE TABLE {schema}.old_orders_{index:03d} AS SELECT {index} AS id" + for index in range(count) + ) + execute_duckdb( + db_path=db_path, + sql=f'{tables}; CREATE TABLE {schema}."{EXPIRED_ARCHIVE_NAME}" AS SELECT 1 AS id', + ) + (project_dir / "sqlbuild_local.toml").write_text( + f'adapter = "{PROBE_JANITOR_ADAPTER_NAME}"\n', encoding="utf-8" + ) + return project_dir + + +def _build(project_dir: Path) -> None: + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def write_probe_settings( + *, + project_dir: Path, + concurrent_ddl_limit: int = 1, + fail_rename: str | None = None, + identifier_limit: int | None = None, +) -> None: + """Enable statement logging and set the probe adapter's concurrency and fault switches.""" + + import json + + payload: dict[str, object] = {"log": True, "concurrent_ddl_limit": concurrent_ddl_limit} + if fail_rename is not None: + payload["fail_rename"] = fail_rename + if identifier_limit is not None: + payload["identifier_limit"] = identifier_limit + (project_dir / PROBE_JANITOR_SETTINGS_FILENAME).write_text( + json.dumps(payload), encoding="utf-8" + ) + + +def read_probe_statements(*, project_dir: Path) -> tuple[str, ...]: + """Return every statement the probe adapter logged, in order.""" + + log: Path = project_dir / PROBE_STATEMENT_LOG_FILENAME + if not log.exists(): + return () + return tuple(log.read_text(encoding="utf-8").splitlines()) + + +def janitor_end_state(*, db_path: Path) -> tuple[tuple[object, ...], ...]: + """Every relation with archive timestamps masked, then every audit event without its run.""" + + relations: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT table_schema, regexp_replace(table_name, '[0-9]{8}t[0-9]{6}z', 'ts') " + "FROM information_schema.tables WHERE table_schema NOT IN ('information_schema') " + "ORDER BY 1, 2" + ), + ) + events: list[tuple[object, ...]] = [] + for schema in sorted({str(row[0]) for row in relations}): + if (schema, "_sqlbuild_janitor_events") not in relations: + continue + events.extend( + query_duckdb( + db_path=db_path, + sql=( + "SELECT relation_schema, event_type, original_name, " + "regexp_replace(archive_name, '[0-9]{8}t[0-9]{6}z', 'ts') " + f"FROM {schema}._sqlbuild_janitor_events ORDER BY 1, 2, 3, 4" + ), + ) + ) + return (*relations, *events) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 35f6ba4e68..f4896cd156 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -72,7 +72,7 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( JanitorCleanupE2ETestCase( description="tracked-only direct janitor archives only tracked stale relations", build_command=("--no-color", "build", "--full-refresh"), - janitor_command=("janitor", "--auto-approve"), + janitor_command=("janitor", "--auto-approve", "--verbose"), expected_exit_code=0, expected_stdout_fragments=( "relations to archive 1", @@ -311,7 +311,7 @@ def test_given_source_in_managed_schema_when_running_direct_janitor_then_blocks_ JanitorCleanupE2ETestCase( description="tracked-only direct janitor archives strict scenario artifacts", build_command=("--no-color", "build", "--full-refresh"), - janitor_command=("janitor", "--auto-approve"), + janitor_command=("janitor", "--auto-approve", "--verbose"), expected_exit_code=0, expected_stdout_fragments=( "relations to archive 2", @@ -395,7 +395,7 @@ def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_t "Eligible direct state pruning", "main._sqlbuild_fingerprints keep latest 2", "main._sqlbuild_source_freshness keep latest 2", - "pruned 2 direct state tables", + "Pruned 2 direct state tables.", ), expected_fingerprint_count_before=5, expected_fingerprint_count_after=3, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py index dbd110d627..1b830691c3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py @@ -47,7 +47,7 @@ [ JanitorArchiveRetentionE2ETestCase( description="retired tracked relation is archived after retention and then retained", - janitor_command=("--no-color", "janitor", "--auto-approve"), + janitor_command=("--no-color", "janitor", "--auto-approve", "--verbose"), retired_model_ages_days={"customers": 20, "products": 3}, expected_archived_names=("customers",), expected_retained_names=("products",), @@ -136,7 +136,7 @@ def test_given_retired_tracked_relation_past_retention_when_running_janitor_then [ JanitorArchiveExpiryE2ETestCase( description="only expired strict archives in managed schemas are deleted", - janitor_command=("--no-color", "janitor", "--auto-approve"), + janitor_command=("--no-color", "janitor", "--auto-approve", "--verbose"), expected_deleted_names=( "_sqb_archive__20200101t000000z__old_customers_view", "_sqb_archive__20200101t000000z__old_products", @@ -163,7 +163,7 @@ def test_given_retired_tracked_relation_past_retention_when_running_janitor_then "main._sqb_archive_notes name resembles a janitor archive", "main._SQB_ARCHIVE__20200101T000000Z__legacy_orders relation name is not a plain " "lowercase identifier and may require quoting; janitor does not act on it", - "Deleted 2 objects", + "\u2713 Janitor completed archived 0, deleted 2", ), expected_delete_event_count=2, expected_total_event_count=5, @@ -262,7 +262,7 @@ def test_given_archive_named_relations_when_running_janitor_then_deletes_only_ex "archived in this run", "Eligible direct state pruning", "main._sqlbuild_fingerprints keep latest 2", - "Archived 2 relations. Deleted 2 objects", + "\u2713 Janitor completed archived 2, deleted 2", ), expected_event_types=( ("archive", "stale_customers_view"), @@ -405,7 +405,7 @@ def test_given_small_identifier_limit_when_archiving_long_name_then_fits_name_an expected_second_stdout_fragments=( "archives to delete 1", "Archives to delete", - "Deleted 1 object", + "\u2713 Janitor completed archived 0, deleted 1", ), ) ], @@ -478,7 +478,7 @@ def test_given_audit_write_interrupted_after_rename_when_janitor_reruns_then_arc [ JanitorUnaddressableRelationE2ETestCase( description="quoted mixed-case relation is reported and left while plain one archives", - janitor_command=("--no-color", "janitor", "--auto-approve"), + janitor_command=("--no-color", "janitor", "--auto-approve", "--verbose"), plain_relation="old_orders", quoted_relation="LegacyCustomers", expected_stdout_fragments=( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py new file mode 100644 index 0000000000..bf2f470fa9 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py @@ -0,0 +1,379 @@ +"""E2E tests for janitor progress output, inspection reads, concurrency and audit batching.""" + +from __future__ import annotations + +import json +import math +import re +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorConcurrencyE2ETestCase, + JanitorInspectionCountE2ETestCase, + JanitorJsonE2ETestCase, + JanitorProgressE2ETestCase, + JanitorShortenedArchiveE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( + janitor_end_state, + prepare_probe_janitor_project, + read_probe_statements, + write_probe_settings, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + query_duckdb, + run_sqb, + table_exists, +) + +ITEM_LINE: re.Pattern[str] = re.compile(r"^\s+(?P\d+)/(?P\d+) (?P\w+)\s") + + +def _assert_in_order(output: str, lines: tuple[str, ...]) -> None: + position: int = 0 + line: str + for line in lines: + found: int = output.find(line, position) + assert found >= 0, f"{line!r} missing after position {position} in:\n{output}" + position = found + len(line) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorProgressE2ETestCase( + description="each phase announces its work, streams each action and closes", + janitor_command=("--no-color", "janitor", "--auto-approve"), + stale_tables=12, + expected_exit_code=0, + expected_ordered_lines=( + "Janitor preview default", + "Relations to archive", + " main.old_orders_009 -> main._sqb_archive__", + " ... and 2 more (use --verbose to show all)", + "Skipped objects (", + "Archiving 12 relations...", + " 1/12 archived main.old_orders_000 -> main._sqb_archive__", + " OK ", + " 12/12 archived main.old_orders_011 -> main._sqb_archive__", + "Archived 12 relations. (", + "Deleting 1 expired archive...", + " 1/1 deleted main._sqb_archive__20200101t000000z__old_products OK", + "Deleted 1 expired archive. (", + "Pruning 1 direct state table...", + " 1/1 pruned main._sqlbuild_fingerprints OK", + "Pruned 1 direct state table. (", + "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", + ), + unexpected_fragments=( + " main.old_orders_010 -> ", + "main._sqlbuild_fingerprints relation matches exclude pattern", + ), + ), + JanitorProgressE2ETestCase( + description="verbose preview lists every relation and skipped object", + janitor_command=("--no-color", "janitor", "--auto-approve", "--verbose"), + stale_tables=12, + expected_exit_code=0, + expected_ordered_lines=( + " main.old_orders_011 -> main._sqb_archive__", + "Skipped objects\n main._sqlbuild_fingerprints relation matches exclude pattern", + "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", + ), + unexpected_fragments=("... and", "Skipped objects ("), + ), + JanitorProgressE2ETestCase( + description="a failing rename keeps earlier lines, marks the item and fails the run", + janitor_command=("--no-color", "janitor", "--auto-approve"), + stale_tables=12, + expected_exit_code=1, + fail_rename="old_orders_005", + expected_ordered_lines=( + "Archiving 12 relations...", + " 5/12 archived main.old_orders_004 -> main._sqb_archive__", + " 6/12 archived main.old_orders_005 -> main._sqb_archive__", + " FAIL ", + "simulated rename failure for old_orders_005", + "\u2717 Archiving failed after 5 of 12 relations (", + "\u2717 Janitor failed archived 5, deleted 0, pruned 0 (", + ), + unexpected_fragments=("7/12", "Deleting 1 expired archive", "Janitor completed"), + expected_archive_events=5, + ), + ], + ids=lambda case: case.description, +) +def test_given_stale_relations_when_running_janitor_then_progress_streams_in_order( + test_case: JanitorProgressE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, + project_name="janitor_progress", + stale_tables={"main": test_case.stale_tables}, + ) + write_probe_settings(project_dir=project_dir, fail_rename=test_case.fail_rename) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + _assert_in_order(result.stdout, test_case.expected_ordered_lines) + for fragment in test_case.unexpected_fragments: + assert fragment not in result.stdout + if test_case.expected_archive_events is not None: + db_path: Path = project_dir / "janitor.duckdb" + assert query_duckdb( + db_path=db_path, + sql="SELECT count(*) FROM main._sqlbuild_janitor_events WHERE event_type = 'archive'", + ) == [(test_case.expected_archive_events,)] + assert table_exists(db_path=db_path, table_name=test_case.fail_rename or "") + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorJsonE2ETestCase( + description="stdout is one JSON document and progress goes to stderr", + janitor_command=("janitor", "--auto-approve", "--json"), + stale_tables=3, + expected_status="completed", + expected_archived=3, + expected_stderr_fragments=( + "Janitor preview default", + "Archiving 3 relations...", + " 1/3 archived main.old_orders_000 -> main._sqb_archive__", + "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_json_flag_when_running_janitor_then_stdout_holds_plan_and_result( + test_case: JanitorJsonE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, + project_name="janitor_json", + stale_tables={"main": test_case.stale_tables}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["status"] == test_case.expected_status + assert len(payload["plan"]["relations_to_archive"]) == test_case.stale_tables + assert payload["result"]["counts"]["archived"] == test_case.expected_archived + assert [item["status"] for item in payload["result"]["items"]] == ["ok"] * ( + test_case.stale_tables + 2 + ) + assert payload["error"] is None + _assert_in_order(result.stderr, test_case.expected_stderr_fragments) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorInspectionCountE2ETestCase( + description="each schema is listed once, its views read once, and events chunked", + stale_tables={"main": 150, "mart": 3, "staging": 3}, + renamed_models=("revenue", "customers"), + expected_listings_per_schema=1, + expected_view_definition_reads=3, + expected_event_inserts={"main": 3, "mart": 2, "staging": 2}, + ), + ], + ids=lambda case: case.description, +) +def test_given_several_schemas_when_running_janitor_then_reads_and_writes_scale_per_schema( + test_case: JanitorInspectionCountE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, + project_name="janitor_counts", + stale_tables=test_case.stale_tables, + renamed_models=test_case.renamed_models, + ) + write_probe_settings(project_dir=project_dir) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "janitor", "--auto-approve"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + statements: tuple[str, ...] = read_probe_statements(project_dir=project_dir) + schema: str + for schema in test_case.stale_tables: + listings: tuple[str, ...] = tuple( + statement + for statement in statements + if "FROM information_schema.tables" in statement + and f"table_schema IN ('{schema}')" in statement + and "table_catalog" not in statement + ) + assert len(listings) == test_case.expected_listings_per_schema, listings + existence: tuple[str, ...] = tuple( + statement + for statement in statements + if statement.startswith(f"SELECT event_id FROM {schema}._sqlbuild_janitor_events") + ) + assert len(existence) == 1, existence + inserts: int = sum( + statement.startswith(f"INSERT INTO {schema}._sqlbuild_janitor_events") + for statement in statements + ) + archived: int = test_case.stale_tables[schema] + assert inserts == test_case.expected_event_inserts[schema] == math.ceil(archived / 100) + 1 + assert not any( + "FROM information_schema.tables" in statement and "', '" in statement + for statement in statements + ) + view_reads: tuple[str, ...] = tuple( + statement for statement in statements if "FROM duckdb_views()" in statement + ) + assert len(view_reads) == test_case.expected_view_definition_reads, view_reads + assert all(" IN (" in statement for statement in view_reads) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorConcurrencyE2ETestCase( + description="concurrent actions leave the same end state as sequential ones", + stale_tables={"main": 30, "staging": 5}, + concurrency=4, + fail_rename=None, + expected_exit_code=0, + ), + JanitorConcurrencyE2ETestCase( + description="a failing concurrent rename is reported alone and events stay complete", + stale_tables={"main": 30}, + concurrency=4, + fail_rename="old_orders_012", + expected_exit_code=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_adapter_when_running_janitor_then_matches_sequential_end_state( + test_case: JanitorConcurrencyE2ETestCase, tmp_path: Path +) -> None: + sequential_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path / "sequential", + project_name="janitor_sequential", + stale_tables=test_case.stale_tables, + ) + concurrent_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path / "concurrent", + project_name="janitor_concurrent", + stale_tables=test_case.stale_tables, + concurrency=test_case.concurrency, + ) + write_probe_settings(project_dir=sequential_dir) + write_probe_settings( + project_dir=concurrent_dir, + concurrent_ddl_limit=test_case.concurrency, + fail_rename=test_case.fail_rename, + ) + command: tuple[str, ...] = ("--no-color", "janitor", "--auto-approve") + + sequential: subprocess.CompletedProcess[str] = run_sqb( + command=command, project_dir=sequential_dir + ) + concurrent: subprocess.CompletedProcess[str] = run_sqb( + command=command, project_dir=concurrent_dir + ) + + assert sequential.returncode == 0, sequential.stdout + sequential.stderr + assert concurrent.returncode == test_case.expected_exit_code, ( + concurrent.stdout + concurrent.stderr + ) + total: int = sum(test_case.stale_tables.values()) + archive_lines: list[re.Match[str]] = [ + match + for line in concurrent.stdout.splitlines() + if (match := ITEM_LINE.match(line)) is not None and match["action"] == "archived" + ] + assert [int(match["index"]) for match in archive_lines] == list( + range(1, len(archive_lines) + 1) + ) + assert {int(match["total"]) for match in archive_lines} == {total} + assert f"Connecting to probe_janitor_duckdb ({test_case.concurrency - 1} connections)" in ( + concurrent.stdout + ) + concurrent_db: Path = concurrent_dir / "janitor.duckdb" + if test_case.fail_rename is None: + assert len(archive_lines) == total + assert janitor_end_state(db_path=concurrent_db) == janitor_end_state( + db_path=sequential_dir / "janitor.duckdb" + ) + return + assert sum(" FAIL " in line for line in concurrent.stdout.splitlines()) == 1 + assert "\u2717 Janitor failed archived" in concurrent.stdout + assert table_exists(db_path=concurrent_db, table_name=test_case.fail_rename) + renamed: list[tuple[object, ...]] = query_duckdb( + db_path=concurrent_db, + sql=( + "SELECT count(*) FROM information_schema.tables " + "WHERE table_schema = 'main' AND table_name LIKE '\\_sqb\\_archive\\_\\_%old\\_orders%' " + "ESCAPE '\\'" + ), + ) + assert query_duckdb( + db_path=concurrent_db, + sql="SELECT count(*) FROM main._sqlbuild_janitor_events WHERE event_type = 'archive'", + ) == renamed + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorShortenedArchiveE2ETestCase( + description="a shortened archive name gets its event right after its rename", + long_relation_name="legacy_orders_with_a_very_long_descriptive_name_kept_for_audit", + identifier_limit=48, + ), + ], + ids=lambda case: case.description, +) +def test_given_shortened_archive_name_when_archiving_then_event_is_written_immediately( + test_case: JanitorShortenedArchiveE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_shortened", stale_tables={"main": 3} + ) + execute_duckdb( + db_path=project_dir / "janitor.duckdb", + sql=f"CREATE TABLE main.{test_case.long_relation_name} AS SELECT 1 AS id", + ) + write_probe_settings(project_dir=project_dir, identifier_limit=test_case.identifier_limit) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "janitor", "--auto-approve"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + statements: tuple[str, ...] = read_probe_statements(project_dir=project_dir) + rename: int = next( + index + for index, statement in enumerate(statements) + if statement.startswith(f"ALTER TABLE main.{test_case.long_relation_name} RENAME") + ) + assert statements[rename + 1].startswith("INSERT INTO main._sqlbuild_janitor_events") + assert f"'{test_case.long_relation_name}'" in statements[rename + 1] + assert statements[rename + 1].count("), (") == 0 + assert query_duckdb( + db_path=project_dir / "janitor.duckdb", + sql=( + "SELECT count(*) FROM main._sqlbuild_janitor_events " + f"WHERE original_name = '{test_case.long_relation_name}'" + ), + ) == [(1,)] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py index dc0169530d..75a481aa83 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -162,7 +162,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview prod", "prod.customers -> prod._sqb_archive__", - "Archived 1 relation. Deleted 1 object and pruned 2 direct state tables.", + "\u2713 Janitor completed archived 1, deleted 1, pruned 2", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), @@ -176,7 +176,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview dev", "dev.customers -> dev._sqb_archive__", - "Archived 1 relation. Deleted 1 object and pruned 2 direct state tables.", + "\u2713 Janitor completed archived 1, deleted 1, pruned 2", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py index f890b505b4..ce18fc8673 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_janitor_pending_origin.py @@ -27,7 +27,7 @@ [ PendingOriginJanitorE2ETestCase( description="janitor keeps the origin until the rename is migrated", - janitor_args=("janitor", "--retention-days", "0", "--auto-approve"), + janitor_args=("janitor", "--retention-days", "0", "--auto-approve", "--verbose"), expected_order_ids=(1, 2, 3, 4), ), ], diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 4637b11380..2ff4b3fc2f 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -284,3 +284,13 @@ class SnowflakeStateTableSchemaFallbackTestCase: rejecting_schema: str accepting_schema: str expected_executed: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class SnowflakeViewDefinitionsTestCase: + description: str + stored_rows: tuple[tuple[str, str], ...] + prefetched_names: tuple[str, ...] + read_names: tuple[str, ...] + expected_definitions: tuple[str | None, ...] + expected_query_count: int diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py new file mode 100644 index 0000000000..501548a70b --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py @@ -0,0 +1,85 @@ +"""Snowflake view definitions read one schema at a time through the inspection catalog.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog +from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeViewDefinitionsTestCase, +) + + +class _Rows: + def __init__(self, rows: list[tuple[str, ...]]) -> None: + self._rows: list[tuple[str, ...]] = rows + + def fetchall(self) -> list[tuple[str, ...]]: + return self._rows + + +class _ViewDefinitionAdapter(SnowflakeAdapter): + """Answers INFORMATION_SCHEMA.VIEWS reads from stored rows, matching names like Snowflake.""" + + def __init__(self, *, stored: tuple[tuple[str, str], ...]) -> None: + super().__init__() + self.stored: tuple[tuple[str, str], ...] = stored + self.queries: list[str] = [] + + def execute(self, *, connection: Any, sql: str) -> Any: + del connection + self.queries.append(sql) + if sql.startswith("SELECT TABLE_NAME, VIEW_DEFINITION"): + return _Rows([row for row in self.stored if f"UPPER('{row[0]}')" in sql.upper()]) + return _Rows( + [(definition,) for name, definition in self.stored if f"'{name.upper()}'" in sql.upper()] + ) + + +@pytest.mark.parametrize( + "test_case", + ( + SnowflakeViewDefinitionsTestCase( + description="prefetched views are served without further queries, absent ones as none", + stored_rows=(("ORDERS_VIEW", "SELECT 1"), ("CUSTOMERS_VIEW", "SELECT 2")), + prefetched_names=("orders_view", "customers_view", "missing_view"), + read_names=("orders_view", "customers_view", "missing_view"), + expected_definitions=("SELECT 1", "SELECT 2", None), + expected_query_count=1, + ), + SnowflakeViewDefinitionsTestCase( + description="a view outside the prefetch is still read on its own", + stored_rows=(("ORDERS_VIEW", "SELECT 1"), ("PRODUCTS_VIEW", "SELECT 3")), + prefetched_names=("orders_view",), + read_names=("orders_view", "products_view"), + expected_definitions=("SELECT 1", "SELECT 3"), + expected_query_count=2, + ), + ), + ids=lambda case: case.description, +) +def test_given_prefetched_schema_when_reading_view_definitions_then_one_query_serves_them( + test_case: SnowflakeViewDefinitionsTestCase, +) -> None: + adapter: _ViewDefinitionAdapter = _ViewDefinitionAdapter(stored=test_case.stored_rows) + connection: object = object() + + catalog: InspectionCatalog + with open_inspection_catalog(adapter=adapter, connection=connection) as catalog: + catalog.prefetch_view_definitions( + requests=(("analytics_db", "analytics", test_case.prefetched_names),) + ) + definitions: tuple[str | None, ...] = tuple( + adapter.read_view_definition( + connection=connection, database="analytics_db", schema="analytics", name=name + ) + for name in test_case.read_names + ) + + assert definitions == test_case.expected_definitions + assert len(adapter.queries) == test_case.expected_query_count + assert "UPPER(TABLE_NAME) IN (" in adapter.queries[0] From 24fba2736ec88b003419ac28d1726159fffce46a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:36:08 +0100 Subject: [PATCH 06/10] docs: describe janitor progress output, --verbose, --json and faster cleanup --- .../sqlbuild/references/docs/cli/janitor.md | 43 ++++++++++++++++++- .../docs/concepts/project-configuration.md | 2 +- website/src/content/docs/docs/cli/janitor.mdx | 41 +++++++++++++++++- .../docs/concepts/project-configuration.mdx | 2 +- 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 7d23b51609..25074a0d76 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -10,6 +10,8 @@ Online: https://sqlbuild.com/docs/cli/janitor/ - Usage - Flags +- Output +- Speed - Targets - Lifecycle - Compatibility views @@ -39,14 +41,49 @@ sqb --project-dir janitor [flags] | `--auto-approve` | Skip the confirmation prompt and run the planned actions immediately | | `--retention-days` | Override the configured retention period (days) before a stale relation is archived | | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | +| `--verbose`, `-v` | List every entry of each preview section and every skipped object with its reason. Without it, each section shows its first 10 entries and skipped objects are counted by reason | +| `--json` | Print the plan and the result as one JSON document on stdout; the preview, prompt and progress go to stderr | | `--drop-old-name-view ` | Drop the [compatibility view](../concepts/models/migrations.md#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | +## Output + +After the preview and confirmation, each phase with work announces how many actions it runs, prints one line per action as it finishes, and closes with its duration. The run ends with one line of totals: + +``` +Archiving 3 relations... + 1/3 archived analytics.old_orders -> analytics._sqb_archive__20261022t101500z__old_orders OK 0.21s + 2/3 archived analytics.old_customers -> analytics._sqb_archive__20261022t101500z__old_customers OK 0.19s + 3/3 archived analytics.old_products -> analytics._sqb_archive__20261022t101500z__old_products OK 0.24s +Archived 3 relations. (0.64s) +Deleting 1 expired archive... + 1/1 deleted analytics._sqb_archive__20261001t080000z__old_inventory OK 0.12s +Deleted 1 expired archive. (0.12s) + +✓ Janitor completed archived 3, deleted 1, pruned 0 (0.81s) +``` + +The phases are archiving stale relations, deleting expired archives, dropping old-name views, dropping expired query artifacts and pruning direct state, in that order. If an action fails, its line shows `FAIL` and the error, no further action starts, and the run ends with `✗ Janitor failed` and the totals completed before the failure, exiting with `1`. + +Long preview sections show their first 10 entries and then `... and N more (use --verbose to show all)`. Skipped objects are counted by reason: + +``` +Skipped objects (385) + 210 relation is newer than 14 days + 175 relation is not tracked by SQLBuild +``` + +With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. + +## Speed + +The janitor reads each schema's relations once per run, and the definitions of compatibility views with one read per schema. On warehouses that run DDL concurrently (Snowflake, BigQuery and Databricks), the independent actions of each phase run on up to the target's [`concurrency`](../concepts/project-configuration.md#target-concurrency) connections at once; phases still run one after another. DuckDB, SQLite and the other adapters apply one action at a time. + ## Targets By default the janitor cleans up the active target. `--target ` selects another configured target, exactly like `sqb build --target`: the janitor connects with that target's connection and inspects and cleans that target's schemas and state tables. Other targets are never touched. -`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](plan.md). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists every relation it would archive, every archive it would delete, every compatibility view it would drop and every state table it would prune, then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. +`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](plan.md). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists the relations it would archive, the archives it would delete, the compatibility views it would drop and the state tables it would prune (all of them with `--verbose`), then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. ``` Previewing janitor as target 'prod' through the connection of target 'dev' (inspection only). @@ -123,6 +160,8 @@ Ages are only looked up for relations that are otherwise candidates, and not at Every archive and every delete is recorded in an append-only `_sqlbuild_janitor_events` table in the affected schema. It records the original name, the archive name, the time and the invocation. The janitor never reads this table to decide what to do. It exists for auditing, and for restoring archives whose names were shortened. +Events are written in batches of up to 100 per schema, after each batch of actions and after a failure. An archive whose name was shortened gets its event right after its rename. If the process is killed, at most one batch of actions in a schema can lack events; those archives still expire by the timestamp in their names. + ## Restoring an archive Rename the archive back to its original name before it expires, using your warehouse's rename statement. For example, on Snowflake: @@ -173,4 +212,4 @@ In addition to archiving stale relations, the janitor prunes old rows from `_sql ## Safety -The janitor shows every planned archive and delete, with exact names, ages and expiry, then asks you to type an exact confirmation phrase. Relations whose names differ only by case from another relation in the same schema are reported and never dropped, and exclude patterns match regardless of case. Use `--auto-approve` only in automation, or when you're certain. +The janitor shows the planned archives and deletes, with exact names, ages and expiry (every one of them with `--verbose`, or in the `--json` plan), then asks you to type an exact confirmation phrase. Relations whose names differ only by case from another relation in the same schema are reported and never dropped, and exclude patterns match regardless of case. Use `--auto-approve` only in automation, or when you're certain. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index c64f5cdca5..25e52f4ea7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -689,7 +689,7 @@ delta_compile = true | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](models/configuration.md#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](models/contracts.md). | -| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`, and for `sqb janitor` actions on warehouses that run DDL concurrently; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](python-nodes/loaders.md). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 91089bee3f..6ac27db06b 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -22,14 +22,49 @@ sqb --project-dir janitor [flags] | `--auto-approve` | Skip the confirmation prompt and run the planned actions immediately | | `--retention-days` | Override the configured retention period (days) before a stale relation is archived | | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | +| `--verbose`, `-v` | List every entry of each preview section and every skipped object with its reason. Without it, each section shows its first 10 entries and skipped objects are counted by reason | +| `--json` | Print the plan and the result as one JSON document on stdout; the preview, prompt and progress go to stderr | | `--drop-old-name-view ` | Drop the [compatibility view](/docs/concepts/models/migrations#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | +## Output + +After the preview and confirmation, each phase with work announces how many actions it runs, prints one line per action as it finishes, and closes with its duration. The run ends with one line of totals: + +``` +Archiving 3 relations... + 1/3 archived analytics.old_orders -> analytics._sqb_archive__20261022t101500z__old_orders OK 0.21s + 2/3 archived analytics.old_customers -> analytics._sqb_archive__20261022t101500z__old_customers OK 0.19s + 3/3 archived analytics.old_products -> analytics._sqb_archive__20261022t101500z__old_products OK 0.24s +Archived 3 relations. (0.64s) +Deleting 1 expired archive... + 1/1 deleted analytics._sqb_archive__20261001t080000z__old_inventory OK 0.12s +Deleted 1 expired archive. (0.12s) + +✓ Janitor completed archived 3, deleted 1, pruned 0 (0.81s) +``` + +The phases are archiving stale relations, deleting expired archives, dropping old-name views, dropping expired query artifacts and pruning direct state, in that order. If an action fails, its line shows `FAIL` and the error, no further action starts, and the run ends with `✗ Janitor failed` and the totals completed before the failure, exiting with `1`. + +Long preview sections show their first 10 entries and then `... and N more (use --verbose to show all)`. Skipped objects are counted by reason: + +``` +Skipped objects (385) + 210 relation is newer than 14 days + 175 relation is not tracked by SQLBuild +``` + +With `--json`, stdout holds one document with `status` (`completed`, `failed`, `cancelled`, `blocked`, `nothing_to_do`, `previewed` or `disabled`), the full `plan`, the `result` with counts and every action, and `error`. Errors before the cleanup starts, such as a refused confirmation, are reported on stderr only. + +## Speed + +The janitor reads each schema's relations once per run, and the definitions of compatibility views with one read per schema. On warehouses that run DDL concurrently (Snowflake, BigQuery and Databricks), the independent actions of each phase run on up to the target's [`concurrency`](/docs/concepts/project-configuration#target-concurrency) connections at once; phases still run one after another. DuckDB, SQLite and the other adapters apply one action at a time. + ## Targets By default the janitor cleans up the active target. `--target ` selects another configured target, exactly like `sqb build --target`: the janitor connects with that target's connection and inspects and cleans that target's schemas and state tables. Other targets are never touched. -`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](/docs/cli/plan). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists every relation it would archive, every archive it would delete, every compatibility view it would drop and every state table it would prune, then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. +`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](/docs/cli/plan). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists the relations it would archive, the archives it would delete, the compatibility views it would drop and the state tables it would prune (all of them with `--verbose`), then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. ``` Previewing janitor as target 'prod' through the connection of target 'dev' (inspection only). @@ -106,6 +141,8 @@ Ages are only looked up for relations that are otherwise candidates, and not at Every archive and every delete is recorded in an append-only `_sqlbuild_janitor_events` table in the affected schema. It records the original name, the archive name, the time and the invocation. The janitor never reads this table to decide what to do. It exists for auditing, and for restoring archives whose names were shortened. +Events are written in batches of up to 100 per schema, after each batch of actions and after a failure. An archive whose name was shortened gets its event right after its rename. If the process is killed, at most one batch of actions in a schema can lack events; those archives still expire by the timestamp in their names. + ## Restoring an archive Rename the archive back to its original name before it expires, using your warehouse's rename statement. For example, on Snowflake: @@ -156,4 +193,4 @@ In addition to archiving stale relations, the janitor prunes old rows from `_sql ## Safety -The janitor shows every planned archive and delete, with exact names, ages and expiry, then asks you to type an exact confirmation phrase. Relations whose names differ only by case from another relation in the same schema are reported and never dropped, and exclude patterns match regardless of case. Use `--auto-approve` only in automation, or when you're certain. +The janitor shows the planned archives and deletes, with exact names, ages and expiry (every one of them with `--verbose`, or in the `--json` plan), then asks you to type an exact confirmation phrase. Relations whose names differ only by case from another relation in the same schema are reported and never dropped, and exclude patterns match regardless of case. Use `--auto-approve` only in automation, or when you're certain. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 22e328e7ab..ef2438826b 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -666,7 +666,7 @@ delta_compile = true | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](/docs/concepts/models/configuration#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](/docs/concepts/models/contracts). | -| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`, and for `sqb janitor` actions on warehouses that run DDL concurrently; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](/docs/concepts/python-nodes/loaders). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | From 6c2a058ad7d19ed88c15a471b963e8c6275da48e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:51:46 +0100 Subject: [PATCH 07/10] refactor: split janitor worker and phase helpers to follow repository policy --- .../src/janitor_text/_helpers/phases.rs | 5 +- .../src/janitor_text/_helpers/plan_json.rs | 18 +- .../janitor_text/_helpers/plan_sections.rs | 5 +- .../janitor_text/main/format_janitor_json.rs | 5 +- .../main/render_janitor_completion.rs | 5 +- .../janitor_text/main/render_janitor_plan.rs | 6 +- .../sqlbuild-core/src/janitor_text/tests.rs | 4 +- .../tests/test_janitor_progress_text.rs | 15 +- .../_helpers/janitor/janitor_command.rs | 36 ++- .../_helpers/janitor/janitor_facts.rs | 5 +- .../_helpers/janitor/janitor_old_names.rs | 4 +- .../_helpers/janitor/janitor_phases.rs | 63 ++--- fensu.toml | 6 - .../adapter/contract/classes/base_adapter.py | 2 - .../relations/classes/inspection_catalog.py | 6 +- .../snowflake/classes/snowflake_adapter.py | 11 +- .../janitor/_helpers/archive_execution.py | 78 +++--- .../executor/janitor/_helpers/deletion.py | 38 +-- .../executor/janitor/_helpers/phase_steps.py | 63 +++++ .../executor/janitor/_helpers/worker_runs.py | 98 +++++++ .../executor/janitor/_helpers/workers.py | 58 ++++ .../janitor/classes/janitor_item_reporter.py | 44 +++ .../janitor/classes/janitor_workers.py | 95 ++----- .../janitor/classes/pooled_janitor_attempt.py | 25 ++ .../janitor/main/_run_janitor_phase.py | 123 ++++----- src/sqlbuild/executor/janitor/main/execute.py | 85 ++---- src/sqlbuild/executor/janitor/models.py | 17 ++ .../classes/janitor_event_writer.py | 98 +++---- .../executor/janitor_events/constants.py | 2 - .../old_name_views/_helpers/janitor.py | 51 ++-- .../main/_drop_old_name_views.py | 2 +- .../main/_close_worker_connection_pool.py | 16 ++ .../main/_open_worker_connection_pool.py | 18 ++ .../cli/commands/main/janitor/_test_types.py | 25 +- .../cli/commands/main/janitor/helpers.py | 150 +++++++---- .../main/janitor/test_janitor_progress.py | 251 ++++++++++-------- .../janitor_events/classes/_test_types.py | 2 +- .../janitor_events/classes/helpers.py | 65 +++++ .../classes/test_janitor_event_writer.py | 91 ++----- .../snowflake/test_view_definitions.py | 6 +- 40 files changed, 1031 insertions(+), 666 deletions(-) create mode 100644 src/sqlbuild/executor/janitor/_helpers/phase_steps.py create mode 100644 src/sqlbuild/executor/janitor/_helpers/worker_runs.py create mode 100644 src/sqlbuild/executor/janitor/_helpers/workers.py create mode 100644 src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py create mode 100644 src/sqlbuild/executor/janitor/classes/pooled_janitor_attempt.py create mode 100644 src/sqlbuild/executor/pipeline/main/_close_worker_connection_pool.py create mode 100644 src/sqlbuild/executor/pipeline/main/_open_worker_connection_pool.py create mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/classes/helpers.py diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs index c6a49b3d54..2b44fe653a 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs @@ -40,5 +40,8 @@ pub(crate) fn duration(seconds: f64) -> String { /// `name -> destination`, or the name alone. pub(crate) fn flow(name: &str, destination: Option<&str>) -> String { - destination.map_or_else(|| name.to_owned(), |destination| format!("{name} -> {destination}")) + destination.map_or_else( + || name.to_owned(), + |destination| format!("{name} -> {destination}"), + ) } diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs index b07123f4ff..d9575dfa4e 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -99,7 +99,10 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { object(vec![ ("direct_mode", JsonValue::Bool(plan.direct_mode)), ("retention_days", integer(plan.retention_days)), - ("archive_retention_days", integer(plan.archive_retention_days)), + ( + "archive_retention_days", + integer(plan.archive_retention_days), + ), ( "age_metadata_supported", JsonValue::Bool(plan.age_metadata_supported), @@ -208,8 +211,14 @@ fn counts_payload(counts: &JanitorResultCounts) -> JsonValue { ("archived", integer(counts.archived)), ("deleted", integer(counts.deleted)), ("deleted_archives", integer(counts.deleted_archives)), - ("deleted_query_artifacts", integer(counts.deleted_query_artifacts)), - ("dropped_old_name_views", integer(counts.dropped_old_name_views)), + ( + "deleted_query_artifacts", + integer(counts.deleted_query_artifacts), + ), + ( + "dropped_old_name_views", + integer(counts.dropped_old_name_views), + ), ("pruned_direct_state", integer(counts.pruned_direct_state)), ]) } @@ -227,7 +236,8 @@ fn item_payload(item: &JanitorItemText) -> JsonValue { ("error", optional_text(item.error.as_deref())), ( "duration_seconds", - item.duration_seconds.map_or(JsonValue::Null, JsonValue::Float), + item.duration_seconds + .map_or(JsonValue::Null, JsonValue::Float), ), ]) } diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs index 1c0235c6d2..d81ae6596a 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs @@ -345,7 +345,10 @@ pub(crate) fn skipped_objects( } let mut groups: Vec<(usize, &str)> = Vec::new(); for (_, reason) in &plan.skipped_relations { - match groups.iter_mut().find(|(_, known)| *known == reason.as_str()) { + match groups + .iter_mut() + .find(|(_, known)| *known == reason.as_str()) + { Some((count, _)) => *count += 1, None => groups.push((1, reason.as_str())), } diff --git a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs index da23689da9..6660340d72 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs @@ -25,7 +25,10 @@ pub fn format_janitor_json( "status".to_owned(), JsonValue::String(outcome.value().to_owned()), ), - ("plan".to_owned(), plan.map_or(JsonValue::Null, plan_payload)), + ( + "plan".to_owned(), + plan.map_or(JsonValue::Null, plan_payload), + ), ( "result".to_owned(), run.map_or(JsonValue::Null, run_payload), diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs index 410107822c..391e6e5d0a 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs @@ -28,7 +28,10 @@ pub fn render_janitor_completion( .collect::>() .join(", "); if let Some(seconds) = elapsed_seconds { - summary.push_str(&format!(" {}", style.muted(&format!("({})", duration(seconds))))); + summary.push_str(&format!( + " {}", + style.muted(&format!("({})", duration(seconds))) + )); } let (state, label) = if failed { (CompletionState::Fail, "Janitor failed") diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs index 19163ae34d..7e83fbcfb6 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs @@ -34,7 +34,11 @@ pub fn render_janitor_plan(plan: &JanitorPlanText, verbose: bool, style: CliStyl ) }) .collect(); - text.push_str(&listing((style, limit), "Skipped schemas", &skipped_schemas)); + text.push_str(&listing( + (style, limit), + "Skipped schemas", + &skipped_schemas, + )); if !plan.direct_mode { let candidates: Vec<(String, Option)> = plan .candidates diff --git a/crates/sqlbuild-core/src/janitor_text/tests.rs b/crates/sqlbuild-core/src/janitor_text/tests.rs index ef148abc3f..3735ed1a78 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests.rs @@ -1,8 +1,8 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_janitor_progress_text.rs"] +mod janitor_progress_text; #[path = "tests/test_janitor_text.rs"] mod janitor_text; #[path = "tests/test_types.rs"] mod test_types; -#[path = "tests/test_janitor_progress_text.rs"] -mod janitor_progress_text; diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs index 5bdf2e3534..3b75ffde67 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs @@ -209,8 +209,16 @@ fn given_finished_actions_when_counting_then_only_successes_count_by_kind() { ), item(JanitorPhase::DeleteArchives, ("deleted", "e", None), None), item(JanitorPhase::DropOldNameViews, ("dropped", "f", None), None), - item(JanitorPhase::DropOldNameViews, ("recorded", "g", None), None), - item(JanitorPhase::DropQueryArtifacts, ("dropped", "h", None), None), + item( + JanitorPhase::DropOldNameViews, + ("recorded", "g", None), + None, + ), + item( + JanitorPhase::DropQueryArtifacts, + ("dropped", "h", None), + None, + ), item(JanitorPhase::PruneDirectState, ("pruned", "i", None), None), ]; @@ -340,7 +348,6 @@ fn given_outcome_when_formatting_json_then_document_carries_plan_result_and_erro } } assert!( - format_janitor_json(None, JanitorOutcome::Disabled, None, None) - .contains("\"plan\": null") + format_janitor_json(None, JanitorOutcome::Disabled, None, None).contains("\"plan\": null") ); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 15a35c0352..0e72916cd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -34,8 +34,8 @@ use crate::bindings::_helpers::janitor::janitor_phases::{ JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, output_stream, }; -use crate::bindings::_helpers::janitor::janitor_progress::JanitorProgressPrinter; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; +use crate::bindings::_helpers::janitor::janitor_progress::JanitorProgressPrinter; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::error_class; @@ -246,7 +246,12 @@ fn apply( render_janitor_completion(&run.counts, Some(elapsed_seconds), false, style) ), )?; - write_json((request, Some(plan)), JanitorOutcome::Completed, Some(&run), None)?; + write_json( + (request, Some(plan)), + JanitorOutcome::Completed, + Some(&run), + None, + )?; Ok(0) } Err(error) => { @@ -297,7 +302,12 @@ fn clean( &stream, &janitor_preview_complete_text(&as_target, active.as_deref(), style), )?; - write_json((request, Some(&plan)), JanitorOutcome::Previewed, None, None)?; + write_json( + (request, Some(&plan)), + JanitorOutcome::Previewed, + None, + None, + )?; return Ok(blocked); } if blocked != 0 { @@ -306,12 +316,22 @@ fn clean( } if !janitor_plan_has_work(&plan) { write(&stream, &janitor_nothing_to_do_text(style))?; - write_json((request, Some(&plan)), JanitorOutcome::NothingToDo, None, None)?; + write_json( + (request, Some(&plan)), + JanitorOutcome::NothingToDo, + None, + None, + )?; return Ok(0); } if !native_field(request, "auto_approve")?.extract::()? && !confirmed(&stream, &plan)? { write(&stream, "Janitor cancelled.\n")?; - write_json((request, Some(&plan)), JanitorOutcome::Cancelled, None, None)?; + write_json( + (request, Some(&plan)), + JanitorOutcome::Cancelled, + None, + None, + )?; return Ok(1); } apply( @@ -350,11 +370,7 @@ pub(crate) fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> } let compiled: JanitorCompiled<'_> = compile_janitor_project(&invocation)?; let connection: Bound<'_, PyAny> = connect_janitor_warehouse(&invocation, &compiled)?; - let outcome: PyResult = clean( - request, - (&invocation, &settings), - (&compiled, &connection), - ); + let outcome: PyResult = clean(request, (&invocation, &settings), (&compiled, &connection)); let closed: PyResult<()> = close_janitor_warehouse(&compiled, &connection); let code: i32 = outcome?; closed?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 5d368b488a..fdda91371b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -271,7 +271,10 @@ fn list_target_schema_relations<'py>( let requests: Vec>> = if concrete.is_empty() { vec![None] } else { - concrete.into_iter().map(|schema| Some(vec![schema])).collect() + concrete + .into_iter() + .map(|schema| Some(vec![schema])) + .collect() }; for schemas in requests { let listed: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index d55d951693..161a1b5c0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -288,7 +288,9 @@ pub(crate) fn plan_old_name_views<'py>( .find(|(database, listed, _)| *database == key.database && listed == schema) { Some((_, _, names)) => names.push(key.name.clone()), - None => definitions.push((key.database.clone(), schema.clone(), vec![key.name.clone()])), + None => { + definitions.push((key.database.clone(), schema.clone(), vec![key.name.clone()])) + } } } warehouse.prefetch_view_definitions(&definitions)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index e81060cedf..fc6753ef99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -228,7 +228,8 @@ pub(crate) fn build_janitor_execution_plan<'py>( ) -> PyResult> { let py: Python<'py> = invocation.py(); let started: Instant = Instant::now(); - let status: Bound<'py, PyAny> = status_reporter(&output_stream(invocation)?, use_color(invocation)?)?; + let status: Bound<'py, PyAny> = + status_reporter(&output_stream(invocation)?, use_color(invocation)?)?; status.call_method1("start", ("Inspecting warehouse state...",))?; let janitor: Bound<'py, PyAny> = native_field(invocation, "discovered_inputs")? .getattr("project_config")? @@ -243,36 +244,38 @@ pub(crate) fn build_janitor_execution_plan<'py>( ("connection", connection.clone()), ], )?; - let plan: Bound<'py, PyAny> = within(&catalog, |_| build_plan( - ( - &compiled.project, - &Warehouse { - adapter: compiled.adapter.clone(), - connection: connection.clone(), - }, - ), - &JanitorSettings { - retention_days: native_field(settings, "retention_days")?.extract()?, - delete_tracked_only: native_field(&janitor, "delete_tracked_only")? - .is_truthy()?, - exclude_patterns: native_field(&janitor, "exclude_patterns")?.extract()?, - direct_mode: true, - state_history_versions: native_field( - settings, - "direct_state_history_versions", - )? - .extract()?, - archive_retention_days: native_field(settings, "archive_retention_days")? - .extract()?, - early_old_name_view_drops: native_field(settings, "drop_old_name_views")? + let plan: Bound<'py, PyAny> = within(&catalog, |_| { + build_plan( + ( + &compiled.project, + &Warehouse { + adapter: compiled.adapter.clone(), + connection: connection.clone(), + }, + ), + &JanitorSettings { + retention_days: native_field(settings, "retention_days")?.extract()?, + delete_tracked_only: native_field(&janitor, "delete_tracked_only")? + .is_truthy()?, + exclude_patterns: native_field(&janitor, "exclude_patterns")?.extract()?, + direct_mode: true, + state_history_versions: native_field( + settings, + "direct_state_history_versions", + )? .extract()?, - }, - &RelationScope { - scan_keys: Vec::new(), - protected_keys: Vec::new(), - reasons: Vec::new(), - }, - ))?; + archive_retention_days: native_field(settings, "archive_retention_days")? + .extract()?, + early_old_name_view_drops: native_field(settings, "drop_old_name_views")? + .extract()?, + }, + &RelationScope { + scan_keys: Vec::new(), + protected_keys: Vec::new(), + reasons: Vec::new(), + }, + ) + })?; let metadata: Bound<'py, PyDict> = PyDict::new(py); metadata.set_item("item_count", candidate_count(&plan)?)?; let options: Bound<'py, PyDict> = PyDict::new(py); diff --git a/fensu.toml b/fensu.toml index 7acee3cbaf..3d55af042f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1611,12 +1611,6 @@ path = "src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.p symbols = ["invoke_lifecycle_event_sink"] reason = "The delivery thread claims each call in the dict it shares with the native dispatcher; the claim is the mutation, atomic under the GIL, so a cancelled call is never both run and counted dropped." -[[targets.python.rule_exceptions]] -rule = "XSB051" -path = "src/sqlbuild/executor/old_name_views/_helpers/janitor.py" -symbols = ["apply_old_name_view_drops"] -reason = "Janitor reads each compatibility view's stored definition right before deciding to drop it, so a relation that replaced the view is never dropped; a project has few live old names." - [targets.rust.rule_options.XSB072] edge_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/native_host", diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 953d8e654d..db9ef689a7 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -114,8 +114,6 @@ class BaseAdapter(RetentionAdapterMixin, StrictAdapter): statement_cancellation_is_asynchronous: ClassVar[bool] = False statement_limit_excludes_queueing: ClassVar[bool] = False metadata_inspection_concurrency: ClassVar[int] = 1 - # Independent DDL statements (renames, drops) this warehouse runs at once on separate - # connections; single-writer warehouses keep 1. concurrent_ddl_limit: ClassVar[int] = 1 supports_session_warehouse: ClassVar[bool] = False state_timestamp_type: ClassVar[str | None] = None diff --git a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py index b2f60283e8..078f055e3b 100644 --- a/src/sqlbuild/adapter/relations/classes/inspection_catalog.py +++ b/src/sqlbuild/adapter/relations/classes/inspection_catalog.py @@ -135,11 +135,7 @@ def prefetch_schema_listings( def prefetch_view_definitions( self, *, requests: tuple[tuple[str | None, str, tuple[str, ...]], ...] ) -> None: - """Read the definitions of these views with one batched read per schema. - - Each request is a database, schema and view names. Adapters without - ``read_view_definitions`` keep reading each definition on demand. - """ + """Read each (database, schema, names) request's view definitions in one batched read.""" read_batch: ViewDefinitionBatchReader | None = getattr( self.adapter, "read_view_definitions", None diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index b916ee583c..1e17650c96 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -101,9 +101,9 @@ from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog from sqlbuild.adapter.relations.constants import INSPECTION_IN_LIST_LIMIT from sqlbuild.adapter.relations.main.active_inspection_catalog import active_inspection_catalog +from sqlbuild.adapter.relations.main.cached_view_definition import cached_view_definition from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.relations.main.render_inspection_sql import render_inspection_sql -from sqlbuild.adapter.relations.main.cached_view_definition import cached_view_definition from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections from sqlbuild.adapter.relations.main.run_recorded_inspection_query import ( run_recorded_inspection_query, @@ -3042,10 +3042,7 @@ def read_view_definition( def read_view_definitions( self, *, connection: Any, database: str | None, schema: str, names: tuple[str, ...] ) -> dict[str, str]: - """Read the stored definitions of several views in one schema with one query. - - Names match case-insensitively, as ``read_view_definition`` matches one name. - """ + """Read several views' definitions in one schema, matching names case-insensitively.""" if not names: return {} @@ -3054,9 +3051,7 @@ def read_view_definitions( if database is None else (f"{database}.INFORMATION_SCHEMA.VIEWS") ) - literals: str = ", ".join( - "UPPER('" + name.replace("'", "''") + "')" for name in names - ) + literals: str = ", ".join("UPPER('" + name.replace("'", "''") + "')" for name in names) rows: list[tuple[Any, ...]] = self.execute( connection=connection, sql=( diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py index 4a277524c8..c7b2dd52ee 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py @@ -18,6 +18,7 @@ from sqlbuild.executor.janitor.constants import ARCHIVE_NAME_SEPARATOR from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( + JanitorActionDone, JanitorArchiveCandidate, JanitorArchivedRelation, JanitorPlan, @@ -56,12 +57,7 @@ def apply_direct_archives( run_id: str, progress: JanitorProgress | None = None, ) -> tuple[tuple[JanitorArchiveCandidate, ...], tuple[JanitorArchivedRelation, ...]]: - """Rename stale relations to archives, then drop expired archives by name. - - Audit events are written after each chunk of actions in a schema, and after a failure for - every action that succeeded. An archive whose name was shortened gets its event right after - its rename, because the event is the only record of its original name. - """ + """Rename stale relations to archives, then drop expired archives, auditing each chunk.""" schemas: tuple[tuple[str | None, str], ...] = _create_event_tables( adapter=adapter, connection=workers.connection, plan=plan @@ -74,15 +70,12 @@ def apply_direct_archives( if schemas: events.read_existing(connection=workers.connection, schemas=schemas) - def flush_events() -> None: - events.flush(connection=workers.connection) - - def archive_relation(candidate: JanitorArchiveCandidate, connection: Any) -> None: + def archive_relation(*, item: JanitorArchiveCandidate, connection: Any) -> JanitorActionDone: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): _rename_relation( adapter=adapter, connection=connection, - candidate=candidate, + candidate=item, recorder=recorder, ) record: JanitorEventRecord = _event_record( @@ -90,19 +83,19 @@ def archive_relation(candidate: JanitorArchiveCandidate, connection: Any) -> Non event_type=JanitorEventType.ARCHIVE, run_id=run_id, archive=JanitorArchivedRelation( - key=candidate.archive_key, - relation_type=candidate.relation.relation_type, - archived_at=candidate.archived_at, - expires_at=candidate.expires_at, - original_key=candidate.key, + key=item.archive_key, + relation_type=item.relation.relation_type, + archived_at=item.archived_at, + expires_at=item.expires_at, + original_key=item.key, ), ) - if _shortened(candidate): - events.write_now(connection=connection, record=record) - else: - events.add(record) + if not _shortened(item): + return JanitorActionDone(event=record) + events.write(connection=connection, records=(record,)) + return JanitorActionDone() - archived: tuple[JanitorArchiveCandidate, ...] = run_janitor_phase( + archived: tuple[tuple[JanitorArchiveCandidate, JanitorActionDone], ...] = run_janitor_phase( progress=progress, phase=JanitorPhase.ARCHIVE, items=plan.archive_candidates, @@ -114,24 +107,24 @@ def archive_relation(candidate: JanitorArchiveCandidate, connection: Any) -> Non apply=archive_relation, workers=workers, chunks=_event_chunks(tuple(candidate.archive_key for candidate in plan.archive_candidates)), - after_chunk=flush_events, + after_chunk=lambda finished: events.write( + connection=workers.connection, records=_chunk_events(finished) + ), ) - def delete_archive(archive: JanitorArchivedRelation, connection: Any) -> None: + def delete_archive(*, item: JanitorArchivedRelation, connection: Any) -> JanitorActionDone: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): - _drop_archive( - adapter=adapter, connection=connection, archive=archive, recorder=recorder - ) - events.add( - _event_record( + _drop_archive(adapter=adapter, connection=connection, archive=item, recorder=recorder) + return JanitorActionDone( + event=_event_record( adapter=adapter, event_type=JanitorEventType.DELETE, run_id=run_id, - archive=archive, + archive=item, ) ) - deleted: tuple[JanitorArchivedRelation, ...] = run_janitor_phase( + deleted: tuple[tuple[JanitorArchivedRelation, JanitorActionDone], ...] = run_janitor_phase( progress=progress, phase=JanitorPhase.DELETE_ARCHIVES, items=plan.archive_deletion_candidates, @@ -141,9 +134,18 @@ def delete_archive(archive: JanitorArchivedRelation, connection: Any) -> None: apply=delete_archive, workers=workers, chunks=_event_chunks(tuple(archive.key for archive in plan.archive_deletion_candidates)), - after_chunk=flush_events, + after_chunk=lambda finished: events.write( + connection=workers.connection, records=_chunk_events(finished) + ), ) - return archived, deleted + return ( + tuple(candidate for candidate, _ in archived), + tuple(archive for archive, _ in deleted), + ) + + +def _chunk_events(finished: tuple[JanitorActionDone, ...]) -> tuple[JanitorEventRecord, ...]: + return tuple(done.event for done in finished if done.event is not None) def _shortened(candidate: JanitorArchiveCandidate) -> bool: @@ -162,11 +164,13 @@ def _event_chunks(keys: tuple[JanitorRelationKey, ...]) -> tuple[tuple[int, ...] key: JanitorRelationKey for index, key in enumerate(keys): by_schema.setdefault((key.database, key.schema), []).append(index) - return tuple( - tuple(positions[start : start + JANITOR_EVENT_CHUNK_SIZE]) - for positions in by_schema.values() - for start in range(0, len(positions), JANITOR_EVENT_CHUNK_SIZE) - ) + chunks: list[tuple[int, ...]] = [] + positions: list[int] + for positions in by_schema.values(): + start: int + for start in range(0, len(positions), JANITOR_EVENT_CHUNK_SIZE): + chunks.append(tuple(positions[start : start + JANITOR_EVENT_CHUNK_SIZE])) + return tuple(chunks) def _required_schema(key: JanitorRelationKey) -> str: diff --git a/src/sqlbuild/executor/janitor/_helpers/deletion.py b/src/sqlbuild/executor/janitor/_helpers/deletion.py index d7615e3362..c68b1905b9 100644 --- a/src/sqlbuild/executor/janitor/_helpers/deletion.py +++ b/src/sqlbuild/executor/janitor/_helpers/deletion.py @@ -7,7 +7,11 @@ from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase -from sqlbuild.executor.janitor.models import JanitorDirectStatePruneCandidate, JanitorProgressItem +from sqlbuild.executor.janitor.models import ( + JanitorActionDone, + JanitorDirectStatePruneCandidate, + JanitorProgressItem, +) from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -15,27 +19,29 @@ def apply_janitor_deletions( *, candidates: tuple[JanitorDirectStatePruneCandidate, ...], - delete: Callable[[JanitorDirectStatePruneCandidate, Any], object] | None, + delete: Callable[..., object] | None, workers: JanitorWorkers, progress: JanitorProgress | None = None, ) -> tuple[JanitorDirectStatePruneCandidate, ...]: - """Run `delete(candidate, connection)` for each candidate, returning the deleted ones.""" + """Run `delete(candidate=, connection=)` for each candidate, returning the deleted ones.""" if delete is None: return () - def prune(candidate: JanitorDirectStatePruneCandidate, connection: Any) -> None: + def prune(*, item: JanitorDirectStatePruneCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): - _ = delete(candidate, connection) - - run_janitor_phase( - progress=progress, - phase=JanitorPhase.PRUNE_DIRECT_STATE, - items=candidates, - describe=lambda candidate: JanitorProgressItem( - action=JanitorAction.PRUNED, name=candidate.display_name() - ), - apply=prune, - workers=workers, + _ = delete(candidate=item, connection=connection) + + pruned: tuple[tuple[JanitorDirectStatePruneCandidate, JanitorActionDone], ...] = ( + run_janitor_phase( + progress=progress, + phase=JanitorPhase.PRUNE_DIRECT_STATE, + items=candidates, + describe=lambda candidate: JanitorProgressItem( + action=JanitorAction.PRUNED, name=candidate.display_name() + ), + apply=prune, + workers=workers, + ) ) - return candidates + return tuple(candidate for candidate, _ in pruned) diff --git a/src/sqlbuild/executor/janitor/_helpers/phase_steps.py b/src/sqlbuild/executor/janitor/_helpers/phase_steps.py new file mode 100644 index 0000000000..295878be0b --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/phase_steps.py @@ -0,0 +1,63 @@ +"""Steps of one janitor apply phase: announcements, chunk positions and finished actions.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from sqlbuild.executor.janitor.models import ( + JanitorActionDone, + JanitorActionOutcome, + JanitorChunkOutcome, +) +from sqlbuild.executor.janitor.types import JanitorPhase, JanitorProgress + + +def announce_phase(*, progress: JanitorProgress | None, phase: JanitorPhase, total: int) -> None: + """Report that a phase starts `total` actions.""" + + if progress is not None: + progress.phase_started(phase=phase.value, total=total) + + +def close_phase(*, progress: JanitorProgress | None, phase: JanitorPhase, failed: bool) -> None: + """Report that a phase finished or failed.""" + + if progress is not None: + progress.phase_finished(phase=phase.value, failed=failed) + + +def chunk_positions( + *, total: int, chunks: Sequence[Sequence[int]] | None +) -> tuple[tuple[int, ...], ...]: + """The item positions of each chunk, or every position as one chunk.""" + + if chunks is None: + return (tuple(range(total)),) + return tuple(tuple(chunk) for chunk in chunks) + + +def finished_actions( + *, + positions: tuple[int, ...], + chunk: JanitorChunkOutcome[JanitorActionDone | None], +) -> tuple[tuple[int, JanitorActionDone], ...]: + """Each succeeded item's phase position and what it did.""" + + pairs: list[tuple[int, JanitorActionDone]] = [] + position: int + outcome: JanitorActionOutcome[JanitorActionDone | None] | None + for position, outcome in zip(positions, chunk.outcomes, strict=True): + if outcome is not None and outcome.error is None: + pairs.append((position, outcome.result or JanitorActionDone())) + return tuple(pairs) + + +def chunk_failure(chunk: JanitorChunkOutcome[JanitorActionDone | None]) -> BaseException | None: + """The interrupt that stopped the chunk, else the earliest item's failure.""" + + if chunk.interruption is not None: + return chunk.interruption + return next( + (outcome.error for outcome in chunk.outcomes if outcome is not None and outcome.error), + None, + ) diff --git a/src/sqlbuild/executor/janitor/_helpers/worker_runs.py b/src/sqlbuild/executor/janitor/_helpers/worker_runs.py new file mode 100644 index 0000000000..7008c798d6 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/worker_runs.py @@ -0,0 +1,98 @@ +"""Attempt janitor actions and collect concurrent outcomes in completion order.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from contextvars import copy_context +from time import perf_counter +from typing import Any, cast + +from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorChunkOutcome + +type _Pending[ResultT] = set[Future[JanitorActionOutcome[ResultT]]] + + +def attempt_janitor_action[ResultT]( + *, action: Callable[..., ResultT], item: object, connection: Any +) -> JanitorActionOutcome[ResultT]: + """Run `action(item=, connection=)`, capturing its result or error and its duration.""" + + started: float = perf_counter() + try: + result: ResultT = action(item=item, connection=connection) + except BaseException as error: + return JanitorActionOutcome(error=error, duration_seconds=perf_counter() - started) + return JanitorActionOutcome(result=result, duration_seconds=perf_counter() - started) + + +def collect_janitor_outcomes[ItemT, ResultT]( + *, + pool: ThreadPoolExecutor, + items: Sequence[ItemT], + attempt: Callable[[ItemT], JanitorActionOutcome[ResultT]], + on_done: Callable[..., None], +) -> JanitorChunkOutcome[ResultT]: + """Report outcomes as they finish; after a failure or interrupt, start nothing more.""" + + futures: dict[Future[JanitorActionOutcome[ResultT]], int] = {} + index: int + item: ItemT + for index, item in enumerate(items): + submitted: Future[Any] = pool.submit(copy_context().run, attempt, item) + futures[cast(Future[JanitorActionOutcome[ResultT]], submitted)] = index + outcomes: dict[int, JanitorActionOutcome[ResultT]] = {} + pending: _Pending[ResultT] = set(futures) + try: + while pending: + done: _Pending[ResultT] + done, pending = wait(pending, return_when=FIRST_COMPLETED) + reported: dict[int, JanitorActionOutcome[ResultT]] + reported, pending = _report( + done=done, pending=pending, futures=futures, on_done=on_done + ) + outcomes.update(reported) + except BaseException as interruption: + _cancel(pending=pending) + finished: dict[int, JanitorActionOutcome[ResultT]] = _report( + done=wait(pending).done, pending=set(), futures=futures, on_done=on_done + )[0] + outcomes.update(finished) + return JanitorChunkOutcome( + outcomes=_in_order(outcomes=outcomes, count=len(items)), interruption=interruption + ) + return JanitorChunkOutcome(outcomes=_in_order(outcomes=outcomes, count=len(items))) + + +def _report[ResultT]( + *, + done: _Pending[ResultT], + pending: _Pending[ResultT], + futures: dict[Future[JanitorActionOutcome[ResultT]], int], + on_done: Callable[..., None], +) -> tuple[dict[int, JanitorActionOutcome[ResultT]], _Pending[ResultT]]: + """Report finished outcomes in item order and cancel the rest once one failed.""" + + reported: dict[int, JanitorActionOutcome[ResultT]] = {} + future: Future[JanitorActionOutcome[ResultT]] + for future in sorted(done, key=futures.__getitem__): + if future.cancelled(): + continue + outcome: JanitorActionOutcome[ResultT] = future.result() + reported[futures[future]] = outcome + _ = on_done(index=futures[future], outcome=outcome) + if outcome.error is not None: + _cancel(pending=pending) + return reported, {future for future in pending if not future.cancelled()} + + +def _cancel[ResultT](*, pending: _Pending[ResultT]) -> None: + future: Future[JanitorActionOutcome[ResultT]] + for future in pending: + _ = future.cancel() + + +def _in_order[ResultT]( + *, outcomes: dict[int, JanitorActionOutcome[ResultT]], count: int +) -> tuple[JanitorActionOutcome[ResultT] | None, ...]: + return tuple(outcomes.get(index) for index in range(count)) diff --git a/src/sqlbuild/executor/janitor/_helpers/workers.py b/src/sqlbuild/executor/janitor/_helpers/workers.py new file mode 100644 index 0000000000..c374453835 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/workers.py @@ -0,0 +1,58 @@ +"""Size and open the extra warehouse connections of concurrent janitor actions.""" + +from __future__ import annotations + +import time +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.janitor.models import JanitorPlan +from sqlbuild.executor.janitor.types import JanitorConnectionProgress +from sqlbuild.executor.pipeline.main._open_worker_connection_pool import ( + open_worker_connection_pool, +) + + +def janitor_worker_count(*, plan: JanitorPlan, adapter: BaseAdapter, concurrency: int) -> int: + """Connections the largest phase can use: the concurrency, capped by the adapter's DDL limit.""" + + largest_phase: int = max( + 0 if plan.direct_mode else len(plan.candidates), + len(plan.archive_candidates) if plan.direct_mode else 0, + len(plan.archive_deletion_candidates) if plan.direct_mode else 0, + len(plan.old_name_views.drops) + len(plan.old_name_views.missing), + len(plan.query_diff_artifact_candidates), + len(plan.direct_state_prune_candidates), + ) + return max(1, min(concurrency, int(adapter.concurrent_ddl_limit), largest_phase)) + + +def open_janitor_worker_connections( + *, + adapter: BaseAdapter, + connection_config: dict[str, object] | None, + count: int, + progress: JanitorConnectionProgress | None, +) -> tuple[Any, ...]: + """Open `count` extra connections, reporting the connect phase to `progress`.""" + + if count <= 0 or connection_config is None: + return () + if progress is not None: + progress.on_connection_start(count) + started: float = time.monotonic() + try: + connections: tuple[Any, ...] = open_worker_connection_pool( + adapter=adapter, connection_config=connection_config, connection_count=count + ) + except BaseException: + if progress is not None: + progress.on_connection_error( + connection_count=count, elapsed_seconds=time.monotonic() - started + ) + raise + if progress is not None: + progress.on_connection_complete( + connection_count=count, elapsed_seconds=time.monotonic() - started + ) + return connections diff --git a/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py b/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py new file mode 100644 index 0000000000..c0f1895f29 --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py @@ -0,0 +1,44 @@ +"""Report each finished janitor action of one chunk to the progress output.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.executor.janitor.models import ( + JanitorActionDone, + JanitorActionOutcome, + JanitorProgressItem, +) +from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress + + +class JanitorItemReporter: + """Report one chunk's finished actions by their position in the chunk.""" + + def __init__( + self, + *, + progress: JanitorProgress | None, + phase: JanitorPhase, + labels: tuple[JanitorProgressItem, ...], + ) -> None: + self._progress: JanitorProgress | None = progress + self._phase: JanitorPhase = phase + self._labels: tuple[JanitorProgressItem, ...] = labels + + def __call__( + self, *, index: int, outcome: JanitorActionOutcome[JanitorActionDone | None] + ) -> None: + if self._progress is None: + return + done: JanitorActionDone | None = outcome.result + taken: JanitorAction | None = None if done is None else done.taken + label: JanitorProgressItem = self._labels[index] + self._progress.item_finished( + phase=self._phase.value, + item=label if taken is None else replace(label, action=taken), + error=None + if outcome.error is None + else str(outcome.error) or type(outcome.error).__name__, + duration_seconds=outcome.duration_seconds, + ) diff --git a/src/sqlbuild/executor/janitor/classes/janitor_workers.py b/src/sqlbuild/executor/janitor/classes/janitor_workers.py index da128d0115..05b333d6c9 100644 --- a/src/sqlbuild/executor/janitor/classes/janitor_workers.py +++ b/src/sqlbuild/executor/janitor/classes/janitor_workers.py @@ -4,12 +4,15 @@ import queue from collections.abc import Callable, Sequence -from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait -from contextvars import copy_context -from time import perf_counter +from concurrent.futures import ThreadPoolExecutor from typing import Any -from sqlbuild.executor.janitor.models import JanitorActionOutcome +from sqlbuild.executor.janitor._helpers.worker_runs import ( + attempt_janitor_action, + collect_janitor_outcomes, +) +from sqlbuild.executor.janitor.classes.pooled_janitor_attempt import PooledJanitorAttempt +from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorChunkOutcome class JanitorWorkers: @@ -28,14 +31,10 @@ def run[ItemT, ResultT]( self, *, items: Sequence[ItemT], - action: Callable[[ItemT, Any], ResultT], - on_done: Callable[[int, JanitorActionOutcome[ResultT]], None], - ) -> BaseException | None: - """Run `action` on every item, reporting each on the calling thread as it finishes. - - After a failure no further action starts; running ones finish and are reported. The - failure of the earliest item is returned. - """ + action: Callable[..., ResultT], + on_done: Callable[..., None], + ) -> JanitorChunkOutcome[ResultT]: + """Run `action(item=, connection=)` per item until one fails, calling `on_done` per item.""" if len(self.connections) == 1 or len(items) <= 1: return self._run_in_order(items=items, action=action, on_done=on_done) @@ -43,19 +42,16 @@ def run[ItemT, ResultT]( connection: Any for connection in self.connections: connections.put(connection) - - def attempt(item: ItemT) -> JanitorActionOutcome[ResultT]: - borrowed: Any = connections.get() - try: - return _attempt(action=action, item=item, connection=borrowed) - finally: - connections.put(borrowed) - pool: ThreadPoolExecutor = ThreadPoolExecutor( max_workers=min(len(self.connections), len(items)) ) try: - return _collect(pool=pool, items=items, attempt=attempt, on_done=on_done) + return collect_janitor_outcomes( + pool=pool, + items=items, + attempt=PooledJanitorAttempt(action=action, connections=connections), + on_done=on_done, + ) finally: pool.shutdown(wait=True, cancel_futures=True) @@ -63,57 +59,18 @@ def _run_in_order[ItemT, ResultT]( self, *, items: Sequence[ItemT], - action: Callable[[ItemT, Any], ResultT], - on_done: Callable[[int, JanitorActionOutcome[ResultT]], None], - ) -> BaseException | None: + action: Callable[..., ResultT], + on_done: Callable[..., None], + ) -> JanitorChunkOutcome[ResultT]: + outcomes: list[JanitorActionOutcome[ResultT] | None] = [None] * len(items) index: int item: ItemT for index, item in enumerate(items): - outcome: JanitorActionOutcome[ResultT] = _attempt( + outcome: JanitorActionOutcome[ResultT] = attempt_janitor_action( action=action, item=item, connection=self.connection ) - on_done(index, outcome) + outcomes[index] = outcome + on_done(index=index, outcome=outcome) if outcome.error is not None: - return outcome.error - return None - - -def _attempt[ItemT, ResultT]( - *, action: Callable[[ItemT, Any], ResultT], item: ItemT, connection: Any -) -> JanitorActionOutcome[ResultT]: - started: float = perf_counter() - try: - result: ResultT = action(item, connection) - except BaseException as error: - return JanitorActionOutcome(error=error, duration_seconds=perf_counter() - started) - return JanitorActionOutcome(result=result, duration_seconds=perf_counter() - started) - - -def _collect[ItemT, ResultT]( - *, - pool: ThreadPoolExecutor, - items: Sequence[ItemT], - attempt: Callable[[ItemT], JanitorActionOutcome[ResultT]], - on_done: Callable[[int, JanitorActionOutcome[ResultT]], None], -) -> BaseException | None: - futures: dict[Future[JanitorActionOutcome[ResultT]], int] = { - pool.submit(copy_context().run, attempt, item): index for index, item in enumerate(items) - } - failures: dict[int, BaseException] = {} - pending: set[Future[JanitorActionOutcome[ResultT]]] = set(futures) - while pending: - done: set[Future[JanitorActionOutcome[ResultT]]] - done, pending = wait(pending, return_when=FIRST_COMPLETED) - future: Future[JanitorActionOutcome[ResultT]] - for future in sorted(done, key=futures.__getitem__): - if future.cancelled(): - continue - outcome: JanitorActionOutcome[ResultT] = future.result() - on_done(futures[future], outcome) - if outcome.error is None: - continue - if not failures: - for waiting in pending: - _ = waiting.cancel() - failures[futures[future]] = outcome.error - return failures[min(failures)] if failures else None + break + return JanitorChunkOutcome(outcomes=tuple(outcomes)) diff --git a/src/sqlbuild/executor/janitor/classes/pooled_janitor_attempt.py b/src/sqlbuild/executor/janitor/classes/pooled_janitor_attempt.py new file mode 100644 index 0000000000..74fba835a3 --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/pooled_janitor_attempt.py @@ -0,0 +1,25 @@ +"""One janitor action attempt on a connection borrowed from a worker pool.""" + +from __future__ import annotations + +import queue +from collections.abc import Callable +from typing import Any + +from sqlbuild.executor.janitor._helpers.worker_runs import attempt_janitor_action +from sqlbuild.executor.janitor.models import JanitorActionOutcome + + +class PooledJanitorAttempt[ResultT]: + """Run an action on a borrowed connection and return the connection afterwards.""" + + def __init__(self, *, action: Callable[..., ResultT], connections: queue.Queue[Any]) -> None: + self._action: Callable[..., ResultT] = action + self._connections: queue.Queue[Any] = connections + + def __call__(self, item: object) -> JanitorActionOutcome[ResultT]: + borrowed: Any = self._connections.get() + try: + return attempt_janitor_action(action=self._action, item=item, connection=borrowed) + finally: + self._connections.put(borrowed) diff --git a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py index 9baf8a5b5c..3dd8eb8d34 100644 --- a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py +++ b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py @@ -3,12 +3,22 @@ from __future__ import annotations from collections.abc import Callable, Sequence -from dataclasses import replace -from typing import Any +from sqlbuild.executor.janitor._helpers.phase_steps import ( + announce_phase, + chunk_failure, + chunk_positions, + close_phase, + finished_actions, +) +from sqlbuild.executor.janitor.classes.janitor_item_reporter import JanitorItemReporter from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers -from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorProgressItem -from sqlbuild.executor.janitor.types import JanitorAction, JanitorPhase, JanitorProgress +from sqlbuild.executor.janitor.models import ( + JanitorActionDone, + JanitorChunkOutcome, + JanitorProgressItem, +) +from sqlbuild.executor.janitor.types import JanitorPhase, JanitorProgress def run_janitor_phase[T]( @@ -17,82 +27,45 @@ def run_janitor_phase[T]( phase: JanitorPhase, items: Sequence[T], describe: Callable[[T], JanitorProgressItem], - apply: Callable[[T, Any], JanitorAction | None], + apply: Callable[..., JanitorActionDone | None], workers: JanitorWorkers, chunks: Sequence[Sequence[int]] | None = None, - after_chunk: Callable[[], None] | None = None, -) -> tuple[T, ...]: - """Apply `apply(item, connection)` to every item and return the items that succeeded. - - Items run concurrently on `workers`, a chunk of item positions at a time (all items when - `chunks` is none); `after_chunk` runs after every chunk - and once more after a failure or interruption, before the error is raised. A failing item is - reported and no later action starts. `apply` may return the action actually taken when it - differs from the described one. - """ + after_chunk: Callable[[tuple[JanitorActionDone, ...]], None] | None = None, +) -> tuple[tuple[T, JanitorActionDone], ...]: + """Apply `apply(item=, connection=)` chunk by chunk; return succeeded items in plan order.""" if not items: return () - if progress is not None: - progress.phase_started(phase=phase.value, total=len(items)) - succeeded: set[int] = set() - - def report( - chunk: Sequence[int], - ) -> Callable[[int, JanitorActionOutcome[JanitorAction | None]], None]: - def on_done(index: int, outcome: JanitorActionOutcome[JanitorAction | None]) -> None: - item: T = items[chunk[index]] - if outcome.error is None: - succeeded.add(chunk[index]) - if progress is None: - return - label: JanitorProgressItem = describe(item) - progress.item_finished( - phase=phase.value, - item=label if outcome.result is None else replace(label, action=outcome.result), - error=None if outcome.error is None else _message(outcome.error), - duration_seconds=outcome.duration_seconds, - ) - - return on_done - - failure: BaseException | None = None - try: - chunk: Sequence[int] - for chunk in (range(len(items)),) if chunks is None else chunks: - failure = workers.run( - items=tuple(items[index] for index in chunk), - action=apply, - on_done=report(chunk), - ) - if failure is not None: - break + _ = announce_phase(progress=progress, phase=phase, total=len(items)) + finished: list[tuple[int, JanitorActionDone]] = [] + positions: tuple[int, ...] + for positions in chunk_positions(total=len(items), chunks=chunks): + chunk: JanitorChunkOutcome[JanitorActionDone | None] = workers.run( + items=tuple(items[position] for position in positions), + action=apply, + on_done=JanitorItemReporter( + progress=progress, + phase=phase, + labels=tuple(describe(items[position]) for position in positions), + ), + ) + succeeded: tuple[tuple[int, JanitorActionDone], ...] = finished_actions( + positions=positions, chunk=chunk + ) + finished.extend(succeeded) + failure: BaseException | None = chunk_failure(chunk) + try: if after_chunk is not None: - after_chunk() - except BaseException: - _finish_failed(progress=progress, phase=phase, after_chunk=after_chunk) - raise - if failure is not None: - _finish_failed(progress=progress, phase=phase, after_chunk=after_chunk) - raise failure - if progress is not None: - progress.phase_finished(phase=phase.value, failed=False) - return tuple(item for index, item in enumerate(items) if index in succeeded) - - -def _finish_failed( - *, - progress: JanitorProgress | None, - phase: JanitorPhase, - after_chunk: Callable[[], None] | None, -) -> None: - try: - if after_chunk is not None: - after_chunk() - finally: - if progress is not None: - progress.phase_finished(phase=phase.value, failed=True) + _ = after_chunk(tuple(done for _, done in succeeded)) + except BaseException: + _ = close_phase(progress=progress, phase=phase, failed=True) + raise + if failure is not None: + _ = close_phase(progress=progress, phase=phase, failed=True) + raise failure + _ = close_phase(progress=progress, phase=phase, failed=False) + return tuple((items[position], done) for position, done in sorted(finished, key=_position)) -def _message(error: BaseException) -> str: - return str(error) or type(error).__name__ +def _position(pair: tuple[int, JanitorActionDone]) -> int: + return pair[0] diff --git a/src/sqlbuild/executor/janitor/main/execute.py b/src/sqlbuild/executor/janitor/main/execute.py index 8f3865b0ed..95faf450c5 100644 --- a/src/sqlbuild/executor/janitor/main/execute.py +++ b/src/sqlbuild/executor/janitor/main/execute.py @@ -3,7 +3,6 @@ from __future__ import annotations import sys -import time from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -13,6 +12,10 @@ janitor_run_id, ) from sqlbuild.executor.janitor._helpers.deletion import apply_janitor_deletions +from sqlbuild.executor.janitor._helpers.workers import ( + janitor_worker_count, + open_janitor_worker_connections, +) from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( @@ -33,9 +36,8 @@ JanitorProgress, ) from sqlbuild.executor.old_name_views.main._drop_old_name_views import drop_old_name_views -from sqlbuild.executor.pipeline._helpers.connections import ( - close_connections, - open_worker_connections, +from sqlbuild.executor.pipeline.main._close_worker_connection_pool import ( + close_worker_connection_pool, ) from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -50,21 +52,17 @@ def execute_janitor_plan( concurrency: int = 1, connection_progress: JanitorConnectionProgress | None = None, ) -> JanitorExecutionResult: - """Delete all candidates in a janitor plan, reporting each phase and action to `progress`. - - Up to `concurrency` independent actions of one phase run at once, each on its own - connection, when the adapter allows concurrent DDL; phases still run one after another. - """ + """Apply a janitor plan phase by phase, running each phase's actions on up to `concurrency`.""" with OperationLifecycle( operation_kind="janitor", operation_name="janitor_execution", metadata={"item_count": _janitor_action_count(plan)}, ): - extra: tuple[Any, ...] = _open_worker_connections( + extra: tuple[Any, ...] = open_janitor_worker_connections( adapter=adapter, connection_config=connection_config, - count=_worker_count(plan=plan, adapter=adapter, concurrency=concurrency) - 1, + count=janitor_worker_count(plan=plan, adapter=adapter, concurrency=concurrency) - 1, progress=connection_progress, ) try: @@ -75,48 +73,9 @@ def execute_janitor_plan( progress=progress, ) finally: - close_connections(adapter=adapter, connections=extra, active_error=sys.exc_info()[1]) - - -def _worker_count(*, plan: JanitorPlan, adapter: BaseAdapter, concurrency: int) -> int: - largest_phase: int = max( - len(() if plan.direct_mode else plan.candidates), - len(plan.archive_candidates) if plan.direct_mode else 0, - len(plan.archive_deletion_candidates) if plan.direct_mode else 0, - len(plan.old_name_views.drops) + len(plan.old_name_views.missing), - len(plan.query_diff_artifact_candidates), - len(plan.direct_state_prune_candidates), - ) - return max(1, min(concurrency, int(adapter.concurrent_ddl_limit), largest_phase)) - - -def _open_worker_connections( - *, - adapter: BaseAdapter, - connection_config: dict[str, object] | None, - count: int, - progress: JanitorConnectionProgress | None, -) -> tuple[Any, ...]: - if count <= 0 or connection_config is None: - return () - if progress is not None: - progress.on_connection_start(count) - started: float = time.monotonic() - try: - connections: tuple[Any, ...] = open_worker_connections( - adapter=adapter, connection_config=connection_config, connection_count=count - ) - except BaseException: - if progress is not None: - progress.on_connection_error( - connection_count=count, elapsed_seconds=time.monotonic() - started + _ = close_worker_connection_pool( + adapter=adapter, connections=extra, active_error=sys.exc_info()[1] ) - raise - if progress is not None: - progress.on_connection_complete( - connection_count=count, elapsed_seconds=time.monotonic() - started - ) - return connections def _execute_janitor_plan( @@ -128,16 +87,16 @@ def _execute_janitor_plan( ) -> JanitorExecutionResult: recorder: StatementRecorder = StatementRecorder() - def drop_candidate(candidate: JanitorDeleteCandidate, connection: Any) -> None: + def drop_candidate(*, item: JanitorDeleteCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): adapter.drop( connection=connection, - destination=candidate.key.display_name(), + destination=item.key.display_name(), if_exists=True, statement_recorder=recorder, ) - run_janitor_phase( + _ = run_janitor_phase( progress=progress, phase=JanitorPhase.DELETE_RELATIONS, items=() if plan.direct_mode else plan.candidates, @@ -170,25 +129,25 @@ def drop_candidate(candidate: JanitorDeleteCandidate, connection: Any) -> None: progress=progress, ) - def drop_query_artifact(candidate: JanitorQueryDiffArtifactCandidate, connection: Any) -> None: + def drop_query_artifact(*, item: JanitorQueryDiffArtifactCandidate, connection: Any) -> None: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): query_artifact_destination: str | None = adapter.render_qualified_name( - database=candidate.key.database, - schema=candidate.key.schema, - name=candidate.key.name, + database=item.key.database, + schema=item.key.schema, + name=item.key.name, ) adapter.drop( connection=connection, destination=( query_artifact_destination if query_artifact_destination is not None - else candidate.key.display_name() + else item.key.display_name() ), if_exists=True, statement_recorder=recorder, ) - run_janitor_phase( + _ = run_janitor_phase( progress=progress, phase=JanitorPhase.DROP_QUERY_ARTIFACTS, items=plan.query_diff_artifact_candidates, @@ -200,8 +159,8 @@ def drop_query_artifact(candidate: JanitorQueryDiffArtifactCandidate, connection ) pruned_direct_state: tuple[JanitorDirectStatePruneCandidate, ...] = apply_janitor_deletions( candidates=plan.direct_state_prune_candidates, - delete=lambda direct_state_candidate, connection: adapter.execute( - connection=connection, sql=direct_state_candidate.prune_sql + delete=lambda *, candidate, connection: adapter.execute( + connection=connection, sql=candidate.prune_sql ), workers=workers, progress=progress, diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 5d19256490..a345b8de1d 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -8,6 +8,7 @@ from sqlbuild._native import OldNameViewDropReason, OldNameViewHistory from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.executor.janitor.types import JanitorAction +from sqlbuild.executor.janitor_events.models import JanitorEventRecord @dataclass(frozen=True, eq=False) @@ -245,3 +246,19 @@ class JanitorActionOutcome[ResultT]: result: ResultT | None = None error: BaseException | None = None duration_seconds: float = 0.0 + + +@dataclass(frozen=True) +class JanitorChunkOutcome[ResultT]: + """Each item's outcome in one chunk (none if it never started), and any interrupt.""" + + outcomes: tuple[JanitorActionOutcome[ResultT] | None, ...] + interruption: BaseException | None = None + + +@dataclass(frozen=True) +class JanitorActionDone: + """A finished action: the action taken when it differs from the described one, and its event.""" + + taken: JanitorAction | None = None + event: JanitorEventRecord | None = None diff --git a/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py b/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py index d970aeb404..8554afb30c 100644 --- a/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py +++ b/src/sqlbuild/executor/janitor_events/classes/janitor_event_writer.py @@ -1,4 +1,4 @@ -"""Append-only janitor audit events, written in chunks per schema event table.""" +"""Append-only janitor audit events, written in multi-row chunks per schema event table.""" from __future__ import annotations @@ -18,13 +18,7 @@ class JanitorEventWriter: - """Write each event at most once, reading the run's existing event IDs once per schema. - - Events wait in memory until `flush`, which appends each schema's pending events with one - multi-row INSERT per chunk. `write_now` appends one event at once. Event IDs are - deterministic per run, so an ID already present is never inserted again, including when the - same invocation is retried. - """ + """Write each deterministic event ID at most once, even across retries of one invocation.""" def __init__( self, @@ -37,7 +31,6 @@ def __init__( self._render_qualified_name: Callable[..., str | None] = render_qualified_name self._run_id: str = run_id self._written: set[str] = set() - self._pending: dict[_SchemaKey, list[JanitorEventRecord]] = {} self._lock: threading.Lock = threading.Lock() def read_existing(self, *, connection: Any, schemas: tuple[_SchemaKey, ...]) -> None: @@ -55,66 +48,37 @@ def read_existing(self, *, connection: Any, schemas: tuple[_SchemaKey, ...]) -> render_qualified_name=self._render_qualified_name, ), ) - self._written.update(str(row[0]) for row in cursor.fetchall()) - - def add(self, record: JanitorEventRecord) -> None: - """Hold one event until the next `flush`, unless it is already written or held.""" - - with self._lock: - if record.event_id in self._written or any( - held.event_id == record.event_id - for held in self._pending.get(_schema_key(record), ()) - ): - return - self._pending.setdefault(_schema_key(record), []).append(record) - - def write_now(self, *, connection: Any, record: JanitorEventRecord) -> None: - """Append one event immediately on `connection`, unless it is already written.""" - - with self._lock: - if record.event_id in self._written: - return - self._written.add(record.event_id) - try: - self._insert(connection=connection, records=(record,)) - except BaseException: + ids: set[str] = {str(row[0]) for row in cursor.fetchall()} with self._lock: - self._written.discard(record.event_id) - raise + self._written |= ids - def flush(self, *, connection: Any) -> None: - """Append every held event, one INSERT per schema and chunk of rows.""" + def write(self, *, connection: Any, records: tuple[JanitorEventRecord, ...]) -> None: + """Append the unwritten events, one INSERT per schema and chunk of rows.""" with self._lock: - pending: dict[_SchemaKey, list[JanitorEventRecord]] = self._pending - self._pending = {} - try: - key: _SchemaKey - records: list[JanitorEventRecord] - for key, records in list(pending.items()): - while records: - chunk: tuple[JanitorEventRecord, ...] = tuple( - records[:JANITOR_EVENT_CHUNK_SIZE] - ) - self._insert(connection=connection, records=chunk) - del records[: len(chunk)] - with self._lock: - self._written.update(record.event_id for record in chunk) - del pending[key] - finally: - if pending: + unwritten: dict[str, JanitorEventRecord] = { + record.event_id: record + for record in records + if record.event_id not in self._written + } + by_schema: dict[_SchemaKey, list[JanitorEventRecord]] = {} + record: JanitorEventRecord + for record in unwritten.values(): + by_schema.setdefault((record.relation_database, record.relation_schema), []).append( + record + ) + schema_records: list[JanitorEventRecord] + for schema_records in by_schema.values(): + start: int + for start in range(0, len(schema_records), JANITOR_EVENT_CHUNK_SIZE): + chunk: tuple[JanitorEventRecord, ...] = tuple( + schema_records[start : start + JANITOR_EVENT_CHUNK_SIZE] + ) + _ = self._execute( + connection=connection, + sql=build_insert_many_sql( + records=chunk, render_qualified_name=self._render_qualified_name + ), + ) with self._lock: - for key, records in pending.items(): - self._pending.setdefault(key, [])[:0] = records - - def _insert(self, *, connection: Any, records: tuple[JanitorEventRecord, ...]) -> None: - _ = self._execute( - connection=connection, - sql=build_insert_many_sql( - records=records, render_qualified_name=self._render_qualified_name - ), - ) - - -def _schema_key(record: JanitorEventRecord) -> _SchemaKey: - return (record.relation_database, record.relation_schema) + self._written |= {written.event_id for written in chunk} diff --git a/src/sqlbuild/executor/janitor_events/constants.py b/src/sqlbuild/executor/janitor_events/constants.py index dc211ed6ad..f120e39903 100644 --- a/src/sqlbuild/executor/janitor_events/constants.py +++ b/src/sqlbuild/executor/janitor_events/constants.py @@ -56,6 +56,4 @@ COLUMN_ARCHIVED_AT: StateSqlValueType.TEXT_TIMESTAMP, } -# Rows per multi-row INSERT: below every adapter's VALUES-list limit (SQL Server allows 1,000 and -# SQLite 500), and at most this many completed actions can lack events after a crash. JANITOR_EVENT_CHUNK_SIZE: int = 100 diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index b7aacbfece..3e1758eb6a 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -23,6 +23,7 @@ from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase from sqlbuild.executor.janitor.models import ( + JanitorActionDone, JanitorOldNameView, JanitorOldNameViewPlanning, JanitorProgressItem, @@ -43,16 +44,13 @@ def apply_old_name_view_drops( ) -> tuple[JanitorOldNameView, ...]: """Drop expired or requested views, then record every drop and every vanished view.""" - dropped: set[int] = set() - - def apply(entry: tuple[int, JanitorOldNameView, bool], connection: Any) -> JanitorAction | None: - position: int + def apply(*, item: tuple[JanitorOldNameView, bool], connection: Any) -> JanitorActionDone: view: JanitorOldNameView planned_missing: bool - position, view, planned_missing = entry + view, planned_missing = item if planned_missing: _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) - return None + return JanitorActionDone() if not is_compatibility_view(adapter=adapter, connection=connection, view=view): _record_drop( adapter=adapter, @@ -60,7 +58,7 @@ def apply(entry: tuple[int, JanitorOldNameView, bool], connection: Any) -> Janit view=_with_reason(view=view, reason=OldNameViewDropReason.MISSING), run_id=run_id, ) - return JanitorAction.RECORDED + return JanitorActionDone(taken=JanitorAction.RECORDED) adapter.drop_view( connection=connection, destination=resolve_qualified_name_parts( @@ -73,24 +71,29 @@ def apply(entry: tuple[int, JanitorOldNameView, bool], connection: Any) -> Janit statement_recorder=recorder, ) _record_drop(adapter=adapter, connection=connection, view=view, run_id=run_id) - dropped.add(position) - return None - - run_janitor_phase( - progress=progress, - phase=JanitorPhase.DROP_OLD_NAME_VIEWS, - items=( - *((index, view, False) for index, view in enumerate(plan.drops)), - *((len(plan.drops), view, True) for view in plan.missing), - ), - describe=lambda entry: JanitorProgressItem( - action=JanitorAction.RECORDED if entry[2] else JanitorAction.DROPPED, - name=entry[1].key.display_name(), - ), - apply=apply, - workers=workers, + return JanitorActionDone() + + finished: tuple[tuple[tuple[JanitorOldNameView, bool], JanitorActionDone], ...] = ( + run_janitor_phase( + progress=progress, + phase=JanitorPhase.DROP_OLD_NAME_VIEWS, + items=( + *((view, False) for view in plan.drops), + *((view, True) for view in plan.missing), + ), + describe=lambda entry: JanitorProgressItem( + action=JanitorAction.RECORDED if entry[1] else JanitorAction.DROPPED, + name=entry[0].key.display_name(), + ), + apply=apply, + workers=workers, + ) + ) + return tuple( + view + for (view, planned_missing), done in finished + if not planned_missing and done.taken is None ) - return tuple(view for index, view in enumerate(plan.drops) if index in dropped) def _record_drop( diff --git a/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py b/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py index 1e4663db98..1c3a0273d2 100644 --- a/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py +++ b/src/sqlbuild/executor/old_name_views/main/_drop_old_name_views.py @@ -4,8 +4,8 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.executor.janitor.models import JanitorOldNameView, JanitorOldNameViewPlanning from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.models import JanitorOldNameView, JanitorOldNameViewPlanning from sqlbuild.executor.janitor.types import JanitorProgress from sqlbuild.executor.old_name_views._helpers.janitor import apply_old_name_view_drops diff --git a/src/sqlbuild/executor/pipeline/main/_close_worker_connection_pool.py b/src/sqlbuild/executor/pipeline/main/_close_worker_connection_pool.py new file mode 100644 index 0000000000..3423594249 --- /dev/null +++ b/src/sqlbuild/executor/pipeline/main/_close_worker_connection_pool.py @@ -0,0 +1,16 @@ +"""Public entry for closing the extra connections of concurrent workers.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.pipeline._helpers.connections import close_connections + + +def close_worker_connection_pool( + *, adapter: BaseAdapter, connections: tuple[Any, ...], active_error: BaseException | None +) -> None: + """Close every connection, raising a close failure only when no error is already active.""" + + _ = close_connections(adapter=adapter, connections=connections, active_error=active_error) diff --git a/src/sqlbuild/executor/pipeline/main/_open_worker_connection_pool.py b/src/sqlbuild/executor/pipeline/main/_open_worker_connection_pool.py new file mode 100644 index 0000000000..5b04daa14e --- /dev/null +++ b/src/sqlbuild/executor/pipeline/main/_open_worker_connection_pool.py @@ -0,0 +1,18 @@ +"""Public entry for opening the extra connections of concurrent workers.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.pipeline._helpers.connections import open_worker_connections + + +def open_worker_connection_pool( + *, adapter: BaseAdapter, connection_config: dict[str, object], connection_count: int +) -> tuple[Any, ...]: + """Open `connection_count` connections concurrently, closing any opened when one fails.""" + + return open_worker_connections( + adapter=adapter, connection_config=connection_config, connection_count=connection_count + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 6c315e0b1a..b09a59b9b0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -197,10 +197,10 @@ class JanitorProgressE2ETestCase: stale_tables: int expected_exit_code: int expected_ordered_lines: tuple[str, ...] - expected_fragments: tuple[str, ...] = field(default_factory=tuple) + expected_archive_events: int unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) fail_rename: str | None = None - expected_archive_events: int | None = None + expected_remaining_tables: tuple[str, ...] = field(default_factory=tuple) @dataclass(frozen=True) @@ -212,6 +212,7 @@ class JanitorJsonE2ETestCase: stale_tables: int expected_status: str expected_archived: int + expected_item_statuses: tuple[str, ...] expected_stderr_fragments: tuple[str, ...] @@ -223,7 +224,10 @@ class JanitorInspectionCountE2ETestCase: stale_tables: dict[str, int] renamed_models: tuple[str, ...] expected_listings_per_schema: int + expected_multi_schema_listings: int + expected_existence_queries_per_schema: int expected_view_definition_reads: int + expected_batched_view_definition_reads: int expected_event_inserts: dict[str, int] @@ -234,8 +238,21 @@ class JanitorConcurrencyE2ETestCase: description: str stale_tables: dict[str, int] concurrency: int - fail_rename: str | None expected_exit_code: int + expected_connection_line: str + + +@dataclass(frozen=True) +class JanitorConcurrentFailureE2ETestCase: + """Test case for one failing action among concurrent janitor actions.""" + + description: str + stale_tables: dict[str, int] + concurrency: int + fail_rename: str + expected_exit_code: int + expected_failed_lines: int + expected_fragments: tuple[str, ...] @dataclass(frozen=True) @@ -245,3 +262,5 @@ class JanitorShortenedArchiveE2ETestCase: description: str long_relation_name: str identifier_limit: int + expected_rows_in_event_insert: int + expected_event_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index c986c8fcf6..ef611e854e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -2,9 +2,12 @@ from __future__ import annotations +import json +import re import subprocess from collections.abc import Mapping from datetime import UTC, datetime +from itertools import product from pathlib import Path from textwrap import dedent from types import MappingProxyType @@ -233,7 +236,7 @@ def supports_relation_age_metadata(self) -> bool: return True def maximum_identifier_length(self) -> int: - return int(_settings().get("identifier_limit", super().maximum_identifier_length())) + return int(_settings().get("identifier_limit") or super().maximum_identifier_length()) def list_relations( self, @@ -493,7 +496,7 @@ def concurrent_ddl_limit(self) -> int: return int(_settings().get("concurrent_ddl_limit", 1)) def maximum_identifier_length(self) -> int: - return int(_settings().get("identifier_limit", super().maximum_identifier_length())) + return int(_settings().get("identifier_limit") or super().maximum_identifier_length()) def _execute(self, *, connection: Any, sql: str) -> Any: settings = _settings() @@ -507,6 +510,16 @@ def _execute(self, *, connection: Any, sql: str) -> Any: """ +ARCHIVE_ITEM_LINE: re.Pattern[str] = re.compile( + r"^\s+(?P\d+)/(?P\d+) archived ", re.MULTILINE +) + + +def _probe_model_sql(*, schema: str, migrate_from: str) -> str: + migration: str = {"": ""}.get(migrate_from, f', migrate_from "{migrate_from}"') + return f"MODEL (description 'Test model.', schema {schema}{migration});\n\nSELECT 1 AS id\n" + + def prepare_probe_janitor_project( *, tmp_path: Path, @@ -515,18 +528,12 @@ def prepare_probe_janitor_project( renamed_models: tuple[str, ...] = (), concurrency: int = 1, ) -> Path: - """Build a DuckDB project with stale tables, an expired archive and old-name views per schema. - - Every schema in `stale_tables` gets one live model, each name in `renamed_models` renamed - once (leaving a compatibility view at its old name), `stale_tables[schema]` stale tables - and one expired archive. Builds use DuckDB; janitor then runs on the probe adapter, which - logs statements once settings enable it. - """ - - def model(schema: str, migrate_from: str = "") -> str: - migration: str = f', migrate_from "{migrate_from}"' if migrate_from else "" - return f"MODEL (description 'Test model.', schema {schema}{migration});\n\nSELECT 1 AS id\n" + """Build a DuckDB project with stale tables and an expired archive in each schema.""" + models: dict[str, str] = { + f"models/{schema}_{name}.sql": _probe_model_sql(schema=schema, migrate_from="") + for schema, name in product(stale_tables, ("orders", *renamed_models)) + } project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name=project_name, @@ -547,23 +554,13 @@ def model(schema: str, migrate_from: str = "") -> str: 'materialized = "table"\n' ), f"adapters/{PROBE_JANITOR_ADAPTER_NAME}.py": PROBE_JANITOR_ADAPTER_SOURCE, - **{ - f"models/{schema}_{name}.sql": model(schema) - for schema in stale_tables - for name in ("orders", *renamed_models) - }, + **models, }, ) _build(project_dir) - if renamed_models: - for schema in stale_tables: - for name in renamed_models: - (project_dir / "models" / f"{schema}_{name}.sql").unlink() - (project_dir / "models" / f"{schema}_{name}_v2.sql").write_text( - model(schema, migrate_from=f"{schema}.{schema}_{name}"), encoding="utf-8" - ) - _build(project_dir) db_path: Path = project_dir / "janitor.duckdb" + schema: str + count: int for schema, count in stale_tables.items(): tables: str = "; ".join( f"CREATE TABLE {schema}.old_orders_{index:03d} AS SELECT {index} AS id" @@ -573,12 +570,25 @@ def model(schema: str, migrate_from: str = "") -> str: db_path=db_path, sql=f'{tables}; CREATE TABLE {schema}."{EXPIRED_ARCHIVE_NAME}" AS SELECT 1 AS id', ) - (project_dir / "sqlbuild_local.toml").write_text( - f'adapter = "{PROBE_JANITOR_ADAPTER_NAME}"\n', encoding="utf-8" - ) return project_dir +def rename_probe_models( + *, project_dir: Path, schemas: tuple[str, ...], renamed_models: tuple[str, ...] +) -> None: + """Rename each model with `migrate_from` and rebuild, leaving compatibility views behind.""" + + schema: str + name: str + for schema, name in product(schemas, renamed_models): + (project_dir / "models" / f"{schema}_{name}.sql").unlink() + (project_dir / "models" / f"{schema}_{name}_v2.sql").write_text( + _probe_model_sql(schema=schema, migrate_from=f"{schema}.{schema}_{name}"), + encoding="utf-8", + ) + _build(project_dir) + + def _build(project_dir: Path) -> None: result: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build"), project_dir=project_dir @@ -593,31 +603,78 @@ def write_probe_settings( fail_rename: str | None = None, identifier_limit: int | None = None, ) -> None: - """Enable statement logging and set the probe adapter's concurrency and fault switches.""" + """Run janitor on the probe adapter with statement logging and these fault switches.""" - import json - - payload: dict[str, object] = {"log": True, "concurrent_ddl_limit": concurrent_ddl_limit} - if fail_rename is not None: - payload["fail_rename"] = fail_rename - if identifier_limit is not None: - payload["identifier_limit"] = identifier_limit + (project_dir / "sqlbuild_local.toml").write_text( + f'adapter = "{PROBE_JANITOR_ADAPTER_NAME}"\n', encoding="utf-8" + ) + (project_dir / PROBE_STATEMENT_LOG_FILENAME).write_text("", encoding="utf-8") (project_dir / PROBE_JANITOR_SETTINGS_FILENAME).write_text( - json.dumps(payload), encoding="utf-8" + json.dumps( + { + "log": True, + "concurrent_ddl_limit": concurrent_ddl_limit, + "fail_rename": fail_rename, + "identifier_limit": identifier_limit, + } + ), + encoding="utf-8", ) def read_probe_statements(*, project_dir: Path) -> tuple[str, ...]: """Return every statement the probe adapter logged, in order.""" - log: Path = project_dir / PROBE_STATEMENT_LOG_FILENAME - if not log.exists(): - return () - return tuple(log.read_text(encoding="utf-8").splitlines()) + return tuple( + (project_dir / PROBE_STATEMENT_LOG_FILENAME).read_text(encoding="utf-8").splitlines() + ) + + +def count_statements( + *, statements: tuple[str, ...], fragments: tuple[str, ...], excluded: tuple[str, ...] = () +) -> int: + """Count statements containing every fragment and none of the excluded fragments.""" + + return len( + tuple( + filter( + lambda statement: ( + all(fragment in statement for fragment in fragments) + and not any(fragment in statement for fragment in excluded) + ), + statements, + ) + ) + ) + + +def statement_index(*, statements: tuple[str, ...], prefix: str) -> int: + """The position of the first statement starting with `prefix`.""" + + return [statement.startswith(prefix) for statement in statements].index(True) + + +def archive_line_positions(output: str) -> tuple[tuple[int, int], ...]: + """The `n/N` counter of every archived action line, in printed order.""" + + return tuple( + (int(match["index"]), int(match["total"])) for match in ARCHIVE_ITEM_LINE.finditer(output) + ) + + +def assert_in_order(*, output: str, lines: tuple[str, ...]) -> None: + """Assert each line appears in `output` after the previous one.""" + + position: int = 0 + line: str + for line in lines: + found: int = output.find(line, position) + assert found >= 0, f"{line!r} missing after position {position} in:\n{output}" + position = found + len(line) -def janitor_end_state(*, db_path: Path) -> tuple[tuple[object, ...], ...]: - """Every relation with archive timestamps masked, then every audit event without its run.""" +def janitor_end_state(*, db_path: Path, schemas: tuple[str, ...]) -> tuple[tuple[object, ...], ...]: + """Every relation and every audit event in `schemas`, with archive timestamps masked.""" relations: list[tuple[object, ...]] = query_duckdb( db_path=db_path, @@ -628,9 +685,8 @@ def janitor_end_state(*, db_path: Path) -> tuple[tuple[object, ...], ...]: ), ) events: list[tuple[object, ...]] = [] - for schema in sorted({str(row[0]) for row in relations}): - if (schema, "_sqlbuild_janitor_events") not in relations: - continue + schema: str + for schema in schemas: events.extend( query_duckdb( db_path=db_path, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py index bf2f470fa9..79590ae6b3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py @@ -3,8 +3,6 @@ from __future__ import annotations import json -import math -import re import subprocess from pathlib import Path from typing import Any @@ -13,15 +11,21 @@ from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( JanitorConcurrencyE2ETestCase, + JanitorConcurrentFailureE2ETestCase, JanitorInspectionCountE2ETestCase, JanitorJsonE2ETestCase, JanitorProgressE2ETestCase, JanitorShortenedArchiveE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( + archive_line_positions, + assert_in_order, + count_statements, janitor_end_state, prepare_probe_janitor_project, read_probe_statements, + rename_probe_models, + statement_index, write_probe_settings, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( @@ -31,16 +35,7 @@ table_exists, ) -ITEM_LINE: re.Pattern[str] = re.compile(r"^\s+(?P\d+)/(?P\d+) (?P\w+)\s") - - -def _assert_in_order(output: str, lines: tuple[str, ...]) -> None: - position: int = 0 - line: str - for line in lines: - found: int = output.find(line, position) - assert found >= 0, f"{line!r} missing after position {position} in:\n{output}" - position = found + len(line) +APPROVED_JANITOR: tuple[str, ...] = ("--no-color", "janitor", "--auto-approve") @pytest.mark.parametrize( @@ -48,7 +43,7 @@ def _assert_in_order(output: str, lines: tuple[str, ...]) -> None: [ JanitorProgressE2ETestCase( description="each phase announces its work, streams each action and closes", - janitor_command=("--no-color", "janitor", "--auto-approve"), + janitor_command=APPROVED_JANITOR, stale_tables=12, expected_exit_code=0, expected_ordered_lines=( @@ -70,6 +65,7 @@ def _assert_in_order(output: str, lines: tuple[str, ...]) -> None: "Pruned 1 direct state table. (", "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", ), + expected_archive_events=12, unexpected_fragments=( " main.old_orders_010 -> ", "main._sqlbuild_fingerprints relation matches exclude pattern", @@ -77,7 +73,7 @@ def _assert_in_order(output: str, lines: tuple[str, ...]) -> None: ), JanitorProgressE2ETestCase( description="verbose preview lists every relation and skipped object", - janitor_command=("--no-color", "janitor", "--auto-approve", "--verbose"), + janitor_command=(*APPROVED_JANITOR, "--verbose"), stale_tables=12, expected_exit_code=0, expected_ordered_lines=( @@ -85,11 +81,12 @@ def _assert_in_order(output: str, lines: tuple[str, ...]) -> None: "Skipped objects\n main._sqlbuild_fingerprints relation matches exclude pattern", "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", ), + expected_archive_events=12, unexpected_fragments=("... and", "Skipped objects ("), ), JanitorProgressE2ETestCase( description="a failing rename keeps earlier lines, marks the item and fails the run", - janitor_command=("--no-color", "janitor", "--auto-approve"), + janitor_command=APPROVED_JANITOR, stale_tables=12, expected_exit_code=1, fail_rename="old_orders_005", @@ -102,8 +99,9 @@ def _assert_in_order(output: str, lines: tuple[str, ...]) -> None: "\u2717 Archiving failed after 5 of 12 relations (", "\u2717 Janitor failed archived 5, deleted 0, pruned 0 (", ), - unexpected_fragments=("7/12", "Deleting 1 expired archive", "Janitor completed"), expected_archive_events=5, + unexpected_fragments=("7/12", "Deleting 1 expired archive", "Janitor completed"), + expected_remaining_tables=("old_orders_005", "old_orders_011"), ), ], ids=lambda case: case.description, @@ -122,17 +120,19 @@ def test_given_stale_relations_when_running_janitor_then_progress_streams_in_ord command=test_case.janitor_command, project_dir=project_dir ) + db_path: Path = project_dir / "janitor.duckdb" assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - _assert_in_order(result.stdout, test_case.expected_ordered_lines) + assert_in_order(output=result.stdout, lines=test_case.expected_ordered_lines) + fragment: str for fragment in test_case.unexpected_fragments: assert fragment not in result.stdout - if test_case.expected_archive_events is not None: - db_path: Path = project_dir / "janitor.duckdb" - assert query_duckdb( - db_path=db_path, - sql="SELECT count(*) FROM main._sqlbuild_janitor_events WHERE event_type = 'archive'", - ) == [(test_case.expected_archive_events,)] - assert table_exists(db_path=db_path, table_name=test_case.fail_rename or "") + assert query_duckdb( + db_path=db_path, + sql="SELECT count(*) FROM main._sqlbuild_janitor_events WHERE event_type = 'archive'", + ) == [(test_case.expected_archive_events,)] + table: str + for table in test_case.expected_remaining_tables: + assert table_exists(db_path=db_path, table_name=table) @pytest.mark.parametrize( @@ -144,6 +144,7 @@ def test_given_stale_relations_when_running_janitor_then_progress_streams_in_ord stale_tables=3, expected_status="completed", expected_archived=3, + expected_item_statuses=("ok",) * 5, expected_stderr_fragments=( "Janitor preview default", "Archiving 3 relations...", @@ -172,11 +173,11 @@ def test_given_json_flag_when_running_janitor_then_stdout_holds_plan_and_result( assert payload["status"] == test_case.expected_status assert len(payload["plan"]["relations_to_archive"]) == test_case.stale_tables assert payload["result"]["counts"]["archived"] == test_case.expected_archived - assert [item["status"] for item in payload["result"]["items"]] == ["ok"] * ( - test_case.stale_tables + 2 + assert tuple(item["status"] for item in payload["result"]["items"]) == ( + test_case.expected_item_statuses ) assert payload["error"] is None - _assert_in_order(result.stderr, test_case.expected_stderr_fragments) + assert_in_order(output=result.stderr, lines=test_case.expected_stderr_fragments) @pytest.mark.parametrize( @@ -187,7 +188,10 @@ def test_given_json_flag_when_running_janitor_then_stdout_holds_plan_and_result( stale_tables={"main": 150, "mart": 3, "staging": 3}, renamed_models=("revenue", "customers"), expected_listings_per_schema=1, + expected_multi_schema_listings=0, + expected_existence_queries_per_schema=1, expected_view_definition_reads=3, + expected_batched_view_definition_reads=3, expected_event_inserts={"main": 3, "mart": 2, "staging": 2}, ), ], @@ -202,45 +206,44 @@ def test_given_several_schemas_when_running_janitor_then_reads_and_writes_scale_ stale_tables=test_case.stale_tables, renamed_models=test_case.renamed_models, ) + rename_probe_models( + project_dir=project_dir, + schemas=tuple(test_case.stale_tables), + renamed_models=test_case.renamed_models, + ) write_probe_settings(project_dir=project_dir) result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), project_dir=project_dir + command=APPROVED_JANITOR, project_dir=project_dir ) assert result.returncode == 0, result.stdout + result.stderr statements: tuple[str, ...] = read_probe_statements(project_dir=project_dir) schema: str - for schema in test_case.stale_tables: - listings: tuple[str, ...] = tuple( - statement - for statement in statements - if "FROM information_schema.tables" in statement - and f"table_schema IN ('{schema}')" in statement - and "table_catalog" not in statement - ) - assert len(listings) == test_case.expected_listings_per_schema, listings - existence: tuple[str, ...] = tuple( - statement - for statement in statements - if statement.startswith(f"SELECT event_id FROM {schema}._sqlbuild_janitor_events") - ) - assert len(existence) == 1, existence - inserts: int = sum( - statement.startswith(f"INSERT INTO {schema}._sqlbuild_janitor_events") - for statement in statements - ) - archived: int = test_case.stale_tables[schema] - assert inserts == test_case.expected_event_inserts[schema] == math.ceil(archived / 100) + 1 - assert not any( - "FROM information_schema.tables" in statement and "', '" in statement - for statement in statements - ) - view_reads: tuple[str, ...] = tuple( - statement for statement in statements if "FROM duckdb_views()" in statement + inserts: int + for schema, inserts in test_case.expected_event_inserts.items(): + assert count_statements( + statements=statements, + fragments=("FROM information_schema.tables", f"table_schema IN ('{schema}')"), + excluded=("table_catalog",), + ) == (test_case.expected_listings_per_schema) + assert count_statements( + statements=statements, + fragments=(f"SELECT event_id FROM {schema}._sqlbuild_janitor_events WHERE",), + ) == (test_case.expected_existence_queries_per_schema) + assert count_statements( + statements=statements, + fragments=(f"INSERT INTO {schema}._sqlbuild_janitor_events",), + ) == (inserts) + assert count_statements( + statements=statements, fragments=("FROM information_schema.tables", "', '") + ) == (test_case.expected_multi_schema_listings) + assert count_statements(statements=statements, fragments=("FROM duckdb_views()",)) == ( + test_case.expected_view_definition_reads ) - assert len(view_reads) == test_case.expected_view_definition_reads, view_reads - assert all(" IN (" in statement for statement in view_reads) + assert count_statements( + statements=statements, fragments=("FROM duckdb_views()", "view_name IN (") + ) == (test_case.expected_batched_view_definition_reads) @pytest.mark.parametrize( @@ -250,15 +253,8 @@ def test_given_several_schemas_when_running_janitor_then_reads_and_writes_scale_ description="concurrent actions leave the same end state as sequential ones", stale_tables={"main": 30, "staging": 5}, concurrency=4, - fail_rename=None, expected_exit_code=0, - ), - JanitorConcurrencyE2ETestCase( - description="a failing concurrent rename is reported alone and events stay complete", - stale_tables={"main": 30}, - concurrency=4, - fail_rename="old_orders_012", - expected_exit_code=1, + expected_connection_line="Connecting to probe_janitor_duckdb (3 connections)", ), ], ids=lambda case: case.description, @@ -278,59 +274,85 @@ def test_given_concurrent_adapter_when_running_janitor_then_matches_sequential_e concurrency=test_case.concurrency, ) write_probe_settings(project_dir=sequential_dir) - write_probe_settings( - project_dir=concurrent_dir, - concurrent_ddl_limit=test_case.concurrency, - fail_rename=test_case.fail_rename, - ) - command: tuple[str, ...] = ("--no-color", "janitor", "--auto-approve") + write_probe_settings(project_dir=concurrent_dir, concurrent_ddl_limit=test_case.concurrency) sequential: subprocess.CompletedProcess[str] = run_sqb( - command=command, project_dir=sequential_dir + command=APPROVED_JANITOR, project_dir=sequential_dir ) concurrent: subprocess.CompletedProcess[str] = run_sqb( - command=command, project_dir=concurrent_dir + command=APPROVED_JANITOR, project_dir=concurrent_dir ) - assert sequential.returncode == 0, sequential.stdout + sequential.stderr - assert concurrent.returncode == test_case.expected_exit_code, ( - concurrent.stdout + concurrent.stderr - ) total: int = sum(test_case.stale_tables.values()) - archive_lines: list[re.Match[str]] = [ - match - for line in concurrent.stdout.splitlines() - if (match := ITEM_LINE.match(line)) is not None and match["action"] == "archived" - ] - assert [int(match["index"]) for match in archive_lines] == list( - range(1, len(archive_lines) + 1) + schemas: tuple[str, ...] = tuple(test_case.stale_tables) + assert sequential.returncode == test_case.expected_exit_code, sequential.stdout + assert concurrent.returncode == test_case.expected_exit_code, concurrent.stdout + assert test_case.expected_connection_line in concurrent.stdout + assert test_case.expected_connection_line not in sequential.stdout + assert archive_line_positions(concurrent.stdout) == tuple( + (index, total) for index in range(1, total + 1) ) - assert {int(match["total"]) for match in archive_lines} == {total} - assert f"Connecting to probe_janitor_duckdb ({test_case.concurrency - 1} connections)" in ( - concurrent.stdout + assert janitor_end_state(db_path=concurrent_dir / "janitor.duckdb", schemas=schemas) == ( + janitor_end_state(db_path=sequential_dir / "janitor.duckdb", schemas=schemas) ) - concurrent_db: Path = concurrent_dir / "janitor.duckdb" - if test_case.fail_rename is None: - assert len(archive_lines) == total - assert janitor_end_state(db_path=concurrent_db) == janitor_end_state( - db_path=sequential_dir / "janitor.duckdb" - ) - return - assert sum(" FAIL " in line for line in concurrent.stdout.splitlines()) == 1 - assert "\u2717 Janitor failed archived" in concurrent.stdout - assert table_exists(db_path=concurrent_db, table_name=test_case.fail_rename) - renamed: list[tuple[object, ...]] = query_duckdb( - db_path=concurrent_db, - sql=( - "SELECT count(*) FROM information_schema.tables " - "WHERE table_schema = 'main' AND table_name LIKE '\\_sqb\\_archive\\_\\_%old\\_orders%' " - "ESCAPE '\\'" + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorConcurrentFailureE2ETestCase( + description="a failing concurrent rename is reported alone and events stay complete", + stale_tables={"main": 30}, + concurrency=4, + fail_rename="old_orders_012", + expected_exit_code=1, + expected_failed_lines=1, + expected_fragments=( + "simulated rename failure for old_orders_012", + "\u2717 Archiving failed after ", + "\u2717 Janitor failed archived ", + ), ), + ], + ids=lambda case: case.description, +) +def test_given_failing_concurrent_rename_when_running_janitor_then_events_match_renames( + test_case: JanitorConcurrentFailureE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, + project_name="janitor_concurrent_failure", + stale_tables=test_case.stale_tables, + concurrency=test_case.concurrency, ) + write_probe_settings( + project_dir=project_dir, + concurrent_ddl_limit=test_case.concurrency, + fail_rename=test_case.fail_rename, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=APPROVED_JANITOR, project_dir=project_dir + ) + + db_path: Path = project_dir / "janitor.duckdb" + positions: tuple[tuple[int, int], ...] = archive_line_positions(result.stdout) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + assert result.stdout.count(" FAIL ") == test_case.expected_failed_lines + assert_in_order(output=result.stdout, lines=test_case.expected_fragments) + assert tuple(index for index, _ in positions) == tuple(range(1, len(positions) + 1)) + assert table_exists(db_path=db_path, table_name=test_case.fail_rename) assert query_duckdb( - db_path=concurrent_db, + db_path=db_path, sql="SELECT count(*) FROM main._sqlbuild_janitor_events WHERE event_type = 'archive'", - ) == renamed + ) == query_duckdb( + db_path=db_path, + sql=( + "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'main' " + "AND starts_with(table_name, '_sqb_archive__') " + "AND contains(table_name, '__old_orders_')" + ), + ) @pytest.mark.parametrize( @@ -340,6 +362,8 @@ def test_given_concurrent_adapter_when_running_janitor_then_matches_sequential_e description="a shortened archive name gets its event right after its rename", long_relation_name="legacy_orders_with_a_very_long_descriptive_name_kept_for_audit", identifier_limit=48, + expected_rows_in_event_insert=1, + expected_event_count=1, ), ], ids=lambda case: case.description, @@ -357,23 +381,22 @@ def test_given_shortened_archive_name_when_archiving_then_event_is_written_immed write_probe_settings(project_dir=project_dir, identifier_limit=test_case.identifier_limit) result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), project_dir=project_dir + command=APPROVED_JANITOR, project_dir=project_dir ) assert result.returncode == 0, result.stdout + result.stderr statements: tuple[str, ...] = read_probe_statements(project_dir=project_dir) - rename: int = next( - index - for index, statement in enumerate(statements) - if statement.startswith(f"ALTER TABLE main.{test_case.long_relation_name} RENAME") + rename: int = statement_index( + statements=statements, prefix=f"ALTER TABLE main.{test_case.long_relation_name} RENAME" ) - assert statements[rename + 1].startswith("INSERT INTO main._sqlbuild_janitor_events") - assert f"'{test_case.long_relation_name}'" in statements[rename + 1] - assert statements[rename + 1].count("), (") == 0 + event_insert: str = statements[rename + 1] + assert event_insert.startswith("INSERT INTO main._sqlbuild_janitor_events") + assert f"'{test_case.long_relation_name}'" in event_insert + assert event_insert.count("), (") + 1 == test_case.expected_rows_in_event_insert assert query_duckdb( db_path=project_dir / "janitor.duckdb", sql=( "SELECT count(*) FROM main._sqlbuild_janitor_events " f"WHERE original_name = '{test_case.long_relation_name}'" ), - ) == [(1,)] + ) == [(test_case.expected_event_count,)] diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py index dca13b7453..3e4bf29302 100644 --- a/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/janitor_events/classes/_test_types.py @@ -8,7 +8,7 @@ class JanitorEventWriterTestCase: description: str record_count: int written_before: int - write_now_first: bool + write_passes: int expected_row_count: int expected_insert_count: int expected_existence_query_count: int diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/classes/helpers.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/helpers.py new file mode 100644 index 0000000000..6e2c5f368e --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/classes/helpers.py @@ -0,0 +1,65 @@ +"""Synthetic janitor events and a statement-recording executor for event writer tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.janitor_events.classes.janitor_event_writer import JanitorEventWriter +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_SCHEMA_VERSION +from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType + +RUN_ID: str = "run_orders_cleanup" +ARCHIVED_AT: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) + + +def archive_event(index: int) -> JanitorEventRecord: + """The archive event of the synthetic relation `orders_` in schema `main`.""" + + archive_name: str = f"_sqb_archive__20260924t101500z__orders_{index:04d}" + return JanitorEventRecord( + event_id=build_janitor_event_id( + event_type=JanitorEventType.ARCHIVE, + run_id=RUN_ID, + relation_database=None, + relation_schema="main", + archive_name=archive_name, + ), + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=JanitorEventType.ARCHIVE, + occurred_at=ARCHIVED_AT, + run_id=RUN_ID, + relation_database=None, + relation_schema="main", + relation_type="BASE TABLE", + original_name=f"orders_{index:04d}", + original_qualified_name=f"main.orders_{index:04d}", + archive_name=archive_name, + archive_qualified_name=f"main.{archive_name}", + archived_at=ARCHIVED_AT, + ) + + +class RecordingExecute: + """An adapter's `execute` that remembers every statement it ran.""" + + def __init__(self, adapter: DuckDbAdapter) -> None: + self.adapter: DuckDbAdapter = adapter + self.statements: list[str] = [] + + def __call__(self, *, connection: Any, sql: str) -> Any: + self.statements.append(sql) + return self.adapter.execute(connection=connection, sql=sql) + + +def event_writer(execute: RecordingExecute) -> JanitorEventWriter: + """A writer for `RUN_ID` on `execute`.""" + + return JanitorEventWriter( + execute=execute, + render_qualified_name=execute.adapter.render_qualified_name, + run_id=RUN_ID, + ) diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py b/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py index 4574848f6d..8c0b72ad57 100644 --- a/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py +++ b/tests/integration/src/sqlbuild/executor/janitor_events/classes/test_janitor_event_writer.py @@ -2,7 +2,6 @@ from __future__ import annotations -from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -14,58 +13,17 @@ JANITOR_EVENT_COLUMNS, JANITOR_EVENT_SCHEMA_VERSION, ) -from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id from sqlbuild.executor.janitor_events.models import JanitorEventRecord -from sqlbuild.executor.janitor_events.types import JanitorEventType from tests.integration.src.sqlbuild.executor.janitor_events.classes._test_types import ( JanitorEventWriterTestCase, ) - -RUN_ID: str = "run_orders_cleanup" -ARCHIVED_AT: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) - - -def _record(index: int) -> JanitorEventRecord: - archive_name: str = f"_sqb_archive__20260924t101500z__orders_{index:04d}" - return JanitorEventRecord( - event_id=build_janitor_event_id( - event_type=JanitorEventType.ARCHIVE, - run_id=RUN_ID, - relation_database=None, - relation_schema="main", - archive_name=archive_name, - ), - schema_version=JANITOR_EVENT_SCHEMA_VERSION, - event_type=JanitorEventType.ARCHIVE, - occurred_at=ARCHIVED_AT, - run_id=RUN_ID, - relation_database=None, - relation_schema="main", - relation_type="BASE TABLE", - original_name=f"orders_{index:04d}", - original_qualified_name=f"main.orders_{index:04d}", - archive_name=archive_name, - archive_qualified_name=f"main.{archive_name}", - archived_at=ARCHIVED_AT, - ) - - -class _RecordingExecute: - def __init__(self, adapter: DuckDbAdapter) -> None: - self.adapter: DuckDbAdapter = adapter - self.statements: list[str] = [] - - def __call__(self, *, connection: Any, sql: str) -> Any: - self.statements.append(sql) - return self.adapter.execute(connection=connection, sql=sql) - - -def _writer(execute: _RecordingExecute) -> JanitorEventWriter: - return JanitorEventWriter( - execute=execute, - render_qualified_name=execute.adapter.render_qualified_name, - run_id=RUN_ID, - ) +from tests.integration.src.sqlbuild.executor.janitor_events.classes.helpers import ( + ARCHIVED_AT, + RUN_ID, + RecordingExecute, + archive_event, + event_writer, +) @pytest.mark.parametrize( @@ -75,7 +33,7 @@ def _writer(execute: _RecordingExecute) -> JanitorEventWriter: description="events past one chunk are appended with one insert per chunk", record_count=250, written_before=0, - write_now_first=False, + write_passes=1, expected_row_count=250, expected_insert_count=3, expected_existence_query_count=1, @@ -84,7 +42,7 @@ def _writer(execute: _RecordingExecute) -> JanitorEventWriter: description="a retried run skips every event it already wrote", record_count=5, written_before=5, - write_now_first=False, + write_passes=1, expected_row_count=5, expected_insert_count=0, expected_existence_query_count=1, @@ -93,17 +51,17 @@ def _writer(execute: _RecordingExecute) -> JanitorEventWriter: description="a retried run appends only the events it had not written", record_count=5, written_before=3, - write_now_first=False, + write_passes=1, expected_row_count=5, expected_insert_count=1, expected_existence_query_count=1, ), JanitorEventWriterTestCase( - description="an event written at once is not held for the next flush", - record_count=1, + description="writing the same events again in one run appends nothing more", + record_count=4, written_before=0, - write_now_first=True, - expected_row_count=1, + write_passes=3, + expected_row_count=4, expected_insert_count=1, expected_existence_query_count=1, ), @@ -117,25 +75,22 @@ def test_given_janitor_events_when_writing_in_chunks_then_each_event_is_appended adapter: DuckDbAdapter = DuckDbAdapter() connection: Any = adapter.connect({"database": str(tmp_path / "janitor_events.duckdb")}) records: tuple[JanitorEventRecord, ...] = tuple( - _record(index) for index in range(test_case.record_count) + archive_event(index) for index in range(test_case.record_count) ) try: adapter.execute( connection=connection, sql=adapter.render_create_janitor_event_table_sql(database=None, schema="main"), ) - earlier: JanitorEventWriter = _writer(_RecordingExecute(adapter)) - for record in records[: test_case.written_before]: - earlier.add(record) - earlier.flush(connection=connection) - execute: _RecordingExecute = _RecordingExecute(adapter) - writer: JanitorEventWriter = _writer(execute) + event_writer(RecordingExecute(adapter)).write( + connection=connection, records=records[: test_case.written_before] + ) + execute: RecordingExecute = RecordingExecute(adapter) + writer: JanitorEventWriter = event_writer(execute) writer.read_existing(connection=connection, schemas=((None, "main"),)) - if test_case.write_now_first: - writer.write_now(connection=connection, record=records[0]) - for record in records: - writer.add(record) - writer.flush(connection=connection) + _: int + for _ in range(test_case.write_passes): + writer.write(connection=connection, records=records) rows: list[tuple[object, ...]] = connection.execute( f"SELECT {', '.join(JANITOR_EVENT_COLUMNS)} FROM main._sqlbuild_janitor_events " "ORDER BY archive_name" diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py index 501548a70b..5939d278fd 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_view_definitions.py @@ -36,7 +36,11 @@ def execute(self, *, connection: Any, sql: str) -> Any: if sql.startswith("SELECT TABLE_NAME, VIEW_DEFINITION"): return _Rows([row for row in self.stored if f"UPPER('{row[0]}')" in sql.upper()]) return _Rows( - [(definition,) for name, definition in self.stored if f"'{name.upper()}'" in sql.upper()] + [ + (definition,) + for name, definition in self.stored + if f"'{name.upper()}'" in sql.upper() + ] ) From a19892d8e26d22191878e02fdafb7698b8ff66be Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 00:35:38 +0100 Subject: [PATCH 08/10] refactor: keep janitor progress and fensu budgets within native boundary policy --- .../src/janitor_text/_helpers/phases.rs | 4 +- .../src/janitor_text/_helpers/plan_json.rs | 2 +- .../src/janitor_text/constants.rs | 2 + .../janitor_text/main/format_janitor_json.rs | 4 +- .../main/render_janitor_completion.rs | 3 +- .../main/render_janitor_item_line.rs | 3 +- .../janitor_text/main/render_janitor_plan.rs | 4 +- .../src/janitor_text/tests/helpers.rs | 8 +- .../tests/test_janitor_progress_text.rs | 138 ++++++++++-------- .../src/janitor_text/tests/test_types.rs | 9 ++ .../build_progress/transient_status.rs | 14 +- .../_helpers/janitor/janitor_facts.rs | 8 +- .../_helpers/janitor/janitor_phases.rs | 4 +- .../_helpers/janitor/janitor_progress.rs | 76 ++++------ fensu.toml | 6 +- .../janitor/classes/janitor_item_reporter.py | 14 +- src/sqlbuild/executor/janitor/models.py | 4 +- src/sqlbuild/executor/janitor/types.py | 9 +- 18 files changed, 157 insertions(+), 155 deletions(-) diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs index 2b44fe653a..98a0026abd 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs @@ -1,7 +1,7 @@ //! Wording and durations of janitor apply phases. use crate::cli_text::main::format_count_noun::format_count_noun; -use crate::janitor_text::constants::{SECONDS_PER_HOUR, SECONDS_PER_MINUTE}; +use crate::janitor_text::constants::{MINUTE_SECONDS, SECONDS_PER_HOUR, SECONDS_PER_MINUTE}; use crate::janitor_text::models::JanitorPhase; /// The phase's present participle, past tense and singular noun. @@ -23,7 +23,7 @@ pub(crate) fn noun(count: usize, singular: &str) -> String { /// Seconds with two decimals under a minute, then whole minutes and seconds. pub(crate) fn duration(seconds: f64) -> String { - if seconds < 60.0 { + if seconds < MINUTE_SECONDS { return format!("{:.2}s", seconds.max(0.0)); } let whole: u64 = seconds.round() as u64; diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs index d9575dfa4e..3a52cb0afc 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs @@ -81,7 +81,7 @@ fn source_schema(schema: &SourceSchemaText, blocked: bool) -> JsonValue { /// Every planned action, retained archive and skipped object of the plan. pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue { - let views = plan + let views: Vec = plan .old_name_drops .iter() .map(|view| old_name_view(view, "drop")) diff --git a/crates/sqlbuild-core/src/janitor_text/constants.rs b/crates/sqlbuild-core/src/janitor_text/constants.rs index e8bbff3d49..13f8108f08 100644 --- a/crates/sqlbuild-core/src/janitor_text/constants.rs +++ b/crates/sqlbuild-core/src/janitor_text/constants.rs @@ -19,4 +19,6 @@ pub(crate) const ACTION_WIDTH: usize = "archived".len(); pub(crate) const RECORDED_ACTION: &str = "recorded"; pub(crate) const DROPPED_ACTION: &str = "dropped"; pub(crate) const SECONDS_PER_MINUTE: u64 = 60; +/// Durations below this many seconds show seconds with two decimals. +pub(crate) const MINUTE_SECONDS: f64 = 60.0; pub(crate) const SECONDS_PER_HOUR: u64 = 3600; diff --git a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs index 6660340d72..8f644e58ab 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/format_janitor_json.rs @@ -5,9 +5,7 @@ use crate::janitor_text::models::{JanitorOutcome, JanitorPlanText, JanitorRunTex use crate::json::main::dumps::dumps; use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; -/// `json.dumps(payload, indent=2)` of the outcome, the plan, what the run did and its error. -/// -/// The plan is none when janitor is disabled for the project. +/// `json.dumps(payload, indent=2)` of the outcome, the plan (none when disabled), run and error. #[must_use] pub fn format_janitor_json( plan: Option<&JanitorPlanText>, diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs index 391e6e5d0a..8523335723 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_completion.rs @@ -5,8 +5,7 @@ use crate::cli_text::models::{CliStyle, CompletionState}; use crate::janitor_text::_helpers::phases::duration; use crate::janitor_text::models::JanitorResultCounts; -/// `✓ Janitor completed archived N, deleted M, pruned K (5m 02s)`, or `✗ Janitor failed ...` -/// with what was done before the failure. +/// `✓ Janitor completed archived N, deleted M, pruned K (5m 02s)`, or `✗ Janitor failed ...`. #[must_use] pub fn render_janitor_completion( counts: &JanitorResultCounts, diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs index cea767a76a..0dec945eeb 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_item_line.rs @@ -5,8 +5,7 @@ use crate::janitor_text::_helpers::phases::{duration, flow}; use crate::janitor_text::constants::ACTION_WIDTH; use crate::janitor_text::models::JanitorItemText; -/// Position, action, name or `name -> destination` padded to `flow_width`, status and duration, -/// then the error of a failed action. +/// Position, action, padded `name -> destination`, status, duration and any error of one action. #[must_use] pub fn render_janitor_item_line( (index, total): (usize, usize), diff --git a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs index 7e83fbcfb6..4c3747f3be 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/render_janitor_plan.rs @@ -8,9 +8,7 @@ use crate::janitor_text::_helpers::plan_sections::{ use crate::janitor_text::constants::PREVIEW_LIMIT; use crate::janitor_text::models::JanitorPlanText; -/// Every section of the preview, ending with a blank line. -/// -/// Without `verbose`, each list shows its first entries and skipped objects are counted by reason. +/// Every preview section, capped and grouped unless `verbose`, ending with a blank line. #[must_use] pub fn render_janitor_plan(plan: &JanitorPlanText, verbose: bool, style: CliStyle) -> String { let limit: Option = (!verbose).then_some(PREVIEW_LIMIT); diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index bebb87af7d..bc3d89307c 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -53,11 +53,9 @@ pub(super) fn large_plan(count: usize, skipped: usize) -> JanitorPlanText { .collect(), skipped_relations: (0..skipped) .map(|index| { - let reason: &str = if index % 3 == 0 { - "relation is newer than 14 days" - } else { - "not tracked by SQLBuild" - }; + let reasons: [&str; 2] = + ["relation is newer than 14 days", "not tracked by SQLBuild"]; + let reason: &str = reasons[usize::from(index % 3 != 0)]; (format!("dev.recent_{index:03}"), reason.to_owned()) }) .collect(), diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs index 3b75ffde67..cab6bcb186 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs @@ -14,7 +14,8 @@ use crate::janitor_text::models::{ }; use crate::janitor_text::tests::helpers::{direct_archive_plan, item, large_plan}; use crate::janitor_text::tests::test_types::{ - CompletionTestCase, ItemLineTestCase, JsonTestCase, PhaseTextTestCase, PreviewTestCase, + CompletionTestCase, ItemCountsTestCase, ItemLineTestCase, JsonTestCase, PhaseTextTestCase, + PreviewTestCase, }; const PLAIN: CliStyle = CliStyle { use_color: false }; @@ -200,43 +201,55 @@ fn given_run_counts_when_rendering_completion_then_final_line_is_unambiguous() { #[test] fn given_finished_actions_when_counting_then_only_successes_count_by_kind() { - let items = vec![ - item(JanitorPhase::Archive, ("archived", "a", Some("b")), None), - item( - JanitorPhase::Archive, - ("archived", "c", Some("d")), - Some("denied"), - ), - item(JanitorPhase::DeleteArchives, ("deleted", "e", None), None), - item(JanitorPhase::DropOldNameViews, ("dropped", "f", None), None), - item( - JanitorPhase::DropOldNameViews, - ("recorded", "g", None), - None, - ), - item( - JanitorPhase::DropQueryArtifacts, - ("dropped", "h", None), - None, - ), - item(JanitorPhase::PruneDirectState, ("pruned", "i", None), None), - ]; - - assert_eq!( - janitor_item_counts(&items), - JanitorResultCounts { + let test_cases = [ItemCountsTestCase { + description: "failed actions and views already gone are not counted", + items: vec![ + item(JanitorPhase::Archive, ("archived", "a", Some("b")), None), + item( + JanitorPhase::Archive, + ("archived", "c", Some("d")), + Some("denied"), + ), + item(JanitorPhase::DeleteArchives, ("deleted", "e", None), None), + item(JanitorPhase::DropOldNameViews, ("dropped", "f", None), None), + item( + JanitorPhase::DropOldNameViews, + ("recorded", "g", None), + None, + ), + item( + JanitorPhase::DropQueryArtifacts, + ("dropped", "h", None), + None, + ), + item(JanitorPhase::PruneDirectState, ("pruned", "i", None), None), + ], + plan: large_plan(3, 0), + expected_counts: JanitorResultCounts { archived: 1, deleted: 0, deleted_archives: 1, deleted_query_artifacts: 1, dropped_old_name_views: 1, pruned_direct_state: 1, - } - ); - assert_eq!( - janitor_phase_flow_width(&large_plan(3, 0), JanitorPhase::Archive), - 72 - ); + }, + expected_archive_flow_width: 72, + }]; + + for test_case in test_cases { + assert_eq!( + janitor_item_counts(&test_case.items), + test_case.expected_counts, + "{}", + test_case.description + ); + assert_eq!( + janitor_phase_flow_width(&test_case.plan, JanitorPhase::Archive), + test_case.expected_archive_flow_width, + "{}", + test_case.description + ); + } } #[test] @@ -268,20 +281,22 @@ fn given_long_plan_when_previewing_then_lists_are_capped_unless_verbose() { for test_case in test_cases { let text: String = render_janitor_plan(&test_case.plan, test_case.verbose, PLAIN); - for fragment in test_case.expected_fragments { - assert!( - text.contains(fragment), - "{}: missing {fragment:?} in {text}", - test_case.description - ); - } - for fragment in test_case.unexpected_fragments { - assert!( - !text.contains(fragment), - "{}: unexpected {fragment:?}", - test_case.description - ); - } + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: missing fragments in {text}", + test_case.description + ); + assert!( + !test_case + .unexpected_fragments + .iter() + .any(|fragment| text.contains(fragment)), + "{}: unexpected fragments in {text}", + test_case.description + ); } } @@ -302,6 +317,7 @@ fn given_outcome_when_formatting_json_then_document_carries_plan_result_and_erro let test_cases = [ JsonTestCase { description: "a completed run lists its actions", + plan: Some(direct_archive_plan()), outcome: JanitorOutcome::Completed, run: Some(run.clone()), error: None, @@ -318,6 +334,7 @@ fn given_outcome_when_formatting_json_then_document_carries_plan_result_and_erro }, JsonTestCase { description: "a failed run carries its error", + plan: Some(direct_archive_plan()), outcome: JanitorOutcome::Failed, run: Some(run), error: Some("rename denied"), @@ -325,29 +342,36 @@ fn given_outcome_when_formatting_json_then_document_carries_plan_result_and_erro }, JsonTestCase { description: "a preview has no result", + plan: Some(direct_archive_plan()), outcome: JanitorOutcome::Previewed, run: None, error: None, expected_fragments: &["\"status\": \"previewed\"", "\"result\": null"], }, + JsonTestCase { + description: "a disabled janitor has no plan", + plan: None, + outcome: JanitorOutcome::Disabled, + run: None, + error: None, + expected_fragments: &["\"status\": \"disabled\"", "\"plan\": null"], + }, ]; for test_case in test_cases { let text: String = format_janitor_json( - Some(&direct_archive_plan()), + test_case.plan.as_ref(), test_case.outcome, test_case.run.as_ref(), test_case.error, ); - for fragment in test_case.expected_fragments { - assert!( - text.contains(fragment), - "{}: missing {fragment:?} in {text}", - test_case.description - ); - } + assert!( + test_case + .expected_fragments + .iter() + .all(|fragment| text.contains(fragment)), + "{}: missing fragments in {text}", + test_case.description + ); } - assert!( - format_janitor_json(None, JanitorOutcome::Disabled, None, None).contains("\"plan\": null") - ); } diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs index 4201f45ccd..40f69d1b08 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs @@ -49,6 +49,14 @@ pub(super) struct CompletionTestCase { pub(super) expected_text: &'static str, } +pub(super) struct ItemCountsTestCase { + pub(super) description: &'static str, + pub(super) items: Vec, + pub(super) plan: JanitorPlanText, + pub(super) expected_counts: JanitorResultCounts, + pub(super) expected_archive_flow_width: usize, +} + pub(super) struct PreviewTestCase { pub(super) description: &'static str, pub(super) plan: JanitorPlanText, @@ -59,6 +67,7 @@ pub(super) struct PreviewTestCase { pub(super) struct JsonTestCase { pub(super) description: &'static str, + pub(super) plan: Option, pub(super) outcome: JanitorOutcome, pub(super) run: Option, pub(super) error: Option<&'static str>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index 33c6f16149..abc2f86bdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -51,7 +51,7 @@ impl TransientStatusReporter { within(self.lock.bind(py), |_| body()) } - fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { + pub(crate) fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { let options: Bound<'_, PyDict> = PyDict::new(py); options.set_item("stream", self.stream.bind(py))?; options.set_item("text", text)?; @@ -161,7 +161,7 @@ impl TransientStatusReporter { }) } - fn start(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { + pub(crate) fn start(slf: &Bound<'_, Self>, message: String) -> PyResult<()> { let py: Python<'_> = slf.py(); let reporter: &Self = slf.get(); if reporter.enabled { @@ -182,7 +182,7 @@ impl TransientStatusReporter { reporter.write_message(py, &message, true) } - fn update(&self, py: Python<'_>, message: String) -> PyResult<()> { + pub(crate) fn update(&self, py: Python<'_>, message: String) -> PyResult<()> { if self.enabled { let updated: bool = self.locked(py, || { let drawn: bool = { @@ -206,7 +206,11 @@ impl TransientStatusReporter { } #[pyo3(signature = (*, message, blank_line_after = false))] - fn complete(slf: &Bound<'_, Self>, message: &str, blank_line_after: bool) -> PyResult<()> { + pub(crate) fn complete( + slf: &Bound<'_, Self>, + message: &str, + blank_line_after: bool, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); Self::close(slf)?; slf.get().write_message(py, message, true)?; @@ -218,7 +222,7 @@ impl TransientStatusReporter { /// Complete with a pre-styled message written verbatim. #[pyo3(signature = (*, message, blank_line_after = false))] - fn complete_styled( + pub(crate) fn complete_styled( slf: &Bound<'_, Self>, message: &str, blank_line_after: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index fdda91371b..052592d9e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -244,9 +244,7 @@ fn source_schemas( Ok(schemas) } -/// Relations matched case-insensitively and respelled as their target schema. -/// -/// Each schema is listed on its own, so every other planning read of it hits the catalog. +/// Each target schema's relations, listed once through the catalog and respelled as targeted. fn list_target_schema_relations<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -308,9 +306,7 @@ fn list_target_schema_relations<'py>( Ok(result) } -/// Relations present at the given locations, keyed as `RelationLookup.key` folds them. -/// -/// Like `build_relation_lookup`, but each schema is listed through the inspection catalog. +/// Folded keys of relations in the locations' schemas, listed through the inspection catalog. fn relation_lookup( warehouse: &Warehouse<'_>, locations: &[(Option, String, &str)], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index fc6753ef99..ec56d16473 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -302,9 +302,7 @@ pub(crate) fn build_janitor_execution_plan<'py>( Ok(plan) } -/// Execute the warehouse cleanup plan, reporting each phase and action to `progress`. -/// -/// Independent actions of a phase run on up to the command's concurrency of connections. +/// Execute the cleanup plan on up to the command's concurrency, reporting to `progress`. pub(crate) fn execute_janitor_cleanup<'py>( invocation: &Bound<'py, PyAny>, (compiled, connection): (&JanitorCompiled<'py>, &Bound<'py, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs index 7ba1d3e3cd..b5bf9dbf4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs @@ -5,7 +5,6 @@ use std::time::Instant; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyclass, pymethods}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::janitor_text::main::janitor_phase_complete_text::janitor_phase_complete_text; @@ -15,9 +14,9 @@ use sqlbuild_core::janitor_text::main::janitor_phase_start_text::janitor_phase_s use sqlbuild_core::janitor_text::main::render_janitor_item_line::render_janitor_item_line; use sqlbuild_core::janitor_text::models::{JanitorItemText, JanitorPhase, JanitorPlanText}; -use crate::bindings::_helpers::build_progress::terminal::{shared_lines, tty_flag}; +use crate::bindings::_helpers::build_progress::terminal::tty_flag; use crate::bindings::_helpers::build_progress::transient_status::{ - progress_spinners_disabled, status_reporter, + TransientStatusReporter, progress_spinners_disabled, }; use crate::bindings::_helpers::records::fields::field as native_field; @@ -37,10 +36,9 @@ struct ProgressState { } /// Print each janitor phase and action as it finishes, and keep the finished actions. -#[pyclass(module = "sqlbuild._native", frozen)] +#[pyclass(frozen)] pub(crate) struct JanitorProgressPrinter { - stream: Py, - status: Py, + status: Py, style: CliStyle, live: bool, plan: JanitorPlanText, @@ -53,9 +51,12 @@ impl JanitorProgressPrinter { style: CliStyle, plan: JanitorPlanText, ) -> PyResult { + let py: Python<'_> = stream.py(); Ok(Self { - stream: stream.clone().unbind(), - status: status_reporter(stream, style.use_color)?.unbind(), + status: Py::new( + py, + TransientStatusReporter::new(py, stream, style.use_color, true)?, + )?, style, live: tty_flag(stream)? && !progress_spinners_disabled(), plan, @@ -71,14 +72,6 @@ impl JanitorProgressPrinter { pub(crate) fn items(&self) -> Vec { self.state().items.clone() } - - fn write_persistent(&self, py: Python<'_>, text: &str) -> PyResult<()> { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("stream", self.stream.bind(py))?; - options.set_item("text", text)?; - shared_lines(py)?.call_method("write_persistent", (), Some(&options))?; - Ok(()) - } } fn phase(value: &str) -> PyResult { @@ -99,27 +92,19 @@ impl JanitorProgressPrinter { flow_width: janitor_phase_flow_width(&self.plan, phase), started: Instant::now(), }); - self.status.bind(py).call_method1("start", (message,))?; - Ok(()) + TransientStatusReporter::start(self.status.bind(py), message) } - /// Record and print one finished action; `item` is its `JanitorProgressItem`. - #[pyo3(signature = (*, phase, item, error, duration_seconds))] - fn item_finished( - &self, - py: Python<'_>, - phase: &str, - item: &Bound<'_, PyAny>, - error: Option, - duration_seconds: f64, - ) -> PyResult<()> { + /// Record and print one finished action; `item` is its finished `JanitorProgressItem`. + #[pyo3(signature = (*, phase, item))] + fn item_finished(&self, py: Python<'_>, phase: &str, item: &Bound<'_, PyAny>) -> PyResult<()> { let item: JanitorItemText = JanitorItemText { phase: self::phase(phase)?, action: native_field(item, "action")?.str()?.extract()?, name: native_field(item, "name")?.extract()?, destination: native_field(item, "destination")?.extract()?, - error, - duration_seconds: Some(duration_seconds), + error: native_field(item, "error")?.extract()?, + duration_seconds: Some(native_field(item, "duration_seconds")?.extract()?), }; let (line, progress) = { let mut state = self.state(); @@ -139,9 +124,10 @@ impl JanitorProgressPrinter { state.items.push(item); (line, format!("{message} {index}/{total}")) }; - self.write_persistent(py, &format!("{line}\n"))?; + let status: &TransientStatusReporter = self.status.get(); + status.write_persistent(py, &format!("{line}\n"))?; if self.live { - self.status.bind(py).call_method1("update", (progress,))?; + status.update(py, progress)?; } Ok(()) } @@ -156,28 +142,20 @@ impl JanitorProgressPrinter { let (total, elapsed) = active.as_ref().map_or((0, 0.0), |active| { (active.total, active.started.elapsed().as_secs_f64()) }); - let options: Bound<'_, PyDict> = PyDict::new(py); + let status: &Bound<'_, TransientStatusReporter> = self.status.bind(py); if failed { let completed: usize = items .iter() .filter(|item| item.phase == phase && item.error.is_none()) .count(); - options.set_item( - "message", - janitor_phase_failed_text(phase, (completed, total), elapsed, self.style), - )?; - self.status - .bind(py) - .call_method("complete_styled", (), Some(&options))?; - return Ok(()); + let message: String = + janitor_phase_failed_text(phase, (completed, total), elapsed, self.style); + return TransientStatusReporter::complete_styled(status, &message, false); } - options.set_item( - "message", - janitor_phase_complete_text(phase, &items, elapsed), - )?; - self.status - .bind(py) - .call_method("complete", (), Some(&options))?; - Ok(()) + TransientStatusReporter::complete( + status, + &janitor_phase_complete_text(phase, &items, elapsed), + false, + ) } } diff --git a/fensu.toml b/fensu.toml index 3d55af042f..ec0575896d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1904,10 +1904,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs=2", diff --git a/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py b/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py index c0f1895f29..a566904cd2 100644 --- a/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py +++ b/src/sqlbuild/executor/janitor/classes/janitor_item_reporter.py @@ -34,11 +34,15 @@ def __call__( done: JanitorActionDone | None = outcome.result taken: JanitorAction | None = None if done is None else done.taken label: JanitorProgressItem = self._labels[index] + error: str | None = ( + None if outcome.error is None else str(outcome.error) or type(outcome.error).__name__ + ) self._progress.item_finished( phase=self._phase.value, - item=label if taken is None else replace(label, action=taken), - error=None - if outcome.error is None - else str(outcome.error) or type(outcome.error).__name__, - duration_seconds=outcome.duration_seconds, + item=replace( + label, + action=label.action if taken is None else taken, + error=error, + duration_seconds=outcome.duration_seconds, + ), ) diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index a345b8de1d..f722985c28 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -232,11 +232,13 @@ class JanitorExecutionResult: @dataclass(frozen=True) class JanitorProgressItem: - """How the progress output names one janitor action before it runs.""" + """How the progress output names one janitor action, and once finished its error and time.""" action: JanitorAction name: str destination: str | None = None + error: str | None = None + duration_seconds: float = 0.0 @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/janitor/types.py b/src/sqlbuild/executor/janitor/types.py index 4a1ea8ff60..bbf4fad9c3 100644 --- a/src/sqlbuild/executor/janitor/types.py +++ b/src/sqlbuild/executor/janitor/types.py @@ -29,14 +29,7 @@ class JanitorAction(StrEnum): class JanitorProgress(Protocol): def phase_started(self, *, phase: str, total: int) -> None: ... - def item_finished( - self, - *, - phase: str, - item: JanitorProgressItem, - error: str | None, - duration_seconds: float, - ) -> None: ... + def item_finished(self, *, phase: str, item: JanitorProgressItem) -> None: ... def phase_finished(self, *, phase: str, failed: bool) -> None: ... From 9aff97a22aaad0b06cc738a984c13b674e879841 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:22:41 +0100 Subject: [PATCH 09/10] fix: keep janitor audit events when reporting fails, and report every --json failure --- .../janitor_text/_helpers/plan_sections.rs | 8 +- .../src/janitor_text/tests/helpers.rs | 16 +++ .../tests/test_janitor_progress_text.rs | 21 ++- .../src/bindings/_helpers/cli/entry.rs | 5 +- .../bindings/_helpers/cli/machine_output.rs | 18 +++ .../src/bindings/_helpers/cli/mod.rs | 1 + .../_helpers/janitor/janitor_command.rs | 8 +- .../executor/janitor/_helpers/phase_steps.py | 4 +- .../executor/janitor/_helpers/worker_runs.py | 71 +--------- .../executor/janitor/_helpers/workers.py | 7 +- .../classes/janitor_chunk_collector.py | 76 +++++++++++ .../janitor/classes/janitor_workers.py | 48 ++++--- .../janitor/main/_run_janitor_phase.py | 40 +++--- src/sqlbuild/executor/janitor/models.py | 3 +- .../cli/commands/main/janitor/_test_types.py | 24 ++++ .../main/janitor/test_janitor_progress.py | 75 +++++++++++ .../executor/janitor/_helpers/_test_types.py | 8 ++ .../_helpers/test_worker_connections.py | 39 ++++++ .../executor/janitor/main/_test_types.py | 20 +++ .../sqlbuild/executor/janitor/main/helpers.py | 108 ++++++++++++++- .../janitor/main/test_run_janitor_phase.py | 127 ++++++++++++++++++ 21 files changed, 615 insertions(+), 112 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cli/machine_output.rs create mode 100644 src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/_helpers/test_worker_connections.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs index d81ae6596a..776bf890c3 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs @@ -297,7 +297,13 @@ pub(crate) fn old_name_views( style.muted(label) )); } - text.push_str(&overflow(style, hidden, " ")); + if hidden > 0 { + text.push_str(&format!( + "{} {}", + tree_connector(style, true), + overflow(style, hidden, "") + )); + } text } diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index bc3d89307c..48c075193d 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -1,5 +1,6 @@ use crate::janitor_text::models::{ ArchiveCandidateText, ArchivedRelationText, JanitorItemText, JanitorPhase, JanitorPlanText, + OldNameViewText, }; /// A direct-mode plan for `dev` that archives one relation and deletes one archive. @@ -62,3 +63,18 @@ pub(super) fn large_plan(count: usize, skipped: usize) -> JanitorPlanText { ..JanitorPlanText::default() } } + +/// A plan with `count` live old-name views expiring on one day. +pub(super) fn old_name_plan(count: usize) -> JanitorPlanText { + JanitorPlanText { + old_name_live: (0..count) + .map(|index| OldNameViewText { + name: format!("dev.orders_v{index:02}"), + destination_model: format!("orders_{index:02}"), + expires_on: Some("2026-11-05".to_owned()), + ..OldNameViewText::default() + }) + .collect(), + ..JanitorPlanText::default() + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs index cab6bcb186..55f122ea01 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs @@ -12,7 +12,7 @@ use crate::janitor_text::main::render_janitor_plan::render_janitor_plan; use crate::janitor_text::models::{ JanitorOutcome, JanitorPhase, JanitorResultCounts, JanitorRunText, }; -use crate::janitor_text::tests::helpers::{direct_archive_plan, item, large_plan}; +use crate::janitor_text::tests::helpers::{direct_archive_plan, item, large_plan, old_name_plan}; use crate::janitor_text::tests::test_types::{ CompletionTestCase, ItemCountsTestCase, ItemLineTestCase, JsonTestCase, PhaseTextTestCase, PreviewTestCase, @@ -277,6 +277,25 @@ fn given_long_plan_when_previewing_then_lists_are_capped_unless_verbose() { ], unexpected_fragments: &["... and", "Skipped objects (7)"], }, + PreviewTestCase { + description: "a capped old-name view tree closes with its overflow line", + plan: old_name_plan(12), + verbose: false, + expected_fragments: &[ + "\u{251c}\u{2500}\u{2500} dev.orders_v09 -> model:orders_09\n\u{2502} \u{2514}\u{2500}\u{2500} expires", + "\u{2514}\u{2500}\u{2500} ... and 2 more (use --verbose to show all)\n", + ], + unexpected_fragments: &["\u{2514}\u{2500}\u{2500} dev.orders_v09", "dev.orders_v10"], + }, + PreviewTestCase { + description: "an uncapped old-name view tree closes with its last view", + plan: old_name_plan(3), + verbose: false, + expected_fragments: &[ + "\u{2514}\u{2500}\u{2500} dev.orders_v02 -> model:orders_02\n \u{2514}", + ], + unexpected_fragments: &["... and"], + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 61132a215e..3352fc97af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; use crate::bindings::_helpers::cli::launch_state::offer_launch_record; +use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, }; @@ -207,7 +208,7 @@ fn write_compile_machine_error( print_object(&document, None, None) } -/// With `sqb recall … --json` or `sqb plan --json`, the failure as `{"error": {"code", "message", "help"}}` on stdout, so callers parsing the output read the error instead of failing to parse it. +/// With `sqb recall … --json`, `sqb plan --json` or `sqb janitor --json` (before it printed its own document), the failure as `{"error": {"code", "message", "help"}}` on stdout, so callers parsing the output read the error instead of failing to parse it. fn write_machine_error( invocation: &Invocation<'_>, error: &Bound<'_, PyAny>, @@ -220,6 +221,8 @@ fn write_machine_error( .map(|format| format.extract()) .transpose()?; format.as_deref() == Some(JSON_FORMAT) + } else if invocation.command_is("janitor") { + invocation.flag("json")? && !take_machine_output_written() } else { invocation.command_is("plan") && invocation.flag("json")? }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/machine_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/machine_output.rs new file mode 100644 index 0000000000..2b33330d34 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/machine_output.rs @@ -0,0 +1,18 @@ +//! Whether the running command already printed its machine-readable document, so a failure adds no second one. + +use std::cell::Cell; + +thread_local! { + /// Per thread, since an invocation dispatches and reports on the thread that runs it. + static MACHINE_OUTPUT_WRITTEN: Cell = const { Cell::new(false) }; +} + +/// Record that the running command printed its machine-readable document on stdout. +pub(crate) fn report_machine_output_written() { + MACHINE_OUTPUT_WRITTEN.with(|written| written.set(true)); +} + +/// Whether the running command printed its document, cleared for the next invocation. +pub(crate) fn take_machine_output_written() -> bool { + MACHINE_OUTPUT_WRITTEN.with(|written| written.replace(false)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index ee447df5e8..17fbdc632b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod dispatch; pub(crate) mod entry; pub(crate) mod failure_cause; pub(crate) mod launch_state; +pub(crate) mod machine_output; pub(crate) mod namespace; pub(crate) mod observability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 0e72916cd9..5dba134a6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -26,6 +26,9 @@ use sqlbuild_core::janitor_text::main::validate_janitor_settings::validate_janit use sqlbuild_core::janitor_text::models::{JanitorOutcome, JanitorPlanText, JanitorRunText}; use std::time::Instant; +use crate::bindings::_helpers::cli::machine_output::{ + report_machine_output_written, take_machine_output_written, +}; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; @@ -99,7 +102,9 @@ fn write_json( write( &stdout, &format!("{}\n", format_janitor_json(plan, outcome, run, error)), - ) + )?; + report_machine_output_written(); + Ok(()) } /// Discover the project and resolve the target and output context. @@ -346,6 +351,7 @@ fn clean( #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let _ = take_machine_output_written(); let invocation: Bound<'_, PyAny> = invocation(py, request)?; let style: CliStyle = CliStyle { use_color: native_field(&invocation, "use_color")?.extract()?, diff --git a/src/sqlbuild/executor/janitor/_helpers/phase_steps.py b/src/sqlbuild/executor/janitor/_helpers/phase_steps.py index 295878be0b..18db178555 100644 --- a/src/sqlbuild/executor/janitor/_helpers/phase_steps.py +++ b/src/sqlbuild/executor/janitor/_helpers/phase_steps.py @@ -53,11 +53,11 @@ def finished_actions( def chunk_failure(chunk: JanitorChunkOutcome[JanitorActionDone | None]) -> BaseException | None: - """The interrupt that stopped the chunk, else the earliest item's failure.""" + """The chunk's interrupt, else its earliest item failure, else its report error.""" if chunk.interruption is not None: return chunk.interruption return next( (outcome.error for outcome in chunk.outcomes if outcome is not None and outcome.error), - None, + chunk.report_error, ) diff --git a/src/sqlbuild/executor/janitor/_helpers/worker_runs.py b/src/sqlbuild/executor/janitor/_helpers/worker_runs.py index 7008c798d6..6626ea7f21 100644 --- a/src/sqlbuild/executor/janitor/_helpers/worker_runs.py +++ b/src/sqlbuild/executor/janitor/_helpers/worker_runs.py @@ -1,16 +1,14 @@ -"""Attempt janitor actions and collect concurrent outcomes in completion order.""" +"""Attempt janitor actions and submit them to a worker pool.""" from __future__ import annotations from collections.abc import Callable, Sequence -from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from concurrent.futures import Future, ThreadPoolExecutor from contextvars import copy_context from time import perf_counter from typing import Any, cast -from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorChunkOutcome - -type _Pending[ResultT] = set[Future[JanitorActionOutcome[ResultT]]] +from sqlbuild.executor.janitor.models import JanitorActionOutcome def attempt_janitor_action[ResultT]( @@ -26,14 +24,13 @@ def attempt_janitor_action[ResultT]( return JanitorActionOutcome(result=result, duration_seconds=perf_counter() - started) -def collect_janitor_outcomes[ItemT, ResultT]( +def submit_janitor_actions[ItemT, ResultT]( *, pool: ThreadPoolExecutor, items: Sequence[ItemT], attempt: Callable[[ItemT], JanitorActionOutcome[ResultT]], - on_done: Callable[..., None], -) -> JanitorChunkOutcome[ResultT]: - """Report outcomes as they finish; after a failure or interrupt, start nothing more.""" +) -> dict[Future[JanitorActionOutcome[ResultT]], int]: + """Each item's submitted attempt, keyed to the item's position.""" futures: dict[Future[JanitorActionOutcome[ResultT]], int] = {} index: int @@ -41,58 +38,4 @@ def collect_janitor_outcomes[ItemT, ResultT]( for index, item in enumerate(items): submitted: Future[Any] = pool.submit(copy_context().run, attempt, item) futures[cast(Future[JanitorActionOutcome[ResultT]], submitted)] = index - outcomes: dict[int, JanitorActionOutcome[ResultT]] = {} - pending: _Pending[ResultT] = set(futures) - try: - while pending: - done: _Pending[ResultT] - done, pending = wait(pending, return_when=FIRST_COMPLETED) - reported: dict[int, JanitorActionOutcome[ResultT]] - reported, pending = _report( - done=done, pending=pending, futures=futures, on_done=on_done - ) - outcomes.update(reported) - except BaseException as interruption: - _cancel(pending=pending) - finished: dict[int, JanitorActionOutcome[ResultT]] = _report( - done=wait(pending).done, pending=set(), futures=futures, on_done=on_done - )[0] - outcomes.update(finished) - return JanitorChunkOutcome( - outcomes=_in_order(outcomes=outcomes, count=len(items)), interruption=interruption - ) - return JanitorChunkOutcome(outcomes=_in_order(outcomes=outcomes, count=len(items))) - - -def _report[ResultT]( - *, - done: _Pending[ResultT], - pending: _Pending[ResultT], - futures: dict[Future[JanitorActionOutcome[ResultT]], int], - on_done: Callable[..., None], -) -> tuple[dict[int, JanitorActionOutcome[ResultT]], _Pending[ResultT]]: - """Report finished outcomes in item order and cancel the rest once one failed.""" - - reported: dict[int, JanitorActionOutcome[ResultT]] = {} - future: Future[JanitorActionOutcome[ResultT]] - for future in sorted(done, key=futures.__getitem__): - if future.cancelled(): - continue - outcome: JanitorActionOutcome[ResultT] = future.result() - reported[futures[future]] = outcome - _ = on_done(index=futures[future], outcome=outcome) - if outcome.error is not None: - _cancel(pending=pending) - return reported, {future for future in pending if not future.cancelled()} - - -def _cancel[ResultT](*, pending: _Pending[ResultT]) -> None: - future: Future[JanitorActionOutcome[ResultT]] - for future in pending: - _ = future.cancel() - - -def _in_order[ResultT]( - *, outcomes: dict[int, JanitorActionOutcome[ResultT]], count: int -) -> tuple[JanitorActionOutcome[ResultT] | None, ...]: - return tuple(outcomes.get(index) for index in range(count)) + return futures diff --git a/src/sqlbuild/executor/janitor/_helpers/workers.py b/src/sqlbuild/executor/janitor/_helpers/workers.py index c374453835..b4c316a625 100644 --- a/src/sqlbuild/executor/janitor/_helpers/workers.py +++ b/src/sqlbuild/executor/janitor/_helpers/workers.py @@ -11,6 +11,9 @@ from sqlbuild.executor.pipeline.main._open_worker_connection_pool import ( open_worker_connection_pool, ) +from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( + raise_warehouse_connection_failure, +) def janitor_worker_count(*, plan: JanitorPlan, adapter: BaseAdapter, concurrency: int) -> int: @@ -45,12 +48,12 @@ def open_janitor_worker_connections( connections: tuple[Any, ...] = open_worker_connection_pool( adapter=adapter, connection_config=connection_config, connection_count=count ) - except BaseException: + except BaseException as error: if progress is not None: progress.on_connection_error( connection_count=count, elapsed_seconds=time.monotonic() - started ) - raise + raise_warehouse_connection_failure(adapter=adapter, error=error) if progress is not None: progress.on_connection_complete( connection_count=count, elapsed_seconds=time.monotonic() - started diff --git a/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py b/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py new file mode 100644 index 0000000000..15a58943c6 --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py @@ -0,0 +1,76 @@ +"""Keep every finished janitor action of one chunk, even when reporting it or waiting fails.""" + +from __future__ import annotations + +from collections.abc import Callable +from concurrent.futures import FIRST_COMPLETED, Future, wait + +from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorChunkOutcome + + +class JanitorChunkCollector[ResultT]: + """Record each outcome before reporting it; a report error or interrupt stops new work only.""" + + def __init__(self, *, size: int, on_done: Callable[..., None]) -> None: + self._size: int = size + self._on_done: Callable[..., None] = on_done + self._outcomes: dict[int, JanitorActionOutcome[ResultT]] = {} + self._interruption: BaseException | None = None + self._report_error: BaseException | None = None + + @property + def accepting(self) -> bool: + """Whether more actions may start: nothing failed, raised or interrupted yet.""" + + return ( + self._interruption is None + and self._report_error is None + and all(outcome.error is None for outcome in self._outcomes.values()) + ) + + def record(self, *, index: int, outcome: JanitorActionOutcome[ResultT]) -> None: + """Keep `outcome`, then report it unless reporting already failed.""" + + if index in self._outcomes: + return + self._outcomes[index] = outcome + if self._report_error is not None: + return + try: + self._on_done(index=index, outcome=outcome) + except BaseException as error: + self._report_error = error + + def interrupt(self, error: BaseException) -> None: + """Remember the first interrupt; running actions still finish and are recorded.""" + + if self._interruption is None: + self._interruption = error + + def drain(self, *, futures: dict[Future[JanitorActionOutcome[ResultT]], int]) -> None: + """Record every submitted action as it finishes, cancelling unstarted ones once stopped.""" + + pending: set[Future[JanitorActionOutcome[ResultT]]] = set(futures) + while pending: + try: + done: set[Future[JanitorActionOutcome[ResultT]]] = wait( + pending, return_when=FIRST_COMPLETED + ).done + future: Future[JanitorActionOutcome[ResultT]] + for future in sorted(done, key=futures.__getitem__): + if not future.cancelled(): + self.record(index=futures[future], outcome=future.result()) + pending.discard(future) + except BaseException as error: + self.interrupt(error) + if not self.accepting: + pending = {future for future in pending if not future.cancel()} + + def outcome(self) -> JanitorChunkOutcome[ResultT]: + """Every recorded outcome by item position, with the interrupt and report error.""" + + return JanitorChunkOutcome( + outcomes=tuple(self._outcomes.get(index) for index in range(self._size)), + interruption=self._interruption, + report_error=self._report_error, + ) diff --git a/src/sqlbuild/executor/janitor/classes/janitor_workers.py b/src/sqlbuild/executor/janitor/classes/janitor_workers.py index 05b333d6c9..717614b5a4 100644 --- a/src/sqlbuild/executor/janitor/classes/janitor_workers.py +++ b/src/sqlbuild/executor/janitor/classes/janitor_workers.py @@ -9,10 +9,10 @@ from sqlbuild.executor.janitor._helpers.worker_runs import ( attempt_janitor_action, - collect_janitor_outcomes, + submit_janitor_actions, ) +from sqlbuild.executor.janitor.classes.janitor_chunk_collector import JanitorChunkCollector from sqlbuild.executor.janitor.classes.pooled_janitor_attempt import PooledJanitorAttempt -from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorChunkOutcome class JanitorWorkers: @@ -32,12 +32,13 @@ def run[ItemT, ResultT]( *, items: Sequence[ItemT], action: Callable[..., ResultT], - on_done: Callable[..., None], - ) -> JanitorChunkOutcome[ResultT]: - """Run `action(item=, connection=)` per item until one fails, calling `on_done` per item.""" + collector: JanitorChunkCollector[ResultT], + ) -> None: + """Run `action(item=, connection=)` per item into `collector` until it stops accepting.""" if len(self.connections) == 1 or len(items) <= 1: - return self._run_in_order(items=items, action=action, on_done=on_done) + self._run_in_order(items=items, action=action, collector=collector) + return connections: queue.Queue[Any] = queue.Queue() connection: Any for connection in self.connections: @@ -46,11 +47,12 @@ def run[ItemT, ResultT]( max_workers=min(len(self.connections), len(items)) ) try: - return collect_janitor_outcomes( - pool=pool, - items=items, - attempt=PooledJanitorAttempt(action=action, connections=connections), - on_done=on_done, + collector.drain( + futures=submit_janitor_actions( + pool=pool, + items=items, + attempt=PooledJanitorAttempt(action=action, connections=connections), + ) ) finally: pool.shutdown(wait=True, cancel_futures=True) @@ -60,17 +62,19 @@ def _run_in_order[ItemT, ResultT]( *, items: Sequence[ItemT], action: Callable[..., ResultT], - on_done: Callable[..., None], - ) -> JanitorChunkOutcome[ResultT]: - outcomes: list[JanitorActionOutcome[ResultT] | None] = [None] * len(items) + collector: JanitorChunkCollector[ResultT], + ) -> None: index: int item: ItemT for index, item in enumerate(items): - outcome: JanitorActionOutcome[ResultT] = attempt_janitor_action( - action=action, item=item, connection=self.connection - ) - outcomes[index] = outcome - on_done(index=index, outcome=outcome) - if outcome.error is not None: - break - return JanitorChunkOutcome(outcomes=tuple(outcomes)) + if not collector.accepting: + return + try: + collector.record( + index=index, + outcome=attempt_janitor_action( + action=action, item=item, connection=self.connection + ), + ) + except BaseException as error: + collector.interrupt(error) diff --git a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py index 3dd8eb8d34..1c868691fb 100644 --- a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py +++ b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py @@ -11,6 +11,7 @@ close_phase, finished_actions, ) +from sqlbuild.executor.janitor.classes.janitor_chunk_collector import JanitorChunkCollector from sqlbuild.executor.janitor.classes.janitor_item_reporter import JanitorItemReporter from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers from sqlbuild.executor.janitor.models import ( @@ -40,29 +41,38 @@ def run_janitor_phase[T]( finished: list[tuple[int, JanitorActionDone]] = [] positions: tuple[int, ...] for positions in chunk_positions(total=len(items), chunks=chunks): - chunk: JanitorChunkOutcome[JanitorActionDone | None] = workers.run( - items=tuple(items[position] for position in positions), - action=apply, + collector: JanitorChunkCollector[JanitorActionDone | None] = JanitorChunkCollector( + size=len(positions), on_done=JanitorItemReporter( progress=progress, phase=phase, labels=tuple(describe(items[position]) for position in positions), ), ) - succeeded: tuple[tuple[int, JanitorActionDone], ...] = finished_actions( - positions=positions, chunk=chunk - ) - finished.extend(succeeded) - failure: BaseException | None = chunk_failure(chunk) try: - if after_chunk is not None: - _ = after_chunk(tuple(done for _, done in succeeded)) - except BaseException: - _ = close_phase(progress=progress, phase=phase, failed=True) - raise + _ = workers.run( + items=tuple(items[position] for position in positions), + action=apply, + collector=collector, + ) + finally: + chunk: JanitorChunkOutcome[JanitorActionDone | None] = collector.outcome() + succeeded: tuple[tuple[int, JanitorActionDone], ...] = finished_actions( + positions=positions, chunk=chunk + ) + finished.extend(succeeded) + try: + if after_chunk is not None: + _ = after_chunk(tuple(done for _, done in succeeded)) + except BaseException: + _ = close_phase(progress=progress, phase=phase, failed=True) + raise + failure: BaseException | None = chunk_failure(chunk) if failure is not None: - _ = close_phase(progress=progress, phase=phase, failed=True) - raise failure + try: + _ = close_phase(progress=progress, phase=phase, failed=True) + finally: + raise failure _ = close_phase(progress=progress, phase=phase, failed=False) return tuple((items[position], done) for position, done in sorted(finished, key=_position)) diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index f722985c28..5313227c0a 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -252,10 +252,11 @@ class JanitorActionOutcome[ResultT]: @dataclass(frozen=True) class JanitorChunkOutcome[ResultT]: - """Each item's outcome in one chunk (none if it never started), and any interrupt.""" + """Each item's outcome in one chunk (none if never started), interrupt and report error.""" outcomes: tuple[JanitorActionOutcome[ResultT] | None, ...] interruption: BaseException | None = None + report_error: BaseException | None = None @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index b09a59b9b0..7c9005104e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -264,3 +264,27 @@ class JanitorShortenedArchiveE2ETestCase: identifier_limit: int expected_rows_in_event_insert: int expected_event_count: int + + +@dataclass(frozen=True) +class JanitorJsonErrorE2ETestCase: + """Test case for the `--json` error document of a janitor run that fails before cleanup.""" + + description: str + janitor_command: tuple[str, ...] + expected_exit_code: int + expected_error_code: str + expected_stderr_fragment: str + + +@dataclass(frozen=True) +class JanitorJsonFailedRunE2ETestCase: + """Test case for the single `--json` document of a janitor run that fails mid-phase.""" + + description: str + janitor_command: tuple[str, ...] + fail_rename: str + expected_exit_code: int + expected_status: str + expected_error_fragment: str + expected_failed_items: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py index 79590ae6b3..22d535943f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py @@ -14,6 +14,8 @@ JanitorConcurrentFailureE2ETestCase, JanitorInspectionCountE2ETestCase, JanitorJsonE2ETestCase, + JanitorJsonErrorE2ETestCase, + JanitorJsonFailedRunE2ETestCase, JanitorProgressE2ETestCase, JanitorShortenedArchiveE2ETestCase, ) @@ -180,6 +182,79 @@ def test_given_json_flag_when_running_janitor_then_stdout_holds_plan_and_result( assert_in_order(output=result.stderr, lines=test_case.expected_stderr_fragments) +@pytest.mark.parametrize( + "test_case", + [ + JanitorJsonErrorE2ETestCase( + description="a refused non-interactive confirmation prints a JSON error document", + janitor_command=("janitor", "--json"), + expected_exit_code=1, + expected_error_code="C503", + expected_stderr_fragment="Janitor preview default", + ), + JanitorJsonErrorE2ETestCase( + description="an invalid setting prints a JSON error document before any inspection", + janitor_command=("janitor", "--json", "--retention-days", "-1"), + expected_exit_code=1, + expected_error_code="C501", + expected_stderr_fragment="error[C501]", + ), + ], + ids=lambda case: case.description, +) +def test_given_json_flag_when_janitor_fails_before_cleanup_then_stdout_holds_error_document( + test_case: JanitorJsonErrorE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_json_error", stale_tables={"main": 2} + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["error"]["code"] == test_case.expected_error_code + assert test_case.expected_stderr_fragment in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorJsonFailedRunE2ETestCase( + description="a run failing mid-phase prints exactly one JSON document", + janitor_command=("janitor", "--auto-approve", "--json"), + fail_rename="old_orders_001", + expected_exit_code=1, + expected_status="failed", + expected_error_fragment="simulated rename failure for old_orders_001", + expected_failed_items=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_json_flag_when_janitor_fails_mid_phase_then_stdout_holds_one_run_document( + test_case: JanitorJsonFailedRunE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_probe_janitor_project( + tmp_path=tmp_path, project_name="janitor_json_failed", stale_tables={"main": 3} + ) + write_probe_settings(project_dir=project_dir, fail_rename=test_case.fail_rename) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["status"] == test_case.expected_status + assert test_case.expected_error_fragment in payload["error"] + assert [item["status"] for item in payload["result"]["items"]].count("failed") == ( + test_case.expected_failed_items + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py index 4a3bc59c44..e18bc6d880 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py @@ -14,3 +14,11 @@ class ArchiveNameBuildTestCase: expected_length: int expected_logical_name_is_original: bool kind: str | None = None + + +@dataclass(frozen=True) +class JanitorWorkerConnectionFailureTestCase: + description: str + connection_count: int + expected_code: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_worker_connections.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_worker_connections.py new file mode 100644 index 0000000000..aa27f98861 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_worker_connections.py @@ -0,0 +1,39 @@ +"""Worker connection failures surface as the coded warehouse connection error.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.janitor._helpers.workers import open_janitor_worker_connections +from sqlbuild.executor.pipeline.exceptions import WarehouseConnectionError +from tests.unit.src.sqlbuild.executor.janitor._helpers._test_types import ( + JanitorWorkerConnectionFailureTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorWorkerConnectionFailureTestCase( + description="an unopenable worker connection raises the coded connection error", + connection_count=2, + expected_code="C115", + expected_message="could not connect to the duckdb warehouse", + ), + ], + ids=lambda case: case.description, +) +def test_given_unreachable_warehouse_when_opening_worker_connections_then_error_is_coded( + test_case: JanitorWorkerConnectionFailureTestCase, +) -> None: + with pytest.raises(WarehouseConnectionError) as raised: + _ = open_janitor_worker_connections( + adapter=DuckDbAdapter(), + connection_config={"database": "/nonexistent-directory/orders/janitor.duckdb"}, + count=test_case.connection_count, + progress=None, + ) + + assert raised.value.code == test_case.expected_code + assert test_case.expected_message in raised.value.message diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index e0966ea1f1..357cd5d78f 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -159,3 +159,23 @@ class JanitorCaseSafetyTestCase: exclude_patterns: tuple[str, ...] expected_skipped_relations: tuple[tuple[str, str], ...] expected_dropped_targets: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorPhaseEventSafetyTestCase: + description: str + connection_count: int + item_count: int + failing_report: int + report_error: type[BaseException] + expected_error: type[BaseException] + expected_minimum_completed: int + + +@dataclass(frozen=True) +class JanitorPhaseInterruptTestCase: + description: str + connection_count: int + item_count: int + expected_error: type[BaseException] + expected_minimum_completed: int diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 402c648ef6..400d9cbf53 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -5,7 +5,7 @@ import base64 from collections.abc import Callable from dataclasses import replace -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from typing import Any, ClassVar, cast @@ -39,7 +39,14 @@ CompiledProjectView, CompiledSourceView, ) -from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME +from sqlbuild.executor.janitor.models import JanitorActionDone, JanitorProgressItem +from sqlbuild.executor.janitor.types import JanitorAction +from sqlbuild.executor.janitor_events.constants import ( + JANITOR_EVENT_SCHEMA_VERSION, + JANITOR_EVENTS_TABLE_NAME, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS UNQUOTED_ADDRESSING_REASON: str = ( @@ -610,3 +617,100 @@ def _ignore_relation(selected: list[RelationInfo], relation: RelationInfo) -> No True: _append_relation, False: _ignore_relation, } + + +PHASE_EVENT_TIME: datetime = datetime(2026, 9, 24, 10, 15, tzinfo=UTC) + + +def phase_event(index: int) -> JanitorEventRecord: + """The synthetic audit event of the archive action at `index`.""" + + archive_name: str = f"_sqb_archive__20260924t101500z__orders_{index:04d}" + return JanitorEventRecord( + event_id=f"event_{index:04d}", + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=JanitorEventType.ARCHIVE, + occurred_at=PHASE_EVENT_TIME, + run_id="run_orders_cleanup", + relation_database=None, + relation_schema="analytics", + relation_type="BASE TABLE", + original_name=f"orders_{index:04d}", + original_qualified_name=f"analytics.orders_{index:04d}", + archive_name=archive_name, + archive_qualified_name=f"analytics.{archive_name}", + archived_at=PHASE_EVENT_TIME, + ) + + +class PhaseActions: + """Archive actions that each return their event and remember that they completed.""" + + def __init__(self) -> None: + self.completed: list[int] = [] + self.flushed: list[str] = [] + + def apply(self, *, item: int, connection: Any) -> JanitorActionDone: + del connection + self.completed.append(item) + return JanitorActionDone(event=phase_event(item)) + + def flush(self, finished: tuple[JanitorActionDone, ...]) -> None: + self.flushed.extend(cast(JanitorEventRecord, done.event).event_id for done in finished) + + def completed_event_ids(self) -> list[str]: + return sorted(phase_event(item).event_id for item in self.completed) + + +class FailingReportProgress: + """Progress whose `failing_report`th finished-action report raises `error`.""" + + def __init__(self, *, failing_report: int, error: type[BaseException], size: int) -> None: + def quiet() -> None: + return None + + def fail() -> None: + raise error("progress stream closed") + + self._reactions: list[Callable[[], None]] = [ + *([quiet] * (failing_report - 1)), + fail, + *([quiet] * size), + ] + self._calls: int = 0 + self.closed_failed: list[bool] = [] + + def phase_started(self, *, phase: str, total: int) -> None: + del phase, total + + def item_finished(self, *, phase: str, item: object) -> None: + del phase, item + self._calls += 1 + self._reactions[self._calls - 1]() + + def phase_finished(self, *, phase: str, failed: bool) -> None: + del phase + self.closed_failed.append(failed) + + +def interrupting_wait(real_wait: Callable[..., Any]) -> Callable[..., Any]: + """`concurrent.futures.wait` that raises KeyboardInterrupt on its first call only.""" + + def interrupt(*args: object, **kwargs: object) -> Any: + del args, kwargs + raise KeyboardInterrupt + + calls: list[Callable[..., Any]] = [interrupt] + + def wait(*args: Any, **kwargs: Any) -> Any: + reaction: Callable[..., Any] = [*calls, real_wait][0] + calls.clear() + return reaction(*args, **kwargs) + + return wait + + +def phase_label(item: int) -> JanitorProgressItem: + """The progress label of the archive action at `item`.""" + + return JanitorProgressItem(action=JanitorAction.ARCHIVED, name=f"analytics.orders_{item:04d}") diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py new file mode 100644 index 0000000000..c41b67bacc --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py @@ -0,0 +1,127 @@ +"""Every completed janitor action keeps its audit event when reporting or waiting fails.""" + +from __future__ import annotations + +from concurrent.futures import wait + +import pytest + +from sqlbuild.executor.janitor.classes import janitor_chunk_collector +from sqlbuild.executor.janitor.classes.janitor_workers import JanitorWorkers +from sqlbuild.executor.janitor.main._run_janitor_phase import run_janitor_phase +from sqlbuild.executor.janitor.types import JanitorPhase +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorPhaseEventSafetyTestCase, + JanitorPhaseInterruptTestCase, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FailingReportProgress, + PhaseActions, + interrupting_wait, + phase_label, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPhaseEventSafetyTestCase( + description="sequential actions keep their events when the progress stream breaks", + connection_count=1, + item_count=10, + failing_report=3, + report_error=BrokenPipeError, + expected_error=BrokenPipeError, + expected_minimum_completed=3, + ), + JanitorPhaseEventSafetyTestCase( + description="concurrent actions keep their events when the progress stream breaks", + connection_count=3, + item_count=10, + failing_report=3, + report_error=BrokenPipeError, + expected_error=BrokenPipeError, + expected_minimum_completed=3, + ), + JanitorPhaseEventSafetyTestCase( + description="an interrupt while reporting still writes every completed event", + connection_count=3, + item_count=10, + failing_report=2, + report_error=KeyboardInterrupt, + expected_error=KeyboardInterrupt, + expected_minimum_completed=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_progress_when_running_phase_then_completed_actions_keep_events( + test_case: JanitorPhaseEventSafetyTestCase, +) -> None: + actions: PhaseActions = PhaseActions() + progress: FailingReportProgress = FailingReportProgress( + failing_report=test_case.failing_report, + error=test_case.report_error, + size=test_case.item_count, + ) + + with pytest.raises(test_case.expected_error): + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=tuple(range(test_case.item_count)), + describe=phase_label, + apply=actions.apply, + workers=JanitorWorkers( + connections=tuple(object() for _ in range(test_case.connection_count)) + ), + after_chunk=actions.flush, + ) + + assert len(actions.completed) >= test_case.expected_minimum_completed + assert len(actions.completed) < test_case.item_count + assert sorted(actions.flushed) == actions.completed_event_ids() + assert progress.closed_failed == [True] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPhaseInterruptTestCase( + description="an interrupt while waiting for workers still writes completed events", + connection_count=3, + item_count=10, + expected_error=KeyboardInterrupt, + expected_minimum_completed=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupt_while_waiting_when_running_phase_then_completed_actions_keep_events( + test_case: JanitorPhaseInterruptTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + actions: PhaseActions = PhaseActions() + progress: FailingReportProgress = FailingReportProgress( + failing_report=test_case.item_count + 1, + error=BrokenPipeError, + size=test_case.item_count, + ) + monkeypatch.setattr(janitor_chunk_collector, "wait", interrupting_wait(wait)) + + with pytest.raises(test_case.expected_error): + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=tuple(range(test_case.item_count)), + describe=phase_label, + apply=actions.apply, + workers=JanitorWorkers( + connections=tuple(object() for _ in range(test_case.connection_count)) + ), + after_chunk=actions.flush, + ) + + assert len(actions.completed) >= test_case.expected_minimum_completed + assert sorted(actions.flushed) == actions.completed_event_ids() + assert progress.closed_failed == [True] From c2408701b50e199a408c90285f304b82a08b99e3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:46:27 +0100 Subject: [PATCH 10/10] fix: keep janitor events on interrupted submission, chain event write failures, reset the json marker per invocation --- .../main/janitor_events_unwritten_text.rs | 27 ++++ .../src/janitor_text/main/mod.rs | 1 + .../tests/test_janitor_progress_text.rs | 29 ++++- .../src/janitor_text/tests/test_types.rs | 8 ++ .../src/bindings/_helpers/cli/entry.rs | 8 ++ .../_helpers/janitor/janitor_progress.rs | 14 ++ .../executor/janitor/_helpers/phase_steps.py | 61 +++++++++ .../executor/janitor/_helpers/worker_runs.py | 21 ++- .../classes/janitor_chunk_collector.py | 26 +++- .../janitor/classes/janitor_workers.py | 15 +-- .../janitor/main/_run_janitor_phase.py | 41 +++--- src/sqlbuild/executor/janitor/types.py | 2 + .../src/sqlbuild/cli/entry/_test_types.py | 11 ++ .../src/sqlbuild/cli/entry/helpers.py | 16 +++ .../cli/entry/test_machine_output_reset.py | 50 ++++++++ .../executor/janitor/main/_test_types.py | 22 ++++ .../sqlbuild/executor/janitor/main/helpers.py | 62 +++++++++ .../janitor/main/test_run_janitor_phase.py | 121 ++++++++++++++++++ 18 files changed, 490 insertions(+), 45 deletions(-) create mode 100644 crates/sqlbuild-core/src/janitor_text/main/janitor_events_unwritten_text.rs create mode 100644 tests/integration/src/sqlbuild/cli/entry/test_machine_output_reset.py diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_events_unwritten_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_events_unwritten_text.rs new file mode 100644 index 0000000000..6a6e33d139 --- /dev/null +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_events_unwritten_text.rs @@ -0,0 +1,27 @@ +//! The warning that completed janitor actions have no audit events. + +use crate::cli_text::main::format_phase_line::format_phase_line; +use crate::cli_text::models::CliStyle; +use crate::janitor_text::_helpers::phases::noun; + +/// `✗ Audit events not written for 5 completed actions in analytics: `. +#[must_use] +pub fn janitor_events_unwritten_text( + (count, schemas): (usize, &[String]), + error: &str, + style: CliStyle, +) -> String { + format_phase_line( + style, + false, + ( + "Audit events not written", + &format!( + "for {} in {}: {}", + noun(count, "completed action"), + schemas.join(", "), + error.lines().next().unwrap_or_default() + ), + ), + ) +} diff --git a/crates/sqlbuild-core/src/janitor_text/main/mod.rs b/crates/sqlbuild-core/src/janitor_text/main/mod.rs index bb3822d571..eb83932969 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/mod.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/mod.rs @@ -2,6 +2,7 @@ pub mod format_janitor_json; pub mod janitor_confirmation_phrase; pub mod janitor_confirmation_prompt; pub mod janitor_disabled_text; +pub mod janitor_events_unwritten_text; pub mod janitor_item_counts; pub mod janitor_nothing_to_do_text; pub mod janitor_phase_complete_text; diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs index 55f122ea01..ae0fa83a86 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_progress_text.rs @@ -1,5 +1,6 @@ use crate::cli_text::models::CliStyle; use crate::janitor_text::main::format_janitor_json::format_janitor_json; +use crate::janitor_text::main::janitor_events_unwritten_text::janitor_events_unwritten_text; use crate::janitor_text::main::janitor_item_counts::janitor_item_counts; use crate::janitor_text::main::janitor_nothing_to_do_text::janitor_nothing_to_do_text; use crate::janitor_text::main::janitor_phase_complete_text::janitor_phase_complete_text; @@ -14,8 +15,8 @@ use crate::janitor_text::models::{ }; use crate::janitor_text::tests::helpers::{direct_archive_plan, item, large_plan, old_name_plan}; use crate::janitor_text::tests::test_types::{ - CompletionTestCase, ItemCountsTestCase, ItemLineTestCase, JsonTestCase, PhaseTextTestCase, - PreviewTestCase, + CompletionTestCase, EventsUnwrittenTestCase, ItemCountsTestCase, ItemLineTestCase, + JsonTestCase, PhaseTextTestCase, PreviewTestCase, }; const PLAIN: CliStyle = CliStyle { use_color: false }; @@ -394,3 +395,27 @@ fn given_outcome_when_formatting_json_then_document_carries_plan_result_and_erro ); } } + +#[test] +fn given_failed_event_write_when_warning_then_line_counts_actions_per_schema() { + let test_cases = [EventsUnwrittenTestCase { + description: "the warning names the schemas and the first line of the error", + count: 5, + schemas: vec!["analytics".to_owned(), "staging".to_owned()], + error: "events table is locked\nquery id 7", + expected_text: "\u{2717} Audit events not written for 5 completed actions in analytics, staging: events table is locked", + }]; + + for test_case in test_cases { + assert_eq!( + janitor_events_unwritten_text( + (test_case.count, &test_case.schemas), + test_case.error, + PLAIN + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs index 40f69d1b08..ae6072dc7c 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_types.rs @@ -73,3 +73,11 @@ pub(super) struct JsonTestCase { pub(super) error: Option<&'static str>, pub(super) expected_fragments: &'static [&'static str], } + +pub(super) struct EventsUnwrittenTestCase { + pub(super) description: &'static str, + pub(super) count: usize, + pub(super) schemas: Vec, + pub(super) error: &'static str, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 3352fc97af..73f7aced98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -602,6 +602,14 @@ fn observed_dispatch<'py>( /// Dispatch once and project every expected failure to its coded error and exit code. fn handled_dispatch<'py>(invocation: &Invocation<'py>) -> PyResult> { + let _ = take_machine_output_written(); + let handled: PyResult> = report_dispatch(invocation); + let _ = take_machine_output_written(); + handled +} + +/// Dispatch the command and report its failure, writing any machine-readable error document. +fn report_dispatch<'py>(invocation: &Invocation<'py>) -> PyResult> { let py: Python<'py> = invocation.py(); let policy: InvocationPolicy = invocation_policy(&invocation.facts); report_skill_freshness(invocation, policy)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs index b5bf9dbf4e..6a717a5964 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_progress.rs @@ -7,6 +7,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::{pyclass, pymethods}; use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::janitor_text::main::janitor_events_unwritten_text::janitor_events_unwritten_text; use sqlbuild_core::janitor_text::main::janitor_phase_complete_text::janitor_phase_complete_text; use sqlbuild_core::janitor_text::main::janitor_phase_failed_text::janitor_phase_failed_text; use sqlbuild_core::janitor_text::main::janitor_phase_flow_width::janitor_phase_flow_width; @@ -132,6 +133,19 @@ impl JanitorProgressPrinter { Ok(()) } + /// Warn that the audit events of `count` completed actions in `schemas` were not written. + #[pyo3(signature = (*, count, schemas, error))] + fn events_unwritten( + &self, + py: Python<'_>, + count: usize, + schemas: Vec, + error: &str, + ) -> PyResult<()> { + let line: String = janitor_events_unwritten_text((count, &schemas), error, self.style); + self.status.get().write_persistent(py, &format!("{line}\n")) + } + #[pyo3(signature = (*, phase, failed))] fn phase_finished(&self, py: Python<'_>, phase: &str, failed: bool) -> PyResult<()> { let phase: JanitorPhase = self::phase(phase)?; diff --git a/src/sqlbuild/executor/janitor/_helpers/phase_steps.py b/src/sqlbuild/executor/janitor/_helpers/phase_steps.py index 18db178555..b25801dda2 100644 --- a/src/sqlbuild/executor/janitor/_helpers/phase_steps.py +++ b/src/sqlbuild/executor/janitor/_helpers/phase_steps.py @@ -10,6 +10,7 @@ JanitorChunkOutcome, ) from sqlbuild.executor.janitor.types import JanitorPhase, JanitorProgress +from sqlbuild.executor.janitor_events.models import JanitorEventRecord def announce_phase(*, progress: JanitorProgress | None, phase: JanitorPhase, total: int) -> None: @@ -61,3 +62,63 @@ def chunk_failure(chunk: JanitorChunkOutcome[JanitorActionDone | None]) -> BaseE (outcome.error for outcome in chunk.outcomes if outcome is not None and outcome.error), chunk.report_error, ) + + +def report_unwritten_events( + *, + progress: JanitorProgress | None, + finished: tuple[JanitorActionDone, ...], + error: BaseException, +) -> None: + """Warn which completed actions' audit events a failed write left out, by schema.""" + + events: tuple[JanitorEventRecord, ...] = tuple( + done.event for done in finished if done.event is not None + ) + if progress is None or not events: + return + schemas: tuple[str, ...] = tuple(sorted({_schema_name(event) for event in events})) + progress.events_unwritten( + count=len(events), schemas=schemas, error=str(error) or type(error).__name__ + ) + + +def raise_phase_failure( + *, + progress: JanitorProgress | None, + phase: JanitorPhase, + failure: BaseException | None, + flush_error: BaseException | None, +) -> None: + """Close the phase as failed and raise its failure; an interrupt wins, the other is chained.""" + + primary: BaseException | None + secondary: BaseException | None + primary, secondary = _ranked(failure=failure, flush_error=flush_error) + if primary is None: + return + if secondary is not None: + primary.__context__ = secondary + if flush_error is not None and primary is not flush_error: + primary.add_note(f"Audit events of completed actions were not written: {flush_error}") + try: + close_phase(progress=progress, phase=phase, failed=True) + finally: + raise primary + + +def _ranked( + *, failure: BaseException | None, flush_error: BaseException | None +) -> tuple[BaseException | None, BaseException | None]: + """The error to raise and the one to chain: an interrupt first, else the original failure.""" + + flush_interrupted: bool = flush_error is not None and not isinstance(flush_error, Exception) + if failure is None or (flush_interrupted and isinstance(failure, Exception)): + return flush_error, failure + return failure, flush_error + + +def _schema_name(event: JanitorEventRecord) -> str: + if event.relation_database is None: + return event.relation_schema + return f"{event.relation_database}.{event.relation_schema}" diff --git a/src/sqlbuild/executor/janitor/_helpers/worker_runs.py b/src/sqlbuild/executor/janitor/_helpers/worker_runs.py index 6626ea7f21..db7d70b3ca 100644 --- a/src/sqlbuild/executor/janitor/_helpers/worker_runs.py +++ b/src/sqlbuild/executor/janitor/_helpers/worker_runs.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Callable, Sequence +from collections.abc import Callable from concurrent.futures import Future, ThreadPoolExecutor from contextvars import copy_context from time import perf_counter @@ -24,18 +24,13 @@ def attempt_janitor_action[ResultT]( return JanitorActionOutcome(result=result, duration_seconds=perf_counter() - started) -def submit_janitor_actions[ItemT, ResultT]( +def submit_janitor_action[ItemT, ResultT]( *, pool: ThreadPoolExecutor, - items: Sequence[ItemT], + item: ItemT, attempt: Callable[[ItemT], JanitorActionOutcome[ResultT]], -) -> dict[Future[JanitorActionOutcome[ResultT]], int]: - """Each item's submitted attempt, keyed to the item's position.""" - - futures: dict[Future[JanitorActionOutcome[ResultT]], int] = {} - index: int - item: ItemT - for index, item in enumerate(items): - submitted: Future[Any] = pool.submit(copy_context().run, attempt, item) - futures[cast(Future[JanitorActionOutcome[ResultT]], submitted)] = index - return futures +) -> Future[JanitorActionOutcome[ResultT]]: + """The submitted attempt of one item, run in a copy of the caller's context.""" + + submitted: Future[Any] = pool.submit(copy_context().run, attempt, item) + return cast(Future[JanitorActionOutcome[ResultT]], submitted) diff --git a/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py b/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py index 15a58943c6..ea0d8bb598 100644 --- a/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py +++ b/src/sqlbuild/executor/janitor/classes/janitor_chunk_collector.py @@ -2,9 +2,10 @@ from __future__ import annotations -from collections.abc import Callable -from concurrent.futures import FIRST_COMPLETED, Future, wait +from collections.abc import Callable, Sequence +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from sqlbuild.executor.janitor._helpers.worker_runs import submit_janitor_action from sqlbuild.executor.janitor.models import JanitorActionOutcome, JanitorChunkOutcome @@ -47,6 +48,27 @@ def interrupt(self, error: BaseException) -> None: if self._interruption is None: self._interruption = error + def collect[ItemT]( + self, + *, + pool: ThreadPoolExecutor, + items: Sequence[ItemT], + attempt: Callable[[ItemT], JanitorActionOutcome[ResultT]], + ) -> None: + """Submit items until stopped, then record every submitted action as it finishes.""" + + futures: dict[Future[JanitorActionOutcome[ResultT]], int] = {} + index: int + item: ItemT + for index, item in enumerate(items): + if not self.accepting: + break + try: + futures[submit_janitor_action(pool=pool, item=item, attempt=attempt)] = index + except BaseException as error: + self.interrupt(error) + self.drain(futures=futures) + def drain(self, *, futures: dict[Future[JanitorActionOutcome[ResultT]], int]) -> None: """Record every submitted action as it finishes, cancelling unstarted ones once stopped.""" diff --git a/src/sqlbuild/executor/janitor/classes/janitor_workers.py b/src/sqlbuild/executor/janitor/classes/janitor_workers.py index 717614b5a4..1116f4754d 100644 --- a/src/sqlbuild/executor/janitor/classes/janitor_workers.py +++ b/src/sqlbuild/executor/janitor/classes/janitor_workers.py @@ -7,10 +7,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import Any -from sqlbuild.executor.janitor._helpers.worker_runs import ( - attempt_janitor_action, - submit_janitor_actions, -) +from sqlbuild.executor.janitor._helpers.worker_runs import attempt_janitor_action from sqlbuild.executor.janitor.classes.janitor_chunk_collector import JanitorChunkCollector from sqlbuild.executor.janitor.classes.pooled_janitor_attempt import PooledJanitorAttempt @@ -47,12 +44,10 @@ def run[ItemT, ResultT]( max_workers=min(len(self.connections), len(items)) ) try: - collector.drain( - futures=submit_janitor_actions( - pool=pool, - items=items, - attempt=PooledJanitorAttempt(action=action, connections=connections), - ) + collector.collect( + pool=pool, + items=items, + attempt=PooledJanitorAttempt(action=action, connections=connections), ) finally: pool.shutdown(wait=True, cancel_futures=True) diff --git a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py index 1c868691fb..dab78408dc 100644 --- a/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py +++ b/src/sqlbuild/executor/janitor/main/_run_janitor_phase.py @@ -10,6 +10,8 @@ chunk_positions, close_phase, finished_actions, + raise_phase_failure, + report_unwritten_events, ) from sqlbuild.executor.janitor.classes.janitor_chunk_collector import JanitorChunkCollector from sqlbuild.executor.janitor.classes.janitor_item_reporter import JanitorItemReporter @@ -55,24 +57,27 @@ def run_janitor_phase[T]( action=apply, collector=collector, ) - finally: - chunk: JanitorChunkOutcome[JanitorActionDone | None] = collector.outcome() - succeeded: tuple[tuple[int, JanitorActionDone], ...] = finished_actions( - positions=positions, chunk=chunk - ) - finished.extend(succeeded) - try: - if after_chunk is not None: - _ = after_chunk(tuple(done for _, done in succeeded)) - except BaseException: - _ = close_phase(progress=progress, phase=phase, failed=True) - raise - failure: BaseException | None = chunk_failure(chunk) - if failure is not None: - try: - _ = close_phase(progress=progress, phase=phase, failed=True) - finally: - raise failure + except BaseException as error: + _ = collector.interrupt(error) + chunk: JanitorChunkOutcome[JanitorActionDone | None] = collector.outcome() + succeeded: tuple[tuple[int, JanitorActionDone], ...] = finished_actions( + positions=positions, chunk=chunk + ) + finished.extend(succeeded) + completed: tuple[JanitorActionDone, ...] = tuple(done for _, done in succeeded) + flush_error: BaseException | None = None + try: + if after_chunk is not None: + _ = after_chunk(completed) + except BaseException as error: + flush_error = error + _ = report_unwritten_events(progress=progress, finished=completed, error=error) + _ = raise_phase_failure( + progress=progress, + phase=phase, + failure=chunk_failure(chunk), + flush_error=flush_error, + ) _ = close_phase(progress=progress, phase=phase, failed=False) return tuple((items[position], done) for position, done in sorted(finished, key=_position)) diff --git a/src/sqlbuild/executor/janitor/types.py b/src/sqlbuild/executor/janitor/types.py index bbf4fad9c3..5549abfb21 100644 --- a/src/sqlbuild/executor/janitor/types.py +++ b/src/sqlbuild/executor/janitor/types.py @@ -33,6 +33,8 @@ def item_finished(self, *, phase: str, item: JanitorProgressItem) -> None: ... def phase_finished(self, *, phase: str, failed: bool) -> None: ... + def events_unwritten(self, *, count: int, schemas: tuple[str, ...], error: str) -> None: ... + class JanitorConnectionProgress(Protocol): def on_connection_start(self, connection_count: int) -> None: ... diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index 5be5472a2a..4df4329ad5 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -124,3 +124,14 @@ class BrokenPipeTestCase: prepare: Callable[[Path], None] expected_exit_code: int expected_stderr_end: bytes + + +@dataclass(frozen=True) +class MachineOutputResetTestCase: + """A successful machine-output command followed in-process by an early failure.""" + + description: str + first_args: tuple[str, ...] + failing_args: tuple[str, ...] + failing_extra_toml: str + expected_error_code: str diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index c479d5d40f..33c4425cfd 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -338,3 +338,19 @@ def run_without_reader(program: Path, *, args: tuple[str, ...], cwd: Path) -> La finally: os.close(writer) return completed.returncode, b"", completed.stderr + + +def write_janitor_project(project_dir: Path, *, extra_toml: str) -> Path: + """Write a DuckDB project with one model, janitor left disabled, and extra settings.""" + + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "models").mkdir(exist_ok=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders_shop"\nadapter = "duckdb"\n\n[connection]\n' + f'database = "orders.duckdb"\n{extra_toml}', + encoding="utf-8", + ) + (project_dir / "models" / "orders.sql").write_text( + "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", encoding="utf-8" + ) + return project_dir diff --git a/tests/integration/src/sqlbuild/cli/entry/test_machine_output_reset.py b/tests/integration/src/sqlbuild/cli/entry/test_machine_output_reset.py new file mode 100644 index 0000000000..a609edd14d --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_machine_output_reset.py @@ -0,0 +1,50 @@ +"""A machine-output document printed by one in-process command never hides the next one's error.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import run_cli +from tests.integration.src.sqlbuild.cli.entry._test_types import MachineOutputResetTestCase +from tests.integration.src.sqlbuild.cli.entry.helpers import write_janitor_project + + +@pytest.mark.parametrize( + "test_case", + [ + MachineOutputResetTestCase( + description="a recall settings error after a janitor json run prints its document", + first_args=("janitor", "--json"), + failing_args=("janitor", "--json"), + failing_extra_toml=( + '\n[connections.history]\nadapter = "sqlite"\npath = "history.sqlite3"\n\n' + '[recall]\nwrite_connection = "history"\n' + ), + expected_error_code="C975", + ), + ], + ids=lambda case: case.description, +) +def test_given_janitor_json_success_when_next_command_fails_early_then_error_document_prints( + test_case: MachineOutputResetTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HOME", str(tmp_path / "home")) + succeeding: Path = write_janitor_project(tmp_path / "succeeding", extra_toml="") + failing: Path = write_janitor_project( + tmp_path / "failing", extra_toml=test_case.failing_extra_toml + ) + + _ = run_cli(["--project-dir", str(succeeding), *test_case.first_args]) + first: dict[str, Any] = json.loads(capsys.readouterr().out) + _ = run_cli(["--project-dir", str(failing), *test_case.failing_args]) + second: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert first["status"] == "disabled" + assert second["error"]["code"] == test_case.expected_error_code diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 357cd5d78f..b891ddf619 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -179,3 +179,25 @@ class JanitorPhaseInterruptTestCase: item_count: int expected_error: type[BaseException] expected_minimum_completed: int + + +@dataclass(frozen=True) +class JanitorPhaseSubmitInterruptTestCase: + description: str + connection_count: int + item_count: int + failing_submit: int + expected_error: type[BaseException] + expected_maximum_completed: int + + +@dataclass(frozen=True) +class JanitorPhaseFlushFailureTestCase: + description: str + item_count: int + failing_item: int + action_error: type[BaseException] + flush_error: type[BaseException] + expected_error: type[BaseException] + expected_chained_error: type[BaseException] + expected_unwritten: tuple[tuple[int, tuple[str, ...], str], ...] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 400d9cbf53..d4bc3910a3 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -679,6 +679,7 @@ def fail() -> None: ] self._calls: int = 0 self.closed_failed: list[bool] = [] + self.unwritten: list[tuple[int, tuple[str, ...], str]] = [] def phase_started(self, *, phase: str, total: int) -> None: del phase, total @@ -692,6 +693,9 @@ def phase_finished(self, *, phase: str, failed: bool) -> None: del phase self.closed_failed.append(failed) + def events_unwritten(self, *, count: int, schemas: tuple[str, ...], error: str) -> None: + self.unwritten.append((count, schemas, error)) + def interrupting_wait(real_wait: Callable[..., Any]) -> Callable[..., Any]: """`concurrent.futures.wait` that raises KeyboardInterrupt on its first call only.""" @@ -714,3 +718,61 @@ def phase_label(item: int) -> JanitorProgressItem: """The progress label of the archive action at `item`.""" return JanitorProgressItem(action=JanitorAction.ARCHIVED, name=f"analytics.orders_{item:04d}") + + +def _raiser(error: type[BaseException], message: str) -> Callable[[], None]: + def fail() -> None: + raise error(message) + + return fail + + +def _quiet() -> None: + return None + + +class FailingPhaseActions(PhaseActions): + """Archive actions where the `failing_item`th raises `error`, and whose event flush raises.""" + + def __init__( + self, + *, + failing_item: int, + error: type[BaseException], + flush_error: type[BaseException], + size: int, + ) -> None: + super().__init__() + self._reactions: list[Callable[[], None]] = [ + *([_quiet] * failing_item), + _raiser(error, f"rename failed for analytics.orders_{failing_item:04d}"), + *([_quiet] * size), + ] + self._flush_error: type[BaseException] = flush_error + + def apply(self, *, item: int, connection: Any) -> JanitorActionDone: + self._reactions[item]() + return super().apply(item=item, connection=connection) + + def flush(self, finished: tuple[JanitorActionDone, ...]) -> None: + del finished + raise self._flush_error("events table is locked") + + +def interrupting_submit( + real_submit: Callable[..., Any], *, failing_call: int +) -> Callable[..., Any]: + """`submit_janitor_action` that raises KeyboardInterrupt on its `failing_call`th call.""" + + reactions: list[Callable[[], None]] = [ + *([_quiet] * (failing_call - 1)), + _raiser(KeyboardInterrupt, "interrupted while submitting"), + ] + calls: list[int] = [] + + def submit(**kwargs: Any) -> Any: + calls.append(len(calls)) + [*reactions, *([_quiet] * len(calls))][len(calls) - 1]() + return real_submit(**kwargs) + + return submit diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py index c41b67bacc..be732bd6a8 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_run_janitor_phase.py @@ -12,11 +12,15 @@ from sqlbuild.executor.janitor.types import JanitorPhase from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( JanitorPhaseEventSafetyTestCase, + JanitorPhaseFlushFailureTestCase, JanitorPhaseInterruptTestCase, + JanitorPhaseSubmitInterruptTestCase, ) from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FailingPhaseActions, FailingReportProgress, PhaseActions, + interrupting_submit, interrupting_wait, phase_label, ) @@ -125,3 +129,120 @@ def test_given_interrupt_while_waiting_when_running_phase_then_completed_actions assert len(actions.completed) >= test_case.expected_minimum_completed assert sorted(actions.flushed) == actions.completed_event_ids() assert progress.closed_failed == [True] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPhaseSubmitInterruptTestCase( + description="an interrupt while submitting still writes every completed event", + connection_count=3, + item_count=10, + failing_submit=3, + expected_error=KeyboardInterrupt, + expected_maximum_completed=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupt_while_submitting_when_running_phase_then_completed_actions_keep_events( + test_case: JanitorPhaseSubmitInterruptTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + actions: PhaseActions = PhaseActions() + progress: FailingReportProgress = FailingReportProgress( + failing_report=test_case.item_count + 1, + error=BrokenPipeError, + size=test_case.item_count, + ) + monkeypatch.setattr( + janitor_chunk_collector, + "submit_janitor_action", + interrupting_submit( + janitor_chunk_collector.submit_janitor_action, failing_call=test_case.failing_submit + ), + ) + + with pytest.raises(test_case.expected_error): + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=tuple(range(test_case.item_count)), + describe=phase_label, + apply=actions.apply, + workers=JanitorWorkers( + connections=tuple(object() for _ in range(test_case.connection_count)) + ), + after_chunk=actions.flush, + ) + + assert len(actions.completed) <= test_case.expected_maximum_completed + assert sorted(actions.flushed) == actions.completed_event_ids() + assert progress.closed_failed == [True] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPhaseFlushFailureTestCase( + description="a failed event write keeps the action failure and warns per schema", + item_count=6, + failing_item=3, + action_error=RuntimeError, + flush_error=OSError, + expected_error=RuntimeError, + expected_chained_error=OSError, + expected_unwritten=((3, ("analytics",), "events table is locked"),), + ), + JanitorPhaseFlushFailureTestCase( + description="an interrupted action still wins over a failed event write", + item_count=6, + failing_item=2, + action_error=KeyboardInterrupt, + flush_error=OSError, + expected_error=KeyboardInterrupt, + expected_chained_error=OSError, + expected_unwritten=((2, ("analytics",), "events table is locked"),), + ), + JanitorPhaseFlushFailureTestCase( + description="an interrupted event write wins over the action failure", + item_count=6, + failing_item=4, + action_error=RuntimeError, + flush_error=KeyboardInterrupt, + expected_error=KeyboardInterrupt, + expected_chained_error=RuntimeError, + expected_unwritten=((4, ("analytics",), "events table is locked"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_event_write_when_phase_fails_then_original_error_is_raised_and_chained( + test_case: JanitorPhaseFlushFailureTestCase, +) -> None: + actions: FailingPhaseActions = FailingPhaseActions( + failing_item=test_case.failing_item, + error=test_case.action_error, + flush_error=test_case.flush_error, + size=test_case.item_count, + ) + progress: FailingReportProgress = FailingReportProgress( + failing_report=test_case.item_count + 1, + error=BrokenPipeError, + size=test_case.item_count, + ) + + with pytest.raises(test_case.expected_error) as raised: + _ = run_janitor_phase( + progress=progress, + phase=JanitorPhase.ARCHIVE, + items=tuple(range(test_case.item_count)), + describe=phase_label, + apply=actions.apply, + workers=JanitorWorkers(connections=(object(),)), + after_chunk=actions.flush, + ) + + assert isinstance(raised.value.__context__, test_case.expected_chained_error) + assert progress.unwritten == list(test_case.expected_unwritten) + assert progress.closed_failed == [True]