Skip to content

Commit bc1245c

Browse files
kvlongekvlonge
authored
feat: show janitor progress, inspect and archive faster (#1226)
* feat: stream janitor phases and actions, shorten the preview, add --verbose and --json * perf: list each janitor schema once and batch view definition reads * perf: run janitor actions concurrently and batch audit events * test: cover janitor progress text, event writer and batched audit events * test: cover janitor progress, inspection reads and concurrent actions end to end * docs: describe janitor progress output, --verbose, --json and faster cleanup * refactor: split janitor worker and phase helpers to follow repository policy * refactor: keep janitor progress and fensu budgets within native boundary policy * fix: keep janitor audit events when reporting fails, and report every --json failure * fix: keep janitor events on interrupted submission, chain event write failures, reset the json marker per invocation
1 parent 57f3b39 commit bc1245c

103 files changed

Lines changed: 5187 additions & 541 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎crates/sqlbuild-cli/src/argv/command_table.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5198,6 +5198,7 @@
51985198
"nargs": 0,
51995199
"const": true,
52005200
"default": false,
5201+
"help": "skip the confirmation prompt and run the planned archives, deletes and prunes",
52015202
"group": 1
52025203
},
52035204
{
@@ -5207,6 +5208,7 @@
52075208
],
52085209
"dest": "retention_days",
52095210
"type": "int",
5211+
"help": "days a stale relation must be unchanged before it is archived (overrides janitor.retention_days)",
52105212
"group": 1
52115213
},
52125214
{
@@ -5216,6 +5218,7 @@
52165218
],
52175219
"dest": "direct_state_history_versions",
52185220
"type": "int",
5221+
"help": "state history rows to keep per identity when pruning direct state tables (overrides janitor.direct_state_history_versions)",
52195222
"group": 1
52205223
},
52215224
{
@@ -5226,6 +5229,32 @@
52265229
"dest": "drop_old_name_views",
52275230
"default": [],
52285231
"metavar": "NAME",
5232+
"help": "drop the compatibility view at this old model name before it expires; repeat for several views",
5233+
"group": 1
5234+
},
5235+
{
5236+
"kind": "store_true",
5237+
"options": [
5238+
"--verbose",
5239+
"-v"
5240+
],
5241+
"dest": "verbose",
5242+
"nargs": 0,
5243+
"const": true,
5244+
"default": false,
5245+
"help": "list every entry of each preview section and every skipped object",
5246+
"group": 1
5247+
},
5248+
{
5249+
"kind": "store_true",
5250+
"options": [
5251+
"--json"
5252+
],
5253+
"dest": "json",
5254+
"nargs": 0,
5255+
"const": true,
5256+
"default": false,
5257+
"help": "print the plan and result as JSON on stdout; progress goes to stderr",
52295258
"group": 1
52305259
},
52315260
{
@@ -5234,6 +5263,7 @@
52345263
"--target"
52355264
],
52365265
"dest": "target",
5266+
"help": "clean up this configured target instead of the active target",
52375267
"group": 1
52385268
},
52395269
{

‎crates/sqlbuild-cli/src/dispatch/constants.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -484,7 +484,7 @@ pub(crate) const DEBUG_ROUTE: Route = routed(
484484
},
485485
);
486486

487-
const JANITOR_FIELDS: [Field; 8] = [
487+
const JANITOR_FIELDS: [Field; 10] = [
488488
PROJECT_DIR,
489489
NO_COLOR,
490490
same("auto_approve"),
@@ -496,6 +496,8 @@ const JANITOR_FIELDS: [Field; 8] = [
496496
),
497497
SELECTED_TARGET,
498498
same("as_target"),
499+
same("verbose"),
500+
JSON_OUTPUT,
499501
];
500502
const JANITOR_ARGUMENTS: [Field; 1] = [request(class("JanitorCommandRequest"), &JANITOR_FIELDS)];
501503
pub(crate) const JANITOR_ROUTE: Route = routed(

‎crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
pub(crate) mod counts;
2+
pub(crate) mod phases;
3+
pub(crate) mod plan_json;
24
pub(crate) mod plan_sections;

‎crates/sqlbuild-core/src/janitor_text/_helpers/phases.rs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//! Wording and durations of janitor apply phases.
2+
3+
use crate::cli_text::main::format_count_noun::format_count_noun;
4+
use crate::janitor_text::constants::{MINUTE_SECONDS, SECONDS_PER_HOUR, SECONDS_PER_MINUTE};
5+
use crate::janitor_text::models::JanitorPhase;
6+
7+
/// The phase's present participle, past tense and singular noun.
8+
pub(crate) fn phase_words(phase: JanitorPhase) -> (&'static str, &'static str, &'static str) {
9+
match phase {
10+
JanitorPhase::DeleteRelations => ("Deleting", "Deleted", "relation"),
11+
JanitorPhase::Archive => ("Archiving", "Archived", "relation"),
12+
JanitorPhase::DeleteArchives => ("Deleting", "Deleted", "expired archive"),
13+
JanitorPhase::DropOldNameViews => ("Dropping", "Dropped", "old-name view"),
14+
JanitorPhase::DropQueryArtifacts => ("Dropping", "Dropped", "query artifact"),
15+
JanitorPhase::PruneDirectState => ("Pruning", "Pruned", "direct state table"),
16+
}
17+
}
18+
19+
/// `count` with the singular noun for one and its plural otherwise.
20+
pub(crate) fn noun(count: usize, singular: &str) -> String {
21+
format_count_noun(i64::try_from(count).unwrap_or(i64::MAX), singular, None)
22+
}
23+
24+
/// Seconds with two decimals under a minute, then whole minutes and seconds.
25+
pub(crate) fn duration(seconds: f64) -> String {
26+
if seconds < MINUTE_SECONDS {
27+
return format!("{:.2}s", seconds.max(0.0));
28+
}
29+
let whole: u64 = seconds.round() as u64;
30+
let (hours, minutes, rest) = (
31+
whole / SECONDS_PER_HOUR,
32+
whole % SECONDS_PER_HOUR / SECONDS_PER_MINUTE,
33+
whole % SECONDS_PER_MINUTE,
34+
);
35+
if hours > 0 {
36+
return format!("{hours}h {minutes:02}m {rest:02}s");
37+
}
38+
format!("{minutes}m {rest:02}s")
39+
}
40+
41+
/// `name -> destination`, or the name alone.
42+
pub(crate) fn flow(name: &str, destination: Option<&str>) -> String {
43+
destination.map_or_else(
44+
|| name.to_owned(),
45+
|destination| format!("{name} -> {destination}"),
46+
)
47+
}

‎crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
//! The JSON payload of `sqb janitor --json`.
2+
3+
use crate::janitor_text::constants::{DROP_EARLY, DROP_EXPIRED};
4+
use crate::janitor_text::models::{
5+
JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText, OldNameViewText,
6+
SourceSchemaText,
7+
};
8+
use crate::json::models::{JsonInteger, JsonValue};
9+
10+
fn text(value: &str) -> JsonValue {
11+
JsonValue::String(value.to_owned())
12+
}
13+
14+
fn optional_text(value: Option<&str>) -> JsonValue {
15+
value.map_or(JsonValue::Null, text)
16+
}
17+
18+
fn texts(values: &[String]) -> JsonValue {
19+
JsonValue::Array(values.iter().map(|value| text(value)).collect())
20+
}
21+
22+
fn integer(value: impl ToString) -> JsonValue {
23+
JsonInteger::parse(&value.to_string()).map_or(JsonValue::Null, JsonValue::Integer)
24+
}
25+
26+
fn optional_integer(value: Option<i64>) -> JsonValue {
27+
value.map_or(JsonValue::Null, integer)
28+
}
29+
30+
fn object(fields: Vec<(&str, JsonValue)>) -> JsonValue {
31+
JsonValue::Object(
32+
fields
33+
.into_iter()
34+
.map(|(key, value)| (key.to_owned(), value))
35+
.collect(),
36+
)
37+
}
38+
39+
/// A preview `YYYY-MM-DD HH:MM:SS` UTC time as ISO 8601.
40+
fn utc_timestamp(value: &str) -> JsonValue {
41+
if value.is_empty() {
42+
return JsonValue::Null;
43+
}
44+
text(&format!("{}Z", value.replacen(' ', "T", 1)))
45+
}
46+
47+
fn old_name_view(view: &OldNameViewText, planned: &str) -> JsonValue {
48+
let action: &str = match (planned, view.drop_reason.as_deref()) {
49+
("drop", Some(DROP_EARLY | DROP_EXPIRED)) => "drop",
50+
("live", _) => "keep",
51+
_ => "record_dropped",
52+
};
53+
object(vec![
54+
("name", text(&view.name)),
55+
("destination_model", text(&view.destination_model)),
56+
("expires_on", optional_text(view.expires_on.as_deref())),
57+
("action", text(action)),
58+
("drop_reason", optional_text(view.drop_reason.as_deref())),
59+
("occupied", JsonValue::Bool(view.occupied)),
60+
("claimed_by", optional_text(view.claimed_by.as_deref())),
61+
])
62+
}
63+
64+
fn source_schema(schema: &SourceSchemaText, blocked: bool) -> JsonValue {
65+
let mut fields: Vec<(&str, JsonValue)> = vec![
66+
("name", text(&schema.name)),
67+
("source_names", texts(&schema.source_names)),
68+
];
69+
if blocked {
70+
fields.push((
71+
"suppressed_candidates",
72+
texts(&schema.suppressed_candidates),
73+
));
74+
fields.push((
75+
"suppressed_archive_deletions",
76+
texts(&schema.suppressed_archive_deletions),
77+
));
78+
}
79+
object(fields)
80+
}
81+
82+
/// Every planned action, retained archive and skipped object of the plan.
83+
pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue {
84+
let views: Vec<JsonValue> = plan
85+
.old_name_drops
86+
.iter()
87+
.map(|view| old_name_view(view, "drop"))
88+
.chain(
89+
plan.old_name_live
90+
.iter()
91+
.map(|view| old_name_view(view, "live")),
92+
)
93+
.chain(
94+
plan.old_name_missing
95+
.iter()
96+
.map(|view| old_name_view(view, "missing")),
97+
)
98+
.collect();
99+
object(vec![
100+
("direct_mode", JsonValue::Bool(plan.direct_mode)),
101+
("retention_days", integer(plan.retention_days)),
102+
(
103+
"archive_retention_days",
104+
integer(plan.archive_retention_days),
105+
),
106+
(
107+
"age_metadata_supported",
108+
JsonValue::Bool(plan.age_metadata_supported),
109+
),
110+
("schemas_scanned", integer(plan.scanned_schema_count)),
111+
("eligible_for_deletion", texts(&plan.candidates)),
112+
(
113+
"relations_to_archive",
114+
JsonValue::Array(
115+
plan.archive_candidates
116+
.iter()
117+
.map(|candidate| {
118+
object(vec![
119+
("name", text(&candidate.name)),
120+
("archive_name", text(&candidate.archive_name)),
121+
("age_days", optional_integer(candidate.age_days)),
122+
("expires_at", utc_timestamp(&candidate.expires_at)),
123+
("deleted_in_run", JsonValue::Bool(candidate.expires_now)),
124+
])
125+
})
126+
.collect(),
127+
),
128+
),
129+
(
130+
"archives_to_delete",
131+
JsonValue::Array(
132+
plan.archive_deletions
133+
.iter()
134+
.map(|archive| {
135+
object(vec![
136+
("name", text(&archive.name)),
137+
("archived_in_run", JsonValue::Bool(archive.archived_in_run)),
138+
("archived_at", utc_timestamp(&archive.archived_at)),
139+
("age_days", optional_integer(archive.age_days)),
140+
("expires_at", utc_timestamp(&archive.expires_at)),
141+
])
142+
})
143+
.collect(),
144+
),
145+
),
146+
(
147+
"retained_archives",
148+
JsonValue::Array(
149+
plan.retained_archives
150+
.iter()
151+
.map(|archive| {
152+
object(vec![
153+
("name", text(&archive.name)),
154+
("archived_at", utc_timestamp(&archive.archived_at)),
155+
("expires_at", utc_timestamp(&archive.expires_at)),
156+
])
157+
})
158+
.collect(),
159+
),
160+
),
161+
("old_name_views", JsonValue::Array(views)),
162+
("query_artifacts", texts(&plan.query_artifacts)),
163+
(
164+
"direct_state_prunes",
165+
JsonValue::Array(
166+
plan.state_prunes
167+
.iter()
168+
.map(|(name, versions)| {
169+
object(vec![
170+
("name", text(name)),
171+
("retain_versions", integer(*versions)),
172+
])
173+
})
174+
.collect(),
175+
),
176+
),
177+
(
178+
"skipped_objects",
179+
JsonValue::Array(
180+
plan.skipped_relations
181+
.iter()
182+
.map(|(name, reason)| {
183+
object(vec![("name", text(name)), ("reason", text(reason))])
184+
})
185+
.collect(),
186+
),
187+
),
188+
(
189+
"skipped_schemas",
190+
JsonValue::Array(
191+
plan.skipped_schemas
192+
.iter()
193+
.map(|schema| source_schema(schema, false))
194+
.collect(),
195+
),
196+
),
197+
(
198+
"blocked_schemas",
199+
JsonValue::Array(
200+
plan.blocked_schemas
201+
.iter()
202+
.map(|schema| source_schema(schema, true))
203+
.collect(),
204+
),
205+
),
206+
])
207+
}
208+
209+
fn counts_payload(counts: &JanitorResultCounts) -> JsonValue {
210+
object(vec![
211+
("archived", integer(counts.archived)),
212+
("deleted", integer(counts.deleted)),
213+
("deleted_archives", integer(counts.deleted_archives)),
214+
(
215+
"deleted_query_artifacts",
216+
integer(counts.deleted_query_artifacts),
217+
),
218+
(
219+
"dropped_old_name_views",
220+
integer(counts.dropped_old_name_views),
221+
),
222+
("pruned_direct_state", integer(counts.pruned_direct_state)),
223+
])
224+
}
225+
226+
fn item_payload(item: &JanitorItemText) -> JsonValue {
227+
object(vec![
228+
("phase", text(item.phase.value())),
229+
("action", text(&item.action)),
230+
("name", text(&item.name)),
231+
("destination", optional_text(item.destination.as_deref())),
232+
(
233+
"status",
234+
text(if item.error.is_some() { "failed" } else { "ok" }),
235+
),
236+
("error", optional_text(item.error.as_deref())),
237+
(
238+
"duration_seconds",
239+
item.duration_seconds
240+
.map_or(JsonValue::Null, JsonValue::Float),
241+
),
242+
])
243+
}
244+
245+
/// The counts, actions and duration of a run.
246+
pub(crate) fn run_payload(run: &JanitorRunText) -> JsonValue {
247+
object(vec![
248+
("counts", counts_payload(&run.counts)),
249+
("elapsed_seconds", JsonValue::Float(run.elapsed_seconds)),
250+
(
251+
"items",
252+
JsonValue::Array(run.items.iter().map(item_payload).collect()),
253+
),
254+
])
255+
}

0 commit comments

Comments
 (0)