feat: show which janitor candidates are in use, and measure and cut custom rule overhead

crates/sqlbuild-cli/src/argv/command_table.json

@@ -5273,6 +5273,18 @@
         "help": "drop the compatibility view at this old model name before it expires; repeat for several views",
         "group": 1
        },
+       {
+        "kind": "store_true",
+        "options": [
+         "--keep-in-use"
+        ],
+        "dest": "keep_in_use",
+        "nargs": 0,
+        "const": true,
+        "default": false,
+        "help": "keep relations other queries read within janitor.in_use_window instead of archiving them (overrides janitor.keep_in_use)",
+        "group": 1
+       },
        {
         "kind": "store_true",
         "options": [

crates/sqlbuild-cli/src/argv/constants.rs

@@ -64,7 +64,7 @@ pub(crate) const INT_CONVERTER: &str = "int";
 pub(crate) const PATH_CONVERTER: &str = "Path";
 
 /// Every `CliNamespace` attribute with a default, in declaration order.
-pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 179] = [
+pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 180] = [
     ("command", NamespaceDefault::None),
     ("project_dir", NamespaceDefault::None),
     ("dbt_project_dir", NamespaceDefault::None),
@@ -139,6 +139,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 179] = [
     ("retention_days", NamespaceDefault::None),
     ("direct_state_history_versions", NamespaceDefault::None),
     ("drop_old_name_views", NamespaceDefault::EmptyList),
+    ("keep_in_use", NamespaceDefault::Bool(false)),
     ("bounded", NamespaceDefault::None),
     ("max_column_examples", NamespaceDefault::None),
     ("max_row_only_examples", NamespaceDefault::None),

crates/sqlbuild-cli/src/dispatch/constants.rs

@@ -486,7 +486,7 @@ pub(crate) const DEBUG_ROUTE: Route = routed(
     },
 );
 
-const JANITOR_FIELDS: [Field; 10] = [
+const JANITOR_FIELDS: [Field; 11] = [
     PROJECT_DIR,
     NO_COLOR,
     same("auto_approve"),
@@ -496,6 +496,7 @@ const JANITOR_FIELDS: [Field; 10] = [
         "drop_old_name_views",
         Source::TupleOf("drop_old_name_views"),
     ),
+    same("keep_in_use"),
     SELECTED_TARGET,
     same("as_target"),
     same("verbose"),

crates/sqlbuild-config/src/project/_helpers/sections.rs

@@ -12,13 +12,13 @@ use crate::project::_helpers::targets::seconds_value;
 use crate::project::_helpers::values::{int_below, python_equal};
 use crate::project::constants::{
     AUDIT_SEVERITIES, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY,
-    DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_QUERY_ARTIFACT_TTL,
-    FUTURE_CURSOR_ACTIONS, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS,
-    MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY,
-    REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS,
-    SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS,
-    SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES,
-    USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT,
+    DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_IN_USE_WINDOW,
+    DEFAULT_QUERY_ARTIFACT_TTL, FUTURE_CURSOR_ACTIONS, LEGACY_CONCURRENCY_KEY,
+    LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS,
+    OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY,
+    RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES,
+    SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY,
+    UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT,
 };
 use crate::project::constants::{
     DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE,
@@ -565,6 +565,19 @@ pub(crate) fn load_janitor(payload: Option<&ConfigValue>, file: ConfigFile) -> L
         get(mapping, "exclude_patterns"),
         place(file, "janitor.exclude_patterns"),
     )?;
+    let keep_in_use = optional_bool(mapping, "keep_in_use", false)?;
+    let in_use_window = match get(mapping, "in_use_window") {
+        None => DEFAULT_IN_USE_WINDOW.to_owned(),
+        Some(ConfigValue::String(window)) if fixed_duration_seconds(window, false)?.is_some() => {
+            window.clone()
+        }
+        Some(_) => {
+            return Err(ConfigFailure::at(
+                file,
+                "janitor.in_use_window must be a positive fixed duration such as '7d'",
+            ));
+        }
+    };
     let checks = [
         (retention_days.as_ref(), "retention_days"),
         (Some(&archive), "archive_retention_days"),
@@ -591,6 +604,11 @@ pub(crate) fn load_janitor(payload: Option<&ConfigValue>, file: ConfigFile) -> L
                 Built::Value(ConfigValue::Bool(delete_tracked_only)),
             ),
             ("exclude_patterns", Built::Tuple(exclude)),
+            (
+                "in_use_window",
+                Built::Value(ConfigValue::String(in_use_window)),
+            ),
+            ("keep_in_use", Built::Value(ConfigValue::Bool(keep_in_use))),
         ],
     ))
 }

crates/sqlbuild-config/src/project/constants.rs

@@ -27,6 +27,7 @@ pub(crate) const MODELS_DIRECTORY_NAME: &str = "models";
 pub(crate) const UNLIMITED_DIFF_FULL_ROWS: &str = "unlimited";
 pub(crate) const ZERO_FIXED_DURATION: &str = "0s";
 pub(crate) const DEFAULT_QUERY_ARTIFACT_TTL: &str = "24h";
+pub(crate) const DEFAULT_IN_USE_WINDOW: &str = "7d";
 pub(crate) const BOUNDED_REPLAY_PREFIX: &str = "bounded-";
 pub(crate) const ZERO_FLOAT: f64 = 0.0;
 pub(crate) const SHUTDOWN_TIMEOUT_MAX_SECONDS: u64 = 600;

crates/sqlbuild-config/src/project/tests/test_load_config.rs

@@ -40,6 +40,33 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() {
                 ),
             ),
         },
+        ProjectFailureTestCase {
+            description: "a janitor usage window and keep_in_use load",
+            project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nenabled = true\n\
+                      in_use_window = '36h'\nkeep_in_use = true\n",
+            local: "",
+            expected_failure: None,
+        },
+        ProjectFailureTestCase {
+            description: "a calendar janitor usage window is rejected",
+            project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nin_use_window = '1mo'\n",
+            local: "",
+            expected_failure: failure_at(
+                ConfigFile::Project,
+                "janitor.in_use_window must be a positive fixed duration such as '7d'",
+                None,
+            ),
+        },
+        ProjectFailureTestCase {
+            description: "a janitor usage window that is not a duration is rejected",
+            project: "name = 'orders'\nadapter = 'duckdb'\n[janitor]\nin_use_window = 7\n",
+            local: "",
+            expected_failure: failure_at(
+                ConfigFile::Project,
+                "janitor.in_use_window must be a positive fixed duration such as '7d'",
+                None,
+            ),
+        },
         ProjectFailureTestCase {
             description: "a git base ref loads",
             project: "name = 'orders'\nadapter = 'duckdb'\n[git]\nbase_ref = 'origin/main'\n",

crates/sqlbuild-core/src/janitor_text/_helpers/mod.rs

@@ -2,3 +2,4 @@ pub(crate) mod counts;
 pub(crate) mod phases;
 pub(crate) mod plan_json;
 pub(crate) mod plan_sections;
+pub(crate) mod usage;

crates/sqlbuild-core/src/janitor_text/_helpers/plan_json.rs

@@ -1,9 +1,10 @@
 //! The JSON payload of `sqb janitor --json`.
 
+use crate::janitor_text::_helpers::usage::checked_usage;
 use crate::janitor_text::constants::{DROP_EARLY, DROP_EXPIRED};
 use crate::janitor_text::models::{
-    JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText, OldNameViewText,
-    SourceSchemaText,
+    ArchiveCandidateText, JanitorItemText, JanitorPlanText, JanitorResultCounts, JanitorRunText,
+    OldNameViewText, SourceSchemaText,
 };
 use crate::json::models::{JsonInteger, JsonValue};
 
@@ -61,6 +62,65 @@ fn old_name_view(view: &OldNameViewText, planned: &str) -> JsonValue {
     ])
 }
 
+/// `null` when usage was not checked, `false` when unread, else the reads.
+fn in_use(candidate: &ArchiveCandidateText, checked: bool) -> JsonValue {
+    if !checked {
+        return JsonValue::Null;
+    }
+    candidate
+        .usage
+        .as_ref()
+        .map_or(JsonValue::Bool(false), |usage| {
+            object(vec![
+                ("last_read", utc_timestamp(&usage.last_read_at)),
+                ("readers", texts(&usage.readers)),
+                ("query_count", integer(usage.query_count)),
+                ("via", usage.via.as_deref().map_or(JsonValue::Null, texts)),
+            ])
+        })
+}
+
+fn archive_candidate(candidate: &ArchiveCandidateText, checked: bool) -> JsonValue {
+    object(vec![
+        ("name", text(&candidate.name)),
+        ("archive_name", text(&candidate.archive_name)),
+        ("age_days", optional_integer(candidate.age_days)),
+        ("expires_at", utc_timestamp(&candidate.expires_at)),
+        ("deleted_in_run", JsonValue::Bool(candidate.expires_now)),
+        ("in_use", in_use(candidate, checked)),
+    ])
+}
+
+/// A candidate kept because it is in use: no archive name or expiry, since it stays put.
+fn kept_candidate(candidate: &ArchiveCandidateText) -> JsonValue {
+    object(vec![
+        ("name", text(&candidate.name)),
+        ("age_days", optional_integer(candidate.age_days)),
+        ("in_use", in_use(candidate, true)),
+    ])
+}
+
+/// The top-level `usage_check`: its status and reason, or `null` when it did not run.
+pub(crate) fn usage_check_payload(plan: Option<&JanitorPlanText>) -> JsonValue {
+    let Some(check) = plan.and_then(|plan| plan.usage_check.as_ref()) else {
+        return JsonValue::Null;
+    };
+    object(vec![
+        (
+            "status",
+            text(if check.checked {
+                "checked"
+            } else {
+                "not_checked"
+            }),
+        ),
+        ("reason", optional_text(check.reason.as_deref())),
+        ("window", text(&check.window)),
+        ("keep_in_use", JsonValue::Bool(check.keep_in_use)),
+        ("notices", texts(&check.notices)),
+    ])
+}
+
 fn source_schema(schema: &SourceSchemaText, blocked: bool) -> JsonValue {
     let mut fields: Vec<(&str, JsonValue)> = vec![
         ("name", text(&schema.name)),
@@ -96,6 +156,7 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue {
                 .map(|view| old_name_view(view, "missing")),
         )
         .collect();
+    let checked: bool = checked_usage(plan).is_some();
     object(vec![
         ("direct_mode", JsonValue::Bool(plan.direct_mode)),
         ("retention_days", integer(plan.retention_days)),
@@ -114,18 +175,14 @@ pub(crate) fn plan_payload(plan: &JanitorPlanText) -> JsonValue {
             JsonValue::Array(
                 plan.archive_candidates
                     .iter()
-                    .map(|candidate| {
-                        object(vec![
-                            ("name", text(&candidate.name)),
-                            ("archive_name", text(&candidate.archive_name)),
-                            ("age_days", optional_integer(candidate.age_days)),
-                            ("expires_at", utc_timestamp(&candidate.expires_at)),
-                            ("deleted_in_run", JsonValue::Bool(candidate.expires_now)),
-                        ])
-                    })
+                    .map(|candidate| archive_candidate(candidate, checked))
                     .collect(),
             ),
         ),
+        (
+            "kept_in_use",
+            JsonValue::Array(plan.kept_in_use.iter().map(kept_candidate).collect()),
+        ),
         (
             "archives_to_delete",
             JsonValue::Array(

crates/sqlbuild-core/src/janitor_text/_helpers/plan_sections.rs

@@ -1,11 +1,14 @@
 //! Sections of the janitor plan preview.
 
 use crate::cli_text::models::CliStyle;
+use crate::janitor_text::_helpers::usage::{
+    checked_usage, in_use_candidates, usage_detail, usage_notices,
+};
 use crate::janitor_text::constants::{
     DROP_EARLY, DROP_EXPIRED, DROP_MISSING, LABEL_WIDTH, TREE_BLANK, TREE_BRANCH_GLYPH,
     TREE_CONTINUATION, TREE_LAST_GLYPH, VERBOSE_FLAG,
 };
-use crate::janitor_text::models::{JanitorPlanText, OldNameViewText};
+use crate::janitor_text::models::{ArchiveCandidateText, JanitorPlanText, OldNameViewText};
 
 /// The first `limit` items, or all of them, and how many are hidden.
 fn visible<T>(items: &[T], limit: Option<usize>) -> (&[T], usize) {
@@ -91,6 +94,13 @@ pub(crate) fn summary(plan: &JanitorPlanText, style: CliStyle) -> String {
             "relations to archive",
             &plan.archive_candidates,
         ));
+        if let Some(check) = checked_usage(plan) {
+            if check.keep_in_use {
+                text.push_str(&count_row(style, "kept (in use)", &plan.kept_in_use));
+            } else {
+                text.push_str(&count_row(style, "in use", &in_use_candidates(plan)));
+            }
+        }
         text.push_str(&count_row(
             style,
             "archives to delete",
@@ -118,9 +128,38 @@ pub(crate) fn summary(plan: &JanitorPlanText, style: CliStyle) -> String {
         "objects skipped",
         &style.accent(&plan.skipped_relations.len().to_string()),
     ));
+    let notices: Vec<String> = usage_notices(plan);
+    if !notices.is_empty() {
+        text.push('\n');
+        for notice in notices {
+            text.push_str(&format!("  {}\n", style.warning(&notice)));
+        }
+    }
     text
 }
 
+/// In-use archive candidates with their readers, or the in-use relations kept instead.
+fn in_use(plan: &JanitorPlanText, limit: Option<usize>, style: CliStyle) -> String {
+    let Some(check) = checked_usage(plan) else {
+        return String::new();
+    };
+    let (title, candidates): (&str, Vec<&ArchiveCandidateText>) = if check.keep_in_use {
+        ("Kept (in use)", plan.kept_in_use.iter().collect())
+    } else {
+        ("In use", in_use_candidates(plan))
+    };
+    let lines: Vec<(String, Option<String>)> = candidates
+        .into_iter()
+        .map(|candidate| {
+            (
+                candidate.name.clone(),
+                usage_detail(candidate, &check.window),
+            )
+        })
+        .collect();
+    listing((style, limit), title, &lines)
+}
+
 /// Schemas whose active sources block every janitor action.
 pub(crate) fn blocked(plan: &JanitorPlanText, limit: Option<usize>, style: CliStyle) -> String {
     if plan.blocked_schemas.is_empty() {
@@ -190,6 +229,7 @@ pub(crate) fn archives(plan: &JanitorPlanText, limit: Option<usize>, style: CliS
         }
         text.push_str(&overflow(style, hidden, "  "));
     }
+    text.push_str(&in_use(plan, limit, style));
     if !plan.archive_deletions.is_empty() {
         text.push_str(&format!("\n{}\n", style.success("Archives to delete")));
         let (deletions, hidden) = visible(&plan.archive_deletions, limit);