diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 6d929df51e..17a1d2c630 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -10,6 +10,7 @@ Online: https://sqlbuild.com/docs/cli/janitor/ - Usage - Flags +- Targets - Lifecycle - Compatibility views - Pending migration origins @@ -33,11 +34,34 @@ sqb --project-dir janitor [flags] | Flag | Description | |------|-------------| +| `--target ` | Clean up a configured target instead of the active/default target, using that target's connection, schemas and state | +| `--as ` | Preview the janitor plan for another configured target's schemas through the active target's connection. Inspection only: nothing is archived, dropped or pruned, and there is no prompt. Cannot be combined with `--auto-approve` | | `--auto-approve` | Skip the confirmation prompt and run the planned actions immediately | | `--retention-days` | Override the configured retention period (days) before a stale relation is archived | | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | | `--drop-old-name-view ` | Drop the [compatibility view](../concepts/models/migrations.md#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | +## Targets + +By default the janitor cleans up the active target. `--target ` selects another configured target, exactly like `sqb build --target`: the janitor connects with that target's connection and inspects and cleans that target's schemas and state tables. Other targets are never touched. + +`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](plan.md). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists every relation it would archive, every archive it would delete, every compatibility view it would drop and every state table it would prune, then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. + +``` +Previewing janitor as target 'prod' through the connection of target 'dev' (inspection only). +... +Relations to archive + prod.customers -> prod._sqb_archive__20261001t101500z__customers age unknown, delete after 2026-10-15 10:15:00 UTC + +Archives to delete + prod._sqb_archive__20200101t000000z__old_products archived 2020-01-01 00:00:00 UTC, age 2465d, expired 2020-01-15 00:00:00 UTC + +Previewed janitor as target 'prod' through the connection of target 'dev'. Nothing was changed. +Rerun with `--target prod` instead of `--as prod` to apply it. +``` + +An unknown target name fails before anything connects. A preview exits with `0`, or with `1` when a schema is blocked, as a normal run would. + ## Lifecycle The janitor never drops a live relation directly. Each stale relation goes through two stages: @@ -134,6 +158,12 @@ sqb janitor --auto-approve # Override retention to 7 days sqb janitor --retention-days 7 + +# Preview the prod cleanup through the current connection, changing nothing +sqb janitor --as prod + +# Clean up the prod target +sqb janitor --target prod ``` ## State history pruning diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 895d4d6c6d..cdb2d20c55 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -406,6 +406,8 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) retention_days=args.retention_days, direct_state_history_versions=args.direct_state_history_versions, drop_old_name_views=tuple(args.drop_old_name_views), + selected_target=args.target, + as_target=args.as_target, ) ) return _dispatch_local_command( diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index d6c353f58c..0ede580a9d 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -630,6 +630,16 @@ def _add_maintenance_parsers( metavar="NAME", dest="drop_old_name_views", ) + janitor_parser.add_argument("--target", default=None) + janitor_parser.add_argument( + "--as", + dest="as_target", + default=None, + help=( + "Preview the janitor plan as another configured target through the active " + "target's connection; inspection only" + ), + ) def _add_workspace_parsers( diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py index 0f7a8f1f63..19053f1c49 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py @@ -129,6 +129,10 @@ def parse_cli_invocation( _validate_scope_args(args=args, parser=parser) if args.command == CliCommand.TEST: _validate_test_args(args=args, parser=parser) + if args.command == CliCommand.JANITOR and args.as_target is not None and args.auto_approve: + parser.error( + "janitor --as is inspection only and cannot be combined with --auto-approve" + ) except SystemExit as error: exit_code: int = error.code if isinstance(error.code, int) else 1 return ParsedCliInvocation(args=None, exit_code=exit_code) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py index 395b79fb6c..db795f3185 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py @@ -26,6 +26,39 @@ def write_janitor_disabled(*, invocation: JanitorInvocation) -> None: write_disabled(stream=sys.stdout, use_color=invocation.use_color) +def write_janitor_preview_start(*, invocation: JanitorInvocation) -> None: + """Announce an inspection-only ``--as`` janitor preview before any work starts.""" + + sys.stdout.write( + f"Previewing janitor as target '{invocation.as_target}' through " + f"{_active_connection_label(invocation=invocation)} (inspection only).\n" + ) + sys.stdout.flush() + + +def write_janitor_preview_complete(*, invocation: JanitorInvocation) -> None: + """Close an ``--as`` janitor preview with an unambiguous nothing-changed state.""" + + style: CliStyle = CliStyle(use_color=invocation.use_color) + sys.stdout.write( + style.success( + f"Previewed janitor as target '{invocation.as_target}' through " + f"{_active_connection_label(invocation=invocation)}. Nothing was changed." + ) + + "\n" + ) + sys.stdout.write( + f"Rerun with `--target {invocation.as_target}` instead of " + f"`--as {invocation.as_target}` to apply it.\n" + ) + + +def _active_connection_label(*, invocation: JanitorInvocation) -> str: + if invocation.active_target_name is None: + return "the active connection" + return f"the connection of target '{invocation.active_target_name}'" + + def write_janitor_plan( *, invocation: JanitorInvocation, planning_result: JanitorPlanningResult ) -> None: diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py index c1e6370b5b..c8d3ab34a2 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py @@ -5,10 +5,11 @@ import sys import time -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import resolve_connection_config +from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( + resolve_adapter_connection_context, +) from sqlbuild.cli.commands.models import ( + AdapterConnectionContext, JanitorCompileContext, JanitorInvocation, ) @@ -16,21 +17,16 @@ from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) def compile_janitor_project(*, invocation: JanitorInvocation) -> JanitorCompileContext: - """Resolve adapter, compile project, and prepare connection config.""" + """Compile for the janitor or ``--as`` target and resolve the active target connection.""" - adapter_name: str = resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, - project_dir=invocation.effective_project_dir, + adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( + discovered_inputs=invocation.discovered_inputs, + effective_project_dir=invocation.effective_project_dir, + selected_target=invocation.selected_target, + cli_vars=None, ) compile_start: float = time.perf_counter() status: PlanningProgressReporter = PlanningProgressReporter( @@ -41,17 +37,18 @@ def compile_janitor_project(*, invocation: JanitorInvocation) -> JanitorCompileC status.on_progress("Compiling project...") project: CompiledProject = compile_project( discovered_inputs=invocation.discovered_inputs, - adapter=adapter, + adapter=adapter_context.adapter, + selected_target=( + invocation.as_target + if invocation.as_target is not None + else invocation.selected_target + ), + resolved_connection=adapter_context.connection_config, ) status.on_progress(f"Compiled project. ({time.perf_counter() - compile_start:.2f}s)") - connection_config: dict[str, object] = resolve_connection_config( - raw_config=project.effective_connection, - project_dir=invocation.effective_project_dir, - adapter_name=adapter_name, - ) return JanitorCompileContext( - adapter_name=adapter_name, - adapter=adapter, + adapter_name=adapter_context.adapter_name, + adapter=adapter_context.adapter, project=project, - connection_config=connection_config, + connection_config=adapter_context.connection_config, ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py index 996f79dc70..229b43313b 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py @@ -4,6 +4,7 @@ from pathlib import Path +from sqlbuild.cli.commands._helpers.runtime.preview_target import validate_preview_target from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( JanitorCommandRequest, @@ -14,6 +15,7 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days from sqlbuild.presentation.main.supports_color import supports_color +from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name def resolve_janitor_invocation(*, request: JanitorCommandRequest) -> JanitorInvocation: @@ -25,10 +27,21 @@ def resolve_janitor_invocation(*, request: JanitorCommandRequest) -> JanitorInvo discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( project_dir=effective_project_dir ) + validate_preview_target( + discovered_inputs=discovered_inputs, as_target=request.as_target, command_name="janitor" + ) + active_target_name: str | None = resolve_target_name( + project_config=discovered_inputs.project_config, + local_config=discovered_inputs.local_config, + selected_target=request.selected_target, + ) return JanitorInvocation( effective_project_dir=effective_project_dir, discovered_inputs=discovered_inputs, use_color=not request.no_color and supports_color(), + selected_target=request.selected_target, + active_target_name=active_target_name, + as_target=request.as_target, ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index 1eb1c37796..9123cc88fc 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -9,7 +9,7 @@ from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.cli.commands._helpers.runtime.preview_target import validate_preview_target from sqlbuild.cli.commands.models import ( AdapterConnectionContext, PlanCommandRequest, @@ -34,7 +34,9 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( project_dir=effective_project_dir, extract_output_column_locations=False ) - _validate_preview_target(discovered_inputs=discovered_inputs, as_target=request.as_target) + validate_preview_target( + discovered_inputs=discovered_inputs, as_target=request.as_target, command_name="plan" + ) adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( discovered_inputs=discovered_inputs, effective_project_dir=effective_project_dir, @@ -64,18 +66,3 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: planning_progress=reporters.planning, should_load_sources=should_load_sources, ) - - -def _validate_preview_target( - *, discovered_inputs: DiscoveredProjectInputs, as_target: str | None -) -> None: - if as_target is None: - return - configured: set[str] = set(discovered_inputs.project_config.targets) | set( - discovered_inputs.local_config.targets - ) - if as_target not in configured: - raise CliUserError( - f"unknown target '{as_target}' for plan --as", - help=f"Configured targets: {', '.join(sorted(configured)) or 'none'}.", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/preview_target.py b/src/sqlbuild/cli/commands/_helpers/runtime/preview_target.py new file mode 100644 index 0000000000..66e1278b31 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/runtime/preview_target.py @@ -0,0 +1,23 @@ +"""Shared validation for inspection-only ``--as`` target previews.""" + +from __future__ import annotations + +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + + +def validate_preview_target( + *, discovered_inputs: DiscoveredProjectInputs, as_target: str | None, command_name: str +) -> None: + """Fail clearly when ``--as`` names a target that is not configured.""" + + if as_target is None: + return + configured: set[str] = set(discovered_inputs.project_config.targets) | set( + discovered_inputs.local_config.targets + ) + if as_target not in configured: + raise CliUserError( + f"unknown target '{as_target}' for {command_name} --as", + help=f"Configured targets: {', '.join(sorted(configured)) or 'none'}.", + ) diff --git a/src/sqlbuild/cli/commands/main/state/_janitor.py b/src/sqlbuild/cli/commands/main/state/_janitor.py index c5cf44e41a..38050952da 100644 --- a/src/sqlbuild/cli/commands/main/state/_janitor.py +++ b/src/sqlbuild/cli/commands/main/state/_janitor.py @@ -9,6 +9,8 @@ write_janitor_completion, write_janitor_disabled, write_janitor_plan, + write_janitor_preview_complete, + write_janitor_preview_start, ) from sqlbuild.cli.commands._helpers.janitor_runtime.compilation import compile_janitor_project from sqlbuild.cli.commands._helpers.janitor_runtime.connections import ( @@ -42,6 +44,8 @@ def run_janitor(request: JanitorCommandRequest) -> int: write_janitor_disabled(invocation=invocation) return 0 settings: JanitorSettings = resolve_janitor_settings(request=request, invocation=invocation) + if invocation.as_target is not None: + write_janitor_preview_start(invocation=invocation) compile_context: JanitorCompileContext = compile_janitor_project(invocation=invocation) connection_context: JanitorConnectionContext = connect_janitor_warehouse( invocation=invocation, @@ -55,6 +59,9 @@ def run_janitor(request: JanitorCommandRequest) -> int: connection_context=connection_context, ) write_janitor_plan(invocation=invocation, planning_result=planning_result) + if invocation.as_target is not None: + write_janitor_preview_complete(invocation=invocation) + return 1 if planning_result.plan.blocked_schemas else 0 if planning_result.plan.blocked_schemas: return 1 if not janitor_plan_has_work(planning_result): diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index be2c086de4..00ec60585e 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -652,6 +652,8 @@ class JanitorCommandRequest: retention_days: int | None = None direct_state_history_versions: int | None = None drop_old_name_views: tuple[str, ...] = () + selected_target: str | None = None + as_target: str | None = None @dataclass(frozen=True) @@ -661,6 +663,9 @@ class JanitorInvocation: effective_project_dir: Path discovered_inputs: DiscoveredProjectInputs use_color: bool + selected_target: str | None = None + active_target_name: str | None = None + as_target: str | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/pipeline/main/project.py b/src/sqlbuild/compiler/pipeline/main/project.py index a8fc5f728e..f44d69461f 100644 --- a/src/sqlbuild/compiler/pipeline/main/project.py +++ b/src/sqlbuild/compiler/pipeline/main/project.py @@ -13,6 +13,8 @@ def compile_project( *, discovered_inputs: DiscoveredProjectInputs, adapter: BaseAdapter, + selected_target: str | None = None, + resolved_connection: dict[str, object] | None = None, no_sql_validation: bool = False, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, ) -> CompiledProject: @@ -21,6 +23,8 @@ def compile_project( return build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, + selected_target=selected_target, + resolved_connection=resolved_connection, no_sql_validation=no_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index ff616913c9..b96a3de3be 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -141,3 +141,37 @@ class JanitorUnaddressableRelationE2ETestCase: plain_relation: str quoted_relation: str expected_stdout_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorTargetPreviewE2ETestCase: + """Test case for an inspection-only janitor --as preview of another target.""" + + description: str + janitor_command: tuple[str, ...] + expected_stdout_fragments: tuple[str, ...] + unexpected_stdout_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorTargetCleanupE2ETestCase: + """Test case for janitor cleanup scoped to one selected target namespace.""" + + description: str + janitor_command: tuple[str, ...] + cleaned_schema: str + untouched_schema: str + expected_stdout_fragments: tuple[str, ...] + expected_deleted_names: tuple[str, ...] + expected_archived_original_names: tuple[str, ...] + expected_fingerprint_probe_count_after: int + + +@dataclass(frozen=True) +class JanitorTargetErrorE2ETestCase: + """Test case for rejected janitor target flags.""" + + description: str + janitor_command: tuple[str, ...] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index ca85dbc78a..68823f71ca 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import subprocess from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path @@ -17,8 +18,10 @@ SourceFreshnessRenderers, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, prepare_inline_project, query_duckdb, + run_sqb, ) @@ -137,7 +140,7 @@ def create_janitor_scenario_relations(*, db_path: Path) -> None: connection.close() -def create_direct_state_history(*, db_path: Path) -> None: +def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: import duckdb adapter: DuckDbAdapter = DuckDbAdapter() @@ -155,12 +158,12 @@ def create_direct_state_history(*, db_path: Path) -> None: connection=connection, execute=lambda *, connection, sql: connection.execute(sql), database=None, - schema="main", + schema=schema, fingerprint=Fingerprint( node_type="model", node_name="janitor_state_probe", target_database=None, - target_schema="main", + target_schema=schema, target_name="janitor_state_probe", run_id=run_id, definition_hash=f"definition_{run_id}", @@ -176,7 +179,7 @@ def create_direct_state_history(*, db_path: Path) -> None: connection=connection, execute=lambda *, connection, sql: connection.execute(sql), database=None, - schema="main", + schema=schema, records=( SourceFreshnessRecord( source_name="raw.janitor_state_probe", @@ -360,3 +363,98 @@ def read_current_janitor_events(*, db_path: Path, run_id: str) -> list[tuple[obj "ORDER BY archive_name" ), ) + + +def prepare_two_target_janitor_project( + *, tmp_path: Path, project_name: str, model_names: tuple[str, ...] +) -> Path: + """Create a DuckDB janitor project with `dev` (default) and `prod` targets in two schemas.""" + + return prepare_inline_project( + tmp_path=tmp_path, + project_name=project_name, + repo_files={ + "sqlbuild_project.toml": ( + f'name = "{project_name}"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + f'database = "{(tmp_path / project_name / "janitor.duckdb").as_posix()}"\n\n' + "[targets.dev]\n" + 'schema = "dev"\n\n' + "[targets.prod]\n" + 'schema = "prod"\n\n' + "[janitor]\n" + "enabled = true\n" + "retention_days = 0\n" + "archive_retention_days = 14\n" + "direct_state_history_versions = 2\n\n" + "[defaults]\n" + 'materialized = "table"\n' + ), + **{ + f"models/{model_name}.sql": f"MODEL ();\n\nSELECT 1 AS {model_name}_id\n" + for model_name in model_names + }, + }, + ) + + +def snapshot_schema_contents( + *, db_path: Path, schemas: tuple[str, ...] +) -> tuple[tuple[str, str, str, tuple[tuple[object, ...], ...]], ...]: + """Return every relation in the schemas with its type and complete sorted contents.""" + + schema_list: str = ", ".join(f"'{schema}'" for schema in schemas) + relations: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT table_schema, table_name, table_type FROM information_schema.tables " + f"WHERE table_schema IN ({schema_list}) ORDER BY 1, 2" + ), + ) + snapshot: list[tuple[str, str, str, tuple[tuple[object, ...], ...]]] = [] + for schema, name, table_type in relations: + rows: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, sql=f'SELECT * FROM "{schema}"."{name}" ORDER BY ALL' + ) + snapshot.append((str(schema), str(name), str(table_type), tuple(rows))) + return tuple(snapshot) + + +EXPIRED_ARCHIVE_NAME: str = "_sqb_archive__20200101t000000z__old_products" + + +def prepare_cruft_in_both_targets(*, tmp_path: Path, project_name: str) -> Path: + """Build both targets, retire one model, and seed an expired archive and state history.""" + + project_dir: Path = prepare_two_target_janitor_project( + tmp_path=tmp_path, project_name=project_name, model_names=("orders", "customers") + ) + for build_command in (("--no-color", "build"), ("--no-color", "build", "--target", "prod")): + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=build_command, project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + (project_dir / "models" / "customers.sql").unlink() + db_path: Path = project_dir / "janitor.duckdb" + for schema in ("dev", "prod"): + execute_duckdb( + db_path=db_path, + sql=f'CREATE TABLE {schema}."{EXPIRED_ARCHIVE_NAME}" AS SELECT 1 AS product_id', + ) + create_direct_state_history(db_path=db_path, schema=schema) + return project_dir + + +def probe_fingerprint_count(*, db_path: Path, schema: str) -> int: + """Return how many seeded probe fingerprint rows remain in one schema.""" + + rows: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + f"SELECT COUNT(*) FROM {schema}._sqlbuild_fingerprints " + "WHERE node_name = 'janitor_state_probe'" + ), + ) + return int(str(rows[0][0])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py new file mode 100644 index 0000000000..8aa3fa0063 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -0,0 +1,286 @@ +"""E2E tests for janitor --target cleanup and inspection-only --as previews.""" + +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorTargetCleanupE2ETestCase, + JanitorTargetErrorE2ETestCase, + JanitorTargetPreviewE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( + EXPIRED_ARCHIVE_NAME, + list_archive_names, + prepare_cruft_in_both_targets, + prepare_two_target_janitor_project, + probe_fingerprint_count, + snapshot_schema_contents, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + run_sqb, + table_exists, +) + +SEEDED_PROBE_FINGERPRINT_COUNT: int = 4 +ARCHIVE_NAME_PATTERN: re.Pattern[str] = re.compile( + r"^_sqb_archive__[0-9]{8}t[0-9]{6}z__(?P.+)$" +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorTargetPreviewE2ETestCase( + description="as prod previews prod cruft through the default dev connection", + janitor_command=("--no-color", "janitor", "--as", "prod"), + expected_stdout_fragments=( + "Previewing janitor as target 'prod' through the connection of target 'dev' " + "(inspection only).", + "Janitor preview prod", + "relations to archive 1", + "archives to delete 1", + "prod.customers -> prod._sqb_archive__", + f"prod.{EXPIRED_ARCHIVE_NAME} archived 2020-01-01 00:00:00", + "prod._sqlbuild_fingerprints keep latest 2", + "Previewed janitor as target 'prod' through the connection of target 'dev'. " + "Nothing was changed.", + "Rerun with `--target prod` instead of `--as prod` to apply it.", + ), + unexpected_stdout_fragments=("dev.", "Type `", "Archived", "Deleted"), + ), + JanitorTargetPreviewE2ETestCase( + description="as dev with target prod previews dev cruft through the prod connection", + janitor_command=("--no-color", "janitor", "--target", "prod", "--as", "dev"), + expected_stdout_fragments=( + "Previewing janitor as target 'dev' through the connection of target 'prod' " + "(inspection only).", + "Janitor preview dev", + "dev.customers -> dev._sqb_archive__", + f"dev.{EXPIRED_ARCHIVE_NAME} archived 2020-01-01 00:00:00", + "Nothing was changed.", + ), + unexpected_stdout_fragments=("prod.", "Type `", "Archived", "Deleted"), + ), + ], + ids=lambda case: case.description, +) +def test_given_cruft_in_two_targets_when_previewing_janitor_as_target_then_nothing_changes( + test_case: JanitorTargetPreviewE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_cruft_in_both_targets( + tmp_path=tmp_path, project_name="janitor_target_preview" + ) + db_path: Path = project_dir / "janitor.duckdb" + before: tuple[object, ...] = snapshot_schema_contents(db_path=db_path, schemas=("dev", "prod")) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + after: tuple[object, ...] = snapshot_schema_contents(db_path=db_path, schemas=("dev", "prod")) + assert result.returncode == 0, result.stdout + result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in result.stdout + for fragment in test_case.unexpected_stdout_fragments: + assert fragment not in result.stdout + assert after == before + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorTargetPreviewE2ETestCase( + description="as prod with an early drop request previews the drop and keeps the view", + janitor_command=( + "--no-color", + "janitor", + "--as", + "prod", + "--drop-old-name-view", + "prod.revenue", + ), + expected_stdout_fragments=( + "└── prod.revenue -> model:daily_revenue\n └── drop now (requested; ", + "Nothing was changed.", + ), + unexpected_stdout_fragments=("Type `", "Deleted"), + ) + ], + ids=lambda case: case.description, +) +def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_is_kept( + test_case: JanitorTargetPreviewE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_two_target_janitor_project( + tmp_path=tmp_path, project_name="janitor_target_old_name", model_names=("revenue",) + ) + origin_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--target", "prod"), project_dir=project_dir + ) + assert origin_build.returncode == 0, origin_build.stdout + origin_build.stderr + (project_dir / "models" / "revenue.sql").unlink() + (project_dir / "models" / "daily_revenue.sql").write_text( + "MODEL (\n migrate_from revenue,\n);\n\nSELECT 1 AS revenue_id\n", encoding="utf-8" + ) + rename_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--target", "prod"), project_dir=project_dir + ) + assert rename_build.returncode == 0, rename_build.stdout + rename_build.stderr + db_path: Path = project_dir / "janitor.duckdb" + before: tuple[object, ...] = snapshot_schema_contents(db_path=db_path, schemas=("prod",)) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in result.stdout + for fragment in test_case.unexpected_stdout_fragments: + assert fragment not in result.stdout + assert snapshot_schema_contents(db_path=db_path, schemas=("prod",)) == before + assert ("prod", "revenue", "VIEW") in {entry[:3] for entry in before} + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorTargetCleanupE2ETestCase( + description="target prod cleans only the prod namespace", + janitor_command=("--no-color", "janitor", "--target", "prod", "--auto-approve"), + cleaned_schema="prod", + untouched_schema="dev", + expected_stdout_fragments=( + "Janitor preview prod", + "prod.customers -> prod._sqb_archive__", + "Archived 1 relation. Deleted 1 object and pruned 2 direct state tables.", + ), + expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), + expected_archived_original_names=("customers",), + expected_fingerprint_probe_count_after=2, + ), + JanitorTargetCleanupE2ETestCase( + description="default janitor cleans only the default dev namespace", + janitor_command=("--no-color", "janitor", "--auto-approve"), + cleaned_schema="dev", + untouched_schema="prod", + expected_stdout_fragments=( + "Janitor preview dev", + "dev.customers -> dev._sqb_archive__", + "Archived 1 relation. Deleted 1 object and pruned 2 direct state tables.", + ), + expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), + expected_archived_original_names=("customers",), + expected_fingerprint_probe_count_after=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_cruft_in_two_targets_when_running_janitor_for_one_target_then_other_is_untouched( + test_case: JanitorTargetCleanupE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_cruft_in_both_targets( + tmp_path=tmp_path, project_name="janitor_target_cleanup" + ) + db_path: Path = project_dir / "janitor.duckdb" + untouched_before: tuple[object, ...] = snapshot_schema_contents( + db_path=db_path, schemas=(test_case.untouched_schema,) + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in result.stdout + assert f"{test_case.untouched_schema}." not in result.stdout + for name in test_case.expected_deleted_names: + assert not table_exists(db_path=db_path, schema=test_case.cleaned_schema, table_name=name) + assert table_exists(db_path=db_path, schema=test_case.cleaned_schema, table_name="orders") + archived: tuple[str, ...] = list_archive_names(db_path=db_path, schema=test_case.cleaned_schema) + assert tuple(ARCHIVE_NAME_PATTERN.sub(r"\g", name) for name in archived) == ( + test_case.expected_archived_original_names + ) + assert ( + probe_fingerprint_count(db_path=db_path, schema=test_case.cleaned_schema) + == test_case.expected_fingerprint_probe_count_after + ) + assert ( + snapshot_schema_contents(db_path=db_path, schemas=(test_case.untouched_schema,)) + == untouched_before + ) + assert ( + probe_fingerprint_count(db_path=db_path, schema=test_case.untouched_schema) + == SEEDED_PROBE_FINGERPRINT_COUNT + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorTargetErrorE2ETestCase( + description="unknown as target fails clearly", + janitor_command=("--no-color", "janitor", "--as", "staging"), + expected_exit_code=1, + expected_output_fragments=( + "unknown target 'staging' for janitor --as", + "Configured targets: dev, prod.", + ), + ), + JanitorTargetErrorE2ETestCase( + description="unknown selected target fails like other commands", + janitor_command=("--no-color", "janitor", "--target", "staging"), + expected_exit_code=1, + expected_output_fragments=("Unknown target 'staging'",), + ), + JanitorTargetErrorE2ETestCase( + description="as with auto approve is a usage error", + janitor_command=("--no-color", "janitor", "--as", "prod", "--auto-approve"), + expected_exit_code=2, + expected_output_fragments=( + "janitor --as is inspection only and cannot be combined with --auto-approve", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_target_flags_when_running_janitor_then_fails_without_changes( + test_case: JanitorTargetErrorE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_two_target_janitor_project( + tmp_path=tmp_path, project_name="janitor_target_errors", model_names=("orders",) + ) + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--target", "prod"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + db_path: Path = project_dir / "janitor.duckdb" + execute_duckdb( + db_path=db_path, + sql=f'CREATE TABLE prod."{EXPIRED_ARCHIVE_NAME}" AS SELECT 1 AS product_id', + ) + before: tuple[object, ...] = snapshot_schema_contents(db_path=db_path, schemas=("prod",)) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + for fragment in test_case.expected_output_fragments: + assert fragment in result.stdout + result.stderr + assert snapshot_schema_contents(db_path=db_path, schemas=("prod",)) == before + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 0c0868e469..70df8b633b 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -17,11 +17,34 @@ sqb --project-dir janitor [flags] | Flag | Description | |------|-------------| +| `--target ` | Clean up a configured target instead of the active/default target, using that target's connection, schemas and state | +| `--as ` | Preview the janitor plan for another configured target's schemas through the active target's connection. Inspection only: nothing is archived, dropped or pruned, and there is no prompt. Cannot be combined with `--auto-approve` | | `--auto-approve` | Skip the confirmation prompt and run the planned actions immediately | | `--retention-days` | Override the configured retention period (days) before a stale relation is archived | | `--direct-state-history-versions` | Override how many state-history versions are kept per identity | | `--drop-old-name-view ` | Drop the [compatibility view](/docs/concepts/models/migrations#old-names) at a renamed model's old name before it expires. Repeat for several views. Fails with `C503` if no compatibility view is recorded at the name | +## Targets + +By default the janitor cleans up the active target. `--target ` selects another configured target, exactly like `sqb build --target`: the janitor connects with that target's connection and inspects and cleans that target's schemas and state tables. Other targets are never touched. + +`--as ` previews what the janitor would do in another target without changing anything, like [`sqb plan --as`](/docs/cli/plan). The project compiles for that target, and its schemas are inspected through the active target's connection, which can be the default target or the one given with `--target`. The janitor lists every relation it would archive, every archive it would delete, every compatibility view it would drop and every state table it would prune, then exits. It drops nothing, writes no audit or state rows, prunes nothing and does not prompt. `--drop-old-name-view` is allowed with `--as` and only previews the drop. + +``` +Previewing janitor as target 'prod' through the connection of target 'dev' (inspection only). +... +Relations to archive + prod.customers -> prod._sqb_archive__20261001t101500z__customers age unknown, delete after 2026-10-15 10:15:00 UTC + +Archives to delete + prod._sqb_archive__20200101t000000z__old_products archived 2020-01-01 00:00:00 UTC, age 2465d, expired 2020-01-15 00:00:00 UTC + +Previewed janitor as target 'prod' through the connection of target 'dev'. Nothing was changed. +Rerun with `--target prod` instead of `--as prod` to apply it. +``` + +An unknown target name fails before anything connects. A preview exits with `0`, or with `1` when a schema is blocked, as a normal run would. + ## Lifecycle The janitor never drops a live relation directly. Each stale relation goes through two stages: @@ -118,6 +141,12 @@ sqb janitor --auto-approve # Override retention to 7 days sqb janitor --retention-days 7 + +# Preview the prod cleanup through the current connection, changing nothing +sqb janitor --as prod + +# Clean up the prod target +sqb janitor --target prod ``` ## State history pruning