From 5fd8d5ce773ba39c4447c1839f62b4e7978a6f07 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:34:41 +0100 Subject: [PATCH 1/8] feat(janitor): archive stale relations before deletion in direct mode Direct-mode janitor now renames eligible stale relations in place to _SQB_ARCHIVE____ after retention_days, and drops relations in managed schemas whose names strictly match the archive grammar once their embedded timestamp is older than archive_retention_days. Malformed look-alike names are reported and never deleted. Deletion decisions use only the name grammar, the embedded timestamp, and managed schema scope. Each archive and delete appends a deterministic, idempotent row to the new _sqlbuild_janitor_events audit table, which is never read for decisions. retention_days now defaults to 14 and the new archive_retention_days defaults to 14; 0/0 archives and deletes in one run. Identifier fitting moves to a shared adapter relation helper, reused by scenario artifacts and archive names. Virtual-environment janitor behaviour is unchanged. --- .../adapter/contract/classes/base_adapter.py | 18 + .../contract/classes/duckdb_backed_adapter.py | 16 + .../contract/classes/strict_adapter.py | 10 + .../relations/_helpers/identifier_fitting.py | 35 ++ src/sqlbuild/adapter/relations/constants.py | 1 + .../main/fit_artifact_logical_name.py | 18 + .../adapter/state_sql/_helpers/table_ddl.py | 32 ++ .../main/render_state_table_create_sql.py | 28 ++ .../bigquery/classes/bigquery_adapter.py | 16 + .../databricks/classes/databricks_adapter.py | 15 + .../postgres/classes/postgres_adapter.py | 16 + .../snowflake/classes/snowflake_adapter.py | 16 + .../sqlserver/classes/sqlserver_adapter.py | 92 ++-- .../_helpers/janitor_output/output.py | 161 ++++-- .../_helpers/janitor_output/outputs.py | 34 +- .../_helpers/janitor_runtime/invocation.py | 3 + .../_helpers/janitor_runtime/planning.py | 2 + src/sqlbuild/cli/commands/models.py | 1 + .../discovery/_helpers/yml/project.py | 8 +- .../_helpers/scenario/artifact_names.py | 31 +- src/sqlbuild/compiler/planner/constants.py | 1 - .../executor/audit_results/_helpers/sql.py | 27 +- .../janitor/_helpers/archive_execution.py | 222 +++++++++ .../janitor/_helpers/archive_names.py | 72 +++ .../janitor/_helpers/archive_planning.py | 187 +++++++ .../janitor/_helpers/classification.py | 10 +- .../janitor/_helpers/schema_planning.py | 4 + src/sqlbuild/executor/janitor/constants.py | 15 + src/sqlbuild/executor/janitor/main/execute.py | 23 + src/sqlbuild/executor/janitor/main/plan.py | 22 +- src/sqlbuild/executor/janitor/models.py | 51 ++ .../executor/janitor_events/__init__.py | 1 + .../janitor_events/_helpers/__init__.py | 1 + .../janitor_events/_helpers/identity.py | 29 ++ .../executor/janitor_events/_helpers/sql.py | 112 +++++ .../executor/janitor_events/constants.py | 57 +++ .../executor/janitor_events/main/__init__.py | 1 + .../janitor_events/main/create_table_sql.py | 27 + .../janitor_events/main/janitor_event_id.py | 25 + .../main/write_janitor_event.py | 35 ++ .../executor/janitor_events/models.py | 27 + src/sqlbuild/executor/janitor_events/types.py | 12 + src/sqlbuild/spec/contracts/models.py | 3 +- .../cli/commands/main/janitor/_test_types.py | 66 +++ .../cli/commands/main/janitor/helpers.py | 168 ++++++- .../cli/commands/main/janitor/test_janitor.py | 40 +- .../main/janitor/test_janitor_archive.py | 468 ++++++++++++++++++ .../executor/janitor_events/__init__.py | 1 + .../executor/janitor_events/main/__init__.py | 1 + .../janitor_events/main/_test_types.py | 11 + .../main/test_write_janitor_event.py | 103 ++++ .../fit_artifact_logical_name/_test_types.py | 21 + .../test_fit_artifact_logical_name.py | 83 ++++ .../cli/commands/main/janitor/_test_types.py | 8 + .../cli/commands/main/janitor/helpers.py | 54 +- .../cli/commands/main/janitor/test_janitor.py | 86 +++- .../discovery/_helpers/_test_types.py | 1 + .../discovery/_helpers/test_yml_project.py | 22 +- .../executor/janitor/_helpers/__init__.py | 1 + .../executor/janitor/_helpers/_test_types.py | 30 ++ .../janitor/_helpers/test_archive_names.py | 133 +++++ .../executor/janitor/main/_test_types.py | 40 ++ .../sqlbuild/executor/janitor/main/helpers.py | 48 +- .../executor/janitor/main/test_execute.py | 149 ++++++ .../executor/janitor/main/test_plan.py | 2 +- .../janitor/main/test_plan_archives.py | 225 +++++++++ 66 files changed, 3123 insertions(+), 155 deletions(-) create mode 100644 src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py create mode 100644 src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py create mode 100644 src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py create mode 100644 src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py create mode 100644 src/sqlbuild/executor/janitor/_helpers/archive_execution.py create mode 100644 src/sqlbuild/executor/janitor/_helpers/archive_names.py create mode 100644 src/sqlbuild/executor/janitor/_helpers/archive_planning.py create mode 100644 src/sqlbuild/executor/janitor_events/__init__.py create mode 100644 src/sqlbuild/executor/janitor_events/_helpers/__init__.py create mode 100644 src/sqlbuild/executor/janitor_events/_helpers/identity.py create mode 100644 src/sqlbuild/executor/janitor_events/_helpers/sql.py create mode 100644 src/sqlbuild/executor/janitor_events/constants.py create mode 100644 src/sqlbuild/executor/janitor_events/main/__init__.py create mode 100644 src/sqlbuild/executor/janitor_events/main/create_table_sql.py create mode 100644 src/sqlbuild/executor/janitor_events/main/janitor_event_id.py create mode 100644 src/sqlbuild/executor/janitor_events/main/write_janitor_event.py create mode 100644 src/sqlbuild/executor/janitor_events/models.py create mode 100644 src/sqlbuild/executor/janitor_events/types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py create mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/__init__.py create mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py create mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py create mode 100644 tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py create mode 100644 tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/_test_types.py create mode 100644 tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/_helpers/__init__.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 174f4b76a0..a61568d86e 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -626,6 +626,9 @@ def render_drop_view(self, *, destination: str, if_exists: bool = True) -> tuple def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER TABLE {origin} RENAME TO {destination}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER VIEW {origin} RENAME TO {destination}",) + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: staging: str = self._with_replaced_relation_name( relation=left, name=f"{self._relation_name(left)}__swap_staging" @@ -2342,6 +2345,21 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the janitor audit event table when it is missing.""" + + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 31f77c7009..61e55dd28e 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1029,6 +1029,18 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1626,6 +1638,10 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: unqualified_destination: str = destination.rsplit(".", 1)[-1] return (f"ALTER TABLE {origin} RENAME TO {unqualified_destination}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + unqualified_destination: str = destination.rsplit(".", 1)[-1] + return (f"ALTER VIEW {origin} RENAME TO {unqualified_destination}",) + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: staging: str = self._with_replaced_relation_name( relation=left, name=f"{self._relation_name(left)}__swap_staging" diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index e76502d06d..c436c727c0 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -312,6 +312,11 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: """Render SQL statements that rename a relation.""" ... + @abstractmethod + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + """Render SQL statements that rename a view relation.""" + ... + @abstractmethod def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: """Render SQL statements that swap two relations.""" @@ -875,6 +880,11 @@ def render_create_audit_result_index_sqls( """Render optional audit result table index DDL statements.""" ... + @abstractmethod + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the janitor audit event table when it is missing.""" + ... + @abstractmethod def render_prune_fingerprint_history_sql( self, diff --git a/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py b/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py new file mode 100644 index 0000000000..f920eac27c --- /dev/null +++ b/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py @@ -0,0 +1,35 @@ +"""Identifier-limit fitting implementations for generated relation names.""" + +from __future__ import annotations + +import hashlib + +from sqlbuild.adapter.relations.constants import SHORTENED_LOGICAL_NAME_HASH_LENGTH +from sqlbuild.errors.contracts.exceptions import SharedInputError + + +def fit_artifact_logical_name_impl( + *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str +) -> str: + """Fit a readable logical component with a deterministic hash suffix.""" + + max_logical_length: int = identifier_limit - len(fixed_prefix) + if max_logical_length < 1: + raise SharedInputError( + f"{artifact_label} prefix '{fixed_prefix}' does not fit within identifier " + f"limit {identifier_limit}" + ) + if len(logical_name) <= max_logical_length: + return logical_name + + suffix_length: int = SHORTENED_LOGICAL_NAME_HASH_LENGTH + 1 + if max_logical_length <= suffix_length: + raise SharedInputError( + f"{artifact_label} name for '{logical_name}' cannot fit within identifier " + f"limit {identifier_limit}" + ) + logical_hash: str = hashlib.sha256(logical_name.encode("utf-8")).hexdigest()[ + :SHORTENED_LOGICAL_NAME_HASH_LENGTH + ] + prefix_length: int = max_logical_length - suffix_length + return f"{logical_name[:prefix_length]}_{logical_hash}" diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 19cede8486..e55b715814 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -1,3 +1,4 @@ """Shared relation metadata limits.""" METADATA_NAME_FILTER_LIMIT: int = 250 +SHORTENED_LOGICAL_NAME_HASH_LENGTH: int = 8 diff --git a/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py b/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py new file mode 100644 index 0000000000..2c601d9173 --- /dev/null +++ b/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py @@ -0,0 +1,18 @@ +"""Generated relation name fitting entrypoint.""" + +from __future__ import annotations + +from sqlbuild.adapter.relations._helpers.identifier_fitting import fit_artifact_logical_name_impl + + +def fit_artifact_logical_name( + *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str +) -> str: + """Fit a readable logical component after a fixed prefix within an identifier limit.""" + + return fit_artifact_logical_name_impl( + logical_name=logical_name, + fixed_prefix=fixed_prefix, + identifier_limit=identifier_limit, + artifact_label=artifact_label, + ) diff --git a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py new file mode 100644 index 0000000000..bbf7c9b285 --- /dev/null +++ b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py @@ -0,0 +1,32 @@ +"""Portable append-only state table DDL implementations.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.sql_values.types import StateSqlValueType + + +def render_state_table_create_sql_impl( + *, + qualified_name: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + required_columns: frozenset[str], + render_framework_type: Callable[[FrameworkType], str], + transient: bool, +) -> str: + """Render create-if-missing DDL for string, integer, and text-timestamp state columns.""" + + rendered_types: dict[StateSqlValueType, str] = { + StateSqlValueType.STRING: render_framework_type(FrameworkType.STRING), + StateSqlValueType.INTEGER: render_framework_type(FrameworkType.INTEGER), + StateSqlValueType.TEXT_TIMESTAMP: render_framework_type(FrameworkType.TIMESTAMP), + } + definitions: list[str] = [] + for column in columns: + required: str = " NOT NULL" if column in required_columns else "" + definitions.append(f"{column} {rendered_types[column_types[column]]}{required}") + table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" + return f"CREATE {table_kind} IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" diff --git a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py new file mode 100644 index 0000000000..7d0a48d69a --- /dev/null +++ b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py @@ -0,0 +1,28 @@ +"""Public append-only state table DDL rendering operation.""" + +from collections.abc import Callable, Mapping + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql._helpers.table_ddl import render_state_table_create_sql_impl +from sqlbuild.sql_values.types import StateSqlValueType + + +def render_state_table_create_sql( + *, + qualified_name: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + required_columns: frozenset[str], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + """Render create-if-missing DDL for one append-only state table.""" + + return render_state_table_create_sql_impl( + qualified_name=qualified_name, + columns=columns, + column_types=column_types, + required_columns=required_columns, + render_framework_type=render_framework_type, + transient=transient, + ) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index d37debc4bd..475e02879b 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -282,6 +282,18 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1825,6 +1837,10 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: destination_name: str = self._strip_identifier_quotes(destination).split(".")[-1] return (f"ALTER TABLE {self._quote_identifier_path(origin)} RENAME TO {destination_name}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + del origin, destination + raise AdapterUserError(message="BigQuery does not support renaming views") + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: raise AdapterUserError(message="BigQuery does not support atomic table swap") diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 743883bbc2..3c84ec3f26 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -376,6 +376,18 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1833,6 +1845,9 @@ def render_delete_insert_cursor( def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER TABLE {origin} RENAME TO {destination}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER VIEW {origin} RENAME TO {destination}",) + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: raise AdapterUserError(message="Databricks does not support atomic table swap") diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 6b123118f8..9ef81a54fa 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1582,6 +1582,18 @@ def render_create_audit_result_index_sqls( f"ON {table_name} (run_id, invocation_id, result_id)", ) + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_read_latest_source_freshness_sql( self, *, @@ -1881,6 +1893,10 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: destination_name: str = destination.split(".")[-1] return (f"ALTER TABLE {origin} RENAME TO {destination_name}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + destination_name: str = destination.split(".")[-1] + return (f"ALTER VIEW {origin} RENAME TO {destination_name}",) + def load_seed( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index cbf3550519..c99c7ec9ea 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -458,6 +458,19 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -2340,6 +2353,9 @@ def render_drop(self, *, destination: str, if_exists: bool = True) -> tuple[str, def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER TABLE {origin} RENAME TO {destination}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER VIEW {origin} RENAME TO {destination}",) + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: return (f"ALTER TABLE {left} SWAP WITH {right}",) diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 4e60c013e1..8fbffec680 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -504,16 +504,12 @@ def render_create_fingerprint_table_sql( f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL" f")" ) - escaped_schema: str = schema.replace("'", "''") - escaped_table: str = FINGERPRINT_TABLE_NAME.replace("'", "''") - exists_sql: str = ( - "SELECT 1 FROM information_schema.tables " - f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=FINGERPRINT_TABLE_NAME, ) - if database is not None: - escaped_database: str = database.replace("'", "''") - exists_sql += f" AND table_catalog = '{escaped_database}'" - return f"IF NOT EXISTS ({exists_sql}) {create_sql}" def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: from sqlbuild.microbatches.constants import MICROBATCH_TABLE_NAME @@ -525,16 +521,12 @@ def render_create_microbatch_state_table_sql(self, *, database: str | None, sche render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, ).replace("CREATE TABLE IF NOT EXISTS ", "CREATE TABLE ", 1) - escaped_schema: str = schema.replace("'", "''") - escaped_table: str = MICROBATCH_TABLE_NAME.replace("'", "''") - exists_sql: str = ( - "SELECT 1 FROM information_schema.tables " - f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=MICROBATCH_TABLE_NAME, ) - if database is not None: - escaped_database: str = database.replace("'", "''") - exists_sql += f" AND table_catalog = '{escaped_database}'" - return f"IF NOT EXISTS ({exists_sql}) {create_sql}" def render_create_fingerprint_index_sqls( self, @@ -676,16 +668,12 @@ def render_create_node_result_table_sql( "status", ): create_sql = create_sql.replace(f"{column} NVARCHAR(MAX)", f"{column} NVARCHAR(450)") - escaped_schema: str = schema.replace("'", "''") - escaped_table: str = NODE_RESULTS_TABLE_NAME.replace("'", "''") - exists_sql: str = ( - "SELECT 1 FROM information_schema.tables " - f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=NODE_RESULTS_TABLE_NAME, ) - if database is not None: - escaped_database: str = database.replace("'", "''") - exists_sql += f" AND table_catalog = '{escaped_database}'" - return f"IF NOT EXISTS ({exists_sql}) {create_sql}" def render_create_node_result_index_sqls( self, @@ -752,16 +740,12 @@ def render_create_audit_result_table_sql(self, *, database: str | None, schema: "outcome", ): create_sql = create_sql.replace(f"{column} NVARCHAR(MAX)", f"{column} NVARCHAR(450)") - escaped_schema: str = schema.replace("'", "''") - escaped_table: str = AUDIT_RESULTS_TABLE_NAME.replace("'", "''") - exists_sql: str = ( - "SELECT 1 FROM information_schema.tables " - f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=AUDIT_RESULTS_TABLE_NAME, ) - if database is not None: - escaped_database: str = database.replace("'", "''") - exists_sql += f" AND table_catalog = '{escaped_database}'" - return f"IF NOT EXISTS ({exists_sql}) {create_sql}" def render_create_audit_result_index_sqls( self, *, database: str | None, schema: str @@ -792,6 +776,25 @@ def render_create_audit_result_index_sqls( "(run_id, invocation_id, result_id)", ) + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + create_sql: str = build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ).replace("CREATE TABLE IF NOT EXISTS", "CREATE TABLE", 1) + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=JANITOR_EVENTS_TABLE_NAME, + ) + def render_read_latest_source_freshness_sql( self, *, @@ -892,6 +895,23 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: destination_name: str = self._unquote_relation_part(destination.split(".")[-1]) return (f"EXEC sp_rename '{origin_name}', '{destination_name}'",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return self.render_rename(origin=origin, destination=destination) + + def _create_table_if_missing_sql( + self, *, create_sql: str, database: str | None, schema: str, table_name: str + ) -> str: + escaped_schema: str = schema.replace("'", "''") + escaped_table: str = table_name.replace("'", "''") + exists_sql: str = ( + "SELECT 1 FROM information_schema.tables " + f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + ) + if database is not None: + escaped_database: str = database.replace("'", "''") + exists_sql += f" AND table_catalog = '{escaped_database}'" + return f"IF NOT EXISTS ({exists_sql}) {create_sql}" + def render_add_columns( self, *, destination: str, columns: tuple[ColumnInfo, ...] ) -> tuple[str, ...]: diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py index 9f479ef0b7..dc6b47f3f4 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py @@ -2,9 +2,12 @@ from __future__ import annotations +from datetime import datetime from typing import TextIO from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, JanitorCheckpointCandidate, JanitorDeleteCandidate, JanitorDetachedVirtualEnvironmentCandidate, @@ -32,7 +35,8 @@ def write_disabled(*, stream: TextIO, use_color: bool = False) -> None: f"Add this block to {style.object_name('sqlbuild_project.toml')}:\n\n" "janitor:\n" " enabled: true\n" - " retention_days: 30\n\n" + " retention_days: 14\n" + " archive_retention_days: 14\n\n" "After enabling, run janitor again to preview cleanup:\n" f" {style.accent('sqb janitor')}\n" ) @@ -46,14 +50,34 @@ def _write_plan_summary(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) - stream.write(f" {'retention':<22} {style.accent(f'{plan.retention_days} days')}\n") if not plan.age_metadata_supported: stream.write(f" {'age metadata':<22} {style.accent('unavailable')}\n") + if plan.direct_mode: + archive_retention: str = f"{plan.archive_retention_days} days" + stream.write(f" {'archive retention':<22} {style.accent(archive_retention)}\n") stream.write(f" {'schemas scanned':<22} {style.accent(str(plan.scanned_schema_count))}\n") stream.write(f" {'schemas skipped':<22} {style.accent(str(len(plan.skipped_schemas)))}\n") - _write_count_row( - stream=stream, - style=style, - label="reported objects" if plan.direct_mode else "eligible for deletion", - items=plan.candidates, - ) + if plan.direct_mode: + _write_count_row( + stream=stream, + style=style, + label="relations to archive", + items=plan.archive_candidates, + ) + _write_count_row( + stream=stream, + style=style, + label="archives to delete", + items=plan.archive_deletion_candidates, + ) + stream.write( + f" {'archives retained':<22} {style.accent(str(len(plan.retained_archives)))}\n" + ) + else: + _write_count_row( + stream=stream, + style=style, + label="eligible for deletion", + items=plan.candidates, + ) _write_count_row( stream=stream, style=style, @@ -123,6 +147,77 @@ def _write_query_artifact_candidates(*, plan: JanitorPlan, stream: TextIO, style stream.write(f" {style.object_name(query_artifact.key.display_name())}\n") +def _write_archive_sections(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: + if plan.archive_candidates: + stream.write(f"\n{style.success('Relations to archive')}\n") + archive_candidate: JanitorArchiveCandidate + for archive_candidate in plan.archive_candidates: + expiry: str = ( + "deleted in this run" + if archive_candidate.expires_at <= archive_candidate.archived_at + else f"delete after {_utc(archive_candidate.expires_at)}" + ) + detail: str = f"{_age(value=archive_candidate.age_timestamp, plan=plan)}, {expiry}" + stream.write( + f" {style.object_name(archive_candidate.key.display_name())} -> " + f"{style.object_name(archive_candidate.archive_key.display_name())} " + f"{style.muted(detail)}\n" + ) + if plan.archive_deletion_candidates: + stream.write(f"\n{style.success('Archives to delete')}\n") + deletion: JanitorArchivedRelation + for deletion in plan.archive_deletion_candidates: + detail = ( + "archived in this run" + if deletion.original_key is not None + else f"archived {_utc(deletion.archived_at)}, " + f"{_age(value=deletion.archived_at, plan=plan)}, " + f"expired {_utc(deletion.expires_at)}" + ) + stream.write( + f" {style.object_name(deletion.key.display_name())} {style.muted(detail)}\n" + ) + if plan.retained_archives: + stream.write(f"\n{style.success('Retained archives')}\n") + retained: JanitorArchivedRelation + for retained in plan.retained_archives: + detail = f"archived {_utc(retained.archived_at)}, expires {_utc(retained.expires_at)}" + stream.write( + f" {style.object_name(retained.key.display_name())} {style.muted(detail)}\n" + ) + + +def _utc(value: datetime) -> str: + return f"{value:%Y-%m-%d %H:%M:%S} UTC" + + +def _age(*, value: datetime | None, plan: JanitorPlan) -> str: + if value is None or plan.planned_at is None: + return "age unknown" + return f"age {max((plan.planned_at - value).days, 0)}d" + + +def _write_blocked_schemas(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: + if not plan.blocked_schemas: + return + stream.write(f"\n{style.error_strong('Janitor blocked')}\n") + stream.write(f" {style.error('Managed target schemas contain active configured sources.')}\n") + suppressed_action: str = "archive" if plan.direct_mode else "deletion" + for blocked_schema in plan.blocked_schemas: + sources: str = ", ".join(blocked_schema.source_names) + stream.write( + f" {style.object_name(blocked_schema.display_name())} " + f"{style.error_muted('active sources: ' + sources)}\n" + ) + for candidate in blocked_schema.suppressed_candidates: + suppressed: str = f"suppressed {suppressed_action}: {candidate.key.display_name()}" + stream.write(f" {style.error_muted(suppressed)}\n") + for archived in blocked_schema.suppressed_archive_deletions: + suppressed = f"suppressed archive deletion: {archived.key.display_name()}" + stream.write(f" {style.error_muted(suppressed)}\n") + stream.write(f" {style.error('No janitor actions will be performed.')}\n") + + def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> None: """Write a janitor preview.""" @@ -132,21 +227,7 @@ def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> stream.write(f"{style.title('Janitor preview')} {rendered_env}\n\n") _write_plan_summary(plan=plan, stream=stream, style=style) - if plan.blocked_schemas: - stream.write(f"\n{style.error_strong('Janitor blocked')}\n") - stream.write( - f" {style.error('Managed target schemas contain active configured sources.')}\n" - ) - for blocked_schema in plan.blocked_schemas: - sources: str = ", ".join(blocked_schema.source_names) - stream.write( - f" {style.object_name(blocked_schema.display_name())} " - f"{style.error_muted('active sources: ' + sources)}\n" - ) - for candidate in blocked_schema.suppressed_candidates: - suppressed: str = f"suppressed deletion: {candidate.key.display_name()}" - stream.write(f" {style.error_muted(suppressed)}\n") - stream.write(f" {style.error('No janitor actions will be performed.')}\n") + _write_blocked_schemas(plan=plan, stream=stream, style=style) if plan.skipped_schemas: stream.write(f"\n{style.success('Skipped schemas')}\n") @@ -158,15 +239,13 @@ def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> f"{style.muted('contains active source ' + sources)}\n" ) - if plan.candidates: - heading: str = "Report-only objects" if plan.direct_mode else "Eligible objects" - stream.write(f"\n{style.success(heading)}\n") + if plan.candidates and not plan.direct_mode: + stream.write(f"\n{style.success('Eligible objects')}\n") candidate: JanitorDeleteCandidate for candidate in plan.candidates: - reason: str = " direct mode does not delete relations" if plan.direct_mode else "" - stream.write( - f" {style.object_name(candidate.key.display_name())}{style.muted(reason)}\n" - ) + stream.write(f" {style.object_name(candidate.key.display_name())}\n") + + _write_archive_sections(plan=plan, stream=stream, style=style) _write_query_artifact_candidates(plan=plan, stream=stream, style=style) @@ -255,16 +334,26 @@ def confirmation_text(plan: JanitorPlan) -> str: + len(plan.direct_state_prune_candidates) + len(plan.virtual_state_prune_candidates) ) + archive_prefix: str = ( + f"archive {len(plan.archive_candidates)} and " if plan.archive_candidates else "" + ) + physical_deletion_count: int = physical_janitor_deletion_count(plan) if state_candidate_count == 0: - deletion_count: int = (0 if plan.direct_mode else len(plan.candidates)) + len( - plan.query_diff_artifact_candidates + return ( + f"{archive_prefix}delete {physical_deletion_count} objects " + f"from {environment_label(plan)}" ) - return f"delete {deletion_count} objects from {environment_label(plan)}" - physical_deletion_count: int = (0 if plan.direct_mode else len(plan.candidates)) + len( - plan.query_diff_artifact_candidates - ) deletion_count: int = physical_deletion_count + state_candidate_count - return f"delete {deletion_count} items from {environment_label(plan)}" + return f"{archive_prefix}delete {deletion_count} items from {environment_label(plan)}" + + +def physical_janitor_deletion_count(plan: JanitorPlan) -> int: + """Count warehouse relations the janitor plan will drop.""" + + relation_count: int = ( + len(plan.archive_deletion_candidates) if plan.direct_mode else len(plan.candidates) + ) + return relation_count + len(plan.query_diff_artifact_candidates) def environment_label(plan: JanitorPlan) -> str: diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py index 8aaedb868b..af1b6480bf 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py @@ -7,6 +7,7 @@ from sqlbuild.cli.commands._helpers.janitor_output.output import ( confirmation_text, environment_label, + physical_janitor_deletion_count, write_disabled, write_plan, ) @@ -38,6 +39,8 @@ def janitor_plan_has_work(planning_result: JanitorPlanningResult) -> bool: plan: JanitorPlan = planning_result.plan return bool( (plan.candidates and not plan.direct_mode) + or plan.archive_candidates + or plan.archive_deletion_candidates or plan.query_diff_artifact_candidates or plan.checkpoint_candidates or plan.detached_virtual_environment_candidates @@ -65,18 +68,24 @@ def confirm_janitor_plan(*, planning_result: JanitorPlanningResult) -> bool: prune_count: int = len(plan.direct_state_prune_candidates) + len( plan.virtual_state_prune_candidates ) + archive_prefix: str = ( + f"archive {len(plan.archive_candidates)} and " if plan.archive_candidates else "" + ) if state_candidate_count or prune_count: - physical_deletion_count: int = (0 if plan.direct_mode else len(plan.candidates)) + len( - plan.query_diff_artifact_candidates - ) + physical_deletion_count: int = physical_janitor_deletion_count(plan) deletion_count: int = physical_deletion_count + state_candidate_count + prune_count sys.stdout.write( - f"Janitor will delete {deletion_count} items from {environment_label(plan)}.\n" + f"Janitor will {archive_prefix}delete {deletion_count} items " + f"from {environment_label(plan)}.\n" ) else: + object_count: int = ( + physical_janitor_deletion_count(plan) + if plan.direct_mode + else len(plan.candidates) + len(plan.query_diff_artifact_candidates) + ) sys.stdout.write( - "Janitor will delete " - f"{len(plan.candidates) + len(plan.query_diff_artifact_candidates)} objects " + f"Janitor will {archive_prefix}delete {object_count} objects " f"from {environment_label(plan)}.\n" ) if plan.retention_days == 0: @@ -84,6 +93,8 @@ def confirm_janitor_plan(*, planning_result: JanitorPlanningResult) -> bool: sys.stdout.write("Age metadata will not be checked.\n") else: sys.stdout.write(f"Retention: {plan.retention_days} days\n") + if plan.direct_mode: + sys.stdout.write(f"Archive retention: {plan.archive_retention_days} days\n") sys.stdout.write(f"\nType `{expected}` to continue: ") sys.stdout.flush() try: @@ -110,6 +121,11 @@ def write_janitor_completion( def _deleted_message(*, result: JanitorExecutionResult) -> str: + archived_prefix: str = f"Archived {len(result.archived)} relations. " if result.archived else "" + return archived_prefix + _deletion_summary(result=result) + + +def _deletion_summary(*, result: JanitorExecutionResult) -> str: deleted_state_count: int = ( len(result.deleted_checkpoints) + len(result.deleted_detached_virtual_environments) @@ -117,7 +133,11 @@ def _deleted_message(*, result: JanitorExecutionResult) -> str: + len(result.deleted_state_backups) + len(result.deleted_expired_locks) ) - deleted_object_count: int = len(result.deleted) + len(result.deleted_query_diff_artifacts) + deleted_object_count: int = ( + len(result.deleted) + + len(result.deleted_archives) + + len(result.deleted_query_diff_artifacts) + ) pruned_state_count: int = len(result.pruned_direct_state) + len(result.pruned_virtual_state) non_checkpoint_state_count: int = deleted_state_count - len(result.deleted_checkpoints) if non_checkpoint_state_count or pruned_state_count: diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py index 7f2bf8288d..1282a4b609 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py @@ -53,4 +53,7 @@ def resolve_janitor_settings( return JanitorSettings( retention_days=retention_days, direct_state_history_versions=direct_state_history_versions, + archive_retention_days=( + invocation.discovered_inputs.project_config.janitor.archive_retention_days + ), ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py index 4b0249cadb..fa126f10e7 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py @@ -191,6 +191,7 @@ def build_janitor_execution_plan( not invocation.discovered_inputs.project_config.settings.virtual_environments ), state_history_versions=settings.direct_state_history_versions, + archive_retention_days=settings.archive_retention_days, ), ) lifecycle.completed(metadata={"item_count": _janitor_candidate_count(plan)}) @@ -206,6 +207,7 @@ def _janitor_candidate_count(plan: JanitorPlan) -> int: len(candidates) for candidates in ( plan.candidates, + plan.archive_deletion_candidates, plan.query_diff_artifact_candidates, plan.checkpoint_candidates, plan.detached_virtual_environment_candidates, diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 98f8c3ec2d..5d8dabd540 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -784,6 +784,7 @@ class JanitorSettings: retention_days: int direct_state_history_versions: int + archive_retention_days: int = 14 @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index 06869d331a..ac7550b8a9 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -1427,7 +1427,10 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: payload=payload, label="janitor", file_path=file_path ) enabled: bool = _optional_bool(mapping=mapping, key="enabled", default=False) - retention_days: int = _optional_int(mapping=mapping, key="retention_days", default=30) + retention_days: int = _optional_int(mapping=mapping, key="retention_days", default=14) + archive_retention_days: int = _optional_int( + mapping=mapping, key="archive_retention_days", default=14 + ) max_checkpoints: int = _optional_int(mapping=mapping, key="max_checkpoints", default=20) direct_state_history_versions: int = _optional_int( mapping=mapping, @@ -1448,6 +1451,8 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: ) if retention_days < 0: raise ProjectConfigError(f"{file_path} janitor.retention_days must be >= 0") + if archive_retention_days < 0: + raise ProjectConfigError(f"{file_path} janitor.archive_retention_days must be >= 0") if max_checkpoints < 1: raise ProjectConfigError(f"{file_path} janitor.max_checkpoints must be >= 1") if direct_state_history_versions < 0: @@ -1455,6 +1460,7 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: return JanitorConfig( enabled=enabled, retention_days=retention_days, + archive_retention_days=archive_retention_days, max_checkpoints=max_checkpoints, direct_state_history_versions=direct_state_history_versions, delete_tracked_only=delete_tracked_only, diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py b/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py index e3ad2d2783..1de215dfe6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py @@ -2,17 +2,15 @@ from __future__ import annotations -import hashlib import re +from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name from sqlbuild.compiler.planner.constants import ( SCENARIO_ARTIFACT_KINDS, SCENARIO_ARTIFACT_PREFIX, SCENARIO_HASH_PREFIX_LENGTH, - SCENARIO_SHORTENED_LOGICAL_HASH_LENGTH, ) from sqlbuild.compiler.planner.models import ParsedScenarioArtifactName -from sqlbuild.errors.contracts.exceptions import SharedInputError _SCENARIO_ARTIFACT_NAME_RE: re.Pattern[str] = re.compile( rf"^{re.escape(SCENARIO_ARTIFACT_PREFIX)}" @@ -62,30 +60,3 @@ def fit_scenario_artifact_logical_name( identifier_limit=identifier_limit, artifact_label="Scenario artifact", ) - - -def fit_artifact_logical_name( - *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str -) -> str: - """Fit a readable logical component with a deterministic hash suffix.""" - - max_logical_length: int = identifier_limit - len(fixed_prefix) - if max_logical_length < 1: - raise SharedInputError( - f"{artifact_label} prefix '{fixed_prefix}' does not fit within identifier " - f"limit {identifier_limit}" - ) - if len(logical_name) <= max_logical_length: - return logical_name - - suffix_length: int = SCENARIO_SHORTENED_LOGICAL_HASH_LENGTH + 1 - if max_logical_length <= suffix_length: - raise SharedInputError( - f"{artifact_label} name for '{logical_name}' cannot fit within identifier " - f"limit {identifier_limit}" - ) - logical_hash: str = hashlib.sha256(logical_name.encode("utf-8")).hexdigest()[ - :SCENARIO_SHORTENED_LOGICAL_HASH_LENGTH - ] - prefix_length: int = max_logical_length - suffix_length - return f"{logical_name[:prefix_length]}_{logical_hash}" diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 5f059d15dd..dc7fa0bb6a 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -43,7 +43,6 @@ METADATA_NAME_FILTER_LIMIT: int = 250 SCENARIO_ARTIFACT_PREFIX: str = "__sqb_" SCENARIO_HASH_PREFIX_LENGTH: int = 12 -SCENARIO_SHORTENED_LOGICAL_HASH_LENGTH: int = 8 SCENARIO_ARTIFACT_KINDS: tuple[str, ...] = ("source", "ref", "seed", "dbt_ref", "model") SCENARIO_DEFAULT_IDENTIFIER_LIMIT: int = 63 SCENARIO_PLAN_INVALID_HASH_PREFIX: str = "S501" diff --git a/src/sqlbuild/executor/audit_results/_helpers/sql.py b/src/sqlbuild/executor/audit_results/_helpers/sql.py index 1aba90f906..6944e12702 100644 --- a/src/sqlbuild/executor/audit_results/_helpers/sql.py +++ b/src/sqlbuild/executor/audit_results/_helpers/sql.py @@ -5,6 +5,9 @@ from collections.abc import Callable, Sequence from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.audit_results.constants import ( AUDIT_RESULT_COLUMN_TYPES, @@ -52,7 +55,6 @@ ) from sqlbuild.executor.audit_results.models import AuditResultRecord from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal -from sqlbuild.sql_values.types import StateSqlValueType _REQUIRED_COLUMNS: frozenset[str] = frozenset( { @@ -95,21 +97,16 @@ def build_create_table_sql( render_framework_type: Callable[[FrameworkType], str], transient: bool = False, ) -> str: - qualified_name: str = build_qualified_table_name( - database=database, schema=schema, render_qualified_name=render_qualified_name + return render_state_table_create_sql( + qualified_name=build_qualified_table_name( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=AUDIT_RESULT_COLUMNS, + column_types=AUDIT_RESULT_COLUMN_TYPES, + required_columns=_REQUIRED_COLUMNS, + render_framework_type=render_framework_type, + transient=transient, ) - rendered_types: dict[StateSqlValueType, str] = { - StateSqlValueType.STRING: render_framework_type(FrameworkType.STRING), - StateSqlValueType.INTEGER: render_framework_type(FrameworkType.INTEGER), - StateSqlValueType.TEXT_TIMESTAMP: render_framework_type(FrameworkType.TIMESTAMP), - } - definitions: list[str] = [] - for column in AUDIT_RESULT_COLUMNS: - required: str = " NOT NULL" if column in _REQUIRED_COLUMNS else "" - column_type: str = rendered_types[AUDIT_RESULT_COLUMN_TYPES[column]] - definitions.append(f"{column} {column_type}{required}") - table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" - return f"CREATE {table_kind} IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" def build_insert_sql( diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py new file mode 100644 index 0000000000..14b48201e4 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py @@ -0,0 +1,222 @@ +"""Direct-mode janitor archive renames, archive drops, and audit events.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import RelationType +from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( + resolve_qualified_name_parts, +) +from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, + JanitorPlan, + JanitorRelationKey, +) +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_SCHEMA_VERSION +from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id +from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType +from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle +from sqlbuild.runtime.observability.main.current_execution_identity import ( + current_execution_identity, +) +from sqlbuild.runtime.observability.models import ExecutionIdentity + + +def janitor_run_id() -> str: + """Return the invocation identity used to group janitor audit events.""" + + identity: ExecutionIdentity | None = current_execution_identity() + return identity.invocation_id if identity is not None else uuid4().hex + + +def apply_direct_archives( + *, + plan: JanitorPlan, + adapter: BaseAdapter, + connection: Any, + recorder: StatementRecorder, + run_id: str, +) -> tuple[tuple[JanitorArchiveCandidate, ...], tuple[JanitorArchivedRelation, ...]]: + """Rename stale relations to archives, then drop expired archives by name.""" + + _create_event_tables(adapter=adapter, connection=connection, plan=plan) + archived: list[JanitorArchiveCandidate] = [] + candidate: JanitorArchiveCandidate + for candidate in plan.archive_candidates: + with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): + _rename_relation( + adapter=adapter, + connection=connection, + candidate=candidate, + recorder=recorder, + ) + archived.append(candidate) + _write_event( + adapter=adapter, + connection=connection, + record=_event_record( + adapter=adapter, + event_type=JanitorEventType.ARCHIVE, + run_id=run_id, + archive=JanitorArchivedRelation( + key=candidate.archive_key, + relation_type=candidate.relation.relation_type, + archived_at=candidate.archived_at, + expires_at=candidate.expires_at, + original_key=candidate.key, + ), + ), + ) + deleted: list[JanitorArchivedRelation] = [] + archive: JanitorArchivedRelation + for archive in plan.archive_deletion_candidates: + with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): + _drop_archive( + adapter=adapter, connection=connection, archive=archive, recorder=recorder + ) + deleted.append(archive) + _write_event( + adapter=adapter, + connection=connection, + record=_event_record( + adapter=adapter, + event_type=JanitorEventType.DELETE, + run_id=run_id, + archive=archive, + ), + ) + return tuple(archived), tuple(deleted) + + +def _required_schema(key: JanitorRelationKey) -> str: + if key.schema is None: + raise ExecutorInputError(f"janitor archive action requires a schema: {key.display_name()}") + return key.schema + + +def _is_view(relation_type: str | None) -> bool: + return relation_type is not None and normalize_relation_type(relation_type) == RelationType.VIEW + + +def _qualified(*, adapter: BaseAdapter, key: JanitorRelationKey) -> str: + return resolve_qualified_name_parts( + adapter=adapter, database=key.database, schema=key.schema, name=key.name + ) + + +def _rename_relation( + *, + adapter: BaseAdapter, + connection: Any, + candidate: JanitorArchiveCandidate, + recorder: StatementRecorder, +) -> None: + origin: str = _qualified(adapter=adapter, key=candidate.key) + destination: str = _qualified(adapter=adapter, key=candidate.archive_key) + if not _is_view(candidate.relation.relation_type): + adapter.rename( + connection=connection, + origin=origin, + destination=destination, + statement_recorder=recorder, + ) + return + statements: tuple[str, ...] = adapter.render_rename_view(origin=origin, destination=destination) + recorder.record_many(statements) + statement: str + for statement in statements: + _ = adapter.execute(connection=connection, sql=statement) + + +def _drop_archive( + *, + adapter: BaseAdapter, + connection: Any, + archive: JanitorArchivedRelation, + recorder: StatementRecorder, +) -> None: + destination: str = _qualified(adapter=adapter, key=archive.key) + if _is_view(archive.relation_type): + adapter.drop_view( + connection=connection, + destination=destination, + if_exists=True, + statement_recorder=recorder, + ) + return + adapter.drop( + connection=connection, + destination=destination, + if_exists=True, + statement_recorder=recorder, + ) + + +def _create_event_tables(*, adapter: BaseAdapter, connection: Any, plan: JanitorPlan) -> None: + keys: tuple[JanitorRelationKey, ...] = ( + *(candidate.key for candidate in plan.archive_candidates), + *(archive.key for archive in plan.archive_deletion_candidates), + ) + schemas: dict[tuple[str | None, str], None] = dict.fromkeys( + (key.database, _required_schema(key)) for key in keys + ) + database: str | None + schema: str + for database, schema in schemas: + _ = adapter.execute( + connection=connection, + sql=adapter.render_create_janitor_event_table_sql(database=database, schema=schema), + ) + + +def _event_record( + *, + adapter: BaseAdapter, + event_type: JanitorEventType, + run_id: str, + archive: JanitorArchivedRelation, +) -> JanitorEventRecord: + schema: str = _required_schema(archive.key) + original: JanitorRelationKey | None = archive.original_key + return JanitorEventRecord( + event_id=build_janitor_event_id( + event_type=event_type, + run_id=run_id, + relation_database=archive.key.database, + relation_schema=schema, + archive_name=archive.key.name, + ), + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=event_type, + occurred_at=datetime.now(UTC), + run_id=run_id, + relation_database=archive.key.database, + relation_schema=schema, + relation_type=archive.relation_type, + original_name=None if original is None else original.name, + original_qualified_name=( + None if original is None else _qualified(adapter=adapter, key=original) + ), + archive_name=archive.key.name, + archive_qualified_name=_qualified(adapter=adapter, key=archive.key), + archived_at=archive.archived_at, + ) + + +def _write_event(*, adapter: BaseAdapter, connection: Any, record: JanitorEventRecord) -> None: + _ = write_janitor_event_record( + connection=connection, + execute=adapter.execute, + record=record, + render_qualified_name=adapter.render_qualified_name, + ) diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_names.py b/src/sqlbuild/executor/janitor/_helpers/archive_names.py new file mode 100644 index 0000000000..45c9081b85 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/archive_names.py @@ -0,0 +1,72 @@ +"""Strict janitor archive relation name grammar.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name +from sqlbuild.executor.janitor.constants import ( + ARCHIVE_ARTIFACT_LABEL, + ARCHIVE_LOOKALIKE_PREFIX, + ARCHIVE_NAME_PREFIX, + ARCHIVE_NAME_SEPARATOR, + ARCHIVE_TIMESTAMP_FORMAT, +) +from sqlbuild.executor.janitor.models import JanitorParsedArchiveName + +_ARCHIVE_NAME_RE: re.Pattern[str] = re.compile( + rf"^{re.escape(ARCHIVE_NAME_PREFIX)}" + r"(?P[0-9]{8}T[0-9]{6}Z)" + rf"{re.escape(ARCHIVE_NAME_SEPARATOR)}" + r"(?P.+)$", + re.IGNORECASE, +) + + +def archive_timestamp(value: datetime) -> datetime: + """Normalize a timestamp to the UTC second resolution embedded in archive names.""" + + aware: datetime = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + return aware.replace(microsecond=0) + + +def build_archive_name(*, original_name: str, archived_at: datetime, identifier_limit: int) -> str: + """Build the fitted archive name, keeping the prefix and timestamp intact.""" + + fixed_prefix: str = ( + f"{ARCHIVE_NAME_PREFIX}" + f"{archive_timestamp(archived_at).strftime(ARCHIVE_TIMESTAMP_FORMAT)}" + f"{ARCHIVE_NAME_SEPARATOR}" + ) + logical_part: str = fit_artifact_logical_name( + logical_name=original_name, + fixed_prefix=fixed_prefix, + identifier_limit=identifier_limit, + artifact_label=ARCHIVE_ARTIFACT_LABEL, + ) + return f"{fixed_prefix}{logical_part}" + + +def parse_archive_name(name: str) -> JanitorParsedArchiveName | None: + """Parse a strict archive name, returning None for anything else.""" + + match: re.Match[str] | None = _ARCHIVE_NAME_RE.fullmatch(name) + if match is None: + return None + try: + archived_at: datetime = datetime.strptime( + match.group("timestamp").upper(), ARCHIVE_TIMESTAMP_FORMAT + ).replace(tzinfo=UTC) + except ValueError: + return None + return JanitorParsedArchiveName( + archived_at=archived_at, + logical_name=match.group("logical_name"), + ) + + +def is_archive_lookalike_name(name: str) -> bool: + """Return whether a relation name is reserved for janitor archive handling.""" + + return name.lower().startswith(ARCHIVE_LOOKALIKE_PREFIX) diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_planning.py b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py new file mode 100644 index 0000000000..42460f9ac2 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py @@ -0,0 +1,187 @@ +"""Direct-mode janitor archive and archive-expiry planning.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timedelta + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.executor.janitor._helpers.archive_names import ( + archive_timestamp, + build_archive_name, + is_archive_lookalike_name, + parse_archive_name, +) +from sqlbuild.executor.janitor._helpers.classification import matching_exclude_pattern +from sqlbuild.executor.janitor._helpers.plan import relation_key as build_relation_key +from sqlbuild.executor.janitor.constants import BUILT_IN_EXCLUDE_PATTERNS, MALFORMED_ARCHIVE_REASON +from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, + JanitorArchivePlanning, + JanitorBlockedSchema, + JanitorDeleteCandidate, + JanitorParsedArchiveName, + JanitorRelationKey, + JanitorSchemaClassification, + JanitorSkippedRelation, + JanitorWarehouseFacts, +) + + +def plan_janitor_archives( + *, + direct_mode: bool, + adapter: BaseAdapter, + schemas: JanitorSchemaClassification, + facts: JanitorWarehouseFacts, + managed_target_schemas: set[tuple[str | None, str | None]], + exclude_patterns: tuple[str, ...], + archive_retention_days: int, + now: datetime, +) -> JanitorArchivePlanning: + """Plan direct-mode archives; virtual mode never archives or expires archives.""" + + if not direct_mode: + return JanitorArchivePlanning( + archive_candidates=(), + archive_deletion_candidates=(), + retained_archives=(), + skipped_relations=(), + blocked_schemas=schemas.blocked_schemas, + ) + return _plan_direct_archives( + candidates=schemas.candidates, + facts=facts, + managed_target_schemas=managed_target_schemas, + blocked_schemas=schemas.blocked_schemas, + exclude_patterns=exclude_patterns, + archive_retention_days=archive_retention_days, + identifier_limit=adapter.maximum_identifier_length(), + now=now, + ) + + +def _plan_direct_archives( + *, + candidates: tuple[JanitorDeleteCandidate, ...], + facts: JanitorWarehouseFacts, + managed_target_schemas: set[tuple[str | None, str | None]], + blocked_schemas: tuple[JanitorBlockedSchema, ...], + exclude_patterns: tuple[str, ...], + archive_retention_days: int, + identifier_limit: int, + now: datetime, +) -> JanitorArchivePlanning: + """Plan archive renames and name-based expiry of existing archives.""" + + archived_at: datetime = archive_timestamp(now) + retention: timedelta = timedelta(days=archive_retention_days) + archive_candidates: tuple[JanitorArchiveCandidate, ...] = tuple( + _archive_candidate( + candidate=candidate, + archived_at=archived_at, + retention=retention, + identifier_limit=identifier_limit, + ) + for candidate in candidates + ) + blocked_schema_keys: frozenset[tuple[str | None, str | None]] = frozenset( + (blocked.database, blocked.schema) for blocked in blocked_schemas + ) + deletions: list[JanitorArchivedRelation] = [] + retained: list[JanitorArchivedRelation] = [] + skipped: list[JanitorSkippedRelation] = [] + suppressed_by_schema: dict[tuple[str | None, str | None], list[JanitorArchivedRelation]] = {} + effective_patterns: tuple[str, ...] = BUILT_IN_EXCLUDE_PATTERNS + exclude_patterns + schema_key: tuple[str | None, str | None] + for schema_key in sorted(managed_target_schemas, key=lambda key: (key[0] or "", key[1] or "")): + relation: RelationInfo + for relation in facts.relations_by_schema.get(schema_key, ()): + key: JanitorRelationKey = build_relation_key(relation) + if key in facts.desired_keys or not is_archive_lookalike_name(key.name): + continue + parsed: JanitorParsedArchiveName | None = parse_archive_name(key.name) + if parsed is None: + skipped.append( + JanitorSkippedRelation( + key=key, relation=relation, reason=MALFORMED_ARCHIVE_REASON + ) + ) + continue + exclude_pattern: str | None = matching_exclude_pattern( + key=key, patterns=effective_patterns + ) + if exclude_pattern is not None: + skipped.append( + JanitorSkippedRelation( + key=key, + relation=relation, + reason=f"relation matches exclude pattern {exclude_pattern!r}", + ) + ) + continue + archived: JanitorArchivedRelation = JanitorArchivedRelation( + key=key, + relation_type=relation.relation_type, + archived_at=parsed.archived_at, + expires_at=parsed.archived_at + retention, + ) + if archived.expires_at > now: + retained.append(archived) + elif schema_key in blocked_schema_keys: + suppressed_by_schema.setdefault(schema_key, []).append(archived) + else: + deletions.append(archived) + same_run_deletions: tuple[JanitorArchivedRelation, ...] = tuple( + JanitorArchivedRelation( + key=candidate.archive_key, + relation_type=candidate.relation.relation_type, + archived_at=candidate.archived_at, + expires_at=candidate.expires_at, + original_key=candidate.key, + ) + for candidate in archive_candidates + if candidate.expires_at <= archived_at + ) + return JanitorArchivePlanning( + archive_candidates=archive_candidates, + archive_deletion_candidates=(*deletions, *same_run_deletions), + retained_archives=tuple(retained), + skipped_relations=tuple(skipped), + blocked_schemas=tuple( + replace( + blocked, + suppressed_archive_deletions=tuple( + suppressed_by_schema.get((blocked.database, blocked.schema), ()) + ), + ) + for blocked in blocked_schemas + ), + ) + + +def _archive_candidate( + *, + candidate: JanitorDeleteCandidate, + archived_at: datetime, + retention: timedelta, + identifier_limit: int, +) -> JanitorArchiveCandidate: + return JanitorArchiveCandidate( + key=candidate.key, + relation=candidate.relation, + age_timestamp=candidate.age_timestamp, + archive_key=JanitorRelationKey( + database=candidate.key.database, + schema=candidate.key.schema, + name=build_archive_name( + original_name=candidate.key.name, + archived_at=archived_at, + identifier_limit=identifier_limit, + ), + ), + archived_at=archived_at, + expires_at=archived_at + retention, + ) diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index 1675314741..1a1cd850c3 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -17,6 +17,7 @@ from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle from sqlbuild.executor.diff.models import QueryDiffArtifactInspection +from sqlbuild.executor.janitor._helpers.archive_names import is_archive_lookalike_name from sqlbuild.executor.janitor._helpers.plan import ( collect_desired_keys, collect_source_schemas, @@ -169,6 +170,7 @@ def classify_janitor_relations( retention_days: int, age_supported: bool, now: datetime, + direct_mode: bool = False, ) -> JanitorRelationClassification: """Split one schema's relations into delete candidates and skipped relations.""" @@ -179,6 +181,8 @@ def classify_janitor_relations( relation_key: JanitorRelationKey = build_relation_key(relation) if relation_key in facts.desired_keys: continue + if direct_mode and is_archive_lookalike_name(relation_key.name): + continue skip_reason: str | None = _relation_skip_reason( relation_key=relation_key, facts=facts, @@ -238,7 +242,7 @@ def _relation_skip_reason( relation_key, "relation is referenced by a retained virtual checkpoint", ) - exclude_pattern: str | None = _matching_exclude_pattern( + exclude_pattern: str | None = matching_exclude_pattern( key=relation_key, patterns=effective_exclude_patterns, ) @@ -269,11 +273,13 @@ def _retention_skip_reason( return None -def _matching_exclude_pattern( +def matching_exclude_pattern( *, key: JanitorRelationKey, patterns: tuple[str, ...], ) -> str | None: + """Return the first exclude pattern matching a relation name or qualified name.""" + display_name: str = key.display_name() pattern: str for pattern in patterns: diff --git a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py index ba5ef885e0..c18f80688f 100644 --- a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py +++ b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py @@ -58,6 +58,7 @@ def classify_target_schemas( retention_days=retention_days, age_supported=age_supported, now=now, + direct_mode=direct_mode, ) blocked_schemas.append( JanitorBlockedSchema( @@ -86,6 +87,7 @@ def classify_target_schemas( retention_days=retention_days, age_supported=age_supported, now=now, + direct_mode=direct_mode, ) candidates.extend(classification.candidates) skipped_relations.extend(classification.skipped_relations) @@ -107,6 +109,7 @@ def _classify_schema( retention_days: int, age_supported: bool, now: datetime, + direct_mode: bool, ) -> JanitorRelationClassification: return classify_janitor_relations( schema_relations=schema_relations, @@ -118,4 +121,5 @@ def _classify_schema( retention_days=retention_days, age_supported=age_supported, now=now, + direct_mode=direct_mode, ) diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 520004962a..517ee7461a 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -4,10 +4,25 @@ from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME +from sqlbuild.executor.audit_results.constants import AUDIT_RESULTS_TABLE_NAME +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME +from sqlbuild.executor.node_results.constants import NODE_RESULTS_TABLE_NAME from sqlbuild.microbatches.constants import MICROBATCH_TABLE_NAME BUILT_IN_EXCLUDE_PATTERNS: tuple[str, ...] = ( FINGERPRINT_TABLE_NAME, MICROBATCH_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, + AUDIT_RESULTS_TABLE_NAME, + JANITOR_EVENTS_TABLE_NAME, +) + +ARCHIVE_NAME_PREFIX: str = "_SQB_ARCHIVE__" +ARCHIVE_NAME_SEPARATOR: str = "__" +ARCHIVE_TIMESTAMP_FORMAT: str = "%Y%m%dT%H%M%SZ" +ARCHIVE_LOOKALIKE_PREFIX: str = "_sqb_archive" +ARCHIVE_ARTIFACT_LABEL: str = "Janitor archive" +MALFORMED_ARCHIVE_REASON: str = ( + "name resembles a janitor archive but does not match the strict archive grammar" ) diff --git a/src/sqlbuild/executor/janitor/main/execute.py b/src/sqlbuild/executor/janitor/main/execute.py index e603f5094a..58ad35c963 100644 --- a/src/sqlbuild/executor/janitor/main/execute.py +++ b/src/sqlbuild/executor/janitor/main/execute.py @@ -7,8 +7,14 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.executor.janitor._helpers.archive_execution import ( + apply_direct_archives, + janitor_run_id, +) from sqlbuild.executor.janitor._helpers.deletion import apply_janitor_deletions from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, JanitorCheckpointCandidate, JanitorDeleteCandidate, JanitorDetachedVirtualEnvironmentCandidate, @@ -87,6 +93,19 @@ def _execute_janitor_plan( if_exists=True, statement_recorder=recorder, ) + archived: tuple[JanitorArchiveCandidate, ...] + deleted_archives: tuple[JanitorArchivedRelation, ...] + archived, deleted_archives = ( + apply_direct_archives( + plan=plan, + adapter=adapter, + connection=connection, + recorder=recorder, + run_id=janitor_run_id(), + ) + if plan.direct_mode + else ((), ()) + ) query_artifact_candidate: JanitorQueryDiffArtifactCandidate for query_artifact_candidate in plan.query_diff_artifact_candidates: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): @@ -137,6 +156,8 @@ def _execute_janitor_plan( ) return JanitorExecutionResult( deleted=() if plan.direct_mode else plan.candidates, + archived=archived, + deleted_archives=deleted_archives, deleted_query_diff_artifacts=plan.query_diff_artifact_candidates, deleted_checkpoints=deleted_checkpoints, deleted_detached_virtual_environments=deleted_detached_virtual_environments, @@ -151,6 +172,8 @@ def _execute_janitor_plan( def _janitor_action_count(plan: JanitorPlan) -> int: relation_count: int = 0 if plan.direct_mode else len(plan.candidates) relation_count += len(plan.query_diff_artifact_candidates) + if plan.direct_mode: + relation_count += len(plan.archive_candidates) + len(plan.archive_deletion_candidates) return relation_count + sum( len(candidates) for candidates in ( diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py index ae7f92df3e..f5354b014a 100644 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ b/src/sqlbuild/executor/janitor/main/plan.py @@ -8,6 +8,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle +from sqlbuild.executor.janitor._helpers.archive_planning import plan_janitor_archives from sqlbuild.executor.janitor._helpers.classification import ( collect_direct_state_prune_candidates, collect_query_diff_artifact_candidates, @@ -16,6 +17,7 @@ from sqlbuild.executor.janitor._helpers.plan import collect_target_schemas from sqlbuild.executor.janitor._helpers.schema_planning import classify_target_schemas from sqlbuild.executor.janitor.models import ( + JanitorArchivePlanning, JanitorDirectModeSettings, JanitorDirectStatePruneCandidate, JanitorPlan, @@ -74,6 +76,7 @@ def build_janitor_plan( target_name=project.effective_target_name, retention_days=retention_days, direct_mode=direct.enabled, + archive_retention_days=direct.archive_retention_days, query_diff_artifact_candidates=query_diff_artifact_candidates, checkpoint_candidates=state.checkpoint_candidates, detached_virtual_environment_candidates=(state.detached_virtual_environment_candidates), @@ -85,6 +88,7 @@ def build_janitor_plan( skipped_relations=query_diff_artifact_skipped, scanned_schema_count=len(query_artifact_schemas), age_metadata_supported=adapter.supports_relation_age_metadata(), + planned_at=now, ) with OperationLifecycle( @@ -119,12 +123,26 @@ def build_janitor_plan( now=now, direct_mode=direct.enabled, ) + archives: JanitorArchivePlanning = plan_janitor_archives( + direct_mode=direct.enabled, + adapter=adapter, + schemas=schemas, + facts=facts, + managed_target_schemas=managed_target_schemas, + exclude_patterns=exclude_patterns, + archive_retention_days=direct.archive_retention_days, + now=now, + ) return JanitorPlan( target_name=project.effective_target_name, retention_days=retention_days, direct_mode=direct.enabled, + archive_retention_days=direct.archive_retention_days, candidates=schemas.candidates, + archive_candidates=archives.archive_candidates, + archive_deletion_candidates=archives.archive_deletion_candidates, + retained_archives=archives.retained_archives, query_diff_artifact_candidates=query_diff_artifact_candidates, checkpoint_candidates=state.checkpoint_candidates, detached_virtual_environment_candidates=state.detached_virtual_environment_candidates, @@ -139,10 +157,12 @@ def build_janitor_plan( for skipped in schemas.skipped_relations if not QueryDiffArtifactLifecycle.is_artifact_name(skipped.key.name) ), + *archives.skipped_relations, *query_diff_artifact_skipped, ), skipped_schemas=schemas.skipped_schemas, - blocked_schemas=schemas.blocked_schemas, + blocked_schemas=archives.blocked_schemas, scanned_schema_count=len(target_schemas | set(query_artifact_schemas)), age_metadata_supported=age_supported, + planned_at=now, ) diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index ebbcdebae1..97c8457654 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -37,6 +37,37 @@ class JanitorDeleteCandidate: age_timestamp: datetime | None +@dataclass(frozen=True) +class JanitorParsedArchiveName: + """Structured parts of one strict janitor archive relation name.""" + + archived_at: datetime + logical_name: str + + +@dataclass(frozen=True) +class JanitorArchiveCandidate: + """One stale direct-mode relation that will be renamed to an archive name.""" + + key: JanitorRelationKey + relation: RelationInfo + age_timestamp: datetime | None + archive_key: JanitorRelationKey + archived_at: datetime + expires_at: datetime + + +@dataclass(frozen=True) +class JanitorArchivedRelation: + """One relation whose name strictly matches the janitor archive grammar.""" + + key: JanitorRelationKey + relation_type: str | None + archived_at: datetime + expires_at: datetime + original_key: JanitorRelationKey | None = None + + @dataclass(frozen=True) class JanitorQueryDiffArtifactCandidate: """One expired, fingerprint-owned query-diff artifact eligible for deletion.""" @@ -156,6 +187,7 @@ class JanitorBlockedSchema: schema: str | None source_names: tuple[str, ...] suppressed_candidates: tuple[JanitorDeleteCandidate, ...] = field(default_factory=tuple) + suppressed_archive_deletions: tuple[JanitorArchivedRelation, ...] = field(default_factory=tuple) def display_name(self) -> str: """Render a schema display name.""" @@ -197,6 +229,17 @@ class JanitorSchemaClassification: blocked_schemas: tuple[JanitorBlockedSchema, ...] +@dataclass(frozen=True) +class JanitorArchivePlanning: + """Direct-mode archive and archive-expiry decisions for one janitor plan.""" + + archive_candidates: tuple[JanitorArchiveCandidate, ...] + archive_deletion_candidates: tuple[JanitorArchivedRelation, ...] + retained_archives: tuple[JanitorArchivedRelation, ...] + skipped_relations: tuple[JanitorSkippedRelation, ...] + blocked_schemas: tuple[JanitorBlockedSchema, ...] + + @dataclass(frozen=True) class JanitorStateCandidates: """Precomputed state-side cleanup candidates for one janitor plan.""" @@ -228,6 +271,7 @@ class JanitorDirectModeSettings: enabled: bool = False state_history_versions: int = 20 + archive_retention_days: int = 14 @dataclass(frozen=True) @@ -237,7 +281,11 @@ class JanitorPlan: target_name: str | None retention_days: int direct_mode: bool = False + archive_retention_days: int = 14 candidates: tuple[JanitorDeleteCandidate, ...] = field(default_factory=tuple) + archive_candidates: tuple[JanitorArchiveCandidate, ...] = field(default_factory=tuple) + archive_deletion_candidates: tuple[JanitorArchivedRelation, ...] = field(default_factory=tuple) + retained_archives: tuple[JanitorArchivedRelation, ...] = field(default_factory=tuple) query_diff_artifact_candidates: tuple[JanitorQueryDiffArtifactCandidate, ...] = field( default_factory=tuple ) @@ -261,6 +309,7 @@ class JanitorPlan: blocked_schemas: tuple[JanitorBlockedSchema, ...] = field(default_factory=tuple) scanned_schema_count: int = 0 age_metadata_supported: bool = False + planned_at: datetime | None = None @dataclass(frozen=True) @@ -268,6 +317,8 @@ class JanitorExecutionResult: """Result from deleting janitor candidates.""" deleted: tuple[JanitorDeleteCandidate, ...] = field(default_factory=tuple) + archived: tuple[JanitorArchiveCandidate, ...] = field(default_factory=tuple) + deleted_archives: tuple[JanitorArchivedRelation, ...] = field(default_factory=tuple) deleted_query_diff_artifacts: tuple[JanitorQueryDiffArtifactCandidate, ...] = field( default_factory=tuple ) diff --git a/src/sqlbuild/executor/janitor_events/__init__.py b/src/sqlbuild/executor/janitor_events/__init__.py new file mode 100644 index 0000000000..58de8b3059 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/__init__.py @@ -0,0 +1 @@ +"""Append-only direct-mode janitor audit events.""" diff --git a/src/sqlbuild/executor/janitor_events/_helpers/__init__.py b/src/sqlbuild/executor/janitor_events/_helpers/__init__.py new file mode 100644 index 0000000000..ca84bc0c46 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/_helpers/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event persistence helpers.""" diff --git a/src/sqlbuild/executor/janitor_events/_helpers/identity.py b/src/sqlbuild/executor/janitor_events/_helpers/identity.py new file mode 100644 index 0000000000..f0cbb41021 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/_helpers/identity.py @@ -0,0 +1,29 @@ +"""Deterministic janitor audit event identity helpers.""" + +from __future__ import annotations + +import hashlib +import json + +from sqlbuild.executor.janitor_events.types import JanitorEventType + + +def build_janitor_event_id_impl( + *, + event_type: JanitorEventType, + run_id: str, + relation_database: str | None, + relation_schema: str, + archive_name: str, +) -> str: + """Return the content-addressed ID for one janitor action within one run.""" + + identity: tuple[object, ...] = ( + event_type.value, + run_id, + relation_database, + relation_schema, + archive_name, + ) + encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py new file mode 100644 index 0000000000..8f9bf5e1da --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -0,0 +1,112 @@ +"""Portable SQL builders for append-only janitor audit events.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor_events.constants import ( + COLUMN_ARCHIVE_NAME, + COLUMN_ARCHIVE_QUALIFIED_NAME, + COLUMN_ARCHIVED_AT, + COLUMN_EVENT_ID, + COLUMN_EVENT_TYPE, + COLUMN_OCCURRED_AT, + COLUMN_ORIGINAL_NAME, + COLUMN_ORIGINAL_QUALIFIED_NAME, + COLUMN_RELATION_DATABASE, + COLUMN_RELATION_SCHEMA, + COLUMN_RELATION_TYPE, + COLUMN_RUN_ID, + COLUMN_SCHEMA_VERSION, + JANITOR_EVENT_COLUMN_TYPES, + JANITOR_EVENT_COLUMNS, + JANITOR_EVENT_REQUIRED_COLUMNS, + JANITOR_EVENTS_TABLE_NAME, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal + + +def build_qualified_table_name( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: + qualified_name: str | None = render_qualified_name( + database=database, schema=schema, name=JANITOR_EVENTS_TABLE_NAME + ) + if qualified_name is None: + raise ExecutorInputError("janitor event table requires a target schema") + return qualified_name + + +def build_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + return render_state_table_create_sql( + qualified_name=build_qualified_table_name( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=JANITOR_EVENT_COLUMNS, + column_types=JANITOR_EVENT_COLUMN_TYPES, + required_columns=JANITOR_EVENT_REQUIRED_COLUMNS, + render_framework_type=render_framework_type, + transient=transient, + ) + + +def build_existing_event_sql( + *, + record: JanitorEventRecord, + render_qualified_name: Callable[..., str | None], +) -> str: + qualified_name: str = build_qualified_table_name( + database=record.relation_database, + schema=record.relation_schema, + render_qualified_name=render_qualified_name, + ) + event_id: str = _column_literal(column=COLUMN_EVENT_ID, value=record.event_id) + return f"SELECT {COLUMN_EVENT_ID} FROM {qualified_name} WHERE {COLUMN_EVENT_ID} = {event_id}" + + +def build_insert_sql( + *, + record: JanitorEventRecord, + render_qualified_name: Callable[..., str | None], +) -> str: + qualified_name: str = build_qualified_table_name( + database=record.relation_database, + schema=record.relation_schema, + render_qualified_name=render_qualified_name, + ) + values: dict[str, object | None] = { + COLUMN_EVENT_ID: record.event_id, + COLUMN_SCHEMA_VERSION: record.schema_version, + COLUMN_EVENT_TYPE: record.event_type.value, + COLUMN_OCCURRED_AT: record.occurred_at, + COLUMN_RUN_ID: record.run_id, + COLUMN_RELATION_DATABASE: record.relation_database, + COLUMN_RELATION_SCHEMA: record.relation_schema, + COLUMN_RELATION_TYPE: record.relation_type, + COLUMN_ORIGINAL_NAME: record.original_name, + COLUMN_ORIGINAL_QUALIFIED_NAME: record.original_qualified_name, + COLUMN_ARCHIVE_NAME: record.archive_name, + COLUMN_ARCHIVE_QUALIFIED_NAME: record.archive_qualified_name, + COLUMN_ARCHIVED_AT: record.archived_at, + } + literals: str = ", ".join( + _column_literal(column=column, value=values[column]) for column in JANITOR_EVENT_COLUMNS + ) + return f"INSERT INTO {qualified_name} ({', '.join(JANITOR_EVENT_COLUMNS)}) VALUES ({literals})" + + +def _column_literal(*, column: str, value: object | None) -> str: + return render_state_sql_literal(value=value, declared_type=JANITOR_EVENT_COLUMN_TYPES[column]) diff --git a/src/sqlbuild/executor/janitor_events/constants.py b/src/sqlbuild/executor/janitor_events/constants.py new file mode 100644 index 0000000000..721f93d887 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/constants.py @@ -0,0 +1,57 @@ +"""Janitor audit event warehouse storage constants.""" + +from sqlbuild.sql_values.types import StateSqlValueType + +JANITOR_EVENTS_TABLE_NAME: str = "_sqlbuild_janitor_events" +JANITOR_EVENT_SCHEMA_VERSION: int = 1 + +COLUMN_EVENT_ID: str = "event_id" +COLUMN_SCHEMA_VERSION: str = "schema_version" +COLUMN_EVENT_TYPE: str = "event_type" +COLUMN_OCCURRED_AT: str = "occurred_at" +COLUMN_RUN_ID: str = "run_id" +COLUMN_RELATION_DATABASE: str = "relation_database" +COLUMN_RELATION_SCHEMA: str = "relation_schema" +COLUMN_RELATION_TYPE: str = "relation_type" +COLUMN_ORIGINAL_NAME: str = "original_name" +COLUMN_ORIGINAL_QUALIFIED_NAME: str = "original_qualified_name" +COLUMN_ARCHIVE_NAME: str = "archive_name" +COLUMN_ARCHIVE_QUALIFIED_NAME: str = "archive_qualified_name" +COLUMN_ARCHIVED_AT: str = "archived_at" + +JANITOR_EVENT_COLUMNS: tuple[str, ...] = ( + COLUMN_EVENT_ID, + COLUMN_SCHEMA_VERSION, + COLUMN_EVENT_TYPE, + COLUMN_OCCURRED_AT, + COLUMN_RUN_ID, + COLUMN_RELATION_DATABASE, + COLUMN_RELATION_SCHEMA, + COLUMN_RELATION_TYPE, + COLUMN_ORIGINAL_NAME, + COLUMN_ORIGINAL_QUALIFIED_NAME, + COLUMN_ARCHIVE_NAME, + COLUMN_ARCHIVE_QUALIFIED_NAME, + COLUMN_ARCHIVED_AT, +) + +JANITOR_EVENT_REQUIRED_COLUMNS: frozenset[str] = frozenset( + { + COLUMN_EVENT_ID, + COLUMN_SCHEMA_VERSION, + COLUMN_EVENT_TYPE, + COLUMN_OCCURRED_AT, + COLUMN_RUN_ID, + COLUMN_RELATION_SCHEMA, + COLUMN_ARCHIVE_NAME, + COLUMN_ARCHIVE_QUALIFIED_NAME, + COLUMN_ARCHIVED_AT, + } +) + +JANITOR_EVENT_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in JANITOR_EVENT_COLUMNS}, + COLUMN_SCHEMA_VERSION: StateSqlValueType.INTEGER, + COLUMN_OCCURRED_AT: StateSqlValueType.TEXT_TIMESTAMP, + COLUMN_ARCHIVED_AT: StateSqlValueType.TEXT_TIMESTAMP, +} diff --git a/src/sqlbuild/executor/janitor_events/main/__init__.py b/src/sqlbuild/executor/janitor_events/main/__init__.py new file mode 100644 index 0000000000..92857c3ea7 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/main/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event persistence entrypoints.""" diff --git a/src/sqlbuild/executor/janitor_events/main/create_table_sql.py b/src/sqlbuild/executor/janitor_events/main/create_table_sql.py new file mode 100644 index 0000000000..4b694023e6 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/main/create_table_sql.py @@ -0,0 +1,27 @@ +"""Janitor audit event table SQL entrypoint.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.executor.janitor_events._helpers.sql import build_create_table_sql + + +def build_janitor_events_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + """Build DDL that creates the janitor audit event table when it is missing.""" + + return build_create_table_sql( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + render_framework_type=render_framework_type, + transient=transient, + ) diff --git a/src/sqlbuild/executor/janitor_events/main/janitor_event_id.py b/src/sqlbuild/executor/janitor_events/main/janitor_event_id.py new file mode 100644 index 0000000000..997fe7811b --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/main/janitor_event_id.py @@ -0,0 +1,25 @@ +"""Janitor audit event identity entrypoint.""" + +from __future__ import annotations + +from sqlbuild.executor.janitor_events._helpers.identity import build_janitor_event_id_impl +from sqlbuild.executor.janitor_events.types import JanitorEventType + + +def build_janitor_event_id( + *, + event_type: JanitorEventType, + run_id: str, + relation_database: str | None, + relation_schema: str, + archive_name: str, +) -> str: + """Return the deterministic ID for one janitor action within one run.""" + + return build_janitor_event_id_impl( + event_type=event_type, + run_id=run_id, + relation_database=relation_database, + relation_schema=relation_schema, + archive_name=archive_name, + ) diff --git a/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py b/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py new file mode 100644 index 0000000000..3dc22a22e4 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py @@ -0,0 +1,35 @@ +"""Janitor audit event warehouse write operation.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.executor.janitor_events._helpers.sql import ( + build_existing_event_sql, + build_insert_sql, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord + + +def write_janitor_event_record( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + record: JanitorEventRecord, + render_qualified_name: Callable[..., str | None], +) -> bool: + """Append one janitor event unless its deterministic ID was already written.""" + + cursor: Any = execute( + connection=connection, + sql=build_existing_event_sql(record=record, render_qualified_name=render_qualified_name), + ) + if cursor.fetchall(): + return False + _ = execute( + connection=connection, + sql=build_insert_sql(record=record, render_qualified_name=render_qualified_name), + ) + return True diff --git a/src/sqlbuild/executor/janitor_events/models.py b/src/sqlbuild/executor/janitor_events/models.py new file mode 100644 index 0000000000..2e99b267ce --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/models.py @@ -0,0 +1,27 @@ +"""Immutable records persisted to the janitor audit table.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from sqlbuild.executor.janitor_events.types import JanitorEventType + + +@dataclass(frozen=True) +class JanitorEventRecord: + """One immutable janitor archive or delete fact.""" + + event_id: str + schema_version: int + event_type: JanitorEventType + occurred_at: datetime + run_id: str + relation_database: str | None + relation_schema: str + relation_type: str | None + original_name: str | None + original_qualified_name: str | None + archive_name: str + archive_qualified_name: str + archived_at: datetime diff --git a/src/sqlbuild/executor/janitor_events/types.py b/src/sqlbuild/executor/janitor_events/types.py new file mode 100644 index 0000000000..2223996f3c --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/types.py @@ -0,0 +1,12 @@ +"""Janitor audit event type declarations.""" + +from __future__ import annotations + +from enum import StrEnum + + +class JanitorEventType(StrEnum): + """Physical janitor actions recorded in the audit table.""" + + ARCHIVE = "archive" + DELETE = "delete" diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index ae9e9d557d..16f03df88e 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -313,7 +313,8 @@ class JanitorConfig: """Janitor command defaults.""" enabled: bool = False - retention_days: int = 30 + retention_days: int = 14 + archive_retention_days: int = 14 max_checkpoints: int = 20 direct_state_history_versions: int = 20 delete_tracked_only: bool = True diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 0ec40455a1..72f5e1d3a0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -26,6 +26,7 @@ class JanitorCleanupE2ETestCase: expected_stdout_fragments: tuple[str, ...] expected_existing_tables: tuple[str, ...] expected_missing_tables: tuple[str, ...] + expected_archived_original_names: tuple[str, ...] = field(default_factory=tuple) @dataclass(frozen=True) @@ -152,3 +153,68 @@ class JanitorDirectStatePruningE2ETestCase: expected_source_freshness_count_after: int expected_fingerprint_run_ids_after: tuple[str, ...] expected_source_freshness_run_ids_after: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorArchiveRetentionE2ETestCase: + """Test case for direct-mode archival after the relation retention period.""" + + description: str + janitor_command: tuple[str, ...] + retired_model_ages_days: dict[str, int] + expected_archived_names: tuple[str, ...] + expected_retained_names: tuple[str, ...] + expected_stdout_fragments: tuple[str, ...] + expected_second_run_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorArchiveExpiryE2ETestCase: + """Test case for name-based deletion of expired archives.""" + + description: str + janitor_command: tuple[str, ...] + expected_deleted_names: tuple[str, ...] + expected_kept_names: tuple[str, ...] + expected_kept_other_schema_names: tuple[str, ...] + expected_stdout_fragments: tuple[str, ...] + expected_delete_event_count: int + expected_total_event_count: int + + +@dataclass(frozen=True) +class JanitorZeroRetentionE2ETestCase: + """Test case for archive and delete in one run.""" + + description: str + janitor_command: tuple[str, ...] + stale_tables: tuple[str, ...] + stale_views: tuple[str, ...] + expected_stdout_fragments: tuple[str, ...] + expected_event_types: tuple[tuple[str, str], ...] + expected_fingerprint_count_after: int + expected_source_freshness_count_after: int + + +@dataclass(frozen=True) +class JanitorArchiveNameFittingE2ETestCase: + """Test case for identifier-fitted archive names.""" + + description: str + janitor_command: tuple[str, ...] + long_relation_name: str + identifier_limit: int + expected_archive_prefix_length: int + + +@dataclass(frozen=True) +class JanitorArchiveInterruptionE2ETestCase: + """Test case for an audit write failure after a rename.""" + + description: str + janitor_command: tuple[str, ...] + stale_table: str + expected_first_exit_code: int + expected_first_output_fragments: tuple[str, ...] + expected_second_exit_code: int + expected_second_stdout_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 6b79ef6612..cfd640db80 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -2,9 +2,11 @@ from __future__ import annotations +from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path from textwrap import dedent +from types import MappingProxyType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.fingerprints.main.write import write_fingerprint @@ -14,7 +16,10 @@ SourceFreshnessRecord, SourceFreshnessRenderers, ) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, +) def prepare_janitor_project( @@ -194,3 +199,164 @@ def create_direct_state_history(*, db_path: Path) -> None: ) finally: connection.close() + + +AGED_JANITOR_ADAPTER_NAME: str = "aged_janitor_duckdb" +JANITOR_TEST_SETTINGS_FILENAME: str = "janitor_test_settings.json" +AGED_JANITOR_ADAPTER_SOURCE: str = """ +import json +from dataclasses import replace +from datetime import datetime +from pathlib import Path +from typing import Any + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter + +_SETTINGS_PATH = Path(__file__).resolve().parent.parent / "janitor_test_settings.json" + + +def _settings() -> dict[str, Any]: + if not _SETTINGS_PATH.exists(): + return {} + return json.loads(_SETTINGS_PATH.read_text(encoding="utf-8")) + + +class AgedJanitorDuckDbAdapter(DuckDbAdapter): + adapter_name = "aged_janitor_duckdb" + + def supports_relation_age_metadata(self) -> bool: + return True + + def maximum_identifier_length(self) -> int: + return int(_settings().get("identifier_limit", super().maximum_identifier_length())) + + def list_relations( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[RelationInfo, ...]: + created_at: dict[str, str] = _settings().get("created_at", {}) + return tuple( + replace(relation, created_at=datetime.fromisoformat(created_at[relation.name])) + if relation.name in created_at + else relation + for relation in super().list_relations( + connection=connection, database=database, schemas=schemas, names=names + ) + ) + + def _execute(self, *, connection: Any, sql: str) -> Any: + if ( + _settings().get("fail_janitor_event_insert") + and sql.startswith("INSERT INTO") + and "_sqlbuild_janitor_events" in sql + ): + raise AdapterUserError("simulated janitor event write failure") + return super()._execute(connection=connection, sql=sql) +""" + + +def prepare_archive_janitor_project( + *, + tmp_path: Path, + project_name: str, + janitor_config: str, + model_names: tuple[str, ...] = ("orders",), + use_aged_adapter: bool = False, +) -> Path: + """Create a DuckDB project whose janitor archives stale relations.""" + + repo_files: dict[str, str] = { + "sqlbuild_project.toml": ( + f'name = "{project_name}"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + f'database = "{(tmp_path / project_name / "janitor.duckdb").as_posix()}"\n\n' + f"[janitor]\n{dedent(janitor_config).strip()}\n\n" + "[defaults]\n" + 'materialized = "table"\n' + ), + **{ + f"models/{model_name}.sql": f"MODEL ();\n\nSELECT 1 AS {model_name}_id\n" + for model_name in model_names + }, + f"adapters/{AGED_JANITOR_ADAPTER_NAME}.py": AGED_JANITOR_ADAPTER_SOURCE, + **{ + False: {}, + True: {"sqlbuild_local.toml": f'adapter = "{AGED_JANITOR_ADAPTER_NAME}"\n'}, + }[use_aged_adapter], + } + return prepare_inline_project( + tmp_path=tmp_path, + project_name=project_name, + repo_files=repo_files, + ) + + +def write_janitor_test_settings( + *, + project_dir: Path, + created_at: Mapping[str, datetime] = MappingProxyType({}), + identifier_limit: int = 255, + fail_janitor_event_insert: bool = False, +) -> None: + """Write relation ages and fault switches consumed by the aged janitor adapter.""" + + import json + + payload: dict[str, object] = { + "created_at": {name: value.isoformat() for name, value in created_at.items()}, + "identifier_limit": identifier_limit, + "fail_janitor_event_insert": fail_janitor_event_insert, + } + (project_dir / JANITOR_TEST_SETTINGS_FILENAME).write_text(json.dumps(payload), encoding="utf-8") + + +def archive_timestamp_text(value: datetime) -> str: + """Render the UTC timestamp component embedded in janitor archive names.""" + + return value.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ") + + +def list_archive_names(*, db_path: Path, schema: str = "main") -> tuple[str, ...]: + """Return archive-prefixed relation names in one schema.""" + + rows: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' AND starts_with(table_name, '_SQB_ARCHIVE__') " + "ORDER BY table_name" + ), + ) + return tuple(str(row[0]) for row in rows) + + +def read_janitor_events(*, db_path: Path) -> list[tuple[object, ...]]: + """Return janitor audit rows in a stable order.""" + + return query_duckdb( + db_path=db_path, + sql=( + "SELECT event_type, original_name, original_qualified_name, archive_name, run_id " + "FROM main._sqlbuild_janitor_events ORDER BY occurred_at, event_type, archive_name" + ), + ) + + +def read_current_janitor_events(*, db_path: Path, run_id: str) -> list[tuple[object, ...]]: + """Return janitor audit rows excluding one seeded historical run.""" + + return query_duckdb( + db_path=db_path, + sql=( + "SELECT event_type, original_name, original_qualified_name, archive_name, run_id " + f"FROM main._sqlbuild_janitor_events WHERE run_id <> '{run_id}' " + "ORDER BY archive_name" + ), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 701ef7e277..9d50387d86 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -81,19 +81,20 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( "test_case", [ JanitorCleanupE2ETestCase( - description="tracked-only direct janitor reports stale relations without deleting", + description="tracked-only direct janitor archives only tracked stale relations", build_command=("--no-color", "build", "--full-refresh"), janitor_command=("janitor", "--auto-approve"), expected_exit_code=0, expected_stdout_fragments=( - "reported objects 1", + "relations to archive 1", + "archives to delete 0", "objects skipped 4", - "main.janitor_tracked_extra", + "main.janitor_tracked_extra -> main._SQB_ARCHIVE__", "main.janitor_untracked_extra relation is not tracked by SQLBuild", "main.partition_state relation matches exclude pattern 'partition_*'", "main._sqlbuild_fingerprints relation matches exclude pattern", "main._sqlbuild_microbatches relation matches exclude pattern", - "direct mode does not delete relations", + "Archived 1 relations.", ), expected_existing_tables=( "orders", @@ -101,14 +102,15 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( "partition_state", "_sqlbuild_fingerprints", "_sqlbuild_microbatches", - "janitor_tracked_extra", + "_sqlbuild_janitor_events", ), - expected_missing_tables=(), + expected_missing_tables=("janitor_tracked_extra",), + expected_archived_original_names=("janitor_tracked_extra",), ) ], ids=lambda case: case.description, ) -def test_given_stale_relations_when_running_direct_janitor_then_it_reports_without_deleting( +def test_given_stale_relations_when_running_direct_janitor_then_it_archives_tracked_relations( test_case: JanitorCleanupE2ETestCase, tmp_path: Path, ) -> None: @@ -150,6 +152,14 @@ def test_given_stale_relations_when_running_direct_janitor_then_it_reports_witho assert table_exists(db_path=db_path, table_name=table_name) for table_name in test_case.expected_missing_tables: assert not table_exists(db_path=db_path, table_name=table_name) + for original_name in test_case.expected_archived_original_names: + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT COUNT(*) FROM information_schema.tables " + f"WHERE table_name LIKE '_SQB_ARCHIVE__%__{original_name}'" + ), + ) == [(1,)] assert "Eligible expired VDEs" not in janitor_result.stdout assert "Eligible state backups" not in janitor_result.stdout assert "Eligible expired locks" not in janitor_result.stdout @@ -166,7 +176,7 @@ def test_given_stale_relations_when_running_direct_janitor_then_it_reports_witho "Janitor blocked", "Managed target schemas contain active configured sources.", "main active sources: raw_events", - "suppressed deletion: main.stale_main", + "suppressed archive: main.stale_main", "No janitor actions will be performed.", ), expected_existing_relations=( @@ -355,30 +365,30 @@ def test_given_virtual_microbatch_history_when_running_janitor_then_events_remai "test_case", [ JanitorCleanupE2ETestCase( - description="tracked-only direct janitor reports strict scenario artifacts", + description="tracked-only direct janitor archives strict scenario artifacts", build_command=("--no-color", "build", "--full-refresh"), janitor_command=("janitor", "--auto-approve"), expected_exit_code=0, expected_stdout_fragments=( - "reported objects 2", + "relations to archive 2", "objects skipped 2", - "main.__sqb_a13f09c2e7b8__model__daily_revenue", - "main.__sqb_a13f09c2e7b8__source__raw_orders", + "main.__sqb_a13f09c2e7b8__model__daily_revenue -> main._SQB_ARCHIVE__", + "main.__sqb_a13f09c2e7b8__source__raw_orders -> main._SQB_ARCHIVE__", "main.__sqb_a13f09c2e7b__model__daily_revenue relation is not tracked by SQLBuild", - "direct mode does not delete relations", ), expected_existing_tables=( "orders", "__sqb_a13f09c2e7b__model__daily_revenue", + ), + expected_missing_tables=( "__sqb_a13f09c2e7b8__source__raw_orders", "__sqb_a13f09c2e7b8__model__daily_revenue", ), - expected_missing_tables=(), ) ], ids=lambda case: case.description, ) -def test_given_scenario_artifacts_when_running_direct_janitor_then_it_reports_without_deleting( +def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_them( test_case: JanitorCleanupE2ETestCase, tmp_path: Path, ) -> None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py new file mode 100644 index 0000000000..271f131ed5 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py @@ -0,0 +1,468 @@ +"""E2E tests for direct-mode janitor archival and archive expiry.""" + +from __future__ import annotations + +import re +import subprocess +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorArchiveExpiryE2ETestCase, + JanitorArchiveInterruptionE2ETestCase, + JanitorArchiveNameFittingE2ETestCase, + JanitorArchiveRetentionE2ETestCase, + JanitorZeroRetentionE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( + archive_timestamp_text, + create_direct_state_history, + list_archive_names, + prepare_archive_janitor_project, + read_current_janitor_events, + read_janitor_events, + write_janitor_test_settings, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + query_duckdb, + run_sqb, + table_exists, +) + +ARCHIVE_NAME_PATTERN: re.Pattern[str] = re.compile( + r"^_SQB_ARCHIVE__(?P[0-9]{8}T[0-9]{6}Z)__(?P.+)$" +) +RECENT_ARCHIVE_NAME: str = ( + f"_SQB_ARCHIVE__{archive_timestamp_text(datetime.now(UTC) - timedelta(days=1))}__recent_orders" +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveRetentionE2ETestCase( + description="retired tracked relation is archived after retention and then retained", + janitor_command=("--no-color", "janitor", "--auto-approve"), + retired_model_ages_days={"customers": 20, "products": 3}, + expected_archived_names=("customers",), + expected_retained_names=("products",), + expected_stdout_fragments=( + "retention 14 days", + "archive retention 14 days", + "relations to archive 1", + "Relations to archive", + "main.customers -> main._SQB_ARCHIVE__", + "age 20d, delete after", + "main.products relation is newer than 14 days", + "Archived 1 relations.", + ), + expected_second_run_fragments=( + "relations to archive 0", + "archives to delete 0", + "archives retained 1", + "Retained archives", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_retired_tracked_relation_past_retention_when_running_janitor_then_archives_it( + test_case: JanitorArchiveRetentionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_archive_retention", + janitor_config="enabled = true\n", + model_names=("orders", *test_case.retired_model_ages_days), + use_aged_adapter=True, + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + now: datetime = datetime.now(UTC) + write_janitor_test_settings( + project_dir=project_dir, + created_at={ + name: now - timedelta(days=age_days) + for name, age_days in test_case.retired_model_ages_days.items() + }, + ) + for model_name in test_case.retired_model_ages_days: + (project_dir / "models" / f"{model_name}.sql").unlink() + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in janitor_result.stdout + archive_names: tuple[str, ...] = list_archive_names(db_path=db_path) + assert tuple(ARCHIVE_NAME_PATTERN.sub(r"\g", name) for name in archive_names) == ( + test_case.expected_archived_names + ) + assert all(ARCHIVE_NAME_PATTERN.fullmatch(name) for name in archive_names) + for name in test_case.expected_archived_names: + assert not table_exists(db_path=db_path, table_name=name) + for name in test_case.expected_retained_names: + assert table_exists(db_path=db_path, table_name=name) + events: list[tuple[object, ...]] = read_janitor_events(db_path=db_path) + assert [(event[0], event[1], event[2], event[3]) for event in events] == [ + ("archive", name, f"main.{name}", archive_name) + for name, archive_name in zip(test_case.expected_archived_names, archive_names, strict=True) + ] + assert all(str(event[4]) for event in events) + + second_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert second_result.returncode == 0, second_result.stdout + second_result.stderr + for fragment in test_case.expected_second_run_fragments: + assert fragment in second_result.stdout + assert list_archive_names(db_path=db_path) == archive_names + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveExpiryE2ETestCase( + description="only expired strict archives in managed schemas are deleted", + janitor_command=("--no-color", "janitor", "--auto-approve"), + expected_deleted_names=( + "_SQB_ARCHIVE__20200101T000000Z__old_customers_view", + "_SQB_ARCHIVE__20200101T000000Z__old_products", + ), + expected_kept_names=( + RECENT_ARCHIVE_NAME, + "_SQB_ARCHIVE__20201399T000000Z__bad_month", + "_SQB_ARCHIVE_20200101T000000Z__one_separator", + "_sqb_archive_notes", + ), + expected_kept_other_schema_names=("_SQB_ARCHIVE__20200101T000000Z__orders",), + expected_stdout_fragments=( + "archives to delete 2", + "archives retained 1", + "Archives to delete", + "main._SQB_ARCHIVE__20200101T000000Z__old_products archived 2020-01-01 00:00:00", + "expired 2020-01-15 00:00:00 UTC", + "Retained archives", + f"main.{RECENT_ARCHIVE_NAME} archived", + "main._SQB_ARCHIVE__20201399T000000Z__bad_month name resembles a janitor archive " + "but does not match the strict archive grammar", + "main._SQB_ARCHIVE_20200101T000000Z__one_separator name resembles a janitor archive", + "main._sqb_archive_notes name resembles a janitor archive", + "Deleted 2 objects", + ), + expected_delete_event_count=2, + expected_total_event_count=5, + ) + ], + ids=lambda case: case.description, +) +def test_given_archive_named_relations_when_running_janitor_then_deletes_only_expired_strict_ones( + test_case: JanitorArchiveExpiryE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_archive_expiry", + janitor_config="enabled = true\narchive_retention_days = 14\n", + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + execute_duckdb( + db_path=db_path, + sql=( + 'CREATE TABLE main."_SQB_ARCHIVE__20200101T000000Z__old_products" AS SELECT 1 AS id; ' + 'CREATE VIEW main."_SQB_ARCHIVE__20200101T000000Z__old_customers_view" AS ' + "SELECT 1 AS id; " + f'CREATE TABLE main."{RECENT_ARCHIVE_NAME}" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_SQB_ARCHIVE__20201399T000000Z__bad_month" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_SQB_ARCHIVE_20200101T000000Z__one_separator" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_sqb_archive_notes" AS SELECT 1 AS id; ' + "CREATE SCHEMA other; " + 'CREATE TABLE other."_SQB_ARCHIVE__20200101T000000Z__orders" AS SELECT 1 AS id; ' + + DuckDbAdapter().render_create_janitor_event_table_sql(database=None, schema="main") + + "; INSERT INTO main._sqlbuild_janitor_events " + "(event_id, schema_version, event_type, occurred_at, run_id, relation_schema, " + "original_name, archive_name, archive_qualified_name, archived_at) VALUES " + "('misleading_delete', 1, 'delete', TIMESTAMP '2020-02-01 00:00:00', 'run_old', " + "'main', 'old_products', '_SQB_ARCHIVE__20200101T000000Z__old_products', " + "'main._SQB_ARCHIVE__20200101T000000Z__old_products', " + "TIMESTAMP '2020-01-01 00:00:00'), " + "('misleading_archive', 1, 'archive', TIMESTAMP '2020-01-01 00:00:00', 'run_old', " + f"'main', 'recent_orders', '{RECENT_ARCHIVE_NAME}', 'main.{RECENT_ARCHIVE_NAME}', " + "TIMESTAMP '2020-01-01 00:00:00'), " + "('misleading_malformed', 1, 'archive', TIMESTAMP '2020-01-01 00:00:00', 'run_old', " + "'main', 'bad_month', '_SQB_ARCHIVE__20201399T000000Z__bad_month', " + "'main._SQB_ARCHIVE__20201399T000000Z__bad_month', TIMESTAMP '2020-01-01 00:00:00')" + ), + ) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in janitor_result.stdout + for name in test_case.expected_deleted_names: + assert not table_exists(db_path=db_path, table_name=name) + for name in test_case.expected_kept_names: + assert table_exists(db_path=db_path, table_name=name) + for name in test_case.expected_kept_other_schema_names: + assert table_exists(db_path=db_path, schema="other", table_name=name) + events: list[tuple[object, ...]] = read_janitor_events(db_path=db_path) + current_events: list[tuple[object, ...]] = read_current_janitor_events( + db_path=db_path, run_id="run_old" + ) + assert len(events) == test_case.expected_total_event_count + assert len(current_events) == test_case.expected_delete_event_count + assert [(event[0], event[1], event[3]) for event in current_events] == [ + ("delete", None, name) for name in sorted(test_case.expected_deleted_names) + ] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorZeroRetentionE2ETestCase( + description="zero retention archives and deletes in one run with unchanged pruning", + janitor_command=( + "--no-color", + "janitor", + "--auto-approve", + "--direct-state-history-versions", + "2", + ), + stale_tables=("stale_orders",), + stale_views=("stale_customers_view",), + expected_stdout_fragments=( + "retention disabled (0 days)", + "archive retention 0 days", + "relations to archive 2", + "archives to delete 2", + "deleted in this run", + "archived in this run", + "Eligible direct state pruning", + "main._sqlbuild_fingerprints keep latest 2", + "Archived 2 relations. Deleted 2 objects", + ), + expected_event_types=( + ("archive", "stale_customers_view"), + ("archive", "stale_orders"), + ("delete", "stale_customers_view"), + ("delete", "stale_orders"), + ), + expected_fingerprint_count_after=3, + expected_source_freshness_count_after=2, + ) + ], + ids=lambda case: case.description, +) +def test_given_zero_retention_settings_when_running_janitor_then_archives_and_deletes_in_one_run( + test_case: JanitorZeroRetentionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_zero_retention", + janitor_config=( + "enabled = true\n" + "retention_days = 0\n" + "archive_retention_days = 0\n" + "delete_tracked_only = false\n" + ), + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + create_direct_state_history(db_path=db_path) + execute_duckdb( + db_path=db_path, + sql="; ".join( + ( + *(f"CREATE TABLE main.{name} AS SELECT 1 AS id" for name in test_case.stale_tables), + *(f"CREATE VIEW main.{name} AS SELECT 1 AS id" for name in test_case.stale_views), + ) + ), + ) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in janitor_result.stdout + for name in (*test_case.stale_tables, *test_case.stale_views): + assert not table_exists(db_path=db_path, table_name=name) + assert list_archive_names(db_path=db_path) == () + events: list[tuple[object, ...]] = read_janitor_events(db_path=db_path) + assert tuple(sorted((str(event[0]), str(event[1])) for event in events)) == ( + test_case.expected_event_types + ) + assert len({event[4] for event in events}) == 1 + assert query_duckdb( + db_path=db_path, sql="SELECT COUNT(*) FROM main._sqlbuild_fingerprints" + ) == [(test_case.expected_fingerprint_count_after,)] + assert query_duckdb( + db_path=db_path, sql="SELECT COUNT(*) FROM main._sqlbuild_source_freshness" + ) == [(test_case.expected_source_freshness_count_after,)] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveNameFittingE2ETestCase( + description="long relation name is fitted to the adapter limit with timestamp intact", + janitor_command=("--no-color", "janitor", "--auto-approve"), + long_relation_name="customer_order_fulfillment_history_snapshot_backup", + identifier_limit=48, + expected_archive_prefix_length=len("_SQB_ARCHIVE__20260101T000000Z__"), + ) + ], + ids=lambda case: case.description, +) +def test_given_small_identifier_limit_when_archiving_long_name_then_fits_name_and_keeps_timestamp( + test_case: JanitorArchiveNameFittingE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_archive_name_fitting", + janitor_config="enabled = true\nretention_days = 0\ndelete_tracked_only = false\n", + use_aged_adapter=True, + ) + db_path: Path = project_dir / "janitor.duckdb" + write_janitor_test_settings( + project_dir=project_dir, identifier_limit=test_case.identifier_limit + ) + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + execute_duckdb( + db_path=db_path, + sql=f"CREATE TABLE main.{test_case.long_relation_name} AS SELECT 1 AS id", + ) + before: datetime = datetime.now(UTC).replace(microsecond=0) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + after: datetime = datetime.now(UTC) + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + archive_names: tuple[str, ...] = list_archive_names(db_path=db_path) + assert len(archive_names) == 1 + archive_name: str = archive_names[0] + match: re.Match[str] | None = ARCHIVE_NAME_PATTERN.match(archive_name) + assert match is not None + archived_at: datetime = datetime.strptime(match.group("timestamp"), "%Y%m%dT%H%M%SZ").replace( + tzinfo=UTC + ) + assert before <= archived_at <= after + logical_name: str = match.group("logical_name") + assert len(archive_name) == test_case.identifier_limit + assert len(archive_name) - len(logical_name) == test_case.expected_archive_prefix_length + assert logical_name != test_case.long_relation_name + assert test_case.long_relation_name.startswith(logical_name.rsplit("_", 1)[0]) + events: list[tuple[object, ...]] = read_janitor_events(db_path=db_path) + assert [(event[0], event[1], event[3]) for event in events] == [ + ("archive", test_case.long_relation_name, archive_name) + ] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveInterruptionE2ETestCase( + description="archive without an audit row is still deleted later by name and age", + janitor_command=("--no-color", "janitor", "--auto-approve"), + stale_table="stale_orders", + expected_first_exit_code=1, + expected_first_output_fragments=("simulated janitor event write failure",), + expected_second_exit_code=0, + expected_second_stdout_fragments=( + "archives to delete 1", + "Archives to delete", + "Deleted 1 objects", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_audit_write_interrupted_after_rename_when_janitor_reruns_then_archive_is_deleted( + test_case: JanitorArchiveInterruptionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_archive_interruption", + janitor_config=( + "enabled = true\n" + "retention_days = 0\n" + "archive_retention_days = 14\n" + "delete_tracked_only = false\n" + ), + use_aged_adapter=True, + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + execute_duckdb( + db_path=db_path, sql=f"CREATE TABLE main.{test_case.stale_table} AS SELECT 1 AS id" + ) + write_janitor_test_settings(project_dir=project_dir, fail_janitor_event_insert=True) + + first_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert first_result.returncode == test_case.expected_first_exit_code, ( + first_result.stdout + first_result.stderr + ) + for fragment in test_case.expected_first_output_fragments: + assert fragment in first_result.stdout + first_result.stderr + stranded_archives: tuple[str, ...] = list_archive_names(db_path=db_path) + assert len(stranded_archives) == 1 + assert not table_exists(db_path=db_path, table_name=test_case.stale_table) + assert read_janitor_events(db_path=db_path) == [] + + write_janitor_test_settings(project_dir=project_dir) + project_file: Path = project_dir / "sqlbuild_project.toml" + project_file.write_text( + project_file.read_text(encoding="utf-8").replace( + "archive_retention_days = 14", "archive_retention_days = 0" + ), + encoding="utf-8", + ) + second_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert second_result.returncode == test_case.expected_second_exit_code, ( + second_result.stdout + second_result.stderr + ) + for fragment in test_case.expected_second_stdout_fragments: + assert fragment in second_result.stdout + assert list_archive_names(db_path=db_path) == () + assert [(event[0], event[1], event[3]) for event in read_janitor_events(db_path=db_path)] == [ + ("delete", None, stranded_archives[0]) + ] diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/__init__.py b/tests/integration/src/sqlbuild/executor/janitor_events/__init__.py new file mode 100644 index 0000000000..3db13f5218 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py new file mode 100644 index 0000000000..65db89d85b --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event write integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py new file mode 100644 index 0000000000..bf162f5406 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class JanitorEventWriteTestCase: + description: str + write_attempts: int + expected_row_count: int + expected_write_results: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py new file mode 100644 index 0000000000..c1ff5d333d --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py @@ -0,0 +1,103 @@ +"""DuckDB integration tests for append-only janitor audit events.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.janitor_events.constants import ( + JANITOR_EVENT_COLUMNS, + JANITOR_EVENT_SCHEMA_VERSION, +) +from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id +from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType +from tests.integration.src.sqlbuild.executor.janitor_events.main._test_types import ( + JanitorEventWriteTestCase, +) + +ARCHIVE_NAME: str = "_SQB_ARCHIVE__20260924T101500Z__orders" + + +@pytest.mark.parametrize( + "test_case", + ( + JanitorEventWriteTestCase( + description="repeated write of one deterministic event appends a single row", + write_attempts=2, + expected_row_count=1, + expected_write_results=(True, False), + ), + ), + ids=lambda case: case.description, +) +def test_given_repeated_janitor_event_when_writing_then_insert_is_idempotent( + test_case: JanitorEventWriteTestCase, + tmp_path: Path, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": str(tmp_path / "janitor_events.duckdb")}) + archived_at: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) + record: JanitorEventRecord = JanitorEventRecord( + event_id=build_janitor_event_id( + event_type=JanitorEventType.ARCHIVE, + run_id="run_orders_cleanup", + relation_database=None, + relation_schema="main", + archive_name=ARCHIVE_NAME, + ), + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=JanitorEventType.ARCHIVE, + occurred_at=archived_at, + run_id="run_orders_cleanup", + relation_database=None, + relation_schema="main", + relation_type="BASE TABLE", + original_name="orders", + original_qualified_name="main.orders", + archive_name=ARCHIVE_NAME, + archive_qualified_name=f"main.{ARCHIVE_NAME}", + archived_at=archived_at, + ) + try: + adapter.execute( + connection=connection, + sql=adapter.render_create_janitor_event_table_sql(database=None, schema="main"), + ) + write_results: tuple[bool, ...] = tuple( + write_janitor_event_record( + connection=connection, + execute=adapter.execute, + record=record, + render_qualified_name=adapter.render_qualified_name, + ) + for _ in range(test_case.write_attempts) + ) + rows: list[tuple[object, ...]] = connection.execute( + f"SELECT {', '.join(JANITOR_EVENT_COLUMNS)} FROM main._sqlbuild_janitor_events" + ).fetchall() + finally: + adapter.close(connection) + + assert write_results == test_case.expected_write_results + assert len(rows) == test_case.expected_row_count + assert rows[0] == ( + record.event_id, + JANITOR_EVENT_SCHEMA_VERSION, + "archive", + archived_at.replace(tzinfo=None), + "run_orders_cleanup", + None, + "main", + "BASE TABLE", + "orders", + "main.orders", + ARCHIVE_NAME, + f"main.{ARCHIVE_NAME}", + archived_at.replace(tzinfo=None), + ) diff --git a/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/_test_types.py b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/_test_types.py new file mode 100644 index 0000000000..f14ad41378 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/_test_types.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class FitArtifactLogicalNameTestCase: + description: str + logical_name: str + fixed_prefix: str + identifier_limit: int + expected_name: str + + +@dataclass(frozen=True) +class FitArtifactLogicalNameErrorTestCase: + description: str + logical_name: str + fixed_prefix: str + identifier_limit: int + expected_error_fragment: str diff --git a/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py new file mode 100644 index 0000000000..ffc2743ab3 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import hashlib + +import pytest + +from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name +from sqlbuild.errors.contracts.exceptions import SharedInputError +from tests.unit.src.sqlbuild.adapter.relations.main.fit_artifact_logical_name._test_types import ( + FitArtifactLogicalNameErrorTestCase, + FitArtifactLogicalNameTestCase, +) + +LONG_NAME: str = "customer_orders_" + "x" * 40 +LONG_NAME_HASH: str = hashlib.sha256(LONG_NAME.encode("utf-8")).hexdigest()[:8] + + +@pytest.mark.parametrize( + "test_case", + [ + FitArtifactLogicalNameTestCase( + description="short name is kept unchanged", + logical_name="orders", + fixed_prefix="__prefix__", + identifier_limit=63, + expected_name="orders", + ), + FitArtifactLogicalNameTestCase( + description="long name is shortened with a deterministic hash suffix", + logical_name=LONG_NAME, + fixed_prefix="__prefix__", + identifier_limit=40, + expected_name=f"{LONG_NAME[:21]}_{LONG_NAME_HASH}", + ), + ], + ids=lambda case: case.description, +) +def test_given_logical_name_when_fitting_artifact_name_then_returns_expected_name( + test_case: FitArtifactLogicalNameTestCase, +) -> None: + fitted: str = fit_artifact_logical_name( + logical_name=test_case.logical_name, + fixed_prefix=test_case.fixed_prefix, + identifier_limit=test_case.identifier_limit, + artifact_label="Test artifact", + ) + + assert fitted == test_case.expected_name + assert len(test_case.fixed_prefix + fitted) <= test_case.identifier_limit + + +@pytest.mark.parametrize( + "test_case", + [ + FitArtifactLogicalNameErrorTestCase( + description="prefix longer than the limit is rejected", + logical_name="orders", + fixed_prefix="__a_prefix_that_is_too_long__", + identifier_limit=10, + expected_error_fragment="does not fit within identifier limit 10", + ), + FitArtifactLogicalNameErrorTestCase( + description="room too small for a hash suffix is rejected", + logical_name=LONG_NAME, + fixed_prefix="__prefix__", + identifier_limit=15, + expected_error_fragment="cannot fit within identifier limit 15", + ), + ], + ids=lambda case: case.description, +) +def test_given_unfittable_name_when_fitting_artifact_name_then_raises_input_error( + test_case: FitArtifactLogicalNameErrorTestCase, +) -> None: + with pytest.raises(SharedInputError) as exc_info: + fit_artifact_logical_name( + logical_name=test_case.logical_name, + fixed_prefix=test_case.fixed_prefix, + identifier_limit=test_case.identifier_limit, + artifact_label="Test artifact", + ) + + assert test_case.expected_error_fragment in str(exc_info.value) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py index dfde23266d..bb20ba08fa 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -23,3 +23,11 @@ class JanitorPlanOutputTestCase: use_color: bool expected_output_fragments: tuple[str, ...] unexpected_output_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class JanitorDirectArchiveConfirmationTestCase: + description: str + typed_response: str + expected_result: bool + expected_output_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py index aa9bbd0754..abbf9b1185 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -1,9 +1,11 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime, timedelta from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, JanitorBlockedSchema, JanitorCheckpointCandidate, JanitorDeleteCandidate, @@ -109,3 +111,53 @@ def build_janitor_plan() -> JanitorPlan: scanned_schema_count=2, age_metadata_supported=True, ) + + +def build_direct_archive_plan() -> JanitorPlan: + archived_at: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) + relation: RelationInfo = RelationInfo( + database=None, + schema="dev", + name="old_orders", + relation_type="table", + ) + original_key: JanitorRelationKey = JanitorRelationKey( + database=None, schema="dev", name="old_orders" + ) + return JanitorPlan( + target_name="dev", + retention_days=14, + direct_mode=True, + archive_retention_days=14, + candidates=( + JanitorDeleteCandidate(key=original_key, relation=relation, age_timestamp=None), + ), + archive_candidates=( + JanitorArchiveCandidate( + key=original_key, + relation=relation, + age_timestamp=archived_at - timedelta(days=20), + archive_key=JanitorRelationKey( + database=None, + schema="dev", + name="_SQB_ARCHIVE__20260924T101500Z__old_orders", + ), + archived_at=archived_at, + expires_at=archived_at + timedelta(days=14), + ), + ), + archive_deletion_candidates=( + JanitorArchivedRelation( + key=JanitorRelationKey( + database=None, + schema="dev", + name="_SQB_ARCHIVE__20260101T000000Z__old_products", + ), + relation_type="table", + archived_at=datetime(2026, 1, 1, tzinfo=UTC), + expires_at=datetime(2026, 1, 15, tzinfo=UTC), + ), + ), + age_metadata_supported=True, + planned_at=archived_at, + ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index a86674f25a..f6cdee93fe 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -18,10 +18,14 @@ ) from tests.unit.src.sqlbuild.cli.commands.main.janitor._test_types import ( JanitorConfirmationInterruptTestCase, + JanitorDirectArchiveConfirmationTestCase, JanitorDisabledOutputTestCase, JanitorPlanOutputTestCase, ) -from tests.unit.src.sqlbuild.cli.commands.main.janitor.helpers import build_janitor_plan +from tests.unit.src.sqlbuild.cli.commands.main.janitor.helpers import ( + build_direct_archive_plan, + build_janitor_plan, +) @pytest.mark.parametrize( @@ -88,7 +92,8 @@ def interrupting_input() -> str: "Add this block to sqlbuild_project.toml:\n\n" "janitor:\n" " enabled: true\n" - " retention_days: 30\n\n" + " retention_days: 14\n" + " archive_retention_days: 14\n\n" "After enabling, run janitor again to preview cleanup:\n" " sqb janitor\n", ) @@ -206,3 +211,80 @@ def test_given_janitor_plan_when_writing_with_color_then_uses_semantic_colors( output: str = stream.getvalue() for fragment in test_case.expected_output_fragments: assert fragment in output + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorDirectArchiveConfirmationTestCase( + description="direct archive plan requires typed archive and delete counts", + typed_response="archive 1 and delete 1 objects from dev", + expected_result=True, + expected_output_fragments=( + "Janitor will archive 1 and delete 1 objects from dev.", + "Retention: 14 days", + "Archive retention: 14 days", + "Type `archive 1 and delete 1 objects from dev` to continue: ", + ), + ), + JanitorDirectArchiveConfirmationTestCase( + description="direct archive plan rejects the virtual-mode phrase", + typed_response="delete 1 objects from dev", + expected_result=False, + expected_output_fragments=("Janitor will archive 1 and delete 1 objects from dev.",), + ), + ], + ids=lambda case: case.description, +) +def test_given_direct_archive_plan_when_confirming_then_requires_exact_archive_phrase( + test_case: JanitorDirectArchiveConfirmationTestCase, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(builtins, "input", lambda: test_case.typed_response) + + result: bool = confirm_janitor_plan( + planning_result=JanitorPlanningResult(plan=build_direct_archive_plan()) + ) + + output: str = capsys.readouterr().out + assert result is test_case.expected_result + for fragment in test_case.expected_output_fragments: + assert fragment in output + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPlanOutputTestCase( + description="direct preview lists exact archive names, ages, and expiry", + use_color=False, + expected_output_fragments=( + " retention 14 days", + " archive retention 14 days", + " relations to archive 1", + " archives to delete 1", + " archives retained 0", + "Relations to archive\n dev.old_orders -> " + "dev._SQB_ARCHIVE__20260924T101500Z__old_orders " + "age 20d, delete after 2026-10-08 10:15:00 UTC", + "Archives to delete\n dev._SQB_ARCHIVE__20260101T000000Z__old_products " + "archived 2026-01-01 00:00:00 UTC, age 266d, expired 2026-01-15 00:00:00 UTC", + ), + unexpected_output_fragments=("eligible for deletion", "Eligible objects"), + ) + ], + ids=lambda case: case.description, +) +def test_given_direct_archive_plan_when_writing_preview_then_lists_archive_actions( + test_case: JanitorPlanOutputTestCase, +) -> None: + stream: StringIO = StringIO() + + write_plan(plan=build_direct_archive_plan(), stream=stream, use_color=test_case.use_color) + + output: str = stream.getvalue() + for fragment in test_case.expected_output_fragments: + assert fragment in output + for fragment in test_case.unexpected_output_fragments: + assert fragment not in output diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index db48cfebc5..44a941efcb 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -271,6 +271,7 @@ class LoadProjectConfigTestCase: expected_function_database: str | None = None expected_function_schema: str | None = None expected_janitor_direct_state_history_versions: int = 20 + expected_janitor_archive_retention_days: int = 14 expected_current_state_full_refresh: str = "deny" expected_historical_full_refresh: str = "require_confirmation" expected_snapshot_schema_change: str = "append_new_columns" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 11adb31dd0..a075007d67 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -667,7 +667,7 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise } }, expected_janitor_enabled=False, - expected_retention_days=30, + expected_retention_days=14, expected_janitor_max_checkpoints=20, expected_janitor_delete_tracked_only=True, expected_janitor_exclude_patterns=(), @@ -735,7 +735,8 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise [janitor] enabled = true -retention_days = 14 +retention_days = 21 +archive_retention_days = 3 max_checkpoints = 3 delete_tracked_only = false exclude_patterns = ["partition_*"] @@ -814,7 +815,8 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise } }, expected_janitor_enabled=True, - expected_retention_days=14, + expected_retention_days=21, + expected_janitor_archive_retention_days=3, expected_janitor_max_checkpoints=3, expected_janitor_delete_tracked_only=False, expected_janitor_exclude_patterns=("partition_*",), @@ -891,6 +893,9 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e } == test_case.expected_targets assert config.janitor.enabled is test_case.expected_janitor_enabled assert config.janitor.retention_days == test_case.expected_retention_days + assert ( + config.janitor.archive_retention_days == test_case.expected_janitor_archive_retention_days + ) assert config.janitor.max_checkpoints == test_case.expected_janitor_max_checkpoints assert config.janitor.delete_tracked_only is test_case.expected_janitor_delete_tracked_only assert config.janitor.exclude_patterns == test_case.expected_janitor_exclude_patterns @@ -1669,6 +1674,17 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed( """.strip(), expected_error_fragment="janitor.direct_state_history_versions must be >= 0", ), + LoadProjectConfigErrorTestCase( + description="raises when janitor archive retention days is negative", + project_file_contents=""" +name = "demo" +adapter = "duckdb" + +[janitor] +archive_retention_days = -1 +""".strip(), + expected_error_fragment="janitor.archive_retention_days must be >= 0", + ), LoadProjectConfigErrorTestCase( description="raises when project settings contain unknown key", project_file_contents=""" diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/__init__.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/__init__.py new file mode 100644 index 0000000000..7acab39e3f --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/__init__.py @@ -0,0 +1 @@ +"""Janitor helper unit tests.""" diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py new file mode 100644 index 0000000000..a3cafb2c94 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class ArchiveNameParseTestCase: + description: str + name: str + expected_archived_at: datetime + expected_logical_name: str + + +@dataclass(frozen=True) +class ArchiveNameRejectTestCase: + description: str + name: str + expected_lookalike: bool + + +@dataclass(frozen=True) +class ArchiveNameBuildTestCase: + description: str + original_name: str + archived_at: datetime + identifier_limit: int + expected_prefix: str + expected_length: int + expected_logical_name_is_original: bool diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py new file mode 100644 index 0000000000..bd2c1d3568 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.executor.janitor._helpers.archive_names import ( + build_archive_name, + is_archive_lookalike_name, + parse_archive_name, +) +from sqlbuild.executor.janitor.models import JanitorParsedArchiveName +from tests.unit.src.sqlbuild.executor.janitor._helpers._test_types import ( + ArchiveNameBuildTestCase, + ArchiveNameParseTestCase, + ArchiveNameRejectTestCase, +) + +ARCHIVED_AT: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) +LONG_NAME: str = "fulfillment_orders_" + "y" * 80 + + +@pytest.mark.parametrize( + "test_case", + [ + ArchiveNameParseTestCase( + description="canonical archive name parses", + name="_SQB_ARCHIVE__20260924T101500Z__orders", + expected_archived_at=ARCHIVED_AT, + expected_logical_name="orders", + ), + ArchiveNameParseTestCase( + description="lowercase folded archive name parses", + name="_sqb_archive__20260924t101500z__orders", + expected_archived_at=ARCHIVED_AT, + expected_logical_name="orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_strict_archive_name_when_parsing_then_returns_timestamp_and_logical_name( + test_case: ArchiveNameParseTestCase, +) -> None: + parsed: JanitorParsedArchiveName | None = parse_archive_name(test_case.name) + + assert parsed == JanitorParsedArchiveName( + archived_at=test_case.expected_archived_at, + logical_name=test_case.expected_logical_name, + ) + assert is_archive_lookalike_name(test_case.name) is True + + +@pytest.mark.parametrize( + "test_case", + [ + ArchiveNameRejectTestCase( + description="impossible calendar timestamp is malformed", + name="_SQB_ARCHIVE__20261324T101500Z__orders", + expected_lookalike=True, + ), + ArchiveNameRejectTestCase( + description="missing timezone marker is malformed", + name="_SQB_ARCHIVE__20260924T101500__orders", + expected_lookalike=True, + ), + ArchiveNameRejectTestCase( + description="single separator after prefix is malformed", + name="_SQB_ARCHIVE_20260924T101500Z__orders", + expected_lookalike=True, + ), + ArchiveNameRejectTestCase( + description="empty original name is malformed", + name="_SQB_ARCHIVE__20260924T101500Z__", + expected_lookalike=True, + ), + ArchiveNameRejectTestCase( + description="ordinary relation is neither archive nor look-alike", + name="orders_archive", + expected_lookalike=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_non_strict_name_when_parsing_archive_name_then_returns_none( + test_case: ArchiveNameRejectTestCase, +) -> None: + parsed: JanitorParsedArchiveName | None = parse_archive_name(test_case.name) + + assert parsed is None + assert is_archive_lookalike_name(test_case.name) is test_case.expected_lookalike + + +@pytest.mark.parametrize( + "test_case", + [ + ArchiveNameBuildTestCase( + description="short name keeps the full original name", + original_name="orders", + archived_at=ARCHIVED_AT, + identifier_limit=255, + expected_prefix="_SQB_ARCHIVE__20260924T101500Z__orders", + expected_length=len("_SQB_ARCHIVE__20260924T101500Z__orders"), + expected_logical_name_is_original=True, + ), + ArchiveNameBuildTestCase( + description="long name is fitted while the timestamp stays intact", + original_name=LONG_NAME, + archived_at=ARCHIVED_AT.replace(microsecond=654321), + identifier_limit=63, + expected_prefix="_SQB_ARCHIVE__20260924T101500Z__fulfillment_orders_", + expected_length=63, + expected_logical_name_is_original=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_original_name_when_building_archive_name_then_round_trips_through_parser( + test_case: ArchiveNameBuildTestCase, +) -> None: + archive_name: str = build_archive_name( + original_name=test_case.original_name, + archived_at=test_case.archived_at, + identifier_limit=test_case.identifier_limit, + ) + + parsed: JanitorParsedArchiveName | None = parse_archive_name(archive_name) + assert archive_name.startswith(test_case.expected_prefix) + assert len(archive_name) == test_case.expected_length + assert parsed is not None + assert parsed.archived_at == ARCHIVED_AT + assert (parsed.logical_name == test_case.original_name) is ( + test_case.expected_logical_name_is_original + ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 3209ab37ae..260daf1b5c 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -72,3 +72,43 @@ def relation_info( created_at=created_at, last_altered_at=last_altered_at, ) + + +@dataclass(frozen=True) +class JanitorArchivePlanTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + direct_mode: bool = True + retention_days: int = 7 + archive_retention_days: int = 14 + source_schema: str | None = None + exclude_patterns: tuple[str, ...] = field(default_factory=tuple) + identifier_limit: int = 255 + expected_archive_source_names: tuple[str, ...] = field(default_factory=tuple) + expected_existing_archive_deletion_names: tuple[str, ...] = field(default_factory=tuple) + expected_same_run_deletion_source_names: tuple[str, ...] = field(default_factory=tuple) + expected_retained_archive_names: tuple[str, ...] = field(default_factory=tuple) + expected_skipped_relations: tuple[tuple[str, str], ...] = field(default_factory=tuple) + expected_suppressed_archive_deletion_names: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorArchiveExecutionTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + archive_retention_days: int + expected_renamed_origins: tuple[str, ...] + expected_view_rename_origins: tuple[str, ...] + expected_dropped_targets: tuple[str, ...] + expected_dropped_view_targets: tuple[str, ...] + expected_event_insert_count: int + expected_event_table_create_count: int + + +@dataclass(frozen=True) +class JanitorArchiveEventFailureTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + expected_error_fragment: str + expected_renamed_origins: tuple[str, ...] + expected_dropped_targets: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 125b93c2bb..603aebfa2d 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -30,6 +30,7 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS from sqlbuild.spec.contracts.models import SchemaSeedEntry, SeedCsvSettings, SourceEntry @@ -43,10 +44,14 @@ def __init__( relation_infos: tuple[RelationInfo, ...], supports_age_metadata: bool = True, tracked_relations: tuple[tuple[str | None, str | None, str], ...] = (), + identifier_limit: int = 255, ) -> None: self.relation_infos: tuple[RelationInfo, ...] = relation_infos self.age_metadata_supported: bool = supports_age_metadata + self.identifier_limit: int = identifier_limit self.dropped_targets: list[str] = [] + self.dropped_view_targets: list[str] = [] + self.renamed_targets: list[tuple[str, str]] = [] self.executed_sql: list[str] = [] self.tracked_relations: tuple[tuple[str | None, str | None, str], ...] = tracked_relations self._tracked_rows: tuple[tuple[Any, ...], ...] = tuple( @@ -101,6 +106,9 @@ def __init__( def supports_relation_age_metadata(self) -> bool: return self.age_metadata_supported + def maximum_identifier_length(self) -> int: + return self.identifier_limit + def default_schema(self) -> str | None: return "analytics" @@ -126,7 +134,9 @@ def _execute(self, connection: Any, sql: str) -> Any: True: (), False: self._tracked_rows, } - return _FakeResult(rows=rows_by_query_kind["LIMIT 0" in sql]) + return _FakeResult( + rows=rows_by_query_kind["LIMIT 0" in sql or JANITOR_EVENTS_TABLE_NAME in sql] + ) def list_relations( self, @@ -213,6 +223,17 @@ def drop( del connection, if_exists, statement_recorder self.dropped_targets.append(destination) + def drop_view( + self, + connection: Any, + *, + destination: str, + if_exists: bool = True, + statement_recorder: StatementRecorder, + ) -> None: + del connection, if_exists, statement_recorder + self.dropped_view_targets.append(destination) + def create_table_as( self, connection: Any, @@ -336,7 +357,8 @@ def rename( destination: str, statement_recorder: StatementRecorder, ) -> None: - raise NotImplementedError + del connection, statement_recorder + self.renamed_targets.append((origin, destination)) def swap( self, @@ -438,6 +460,28 @@ def drop( raise RuntimeError(self.message) +class FailingJanitorEventAdapter(FakeJanitorAdapter): + def _execute(self, connection: Any, sql: str) -> Any: + is_event_insert: bool = sql.startswith("INSERT INTO") and JANITOR_EVENTS_TABLE_NAME in sql + _EVENT_WRITE_GUARDS[is_event_insert](sql) + return super()._execute(connection=connection, sql=sql) + + +def _allow_statement(sql: str) -> None: + del sql + + +def _fail_event_write(sql: str) -> None: + del sql + raise RuntimeError("simulated janitor event write failure") + + +_EVENT_WRITE_GUARDS: dict[bool, Callable[[str], None]] = { + False: _allow_statement, + True: _fail_event_write, +} + + class _FakeResult: def __init__(self, *, rows: tuple[tuple[Any, ...], ...]) -> None: self.rows: tuple[tuple[Any, ...], ...] = rows diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py index 160c2bebb4..ffe82faf5e 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py @@ -2,23 +2,37 @@ from __future__ import annotations +from datetime import UTC, datetime, timedelta + import pytest from sqlbuild.executor.janitor.main.execute import execute_janitor_plan +from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import ( JanitorDeleteCandidate, JanitorDetachedVirtualEnvironmentCandidate, + JanitorDirectModeSettings, + JanitorExecutionResult, JanitorPlan, JanitorRelationKey, ) +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorArchiveEventFailureTestCase, + JanitorArchiveExecutionTestCase, JanitorExecutionOrderTestCase, + relation_info, ) from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( FailingDropAdapter, + FailingJanitorEventAdapter, + FakeJanitorAdapter, + build_project, relation_info_for_test, ) +OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) + @pytest.mark.parametrize( "test_case", @@ -73,3 +87,138 @@ def test_given_physical_drop_failure_when_executing_janitor_then_state_cleanup_i assert test_case.expected_error_fragment in str(exc_info.value) assert tuple(deleted_state_items) == test_case.expected_deleted_state_items assert adapter.dropped_targets == ["dev__sqb_physical.orders__v_old"] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveExecutionTestCase( + description="renames stale table and view, drops expired archive, and audits each", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("old_customers_view", relation_type="VIEW", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products"), + ), + archive_retention_days=14, + expected_renamed_origins=("analytics.old_orders",), + expected_view_rename_origins=("analytics.old_customers_view",), + expected_dropped_targets=("analytics._SQB_ARCHIVE__20200101T000000Z__old_products",), + expected_dropped_view_targets=(), + expected_event_insert_count=3, + expected_event_table_create_count=1, + ), + JanitorArchiveExecutionTestCase( + description="zero archive retention archives then drops in the same run", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("old_customers_view", relation_type="VIEW", created_at=OLD_TIME), + ), + archive_retention_days=0, + expected_renamed_origins=("analytics.old_orders",), + expected_view_rename_origins=("analytics.old_customers_view",), + expected_dropped_targets=("analytics._SQB_ARCHIVE__",), + expected_dropped_view_targets=("analytics._SQB_ARCHIVE__",), + expected_event_insert_count=4, + expected_event_table_create_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_direct_archive_plan_when_executing_then_renames_drops_and_writes_audit_events( + test_case: JanitorArchiveExecutionTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter(relation_infos=test_case.relation_infos) + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings( + enabled=True, archive_retention_days=test_case.archive_retention_days + ), + ) + + result: JanitorExecutionResult = execute_janitor_plan( + plan=plan, adapter=adapter, connection=object() + ) + + assert tuple(origin for origin, _ in adapter.renamed_targets) == ( + test_case.expected_renamed_origins + ) + view_rename_statements: tuple[str, ...] = tuple( + filter(lambda sql: sql.startswith("ALTER VIEW "), adapter.executed_sql) + ) + view_renames: tuple[str, ...] = tuple( + sql.removeprefix("ALTER VIEW ").split(" RENAME TO ")[0] for sql in view_rename_statements + ) + assert view_renames == test_case.expected_view_rename_origins + assert len(adapter.dropped_targets) == len(test_case.expected_dropped_targets) + for dropped, expected_prefix in zip( + adapter.dropped_targets, test_case.expected_dropped_targets, strict=True + ): + assert dropped.startswith(expected_prefix) + assert len(adapter.dropped_view_targets) == len(test_case.expected_dropped_view_targets) + for dropped, expected_prefix in zip( + adapter.dropped_view_targets, test_case.expected_dropped_view_targets, strict=True + ): + assert dropped.startswith(expected_prefix) + event_statements: tuple[str, ...] = tuple( + filter(lambda sql: JANITOR_EVENTS_TABLE_NAME in sql, adapter.executed_sql) + ) + event_inserts: tuple[str, ...] = tuple( + filter(lambda sql: sql.startswith("INSERT INTO"), event_statements) + ) + event_creates: tuple[str, ...] = tuple( + filter(lambda sql: sql.startswith("CREATE TABLE"), event_statements) + ) + assert len(event_inserts) == test_case.expected_event_insert_count + assert len(event_creates) == test_case.expected_event_table_create_count + assert tuple(candidate.key.name for candidate in result.archived) == tuple( + origin.rsplit(".", maxsplit=1)[-1] + for origin in (*test_case.expected_renamed_origins, *test_case.expected_view_rename_origins) + ) + assert len(result.deleted_archives) == len(test_case.expected_dropped_targets) + len( + test_case.expected_dropped_view_targets + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveEventFailureTestCase( + description="audit write failure after rename stops before further actions", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products"), + ), + expected_error_fragment="simulated janitor event write failure", + expected_renamed_origins=("analytics.old_orders",), + expected_dropped_targets=(), + ) + ], + ids=lambda case: case.description, +) +def test_given_audit_write_failure_when_archiving_then_rename_persists_and_error_propagates( + test_case: JanitorArchiveEventFailureTestCase, +) -> None: + adapter: FailingJanitorEventAdapter = FailingJanitorEventAdapter( + relation_infos=test_case.relation_infos + ) + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings(enabled=True), + ) + + with pytest.raises(RuntimeError) as exc_info: + execute_janitor_plan(plan=plan, adapter=adapter, connection=object()) + + assert test_case.expected_error_fragment in str(exc_info.value) + assert tuple(origin for origin, _ in adapter.renamed_targets) == ( + test_case.expected_renamed_origins + ) + assert tuple(adapter.dropped_targets) == test_case.expected_dropped_targets diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index acb1fc712f..cc77f5b520 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -280,7 +280,7 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp expected_dropped_targets=("analytics.old_orders",), ), JanitorExecuteTestCase( - description="direct mode reports eligible candidates without dropping", + description="direct mode archives eligible candidates instead of dropping them", relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), expected_dropped_targets=(), direct_mode=True, diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py new file mode 100644 index 0000000000..63ca92991b --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py @@ -0,0 +1,225 @@ +"""Tests for direct-mode janitor archive planning.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from itertools import chain + +import pytest + +from sqlbuild.executor.janitor.constants import MALFORMED_ARCHIVE_REASON +from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild.executor.janitor.models import ( + JanitorArchivedRelation, + JanitorDirectModeSettings, + JanitorPlan, + JanitorRelationKey, + JanitorSkippedRelation, +) +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorArchivePlanTestCase, + relation_info, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FakeJanitorAdapter, + build_project, +) + +OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) +RECENT_ARCHIVE_NAME: str = ( + f"_SQB_ARCHIVE__{(datetime.now(UTC) - timedelta(days=1)):%Y%m%dT%H%M%SZ}__old_customers" +) +LONG_ORDERS_NAME: str = "orders_" + "x" * 60 + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchivePlanTestCase( + description="stale relation is archived and kept until archive retention", + relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), + expected_archive_source_names=("old_orders",), + ), + JanitorArchivePlanTestCase( + description="zero archive retention deletes the new archive in the same run", + relation_infos=(relation_info("old_orders"),), + retention_days=0, + archive_retention_days=0, + expected_archive_source_names=("old_orders",), + expected_same_run_deletion_source_names=("old_orders",), + ), + JanitorArchivePlanTestCase( + description="expired archive is deleted and recent archive is retained", + relation_infos=( + relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products"), + relation_info(RECENT_ARCHIVE_NAME), + ), + expected_existing_archive_deletion_names=( + "_SQB_ARCHIVE__20200101T000000Z__old_products", + ), + expected_retained_archive_names=(RECENT_ARCHIVE_NAME,), + ), + JanitorArchivePlanTestCase( + description="case-folded archive name is recognized by strict grammar", + relation_infos=(relation_info("_sqb_archive__20200101t000000z__old_products"),), + expected_existing_archive_deletion_names=( + "_sqb_archive__20200101t000000z__old_products", + ), + ), + JanitorArchivePlanTestCase( + description="malformed archive look-alikes are reported and never archived or deleted", + relation_infos=( + relation_info("_SQB_ARCHIVE__20201399T000000Z__orders", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE_20200101T000000Z__orders", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE__2020010T000000Z__orders", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE__20200101T000000Z__", created_at=OLD_TIME), + relation_info("_sqb_archive_notes", created_at=OLD_TIME), + ), + retention_days=0, + archive_retention_days=0, + expected_skipped_relations=( + ("_SQB_ARCHIVE__20201399T000000Z__orders", MALFORMED_ARCHIVE_REASON), + ("_SQB_ARCHIVE_20200101T000000Z__orders", MALFORMED_ARCHIVE_REASON), + ("_SQB_ARCHIVE__2020010T000000Z__orders", MALFORMED_ARCHIVE_REASON), + ("_SQB_ARCHIVE__20200101T000000Z__", MALFORMED_ARCHIVE_REASON), + ("_sqb_archive_notes", MALFORMED_ARCHIVE_REASON), + ), + ), + JanitorArchivePlanTestCase( + description="archive matching an exclude pattern is skipped", + relation_infos=(relation_info("_SQB_ARCHIVE__20200101T000000Z__keep_orders"),), + exclude_patterns=("*keep_*",), + expected_skipped_relations=( + ( + "_SQB_ARCHIVE__20200101T000000Z__keep_orders", + "relation matches exclude pattern '*keep_*'", + ), + ), + ), + JanitorArchivePlanTestCase( + description="blocked schema suppresses expired archive deletion", + relation_infos=(relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products"),), + source_schema="analytics", + expected_suppressed_archive_deletion_names=( + "_SQB_ARCHIVE__20200101T000000Z__old_products", + ), + ), + JanitorArchivePlanTestCase( + description="virtual mode neither archives nor expires archive-named relations", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products", created_at=OLD_TIME), + ), + direct_mode=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_direct_warehouse_when_planning_janitor_then_plans_archives_by_strict_name( + test_case: JanitorArchivePlanTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter( + relation_infos=test_case.relation_infos, + identifier_limit=test_case.identifier_limit, + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(source_schema=test_case.source_schema), + adapter=adapter, + connection=object(), + retention_days=test_case.retention_days, + delete_tracked_only=False, + exclude_patterns=test_case.exclude_patterns, + direct_settings=JanitorDirectModeSettings( + enabled=test_case.direct_mode, + archive_retention_days=test_case.archive_retention_days, + ), + ) + + assert ( + tuple(candidate.key.name for candidate in plan.archive_candidates) + == test_case.expected_archive_source_names + ) + existing_deletions: tuple[JanitorArchivedRelation, ...] = tuple( + filter(lambda archive: archive.original_key is None, plan.archive_deletion_candidates) + ) + same_run_original_keys: tuple[JanitorRelationKey, ...] = tuple( + filter(None, (archive.original_key for archive in plan.archive_deletion_candidates)) + ) + assert ( + tuple(archive.key.name for archive in existing_deletions) + == test_case.expected_existing_archive_deletion_names + ) + assert ( + tuple(key.name for key in same_run_original_keys) + == test_case.expected_same_run_deletion_source_names + ) + assert ( + tuple(archive.key.name for archive in plan.retained_archives) + == test_case.expected_retained_archive_names + ) + archive_skipped: tuple[JanitorSkippedRelation, ...] = tuple( + filter( + lambda skipped: skipped.key.name.lower().startswith("_sqb_archive"), + plan.skipped_relations, + ) + ) + assert ( + tuple((skipped.key.name, skipped.reason) for skipped in archive_skipped) + == test_case.expected_skipped_relations + ) + suppressed: tuple[JanitorArchivedRelation, ...] = tuple( + chain.from_iterable( + blocked.suppressed_archive_deletions for blocked in plan.blocked_schemas + ) + ) + assert ( + tuple(archive.key.name for archive in suppressed) + == test_case.expected_suppressed_archive_deletion_names + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchivePlanTestCase( + description="long relation name is fitted with the archive timestamp intact", + relation_infos=(relation_info(LONG_ORDERS_NAME, created_at=OLD_TIME),), + identifier_limit=48, + expected_archive_source_names=(LONG_ORDERS_NAME,), + ), + ], + ids=lambda case: case.description, +) +def test_given_long_relation_name_when_planning_archive_then_fits_name_with_timestamp_intact( + test_case: JanitorArchivePlanTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter( + relation_infos=test_case.relation_infos, + identifier_limit=test_case.identifier_limit, + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=test_case.retention_days, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings(enabled=True), + ) + + assert ( + tuple(candidate.key.name for candidate in plan.archive_candidates) + == test_case.expected_archive_source_names + ) + archive_name: str = plan.archive_candidates[0].archive_key.name + expected_prefix: str = ( + f"_SQB_ARCHIVE__{plan.archive_candidates[0].archived_at:%Y%m%dT%H%M%SZ}__orders_" + ) + assert len(archive_name) == test_case.identifier_limit + assert archive_name.startswith(expected_prefix) + assert archive_name != f"{expected_prefix[:-7]}{LONG_ORDERS_NAME}" + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From a8616580154c92906272e0082911dad353fb88c0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:51:44 +0100 Subject: [PATCH 2/8] fix(janitor): only act on unquoted-addressable names and keep virtual retention default Direct-mode janitor now renames or drops a relation only when its catalog name is a plain lowercase identifier that does not collide case-insensitively with another relation in the same schema listing. Other names are reported as skipped and never touched. Archive names are generated in case-folded form so the archives the janitor creates always pass this guard. An unset [janitor] retention_days now resolves to 14 days in direct mode and keeps the 30-day default in virtual-environment mode; explicit values and the --retention-days override apply to both modes as before. --- .../_helpers/janitor_runtime/invocation.py | 9 +- .../discovery/_helpers/yml/project.py | 4 +- .../janitor/_helpers/archive_names.py | 4 +- .../janitor/_helpers/archive_planning.py | 39 +++++-- .../janitor/_helpers/classification.py | 11 ++ .../janitor/_helpers/relation_addressing.py | 31 +++++ src/sqlbuild/executor/janitor/constants.py | 10 ++ .../janitor/main/resolve_retention_days.py | 20 ++++ src/sqlbuild/spec/contracts/models.py | 2 +- .../cli/commands/main/janitor/_test_types.py | 11 ++ .../cli/commands/main/janitor/helpers.py | 4 +- .../cli/commands/main/janitor/test_janitor.py | 8 +- .../main/janitor/test_janitor_archive.py | 107 ++++++++++++++---- .../cli/commands/main/janitor/_test_types.py | 9 ++ .../cli/commands/main/janitor/helpers.py | 4 +- .../cli/commands/main/janitor/test_janitor.py | 4 +- .../main/janitor/test_janitor_settings.py | 98 ++++++++++++++++ .../discovery/_helpers/_test_types.py | 2 +- .../discovery/_helpers/test_yml_project.py | 2 +- .../executor/janitor/_helpers/_test_types.py | 8 ++ .../janitor/_helpers/test_archive_names.py | 6 +- .../_helpers/test_relation_addressing.py | 79 +++++++++++++ .../executor/janitor/main/_test_types.py | 20 ++++ .../executor/janitor/main/test_execute.py | 10 +- .../janitor/main/test_plan_archives.py | 100 ++++++++++++++-- .../main/test_resolve_retention_days.py | 70 ++++++++++++ 26 files changed, 602 insertions(+), 70 deletions(-) create mode 100644 src/sqlbuild/executor/janitor/_helpers/relation_addressing.py create mode 100644 src/sqlbuild/executor/janitor/main/resolve_retention_days.py create mode 100644 tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py index 1282a4b609..53e5bee982 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py @@ -12,6 +12,7 @@ ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days from sqlbuild.presentation.main.supports_color import supports_color @@ -36,10 +37,10 @@ def resolve_janitor_settings( ) -> JanitorSettings: """Resolve and validate effective janitor settings.""" - retention_days: int = ( - request.retention_days - if request.retention_days is not None - else invocation.discovered_inputs.project_config.janitor.retention_days + retention_days: int = resolve_janitor_retention_days( + override=request.retention_days, + configured=invocation.discovered_inputs.project_config.janitor.retention_days, + virtual_environments=invocation.discovered_inputs.project_config.settings.virtual_environments, ) if retention_days < 0: raise CliUserError("janitor --retention-days must be >= 0", code="C501") diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index ac7550b8a9..0662eecb27 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -1427,7 +1427,7 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: payload=payload, label="janitor", file_path=file_path ) enabled: bool = _optional_bool(mapping=mapping, key="enabled", default=False) - retention_days: int = _optional_int(mapping=mapping, key="retention_days", default=14) + retention_days: int | None = _optional_nullable_int(mapping=mapping, key="retention_days") archive_retention_days: int = _optional_int( mapping=mapping, key="archive_retention_days", default=14 ) @@ -1449,7 +1449,7 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: file_path=file_path, ) ) - if retention_days < 0: + if retention_days is not None and retention_days < 0: raise ProjectConfigError(f"{file_path} janitor.retention_days must be >= 0") if archive_retention_days < 0: raise ProjectConfigError(f"{file_path} janitor.archive_retention_days must be >= 0") diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_names.py b/src/sqlbuild/executor/janitor/_helpers/archive_names.py index 45c9081b85..b9382bfa27 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_names.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_names.py @@ -32,7 +32,7 @@ def archive_timestamp(value: datetime) -> datetime: def build_archive_name(*, original_name: str, archived_at: datetime, identifier_limit: int) -> str: - """Build the fitted archive name, keeping the prefix and timestamp intact.""" + """Build the fitted, case-folded archive name, keeping the prefix and timestamp intact.""" fixed_prefix: str = ( f"{ARCHIVE_NAME_PREFIX}" @@ -45,7 +45,7 @@ def build_archive_name(*, original_name: str, archived_at: datetime, identifier_ identifier_limit=identifier_limit, artifact_label=ARCHIVE_ARTIFACT_LABEL, ) - return f"{fixed_prefix}{logical_part}" + return f"{fixed_prefix}{logical_part}".lower() def parse_archive_name(name: str) -> JanitorParsedArchiveName | None: diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_planning.py b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py index 42460f9ac2..d0749b5dbd 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_planning.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py @@ -15,6 +15,10 @@ ) from sqlbuild.executor.janitor._helpers.classification import matching_exclude_pattern from sqlbuild.executor.janitor._helpers.plan import relation_key as build_relation_key +from sqlbuild.executor.janitor._helpers.relation_addressing import ( + case_colliding_names, + unaddressable_relation_reason, +) from sqlbuild.executor.janitor.constants import BUILT_IN_EXCLUDE_PATTERNS, MALFORMED_ARCHIVE_REASON from sqlbuild.executor.janitor.models import ( JanitorArchiveCandidate, @@ -97,8 +101,12 @@ def _plan_direct_archives( effective_patterns: tuple[str, ...] = BUILT_IN_EXCLUDE_PATTERNS + exclude_patterns schema_key: tuple[str | None, str | None] for schema_key in sorted(managed_target_schemas, key=lambda key: (key[0] or "", key[1] or "")): + schema_relations: tuple[RelationInfo, ...] = facts.relations_by_schema.get(schema_key, ()) + colliding_names: frozenset[str] = case_colliding_names( + relation.name for relation in schema_relations + ) relation: RelationInfo - for relation in facts.relations_by_schema.get(schema_key, ()): + for relation in schema_relations: key: JanitorRelationKey = build_relation_key(relation) if key in facts.desired_keys or not is_archive_lookalike_name(key.name): continue @@ -110,16 +118,12 @@ def _plan_direct_archives( ) ) continue - exclude_pattern: str | None = matching_exclude_pattern( - key=key, patterns=effective_patterns + skip_reason: str | None = _archive_skip_reason( + key=key, colliding_names=colliding_names, effective_patterns=effective_patterns ) - if exclude_pattern is not None: + if skip_reason is not None: skipped.append( - JanitorSkippedRelation( - key=key, - relation=relation, - reason=f"relation matches exclude pattern {exclude_pattern!r}", - ) + JanitorSkippedRelation(key=key, relation=relation, reason=skip_reason) ) continue archived: JanitorArchivedRelation = JanitorArchivedRelation( @@ -162,6 +166,23 @@ def _plan_direct_archives( ) +def _archive_skip_reason( + *, + key: JanitorRelationKey, + colliding_names: frozenset[str], + effective_patterns: tuple[str, ...], +) -> str | None: + addressing_reason: str | None = unaddressable_relation_reason( + name=key.name, colliding_names=colliding_names + ) + if addressing_reason is not None: + return addressing_reason + exclude_pattern: str | None = matching_exclude_pattern(key=key, patterns=effective_patterns) + if exclude_pattern is not None: + return f"relation matches exclude pattern {exclude_pattern!r}" + return None + + def _archive_candidate( *, candidate: JanitorDeleteCandidate, diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index 1a1cd850c3..78f282f290 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -27,6 +27,10 @@ from sqlbuild.executor.janitor._helpers.plan import ( relation_key as build_relation_key, ) +from sqlbuild.executor.janitor._helpers.relation_addressing import ( + case_colliding_names, + unaddressable_relation_reason, +) from sqlbuild.executor.janitor._helpers.tracking import collect_tracked_relation_keys from sqlbuild.executor.janitor.models import ( JanitorDeleteCandidate, @@ -176,6 +180,9 @@ def classify_janitor_relations( candidates: list[JanitorDeleteCandidate] = [] skipped_relations: list[JanitorSkippedRelation] = [] + colliding_names: frozenset[str] = case_colliding_names( + relation.name for relation in schema_relations + ) relation: RelationInfo for relation in schema_relations: relation_key: JanitorRelationKey = build_relation_key(relation) @@ -191,6 +198,10 @@ def classify_janitor_relations( effective_exclude_patterns=effective_exclude_patterns, delete_tracked_only=delete_tracked_only, ) + if skip_reason is None and direct_mode: + skip_reason = unaddressable_relation_reason( + name=relation_key.name, colliding_names=colliding_names + ) if skip_reason is not None: skipped_relations.append( JanitorSkippedRelation(key=relation_key, relation=relation, reason=skip_reason) diff --git a/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py b/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py new file mode 100644 index 0000000000..4f499785cd --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py @@ -0,0 +1,31 @@ +"""Direct-mode janitor guard for relation names that are safe to address unquoted.""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Iterable + +from sqlbuild.executor.janitor.constants import ( + CASE_COLLISION_REASON, + UNQUOTED_ADDRESSING_REASON, +) + +_PLAIN_LOWERCASE_IDENTIFIER_RE: re.Pattern[str] = re.compile(r"^[a-z_][a-z0-9_$]*$") + + +def case_colliding_names(names: Iterable[str]) -> frozenset[str]: + """Return case-folded names shared by more than one relation in a listing.""" + + counts: Counter[str] = Counter(name.lower() for name in names) + return frozenset(name for name, count in counts.items() if count > 1) + + +def unaddressable_relation_reason(*, name: str, colliding_names: frozenset[str]) -> str | None: + """Return why the janitor must not rename or drop a relation by its catalog name.""" + + if _PLAIN_LOWERCASE_IDENTIFIER_RE.fullmatch(name) is None: + return UNQUOTED_ADDRESSING_REASON + if name.lower() in colliding_names: + return CASE_COLLISION_REASON + return None diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 517ee7461a..5cd2663418 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -18,11 +18,21 @@ JANITOR_EVENTS_TABLE_NAME, ) +DIRECT_DEFAULT_RETENTION_DAYS: int = 14 +VIRTUAL_DEFAULT_RETENTION_DAYS: int = 30 + ARCHIVE_NAME_PREFIX: str = "_SQB_ARCHIVE__" ARCHIVE_NAME_SEPARATOR: str = "__" ARCHIVE_TIMESTAMP_FORMAT: str = "%Y%m%dT%H%M%SZ" ARCHIVE_LOOKALIKE_PREFIX: str = "_sqb_archive" ARCHIVE_ARTIFACT_LABEL: str = "Janitor archive" +UNQUOTED_ADDRESSING_REASON: str = ( + "relation name is not a plain lowercase identifier and may require quoting; " + "janitor does not act on it" +) +CASE_COLLISION_REASON: str = ( + "relation name collides case-insensitively with another relation; janitor does not act on it" +) MALFORMED_ARCHIVE_REASON: str = ( "name resembles a janitor archive but does not match the strict archive grammar" ) diff --git a/src/sqlbuild/executor/janitor/main/resolve_retention_days.py b/src/sqlbuild/executor/janitor/main/resolve_retention_days.py new file mode 100644 index 0000000000..19e6372361 --- /dev/null +++ b/src/sqlbuild/executor/janitor/main/resolve_retention_days.py @@ -0,0 +1,20 @@ +"""Janitor relation retention resolution entrypoint.""" + +from __future__ import annotations + +from sqlbuild.executor.janitor.constants import ( + DIRECT_DEFAULT_RETENTION_DAYS, + VIRTUAL_DEFAULT_RETENTION_DAYS, +) + + +def resolve_janitor_retention_days( + *, override: int | None, configured: int | None, virtual_environments: bool +) -> int: + """Resolve CLI override, explicit config, then the mode-specific default.""" + + if override is not None: + return override + if configured is not None: + return configured + return VIRTUAL_DEFAULT_RETENTION_DAYS if virtual_environments else DIRECT_DEFAULT_RETENTION_DAYS diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index 16f03df88e..8d92ed41af 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -313,7 +313,7 @@ class JanitorConfig: """Janitor command defaults.""" enabled: bool = False - retention_days: int = 14 + retention_days: int | None = None archive_retention_days: int = 14 max_checkpoints: int = 20 direct_state_history_versions: int = 20 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 72f5e1d3a0..d12e339fac 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -218,3 +218,14 @@ class JanitorArchiveInterruptionE2ETestCase: expected_first_output_fragments: tuple[str, ...] expected_second_exit_code: int expected_second_stdout_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorUnaddressableRelationE2ETestCase: + """Test case for relations the janitor cannot safely address unquoted.""" + + description: str + janitor_command: tuple[str, ...] + plain_relation: str + quoted_relation: str + expected_stdout_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index cfd640db80..ca85dbc78a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -320,7 +320,7 @@ def write_janitor_test_settings( def archive_timestamp_text(value: datetime) -> str: """Render the UTC timestamp component embedded in janitor archive names.""" - return value.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ") + return value.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ").lower() def list_archive_names(*, db_path: Path, schema: str = "main") -> tuple[str, ...]: @@ -330,7 +330,7 @@ def list_archive_names(*, db_path: Path, schema: str = "main") -> tuple[str, ... db_path=db_path, sql=( "SELECT table_name FROM information_schema.tables " - f"WHERE table_schema = '{schema}' AND starts_with(table_name, '_SQB_ARCHIVE__') " + f"WHERE table_schema = '{schema}' AND starts_with(table_name, '_sqb_archive__') " "ORDER BY table_name" ), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 9d50387d86..4bc14b255e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -89,7 +89,7 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( "relations to archive 1", "archives to delete 0", "objects skipped 4", - "main.janitor_tracked_extra -> main._SQB_ARCHIVE__", + "main.janitor_tracked_extra -> main._sqb_archive__", "main.janitor_untracked_extra relation is not tracked by SQLBuild", "main.partition_state relation matches exclude pattern 'partition_*'", "main._sqlbuild_fingerprints relation matches exclude pattern", @@ -157,7 +157,7 @@ def test_given_stale_relations_when_running_direct_janitor_then_it_archives_trac db_path=db_path, sql=( "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_name LIKE '_SQB_ARCHIVE__%__{original_name}'" + f"WHERE table_name LIKE '_sqb_archive__%__{original_name}'" ), ) == [(1,)] assert "Eligible expired VDEs" not in janitor_result.stdout @@ -372,8 +372,8 @@ def test_given_virtual_microbatch_history_when_running_janitor_then_events_remai expected_stdout_fragments=( "relations to archive 2", "objects skipped 2", - "main.__sqb_a13f09c2e7b8__model__daily_revenue -> main._SQB_ARCHIVE__", - "main.__sqb_a13f09c2e7b8__source__raw_orders -> main._SQB_ARCHIVE__", + "main.__sqb_a13f09c2e7b8__model__daily_revenue -> main._sqb_archive__", + "main.__sqb_a13f09c2e7b8__source__raw_orders -> main._sqb_archive__", "main.__sqb_a13f09c2e7b__model__daily_revenue relation is not tracked by SQLBuild", ), expected_existing_tables=( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py index 271f131ed5..193fd3d19a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py @@ -15,6 +15,7 @@ JanitorArchiveInterruptionE2ETestCase, JanitorArchiveNameFittingE2ETestCase, JanitorArchiveRetentionE2ETestCase, + JanitorUnaddressableRelationE2ETestCase, JanitorZeroRetentionE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( @@ -34,10 +35,10 @@ ) ARCHIVE_NAME_PATTERN: re.Pattern[str] = re.compile( - r"^_SQB_ARCHIVE__(?P[0-9]{8}T[0-9]{6}Z)__(?P.+)$" + r"^_sqb_archive__(?P[0-9]{8}t[0-9]{6}z)__(?P.+)$" ) RECENT_ARCHIVE_NAME: str = ( - f"_SQB_ARCHIVE__{archive_timestamp_text(datetime.now(UTC) - timedelta(days=1))}__recent_orders" + f"_sqb_archive__{archive_timestamp_text(datetime.now(UTC) - timedelta(days=1))}__recent_orders" ) @@ -55,7 +56,7 @@ "archive retention 14 days", "relations to archive 1", "Relations to archive", - "main.customers -> main._SQB_ARCHIVE__", + "main.customers -> main._sqb_archive__", "age 20d, delete after", "main.products relation is newer than 14 days", "Archived 1 relations.", @@ -137,28 +138,31 @@ def test_given_retired_tracked_relation_past_retention_when_running_janitor_then description="only expired strict archives in managed schemas are deleted", janitor_command=("--no-color", "janitor", "--auto-approve"), expected_deleted_names=( - "_SQB_ARCHIVE__20200101T000000Z__old_customers_view", - "_SQB_ARCHIVE__20200101T000000Z__old_products", + "_sqb_archive__20200101t000000z__old_customers_view", + "_sqb_archive__20200101t000000z__old_products", ), expected_kept_names=( RECENT_ARCHIVE_NAME, - "_SQB_ARCHIVE__20201399T000000Z__bad_month", + "_sqb_archive__20201399t000000z__bad_month", "_SQB_ARCHIVE_20200101T000000Z__one_separator", "_sqb_archive_notes", + "_SQB_ARCHIVE__20200101T000000Z__legacy_orders", ), - expected_kept_other_schema_names=("_SQB_ARCHIVE__20200101T000000Z__orders",), + expected_kept_other_schema_names=("_sqb_archive__20200101t000000z__orders",), expected_stdout_fragments=( "archives to delete 2", "archives retained 1", "Archives to delete", - "main._SQB_ARCHIVE__20200101T000000Z__old_products archived 2020-01-01 00:00:00", + "main._sqb_archive__20200101t000000z__old_products archived 2020-01-01 00:00:00", "expired 2020-01-15 00:00:00 UTC", "Retained archives", f"main.{RECENT_ARCHIVE_NAME} archived", - "main._SQB_ARCHIVE__20201399T000000Z__bad_month name resembles a janitor archive " + "main._sqb_archive__20201399t000000z__bad_month name resembles a janitor archive " "but does not match the strict archive grammar", "main._SQB_ARCHIVE_20200101T000000Z__one_separator name resembles a janitor archive", "main._sqb_archive_notes name resembles a janitor archive", + "main._SQB_ARCHIVE__20200101T000000Z__legacy_orders relation name is not a plain " + "lowercase identifier and may require quoting; janitor does not act on it", "Deleted 2 objects", ), expected_delete_event_count=2, @@ -184,29 +188,30 @@ def test_given_archive_named_relations_when_running_janitor_then_deletes_only_ex execute_duckdb( db_path=db_path, sql=( - 'CREATE TABLE main."_SQB_ARCHIVE__20200101T000000Z__old_products" AS SELECT 1 AS id; ' - 'CREATE VIEW main."_SQB_ARCHIVE__20200101T000000Z__old_customers_view" AS ' + 'CREATE TABLE main."_sqb_archive__20200101t000000z__old_products" AS SELECT 1 AS id; ' + 'CREATE VIEW main."_sqb_archive__20200101t000000z__old_customers_view" AS ' "SELECT 1 AS id; " f'CREATE TABLE main."{RECENT_ARCHIVE_NAME}" AS SELECT 1 AS id; ' - 'CREATE TABLE main."_SQB_ARCHIVE__20201399T000000Z__bad_month" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_sqb_archive__20201399t000000z__bad_month" AS SELECT 1 AS id; ' 'CREATE TABLE main."_SQB_ARCHIVE_20200101T000000Z__one_separator" AS SELECT 1 AS id; ' 'CREATE TABLE main."_sqb_archive_notes" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_SQB_ARCHIVE__20200101T000000Z__legacy_orders" AS SELECT 1 AS id; ' "CREATE SCHEMA other; " - 'CREATE TABLE other."_SQB_ARCHIVE__20200101T000000Z__orders" AS SELECT 1 AS id; ' + 'CREATE TABLE other."_sqb_archive__20200101t000000z__orders" AS SELECT 1 AS id; ' + DuckDbAdapter().render_create_janitor_event_table_sql(database=None, schema="main") + "; INSERT INTO main._sqlbuild_janitor_events " "(event_id, schema_version, event_type, occurred_at, run_id, relation_schema, " "original_name, archive_name, archive_qualified_name, archived_at) VALUES " "('misleading_delete', 1, 'delete', TIMESTAMP '2020-02-01 00:00:00', 'run_old', " - "'main', 'old_products', '_SQB_ARCHIVE__20200101T000000Z__old_products', " - "'main._SQB_ARCHIVE__20200101T000000Z__old_products', " + "'main', 'old_products', '_sqb_archive__20200101t000000z__old_products', " + "'main._sqb_archive__20200101t000000z__old_products', " "TIMESTAMP '2020-01-01 00:00:00'), " "('misleading_archive', 1, 'archive', TIMESTAMP '2020-01-01 00:00:00', 'run_old', " f"'main', 'recent_orders', '{RECENT_ARCHIVE_NAME}', 'main.{RECENT_ARCHIVE_NAME}', " "TIMESTAMP '2020-01-01 00:00:00'), " "('misleading_malformed', 1, 'archive', TIMESTAMP '2020-01-01 00:00:00', 'run_old', " - "'main', 'bad_month', '_SQB_ARCHIVE__20201399T000000Z__bad_month', " - "'main._SQB_ARCHIVE__20201399T000000Z__bad_month', TIMESTAMP '2020-01-01 00:00:00')" + "'main', 'bad_month', '_sqb_archive__20201399t000000z__bad_month', " + "'main._sqb_archive__20201399t000000z__bad_month', TIMESTAMP '2020-01-01 00:00:00')" ), ) @@ -372,9 +377,9 @@ def test_given_small_identifier_limit_when_archiving_long_name_then_fits_name_an archive_name: str = archive_names[0] match: re.Match[str] | None = ARCHIVE_NAME_PATTERN.match(archive_name) assert match is not None - archived_at: datetime = datetime.strptime(match.group("timestamp"), "%Y%m%dT%H%M%SZ").replace( - tzinfo=UTC - ) + archived_at: datetime = datetime.strptime( + match.group("timestamp").upper(), "%Y%m%dT%H%M%SZ" + ).replace(tzinfo=UTC) assert before <= archived_at <= after logical_name: str = match.group("logical_name") assert len(archive_name) == test_case.identifier_limit @@ -466,3 +471,65 @@ def test_given_audit_write_interrupted_after_rename_when_janitor_reruns_then_arc assert [(event[0], event[1], event[3]) for event in read_janitor_events(db_path=db_path)] == [ ("delete", None, stranded_archives[0]) ] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUnaddressableRelationE2ETestCase( + description="quoted mixed-case relation is reported and left while plain one archives", + janitor_command=("--no-color", "janitor", "--auto-approve"), + plain_relation="old_orders", + quoted_relation="LegacyCustomers", + expected_stdout_fragments=( + "relations to archive 1", + "main.old_orders -> main._sqb_archive__", + "main.LegacyCustomers relation name is not a plain lowercase identifier and may " + "require quoting; janitor does not act on it", + "Archived 1 relations.", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_quoted_mixed_case_relation_when_running_janitor_then_it_is_reported_and_untouched( + test_case: JanitorUnaddressableRelationE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_unaddressable_relation", + janitor_config="enabled = true\nretention_days = 0\ndelete_tracked_only = false\n", + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + execute_duckdb( + db_path=db_path, + sql=( + f"CREATE TABLE main.{test_case.plain_relation} AS SELECT 1 AS id; " + f'CREATE TABLE main."{test_case.quoted_relation}" AS SELECT 2 AS id' + ), + ) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in janitor_result.stdout + assert not table_exists(db_path=db_path, table_name=test_case.plain_relation) + assert table_exists(db_path=db_path, table_name=test_case.quoted_relation) + assert query_duckdb( + db_path=db_path, sql=f'SELECT id FROM main."{test_case.quoted_relation}"' + ) == [(2,)] + archive_names: tuple[str, ...] = list_archive_names(db_path=db_path) + assert tuple(ARCHIVE_NAME_PATTERN.sub(r"\g", name) for name in archive_names) == ( + test_case.plain_relation, + ) + assert [(event[0], event[1]) for event in read_janitor_events(db_path=db_path)] == [ + ("archive", test_case.plain_relation) + ] diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py index bb20ba08fa..e87261302d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -31,3 +31,12 @@ class JanitorDirectArchiveConfirmationTestCase: typed_response: str expected_result: bool expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorSettingsResolutionTestCase: + description: str + project_toml: str + cli_retention_days: int | None + expected_retention_days: int + expected_archive_retention_days: int diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py index abbf9b1185..058912dca2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -140,7 +140,7 @@ def build_direct_archive_plan() -> JanitorPlan: archive_key=JanitorRelationKey( database=None, schema="dev", - name="_SQB_ARCHIVE__20260924T101500Z__old_orders", + name="_sqb_archive__20260924t101500z__old_orders", ), archived_at=archived_at, expires_at=archived_at + timedelta(days=14), @@ -151,7 +151,7 @@ def build_direct_archive_plan() -> JanitorPlan: key=JanitorRelationKey( database=None, schema="dev", - name="_SQB_ARCHIVE__20260101T000000Z__old_products", + name="_sqb_archive__20260101t000000z__old_products", ), relation_type="table", archived_at=datetime(2026, 1, 1, tzinfo=UTC), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index f6cdee93fe..04237cafab 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -266,9 +266,9 @@ def test_given_direct_archive_plan_when_confirming_then_requires_exact_archive_p " archives to delete 1", " archives retained 0", "Relations to archive\n dev.old_orders -> " - "dev._SQB_ARCHIVE__20260924T101500Z__old_orders " + "dev._sqb_archive__20260924t101500z__old_orders " "age 20d, delete after 2026-10-08 10:15:00 UTC", - "Archives to delete\n dev._SQB_ARCHIVE__20260101T000000Z__old_products " + "Archives to delete\n dev._sqb_archive__20260101t000000z__old_products " "archived 2026-01-01 00:00:00 UTC, age 266d, expired 2026-01-15 00:00:00 UTC", ), unexpected_output_fragments=("eligible for deletion", "Eligible objects"), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py new file mode 100644 index 0000000000..a97d044d23 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py @@ -0,0 +1,98 @@ +"""Tests for janitor settings resolution from discovered project config.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.commands._helpers.janitor_runtime.invocation import ( + resolve_janitor_invocation, + resolve_janitor_settings, +) +from sqlbuild.cli.commands.models import ( + JanitorCommandRequest, + JanitorInvocation, + JanitorSettings, +) +from tests.unit.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorSettingsResolutionTestCase, +) + +DIRECT_PROJECT_TOML: str = ( + 'name = "orders_direct"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +VIRTUAL_PROJECT_TOML: str = ( + 'name = "orders_virtual"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[settings]\n" + "virtual_environments = true\n\n" + "[connection]\n" + 'database = "warehouse.duckdb"\n\n' + "[targets.dev]\n" + 'schema = "dev"\n\n' + "[targets.dev.state]\n" + 'backend = "duckdb"\n' + 'schema = "sqlbuild_state"\n\n' + "[targets.dev.state.connection]\n" + 'database = "state.duckdb"\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorSettingsResolutionTestCase( + description="direct mode without explicit retention resolves 14", + project_toml=DIRECT_PROJECT_TOML + "\n[janitor]\nenabled = true\n", + cli_retention_days=None, + expected_retention_days=14, + expected_archive_retention_days=14, + ), + JanitorSettingsResolutionTestCase( + description="virtual mode without explicit retention resolves 30", + project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\n", + cli_retention_days=None, + expected_retention_days=30, + expected_archive_retention_days=14, + ), + JanitorSettingsResolutionTestCase( + description="explicit retention applies to direct mode", + project_toml=DIRECT_PROJECT_TOML + + "\n[janitor]\nenabled = true\nretention_days = 9\narchive_retention_days = 2\n", + cli_retention_days=None, + expected_retention_days=9, + expected_archive_retention_days=2, + ), + JanitorSettingsResolutionTestCase( + description="explicit retention applies to virtual mode", + project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\nretention_days = 9\n", + cli_retention_days=None, + expected_retention_days=9, + expected_archive_retention_days=14, + ), + JanitorSettingsResolutionTestCase( + description="cli retention override wins in virtual mode", + project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\nretention_days = 9\n", + cli_retention_days=4, + expected_retention_days=4, + expected_archive_retention_days=14, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_config_when_resolving_janitor_settings_then_uses_mode_specific_default( + test_case: JanitorSettingsResolutionTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_toml, encoding="utf-8") + request: JanitorCommandRequest = JanitorCommandRequest( + project_dir=tmp_path, no_color=True, retention_days=test_case.cli_retention_days + ) + + invocation: JanitorInvocation = resolve_janitor_invocation(request=request) + settings: JanitorSettings = resolve_janitor_settings(request=request, invocation=invocation) + + assert settings.retention_days == test_case.expected_retention_days + assert settings.archive_retention_days == test_case.expected_archive_retention_days diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 44a941efcb..ef91b1d8cd 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -262,7 +262,7 @@ class LoadProjectConfigTestCase: expected_vars: dict[str, str] expected_targets: dict[str, dict[str, object]] expected_janitor_enabled: bool - expected_retention_days: int + expected_retention_days: int | None expected_janitor_max_checkpoints: int expected_janitor_delete_tracked_only: bool expected_janitor_exclude_patterns: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index a075007d67..8205c0b06c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -667,7 +667,7 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise } }, expected_janitor_enabled=False, - expected_retention_days=14, + expected_retention_days=None, expected_janitor_max_checkpoints=20, expected_janitor_delete_tracked_only=True, expected_janitor_exclude_patterns=(), diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py index a3cafb2c94..6a11a18d0d 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py @@ -28,3 +28,11 @@ class ArchiveNameBuildTestCase: expected_prefix: str expected_length: int expected_logical_name_is_original: bool + + +@dataclass(frozen=True) +class RelationAddressingTestCase: + description: str + name: str + listing: tuple[str, ...] + expected_reason: str | None diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py index bd2c1d3568..464e7bd297 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py @@ -98,8 +98,8 @@ def test_given_non_strict_name_when_parsing_archive_name_then_returns_none( original_name="orders", archived_at=ARCHIVED_AT, identifier_limit=255, - expected_prefix="_SQB_ARCHIVE__20260924T101500Z__orders", - expected_length=len("_SQB_ARCHIVE__20260924T101500Z__orders"), + expected_prefix="_sqb_archive__20260924t101500z__orders", + expected_length=len("_sqb_archive__20260924t101500z__orders"), expected_logical_name_is_original=True, ), ArchiveNameBuildTestCase( @@ -107,7 +107,7 @@ def test_given_non_strict_name_when_parsing_archive_name_then_returns_none( original_name=LONG_NAME, archived_at=ARCHIVED_AT.replace(microsecond=654321), identifier_limit=63, - expected_prefix="_SQB_ARCHIVE__20260924T101500Z__fulfillment_orders_", + expected_prefix="_sqb_archive__20260924t101500z__fulfillment_orders_", expected_length=63, expected_logical_name_is_original=False, ), diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py new file mode 100644 index 0000000000..f5bc3a9feb --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.executor.janitor._helpers.relation_addressing import ( + case_colliding_names, + unaddressable_relation_reason, +) +from sqlbuild.executor.janitor.constants import ( + CASE_COLLISION_REASON, + UNQUOTED_ADDRESSING_REASON, +) +from tests.unit.src.sqlbuild.executor.janitor._helpers._test_types import ( + RelationAddressingTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RelationAddressingTestCase( + description="plain lowercase name is addressable", + name="old_orders", + listing=("old_orders", "customers"), + expected_reason=None, + ), + RelationAddressingTestCase( + description="lowercase name with digits dollar and leading underscore is addressable", + name="_orders$2026", + listing=("_orders$2026",), + expected_reason=None, + ), + RelationAddressingTestCase( + description="uppercase name is skipped", + name="OLD_ORDERS", + listing=("OLD_ORDERS",), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + RelationAddressingTestCase( + description="mixed-case name is skipped", + name="OldOrders", + listing=("OldOrders",), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + RelationAddressingTestCase( + description="name with punctuation is skipped", + name="old-orders;drop", + listing=("old-orders;drop",), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + RelationAddressingTestCase( + description="name with a leading digit is skipped", + name="2026_orders", + listing=("2026_orders",), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + RelationAddressingTestCase( + description="lowercase name colliding case-insensitively is skipped", + name="old_orders", + listing=("old_orders", "OLD_ORDERS"), + expected_reason=CASE_COLLISION_REASON, + ), + RelationAddressingTestCase( + description="uppercase side of a case collision is skipped", + name="OLD_ORDERS", + listing=("old_orders", "OLD_ORDERS"), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + ], + ids=lambda case: case.description, +) +def test_given_catalog_name_when_checking_addressing_then_returns_expected_reason( + test_case: RelationAddressingTestCase, +) -> None: + reason: str | None = unaddressable_relation_reason( + name=test_case.name, colliding_names=case_colliding_names(test_case.listing) + ) + + assert reason == test_case.expected_reason diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 260daf1b5c..6d71f85d3c 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -112,3 +112,23 @@ class JanitorArchiveEventFailureTestCase: expected_error_fragment: str expected_renamed_origins: tuple[str, ...] expected_dropped_targets: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorAddressingPlanTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + direct_mode: bool + expected_archive_source_names: tuple[str, ...] = field(default_factory=tuple) + expected_archive_deletion_names: tuple[str, ...] = field(default_factory=tuple) + expected_skipped_relations: tuple[tuple[str, str], ...] = field(default_factory=tuple) + expected_candidate_names: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorRetentionResolutionTestCase: + description: str + override: int | None + configured: int | None + virtual_environments: bool + expected_retention_days: int diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py index ffe82faf5e..394a338b22 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py @@ -97,12 +97,12 @@ def test_given_physical_drop_failure_when_executing_janitor_then_state_cleanup_i relation_infos=( relation_info("old_orders", created_at=OLD_TIME), relation_info("old_customers_view", relation_type="VIEW", created_at=OLD_TIME), - relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products"), + relation_info("_sqb_archive__20200101t000000z__old_products"), ), archive_retention_days=14, expected_renamed_origins=("analytics.old_orders",), expected_view_rename_origins=("analytics.old_customers_view",), - expected_dropped_targets=("analytics._SQB_ARCHIVE__20200101T000000Z__old_products",), + expected_dropped_targets=("analytics._sqb_archive__20200101t000000z__old_products",), expected_dropped_view_targets=(), expected_event_insert_count=3, expected_event_table_create_count=1, @@ -116,8 +116,8 @@ def test_given_physical_drop_failure_when_executing_janitor_then_state_cleanup_i archive_retention_days=0, expected_renamed_origins=("analytics.old_orders",), expected_view_rename_origins=("analytics.old_customers_view",), - expected_dropped_targets=("analytics._SQB_ARCHIVE__",), - expected_dropped_view_targets=("analytics._SQB_ARCHIVE__",), + expected_dropped_targets=("analytics._sqb_archive__",), + expected_dropped_view_targets=("analytics._sqb_archive__",), expected_event_insert_count=4, expected_event_table_create_count=1, ), @@ -190,7 +190,7 @@ def test_given_direct_archive_plan_when_executing_then_renames_drops_and_writes_ description="audit write failure after rename stops before further actions", relation_infos=( relation_info("old_orders", created_at=OLD_TIME), - relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products"), + relation_info("_sqb_archive__20200101t000000z__old_products"), ), expected_error_fragment="simulated janitor event write failure", expected_renamed_origins=("analytics.old_orders",), diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py index 63ca92991b..0a5c041f8d 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py @@ -7,7 +7,11 @@ import pytest -from sqlbuild.executor.janitor.constants import MALFORMED_ARCHIVE_REASON +from sqlbuild.executor.janitor.constants import ( + CASE_COLLISION_REASON, + MALFORMED_ARCHIVE_REASON, + UNQUOTED_ADDRESSING_REASON, +) from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import ( JanitorArchivedRelation, @@ -17,6 +21,7 @@ JanitorSkippedRelation, ) from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorAddressingPlanTestCase, JanitorArchivePlanTestCase, relation_info, ) @@ -28,7 +33,7 @@ OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) RECENT_ARCHIVE_NAME: str = ( f"_SQB_ARCHIVE__{(datetime.now(UTC) - timedelta(days=1)):%Y%m%dT%H%M%SZ}__old_customers" -) +).lower() LONG_ORDERS_NAME: str = "orders_" + "x" * 60 @@ -51,11 +56,11 @@ JanitorArchivePlanTestCase( description="expired archive is deleted and recent archive is retained", relation_infos=( - relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products"), + relation_info("_sqb_archive__20200101t000000z__old_products"), relation_info(RECENT_ARCHIVE_NAME), ), expected_existing_archive_deletion_names=( - "_SQB_ARCHIVE__20200101T000000Z__old_products", + "_sqb_archive__20200101t000000z__old_products", ), expected_retained_archive_names=(RECENT_ARCHIVE_NAME,), ), @@ -69,7 +74,7 @@ JanitorArchivePlanTestCase( description="malformed archive look-alikes are reported and never archived or deleted", relation_infos=( - relation_info("_SQB_ARCHIVE__20201399T000000Z__orders", created_at=OLD_TIME), + relation_info("_sqb_archive__20201399t000000z__orders", created_at=OLD_TIME), relation_info("_SQB_ARCHIVE_20200101T000000Z__orders", created_at=OLD_TIME), relation_info("_SQB_ARCHIVE__2020010T000000Z__orders", created_at=OLD_TIME), relation_info("_SQB_ARCHIVE__20200101T000000Z__", created_at=OLD_TIME), @@ -78,7 +83,7 @@ retention_days=0, archive_retention_days=0, expected_skipped_relations=( - ("_SQB_ARCHIVE__20201399T000000Z__orders", MALFORMED_ARCHIVE_REASON), + ("_sqb_archive__20201399t000000z__orders", MALFORMED_ARCHIVE_REASON), ("_SQB_ARCHIVE_20200101T000000Z__orders", MALFORMED_ARCHIVE_REASON), ("_SQB_ARCHIVE__2020010T000000Z__orders", MALFORMED_ARCHIVE_REASON), ("_SQB_ARCHIVE__20200101T000000Z__", MALFORMED_ARCHIVE_REASON), @@ -87,28 +92,28 @@ ), JanitorArchivePlanTestCase( description="archive matching an exclude pattern is skipped", - relation_infos=(relation_info("_SQB_ARCHIVE__20200101T000000Z__keep_orders"),), + relation_infos=(relation_info("_sqb_archive__20200101t000000z__keep_orders"),), exclude_patterns=("*keep_*",), expected_skipped_relations=( ( - "_SQB_ARCHIVE__20200101T000000Z__keep_orders", + "_sqb_archive__20200101t000000z__keep_orders", "relation matches exclude pattern '*keep_*'", ), ), ), JanitorArchivePlanTestCase( description="blocked schema suppresses expired archive deletion", - relation_infos=(relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products"),), + relation_infos=(relation_info("_sqb_archive__20200101t000000z__old_products"),), source_schema="analytics", expected_suppressed_archive_deletion_names=( - "_SQB_ARCHIVE__20200101T000000Z__old_products", + "_sqb_archive__20200101t000000z__old_products", ), ), JanitorArchivePlanTestCase( description="virtual mode neither archives nor expires archive-named relations", relation_infos=( relation_info("old_orders", created_at=OLD_TIME), - relation_info("_SQB_ARCHIVE__20200101T000000Z__old_products", created_at=OLD_TIME), + relation_info("_sqb_archive__20200101t000000z__old_products", created_at=OLD_TIME), ), direct_mode=False, ), @@ -215,11 +220,82 @@ def test_given_long_relation_name_when_planning_archive_then_fits_name_with_time archive_name: str = plan.archive_candidates[0].archive_key.name expected_prefix: str = ( f"_SQB_ARCHIVE__{plan.archive_candidates[0].archived_at:%Y%m%dT%H%M%SZ}__orders_" - ) + ).lower() assert len(archive_name) == test_case.identifier_limit assert archive_name.startswith(expected_prefix) assert archive_name != f"{expected_prefix[:-7]}{LONG_ORDERS_NAME}" +@pytest.mark.parametrize( + "test_case", + [ + JanitorAddressingPlanTestCase( + description="unaddressable stale relations and archives are skipped, plain ones act", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("LegacyCustomers", created_at=OLD_TIME), + relation_info("old-products", created_at=OLD_TIME), + relation_info("stale_items", created_at=OLD_TIME), + relation_info("STALE_ITEMS", created_at=OLD_TIME), + relation_info("_sqb_archive__20200101t000000z__old_products"), + relation_info("_SQB_ARCHIVE__20200101T000000Z__old_customers"), + relation_info("_sqb_archive__20200101t000000z__old_invoices"), + relation_info("_SQB_ARCHIVE__20200101T000000Z__OLD_INVOICES"), + ), + direct_mode=True, + expected_candidate_names=("old_orders",), + expected_archive_source_names=("old_orders",), + expected_archive_deletion_names=("_sqb_archive__20200101t000000z__old_products",), + expected_skipped_relations=( + ("LegacyCustomers", UNQUOTED_ADDRESSING_REASON), + ("old-products", UNQUOTED_ADDRESSING_REASON), + ("stale_items", CASE_COLLISION_REASON), + ("STALE_ITEMS", UNQUOTED_ADDRESSING_REASON), + ("_SQB_ARCHIVE__20200101T000000Z__old_customers", UNQUOTED_ADDRESSING_REASON), + ("_sqb_archive__20200101t000000z__old_invoices", CASE_COLLISION_REASON), + ("_SQB_ARCHIVE__20200101T000000Z__OLD_INVOICES", UNQUOTED_ADDRESSING_REASON), + ), + ), + JanitorAddressingPlanTestCase( + description="virtual mode keeps deleting mixed-case relations as before", + relation_infos=(relation_info("LegacyCustomers", created_at=OLD_TIME),), + direct_mode=False, + expected_candidate_names=("LegacyCustomers",), + ), + ], + ids=lambda case: case.description, +) +def test_given_unaddressable_names_when_planning_direct_janitor_then_they_are_skipped( + test_case: JanitorAddressingPlanTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter(relation_infos=test_case.relation_infos) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + ) + + assert ( + tuple(candidate.key.name for candidate in plan.archive_candidates) + == test_case.expected_archive_source_names + ) + assert ( + tuple(archive.key.name for archive in plan.archive_deletion_candidates) + == test_case.expected_archive_deletion_names + ) + assert ( + tuple((skipped.key.name, skipped.reason) for skipped in plan.skipped_relations) + == test_case.expected_skipped_relations + ) + assert ( + tuple(candidate.key.name for candidate in plan.candidates) + == test_case.expected_candidate_names + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py new file mode 100644 index 0000000000..69253bd50d --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py @@ -0,0 +1,70 @@ +"""Tests for mode-specific janitor retention resolution.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorRetentionResolutionTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorRetentionResolutionTestCase( + description="unset retention in direct mode defaults to 14", + override=None, + configured=None, + virtual_environments=False, + expected_retention_days=14, + ), + JanitorRetentionResolutionTestCase( + description="unset retention in virtual mode defaults to 30", + override=None, + configured=None, + virtual_environments=True, + expected_retention_days=30, + ), + JanitorRetentionResolutionTestCase( + description="explicit retention applies in direct mode", + override=None, + configured=7, + virtual_environments=False, + expected_retention_days=7, + ), + JanitorRetentionResolutionTestCase( + description="explicit retention applies in virtual mode", + override=None, + configured=7, + virtual_environments=True, + expected_retention_days=7, + ), + JanitorRetentionResolutionTestCase( + description="cli override wins over explicit config", + override=3, + configured=7, + virtual_environments=True, + expected_retention_days=3, + ), + JanitorRetentionResolutionTestCase( + description="cli zero override wins over the mode default", + override=0, + configured=None, + virtual_environments=True, + expected_retention_days=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_retention_inputs_when_resolving_janitor_retention_then_returns_expected_days( + test_case: JanitorRetentionResolutionTestCase, +) -> None: + resolved: int = resolve_janitor_retention_days( + override=test_case.override, + configured=test_case.configured, + virtual_environments=test_case.virtual_environments, + ) + + assert resolved == test_case.expected_retention_days From 5c97c539d5a0d03e7ada2e7df581fbf0e05cedae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:02:43 +0100 Subject: [PATCH 3/8] fix(janitor): show TOML enable guidance without mode-specific retention --- .../cli/commands/_helpers/janitor_output/output.py | 9 ++++----- .../sqlbuild/cli/commands/main/janitor/test_janitor.py | 2 +- .../sqlbuild/cli/commands/main/janitor/test_janitor.py | 10 ++++------ 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py index dc6b47f3f4..a516b1629c 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py @@ -27,16 +27,15 @@ def write_disabled(*, stream: TextIO, use_color: bool = False) -> None: style: CliStyle = CliStyle(use_color=use_color) disabled_detail: str = ( - "Janitor is opt-in. It previews stale warehouse objects and asks before deleting anything." + "Janitor is opt-in. It previews stale warehouse objects and asks before archiving or " + "deleting anything." ) stream.write( f"{style.warning_strong('Janitor is disabled for this project.')}\n\n" f"{style.warning(disabled_detail)}\n\n" f"Add this block to {style.object_name('sqlbuild_project.toml')}:\n\n" - "janitor:\n" - " enabled: true\n" - " retention_days: 14\n" - " archive_retention_days: 14\n\n" + "[janitor]\n" + "enabled = true\n\n" "After enabling, run janitor again to preview cleanup:\n" f" {style.accent('sqb janitor')}\n" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 4bc14b255e..d59b115508 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -51,7 +51,7 @@ expected_exit_code=0, expected_stdout_fragments=( "Janitor is disabled for this project.", - "enabled: true", + "enabled = true", ), ) ], diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 04237cafab..259b2a7962 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -88,12 +88,10 @@ def interrupting_input() -> str: use_color=False, expected_output="Janitor is disabled for this project.\n\n" "Janitor is opt-in. It previews stale warehouse objects and asks before " - "deleting anything.\n\n" + "archiving or deleting anything.\n\n" "Add this block to sqlbuild_project.toml:\n\n" - "janitor:\n" - " enabled: true\n" - " retention_days: 14\n" - " archive_retention_days: 14\n\n" + "[janitor]\n" + "enabled = true\n\n" "After enabling, run janitor again to preview cleanup:\n" " sqb janitor\n", ) @@ -132,7 +130,7 @@ def test_given_disabled_janitor_when_writing_with_color_then_styles_title( assert stream.getvalue().startswith(test_case.expected_prefix) assert ( "\033[33mJanitor is opt-in. It previews stale warehouse objects and asks before " - "deleting anything.\033[0m" + "archiving or deleting anything.\033[0m" ) in stream.getvalue() assert "\033[34msqb janitor\033[0m" in stream.getvalue() From 7d4bee0cc7eb606f74fca3f1e03136adf5428d69 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:24:51 +0100 Subject: [PATCH 4/8] fix(janitor): compare relation keys case-insensitively across folded catalog identifiers Listed relations are matched to target schemas case-insensitively and respelled with the project's schema spelling, so warehouses that fold unquoted identifiers (for example lowercase listings of uppercase schemas) produce janitor candidates. Relation keys, source-schema overlap detection, and scan-schema collection compare case-insensitively while rendering keeps an addressable spelling. --- .../executor/janitor/_helpers/plan.py | 69 +++++- .../janitor/_helpers/schema_planning.py | 9 +- .../janitor/_helpers/source_safety.py | 36 ++-- src/sqlbuild/executor/janitor/main/plan.py | 9 +- src/sqlbuild/executor/janitor/models.py | 19 +- .../executor/janitor/main/_test_types.py | 18 ++ .../sqlbuild/executor/janitor/main/helpers.py | 65 +++++- .../main/test_plan_folded_identifiers.py | 201 ++++++++++++++++++ 8 files changed, 383 insertions(+), 43 deletions(-) create mode 100644 tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py diff --git a/src/sqlbuild/executor/janitor/_helpers/plan.py b/src/sqlbuild/executor/janitor/_helpers/plan.py index 763cf41591..3e8e4c466f 100644 --- a/src/sqlbuild/executor/janitor/_helpers/plan.py +++ b/src/sqlbuild/executor/janitor/_helpers/plan.py @@ -3,10 +3,11 @@ from __future__ import annotations from collections import defaultdict +from dataclasses import replace from datetime import UTC, datetime from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledProject, @@ -42,16 +43,45 @@ def collect_desired_keys(project: CompiledProject) -> set[JanitorRelationKey]: def collect_target_schemas(project: CompiledProject) -> set[tuple[str | None, str | None]]: - """Collect schemas where the compiled project writes relations.""" + """Collect schemas where the compiled project writes relations, one spelling per schema.""" - schemas: set[tuple[str | None, str | None]] = set() + schemas: list[tuple[str | None, str | None]] = [] model: CompiledModel for model in project.models: - schemas.add((model.destination.database, model.destination.schema)) + schemas.append((model.destination.database, model.destination.schema)) seed: CompiledSeed for seed in project.seeds: - schemas.add((seed.destination.database, seed.destination.schema)) - return schemas + schemas.append((seed.destination.database, seed.destination.schema)) + return _dedupe_schema_keys(schemas) + + +def collect_scan_schemas( + *, + managed_target_schemas: set[tuple[str | None, str | None]], + relation_keys: frozenset[JanitorRelationKey], +) -> set[tuple[str | None, str | None]]: + """Add schemas of scanned or protected relations without duplicating folded spellings.""" + + return _dedupe_schema_keys( + [ + *sorted(managed_target_schemas, key=_schema_sort_key), + *sorted(((key.database, key.schema) for key in relation_keys), key=_schema_sort_key), + ] + ) + + +def _dedupe_schema_keys( + schema_keys: list[tuple[str | None, str | None]], +) -> set[tuple[str | None, str | None]]: + kept: dict[tuple[str | None, str | None], tuple[str | None, str | None]] = {} + schema_key: tuple[str | None, str | None] + for schema_key in schema_keys: + kept.setdefault(normalized_schema_key(schema_key), schema_key) + return set(kept.values()) + + +def _schema_sort_key(schema_key: tuple[str | None, str | None]) -> tuple[str, str]: + return (schema_key[0] or "", schema_key[1] or "") def collect_source_schemas( @@ -85,8 +115,11 @@ def list_target_schema_relations( connection: object, target_schemas: set[tuple[str | None, str | None]], ) -> dict[tuple[str | None, str | None], tuple[RelationInfo, ...]]: - """List warehouse relations for the target schemas.""" + """List relations matched case-insensitively and respelled as their target schema.""" + target_by_normalized: dict[tuple[str | None, str | None], tuple[str | None, str | None]] = { + normalized_schema_key(schema_key): schema_key for schema_key in target_schemas + } by_database: dict[str | None, set[str | None]] = defaultdict(set) schema_key: tuple[str | None, str | None] for schema_key in target_schemas: @@ -107,13 +140,27 @@ def list_target_schema_relations( database=database, schemas=concrete_schemas, ): - key: tuple[str | None, str | None] = (relation.database, relation.schema) - if key in target_schemas: - result[key].append(relation) + target_key: tuple[str | None, str | None] | None = target_by_normalized.get( + normalized_schema_key((relation.database, relation.schema)) + ) + if target_key is None: + continue + result[target_key].append( + replace(relation, database=target_key[0], schema=target_key[1]) + ) return {key: tuple(value) for key, value in result.items()} +def normalized_schema_key( + schema_key: tuple[str | None, str | None], +) -> tuple[str | None, str | None]: + """Return the case-insensitive comparison form of one database and schema pair.""" + + database, schema, _ = RelationLookup.key(database=schema_key[0], schema=schema_key[1], name="") + return (database, schema) + + def relation_key(relation: RelationInfo) -> JanitorRelationKey: """Build a janitor relation key from adapter relation metadata.""" @@ -140,4 +187,4 @@ def relation_age_timestamp(relation: RelationInfo) -> datetime | None: def _ensure_aware(value: datetime) -> datetime: if value.tzinfo is None: return value.replace(tzinfo=UTC) - return value + return value.astimezone(UTC) diff --git a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py index c18f80688f..6d84225372 100644 --- a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py +++ b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py @@ -6,7 +6,10 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.executor.janitor._helpers.classification import classify_janitor_relations -from sqlbuild.executor.janitor._helpers.source_safety import blocking_source_names +from sqlbuild.executor.janitor._helpers.source_safety import ( + blocking_source_names, + source_names_for_schema, +) from sqlbuild.executor.janitor.constants import BUILT_IN_EXCLUDE_PATTERNS from sqlbuild.executor.janitor.models import ( JanitorBlockedSchema, @@ -41,7 +44,9 @@ def classify_target_schemas( blocked_schemas: list[JanitorBlockedSchema] = [] for schema_key in sorted(target_schemas, key=lambda key: (key[0] or "", key[1] or "")): schema_relations: tuple[RelationInfo, ...] = facts.relations_by_schema.get(schema_key, ()) - source_names: set[str] | None = facts.source_schema_names.get(schema_key) + source_names: set[str] = source_names_for_schema( + schema_key=schema_key, source_schema_names=facts.source_schema_names + ) blocking_sources: tuple[str, ...] = blocking_source_names( schema_key=schema_key, managed_schema_keys=managed_target_schemas, diff --git a/src/sqlbuild/executor/janitor/_helpers/source_safety.py b/src/sqlbuild/executor/janitor/_helpers/source_safety.py index eaa1beb42e..695061f74c 100644 --- a/src/sqlbuild/executor/janitor/_helpers/source_safety.py +++ b/src/sqlbuild/executor/janitor/_helpers/source_safety.py @@ -2,6 +2,8 @@ from __future__ import annotations +from sqlbuild.executor.janitor._helpers.plan import normalized_schema_key + def blocking_source_names( *, @@ -11,22 +13,26 @@ def blocking_source_names( ) -> tuple[str, ...]: """Return active sources when one managed schema has mixed ownership.""" - normalized: tuple[str | None, str | None] = _normalized_schema_key(schema_key) - if not any( - _normalized_schema_key(candidate) == normalized for candidate in managed_schema_keys - ): + normalized: tuple[str | None, str | None] = normalized_schema_key(schema_key) + if not any(normalized_schema_key(candidate) == normalized for candidate in managed_schema_keys): return () - for candidate, source_names in source_schema_names.items(): - if _normalized_schema_key(candidate) == normalized: - return tuple(sorted(source_names)) - return () + return tuple( + sorted( + source_names_for_schema(schema_key=schema_key, source_schema_names=source_schema_names) + ) + ) -def _normalized_schema_key( +def source_names_for_schema( + *, schema_key: tuple[str | None, str | None], -) -> tuple[str | None, str | None]: - database, schema = schema_key - return ( - None if database is None else database.lower(), - None if schema is None else schema.lower(), - ) + source_schema_names: dict[tuple[str | None, str | None], set[str]], +) -> set[str]: + """Return active source names configured in one schema, compared case-insensitively.""" + + normalized: tuple[str | None, str | None] = normalized_schema_key(schema_key) + names: set[str] = set() + for candidate, source_names in source_schema_names.items(): + if normalized_schema_key(candidate) == normalized: + names.update(source_names) + return names diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py index f5354b014a..123f0b3b1f 100644 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ b/src/sqlbuild/executor/janitor/main/plan.py @@ -14,7 +14,7 @@ collect_query_diff_artifact_candidates, gather_janitor_warehouse_facts, ) -from sqlbuild.executor.janitor._helpers.plan import collect_target_schemas +from sqlbuild.executor.janitor._helpers.plan import collect_scan_schemas, collect_target_schemas from sqlbuild.executor.janitor._helpers.schema_planning import classify_target_schemas from sqlbuild.executor.janitor.models import ( JanitorArchivePlanning, @@ -51,9 +51,10 @@ def build_janitor_plan( ) direct: JanitorDirectModeSettings = direct_settings or JanitorDirectModeSettings() managed_target_schemas: set[tuple[str | None, str | None]] = collect_target_schemas(project) - target_schemas: set[tuple[str | None, str | None]] = set(managed_target_schemas) - target_schemas.update((key.database, key.schema) for key in scope.protected_relation_keys) - target_schemas.update((key.database, key.schema) for key in scope.scan_relation_keys) + target_schemas: set[tuple[str | None, str | None]] = collect_scan_schemas( + managed_target_schemas=managed_target_schemas, + relation_keys=scope.protected_relation_keys | scope.scan_relation_keys, + ) query_artifact_schemas: set[tuple[str | None, str]] = { (database, schema) for database, schema in target_schemas if schema is not None } diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 97c8457654..5601dfcc81 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -5,17 +5,30 @@ from dataclasses import dataclass, field from datetime import datetime -from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -@dataclass(frozen=True) +@dataclass(frozen=True, eq=False) class JanitorRelationKey: - """Physical identity for a warehouse relation considered by janitor.""" + """Physical relation identity compared case-insensitively but rendered as spelled.""" database: str | None schema: str | None name: str + def identity(self) -> tuple[str | None, str | None, str]: + """Return the case-insensitive comparison identity.""" + + return RelationLookup.key(database=self.database, schema=self.schema, name=self.name) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, JanitorRelationKey): + return NotImplemented + return self.identity() == other.identity() + + def __hash__(self) -> int: + return hash(self.identity()) + def display_name(self) -> str: """Render a qualified display name.""" diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 6d71f85d3c..b8c7607fe2 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -132,3 +132,21 @@ class JanitorRetentionResolutionTestCase: configured: int | None virtual_environments: bool expected_retention_days: int + + +@dataclass(frozen=True) +class JanitorFoldedIdentifierPlanTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + direct_mode: bool = True + delete_tracked_only: bool = False + tracked_relations: tuple[tuple[str | None, str | None, str], ...] = field(default_factory=tuple) + source_database: str | None = None + source_schema: str | None = None + protected_relation_keys: frozenset[JanitorRelationKey] = frozenset() + expected_candidate_display_names: tuple[str, ...] = field(default_factory=tuple) + expected_archive_display_names: tuple[str, ...] = field(default_factory=tuple) + expected_archive_deletion_display_names: tuple[str, ...] = field(default_factory=tuple) + expected_skipped_relations: tuple[tuple[str, str], ...] = field(default_factory=tuple) + expected_blocked_sources: tuple[str, ...] = field(default_factory=tuple) + expected_suppressed_names: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 603aebfa2d..35caabbe48 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -4,6 +4,7 @@ import base64 from collections.abc import Callable +from dataclasses import replace from pathlib import Path from typing import Any, ClassVar, cast @@ -460,6 +461,43 @@ def drop( raise RuntimeError(self.message) +class FoldingJanitorAdapter(FakeJanitorAdapter): + """Warehouse double that filters case-insensitively and lists lowercase identifiers.""" + + def list_relations( + self, + connection: Any, + *, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[RelationInfo, ...]: + requested_schemas: frozenset[str] = frozenset(schema.lower() for schema in schemas or ()) + requested_names: frozenset[str] = frozenset(name.lower() for name in names or ()) + matching: tuple[RelationInfo, ...] = tuple( + filter( + lambda relation: ( + (relation.database or "").lower() == (database or "").lower() + and ( + not requested_schemas + or (relation.schema or "").lower() in requested_schemas + ) + and (not requested_names or relation.name.lower() in requested_names) + ), + self._available_relations, + ) + ) + return tuple( + replace( + relation, + database=database, + schema=(relation.schema or "").lower(), + name=relation.name.lower(), + ) + for relation in matching + ) + + class FailingJanitorEventAdapter(FakeJanitorAdapter): def _execute(self, connection: Any, sql: str) -> Any: is_event_insert: bool = sql.startswith("INSERT INTO") and JANITOR_EVENTS_TABLE_NAME in sql @@ -490,7 +528,13 @@ def fetchall(self) -> list[tuple[Any, ...]]: return list(self.rows) -def build_project(*, source_schema: str | None = None) -> CompiledProject: +def build_project( + *, + source_schema: str | None = None, + source_database: str | None = None, + destination_database: str | None = None, + destination_schema: str = "analytics", +) -> CompiledProject: source: CompiledSource = CompiledSource( key=CompiledObjectKey( resource_type=CompiledResourceType.SOURCE, @@ -498,7 +542,12 @@ def build_project(*, source_schema: str | None = None) -> CompiledProject: ), deps=(), name="raw_orders", - source_entry=SourceEntry(name="raw_orders", schema=source_schema or "", table="orders"), + source_entry=SourceEntry( + name="raw_orders", + database=source_database, + schema=source_schema or "", + table="orders", + ), source_file=cast(Any, object()), ) sources: tuple[CompiledSource, ...] = {True: (), False: (source,)}[source_schema is None] @@ -516,10 +565,10 @@ def build_project(*, source_schema: str | None = None) -> CompiledProject: query_sql="select 1", config=CompileModelConfig(), destination=CompiledRelationLocation( - database=None, - schema="analytics", + database=destination_database, + schema=destination_schema, name="orders", - qualified_name="analytics.orders", + qualified_name=f"{destination_schema}.orders", ), ), ), @@ -532,10 +581,10 @@ def build_project(*, source_schema: str | None = None) -> CompiledProject: schema_entry=SchemaSeedEntry(name="countries"), schema_file=cast(Any, object()), destination=CompiledRelationLocation( - database=None, - schema="analytics", + database=destination_database, + schema=destination_schema, name="countries", - qualified_name="analytics.countries", + qualified_name=f"{destination_schema}.countries", ), ), ), diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py new file mode 100644 index 0000000000..b9c009d4ee --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py @@ -0,0 +1,201 @@ +"""Janitor planning against warehouses that report folded identifier case.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime, timedelta +from itertools import chain + +import pytest + +from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild.executor.janitor.models import ( + JanitorDeleteCandidate, + JanitorDirectModeSettings, + JanitorPlan, + JanitorRelationKey, + JanitorRelationScope, +) +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorFoldedIdentifierPlanTestCase, + relation_info, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FoldingJanitorAdapter, + build_project, +) + +OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) +ARCHIVE_TIMESTAMP_RE: re.Pattern[str] = re.compile(r"__[0-9]{8}t[0-9]{6}z__") + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorFoldedIdentifierPlanTestCase( + description="lowercase listing finds stale relation and protects desired relations", + relation_infos=( + relation_info( + "orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + relation_info( + "countries", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + relation_info( + "old_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + ), + expected_candidate_display_names=("ANALYTICS.DEV_ORDERS.old_orders",), + expected_archive_display_names=("ANALYTICS.DEV_ORDERS._sqb_archive____old_orders",), + ), + JanitorFoldedIdentifierPlanTestCase( + description="tracked-only filter matches fingerprints across identifier case", + relation_infos=( + relation_info( + "old_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + relation_info( + "stray_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + ), + delete_tracked_only=True, + tracked_relations=(("ANALYTICS", "DEV_ORDERS", "OLD_ORDERS"),), + expected_candidate_display_names=("ANALYTICS.DEV_ORDERS.old_orders",), + expected_archive_display_names=("ANALYTICS.DEV_ORDERS._sqb_archive____old_orders",), + expected_skipped_relations=( + ("ANALYTICS.DEV_ORDERS.stray_orders", "relation is not tracked by SQLBuild"), + ( + "ANALYTICS.DEV_ORDERS._sqlbuild_fingerprints", + "relation matches exclude pattern '_sqlbuild_fingerprints'", + ), + ), + ), + JanitorFoldedIdentifierPlanTestCase( + description="sources in a different schema of the same database do not block", + relation_infos=( + relation_info( + "old_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + ), + source_database="ANALYTICS", + source_schema="RAW", + expected_candidate_display_names=("ANALYTICS.DEV_ORDERS.old_orders",), + expected_archive_display_names=("ANALYTICS.DEV_ORDERS._sqb_archive____old_orders",), + ), + JanitorFoldedIdentifierPlanTestCase( + description="source in the managed schema blocks even when spelled differently", + relation_infos=( + relation_info( + "old_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + ), + source_database="analytics", + source_schema="dev_orders", + expected_blocked_sources=("raw_orders",), + expected_suppressed_names=("ANALYTICS.DEV_ORDERS.old_orders",), + ), + JanitorFoldedIdentifierPlanTestCase( + description="expired archive is found in a lowercase listing", + relation_infos=( + relation_info( + "_sqb_archive__20200101t000000z__old_products", + database="ANALYTICS", + schema="DEV_ORDERS", + ), + ), + expected_archive_deletion_display_names=( + "ANALYTICS.DEV_ORDERS._sqb_archive__20200101t000000z__old_products", + ), + ), + JanitorFoldedIdentifierPlanTestCase( + description="virtual protection matches physical relations across identifier case", + relation_infos=( + relation_info( + "orders__v1", + database="ANALYTICS", + schema="DEV_ORDERS__SQB_PHYSICAL", + created_at=OLD_TIME, + ), + relation_info( + "orders__v0", + database="ANALYTICS", + schema="DEV_ORDERS__SQB_PHYSICAL", + created_at=OLD_TIME, + ), + ), + direct_mode=False, + protected_relation_keys=frozenset( + ( + JanitorRelationKey( + database="ANALYTICS", + schema="DEV_ORDERS__SQB_PHYSICAL", + name="ORDERS__V1", + ), + ) + ), + expected_candidate_display_names=("ANALYTICS.DEV_ORDERS__SQB_PHYSICAL.orders__v0",), + expected_skipped_relations=( + ( + "ANALYTICS.DEV_ORDERS__SQB_PHYSICAL.orders__v1", + "relation is referenced by a retained virtual checkpoint", + ), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_folded_catalog_identifiers_when_planning_janitor_then_keys_compare_consistently( + test_case: JanitorFoldedIdentifierPlanTestCase, +) -> None: + adapter: FoldingJanitorAdapter = FoldingJanitorAdapter( + relation_infos=test_case.relation_infos, + tracked_relations=test_case.tracked_relations, + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project( + source_database=test_case.source_database, + source_schema=test_case.source_schema, + destination_database="ANALYTICS", + destination_schema="DEV_ORDERS", + ), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=test_case.delete_tracked_only, + relation_scope=JanitorRelationScope( + protected_relation_keys=test_case.protected_relation_keys + ), + direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + ) + + assert ( + tuple(candidate.key.display_name() for candidate in plan.candidates) + == test_case.expected_candidate_display_names + ) + assert ( + tuple( + ARCHIVE_TIMESTAMP_RE.sub("____", candidate.archive_key.display_name()) + for candidate in plan.archive_candidates + ) + == test_case.expected_archive_display_names + ) + assert ( + tuple(archive.key.display_name() for archive in plan.archive_deletion_candidates) + == test_case.expected_archive_deletion_display_names + ) + assert ( + tuple((skipped.key.display_name(), skipped.reason) for skipped in plan.skipped_relations) + == test_case.expected_skipped_relations + ) + blocked_sources: tuple[str, ...] = tuple( + chain.from_iterable(blocked.source_names for blocked in plan.blocked_schemas) + ) + suppressed: tuple[JanitorDeleteCandidate, ...] = tuple( + chain.from_iterable(blocked.suppressed_candidates for blocked in plan.blocked_schemas) + ) + assert blocked_sources == test_case.expected_blocked_sources + assert ( + tuple(candidate.key.display_name() for candidate in suppressed) + == test_case.expected_suppressed_names + ) From cb56814e2d2545b6d8d61ebc88f5b0f107eaa159 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:24:51 +0100 Subject: [PATCH 5/8] fix(snowflake): expose relation age metadata in UTC Snowflake relation listings already read created and last_altered for tables and views. The adapter now reports age metadata support and normalizes both timestamps to UTC so retention-based janitor cleanup can evaluate relation age. --- .../snowflake/classes/snowflake_adapter.py | 16 +++-- .../adapters/snowflake/_test_types.py | 10 +++ .../adapters/snowflake/test_client.py | 66 ++++++++++++++++++- 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index c99c7ec9ea..db3d8c030a 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -6,7 +6,7 @@ import json import logging from dataclasses import replace -from datetime import date, datetime +from datetime import UTC, date, datetime from decimal import Decimal from pathlib import Path from typing import Any, ClassVar @@ -540,7 +540,15 @@ def render_prune_source_freshness_history_sql( ) def supports_relation_age_metadata(self) -> bool: - return False + return True + + @staticmethod + def _utc_timestamp(*, value: object) -> datetime | None: + if not isinstance(value, datetime): + return None + if value.tzinfo is None or value.utcoffset() is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) def supports_table_freshness_metadata(self) -> bool: return True @@ -1823,8 +1831,8 @@ def list_relations( is_transient=( None if row[3] is None else str(row[3]).upper() == TRUE_METADATA_VALUE ), - created_at=None if row[4] is None else row[4], - last_altered_at=None if row[5] is None else row[5], + created_at=self._utc_timestamp(value=row[4]), + last_altered_at=self._utc_timestamp(value=row[5]), retention_days=None if row[6] is None else int(row[6]), ) for row in rows diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 103623818c..48189e6048 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -194,3 +194,13 @@ class SnowflakeBatchedRetentionErrorTestCase: requests: tuple[tuple[str, str], ...] rows: list[tuple[object, ...]] expected_error: str + + +@dataclass(frozen=True) +class SnowflakeRelationAgeMetadataTestCase: + description: str + relation_type: str + created: datetime | None + last_altered: datetime | None + expected_created_at: datetime | None + expected_last_altered_at: datetime | None diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 54bcbf4ef5..d4b7cd9a7e 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta, timezone from pathlib import Path from typing import Any, cast from unittest.mock import patch @@ -36,6 +36,7 @@ SnowflakePruneSqlTestCase, SnowflakeQualifiedColumnInspectionTestCase, SnowflakeQueryColumnNamesTestCase, + SnowflakeRelationAgeMetadataTestCase, SnowflakeRenderCloneTestCase, SnowflakeRenderCursorBoundLiteralTestCase, SnowflakeRenderIdentifierTestCase, @@ -1200,3 +1201,66 @@ def test_given_default_seed_csv_settings_when_loading_seed_then_uses_python_csv_ ) assert cursor.executemany_rows == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeRelationAgeMetadataTestCase( + description="local-offset table timestamps are normalized to UTC", + relation_type="BASE TABLE", + created=datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + last_altered=datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), + expected_created_at=datetime(2026, 8, 1, 10, 0, tzinfo=UTC), + expected_last_altered_at=datetime(2026, 8, 2, 10, 30, tzinfo=UTC), + ), + SnowflakeRelationAgeMetadataTestCase( + description="view timestamps are read and naive values are treated as UTC", + relation_type="VIEW", + created=datetime(2026, 8, 1, 3, 0), + last_altered=datetime(2026, 8, 3, 4, 0), + expected_created_at=datetime(2026, 8, 1, 3, 0, tzinfo=UTC), + expected_last_altered_at=datetime(2026, 8, 3, 4, 0, tzinfo=UTC), + ), + SnowflakeRelationAgeMetadataTestCase( + description="missing timestamps stay unknown", + relation_type="VIEW", + created=None, + last_altered=None, + expected_created_at=None, + expected_last_altered_at=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_relation_timestamps_when_listing_then_exposes_utc_age_metadata( + test_case: SnowflakeRelationAgeMetadataTestCase, +) -> None: + cursor: FakeSnowflakeMetadataCursor = FakeSnowflakeMetadataCursor( + rows=[ + ( + "OLD_ORDERS", + "DEV_ORDERS", + test_case.relation_type, + "NO", + test_case.created, + test_case.last_altered, + None, + ) + ] + ) + adapter: SnowflakeAdapter = SnowflakeAdapter() + + relations: tuple[Any, ...] = adapter.list_relations( + connection=cast(Any, FakeSnowflakeMetadataConnection(cursor)), + database="ANALYTICS", + schemas=("DEV_ORDERS",), + ) + + assert adapter.supports_relation_age_metadata() is True + assert relations[0].created_at == test_case.expected_created_at + assert relations[0].last_altered_at == test_case.expected_last_altered_at + assert (str(relations[0].created_at), str(relations[0].last_altered_at)) == ( + str(test_case.expected_created_at), + str(test_case.expected_last_altered_at), + ) From 46bfab0a6ec83a4756e487f217c4097c6248933b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:44:32 +0100 Subject: [PATCH 6/8] fix(adapters): expose relation age metadata for Databricks and BigQuery Databricks relation listings read created and last_altered from information_schema.tables. BigQuery keeps its list_tables listing and adds an on-demand with_relation_age_metadata hook that reads creation and last-modified times from one __TABLES__ query per dataset; janitor planning calls it, so other commands pay no extra query jobs. Catalog timestamps share one UTC normalizer. SQL Server, Postgres, DuckDB, and MotherDuck document why they do not report relation age. --- .../adapter/contract/classes/base_adapter.py | 8 +++ .../contract/classes/duckdb_backed_adapter.py | 10 +++ .../adapter/contract/classes/schema.py | 10 +++ .../relations/main/relation_age_timestamp.py | 15 ++++ .../bigquery/classes/bigquery_adapter.py | 57 ++++++++++++++- .../databricks/classes/databricks_adapter.py | 17 ++++- .../postgres/classes/postgres_adapter.py | 10 +++ .../snowflake/classes/snowflake_adapter.py | 23 +++--- .../sqlserver/classes/sqlserver_adapter.py | 10 +++ .../executor/janitor/_helpers/plan.py | 9 ++- .../unit/src/sqlbuild/adapters/_test_types.py | 7 ++ .../sqlbuild/adapters/bigquery/_test_types.py | 7 ++ .../sqlbuild/adapters/bigquery/test_client.py | 69 +++++++++++++++++- .../adapters/databricks/_test_types.py | 10 +++ .../adapters/databricks/test_client.py | 66 ++++++++++++++++- .../test_relation_age_metadata_capability.py | 61 ++++++++++++++++ .../executor/janitor/main/_test_types.py | 8 +++ .../sqlbuild/executor/janitor/main/helpers.py | 27 +++++++ .../main/test_plan_separate_age_metadata.py | 72 +++++++++++++++++++ 19 files changed, 476 insertions(+), 20 deletions(-) create mode 100644 src/sqlbuild/adapter/relations/main/relation_age_timestamp.py create mode 100644 tests/unit/src/sqlbuild/adapters/test_relation_age_metadata_capability.py create mode 100644 tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index a61568d86e..d2bf04d48a 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -389,6 +389,14 @@ def relation_exists( ) return cursor.fetchone() is not None + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_relations( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 61e55dd28e..c87ab8f663 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -97,6 +97,8 @@ def supports_durable_clone(self) -> bool: return False def supports_relation_age_metadata(self) -> bool: + """Return False because the DuckDB and MotherDuck catalogs record no relation timestamps.""" + return False def supports_table_freshness_metadata(self) -> bool: @@ -1323,6 +1325,14 @@ def list_relations( for row in rows ) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/schema.py b/src/sqlbuild/adapter/contract/classes/schema.py index 980e41a298..501ccffd76 100644 --- a/src/sqlbuild/adapter/contract/classes/schema.py +++ b/src/sqlbuild/adapter/contract/classes/schema.py @@ -23,6 +23,16 @@ def list_relations( """Return all relations in the given database/schema scope.""" ... + @abstractmethod + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + """Return listed relations with age timestamps that listing does not read inline.""" + ... + @abstractmethod def list_functions( self, diff --git a/src/sqlbuild/adapter/relations/main/relation_age_timestamp.py b/src/sqlbuild/adapter/relations/main/relation_age_timestamp.py new file mode 100644 index 0000000000..77337b3044 --- /dev/null +++ b/src/sqlbuild/adapter/relations/main/relation_age_timestamp.py @@ -0,0 +1,15 @@ +"""Relation catalog timestamp normalization entrypoint.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def relation_age_timestamp_utc(value: object) -> datetime | None: + """Return one catalog timestamp as aware UTC, treating offset-free values as UTC.""" + + if not isinstance(value, datetime): + return None + if value.tzinfo is None or value.utcoffset() is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 475e02879b..95bfc19c6e 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -4,6 +4,7 @@ import json import logging +from dataclasses import replace from datetime import date, datetime from decimal import Decimal from pathlib import Path @@ -64,6 +65,7 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) +from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( render_insert_source_freshness_records_sql, ) @@ -362,7 +364,7 @@ def render_prune_source_freshness_history_sql( ) def supports_relation_age_metadata(self) -> bool: - return False + return True def supports_table_freshness_metadata(self) -> bool: return True @@ -2046,6 +2048,59 @@ def list_relations( raise return tuple(relations) + def with_relation_age_metadata( + self, + *, + connection: _BigQueryConnection, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + timestamps: dict[tuple[str | None, str, str], tuple[datetime | None, datetime | None]] = {} + dataset_key: tuple[str | None, str] + for dataset_key in sorted( + {(relation.database, str(relation.schema)) for relation in relations}, + key=lambda key: (key[0] or "", key[1]), + ): + timestamps.update( + self._dataset_relation_timestamps( + connection=connection, database=dataset_key[0], schema=dataset_key[1] + ) + ) + return tuple( + replace( + relation, + created_at=timestamps.get(self._age_key(relation), (None, None))[0], + last_altered_at=timestamps.get(self._age_key(relation), (None, None))[1], + ) + for relation in relations + ) + + def _dataset_relation_timestamps( + self, *, connection: _BigQueryConnection, database: str | None, schema: str + ) -> dict[tuple[str | None, str, str], tuple[datetime | None, datetime | None]]: + dataset_id: str = self._build_dataset_id(database=database, schema=schema) + try: + cursor: _BigQueryCursor = self.execute( + connection=connection, + sql="SELECT table_id, TIMESTAMP_MILLIS(creation_time), " + f"TIMESTAMP_MILLIS(last_modified_time) FROM `{dataset_id}.__TABLES__`", + ) + rows: list[tuple[Any, ...]] = cursor.fetchall() + except Exception as error: + if not self._is_google_not_found(error): + raise + return {} + return { + (database, schema.lower(), str(row[0]).lower()): ( + relation_age_timestamp_utc(row[1]), + relation_age_timestamp_utc(row[2]), + ) + for row in rows + } + + @staticmethod + def _age_key(relation: RelationInfo) -> tuple[str | None, str, str]: + return (relation.database, str(relation.schema).lower(), relation.name.lower()) + def list_functions( self, *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 3c84ec3f26..e10ec8f6ae 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -63,6 +63,7 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) +from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( render_insert_source_freshness_records_sql, ) @@ -1269,7 +1270,7 @@ def supports_durable_clone(self) -> bool: return True def supports_relation_age_metadata(self) -> bool: - return False + return True def recommended_max_sql_length(self) -> int | None: return 256_000 @@ -1406,8 +1407,8 @@ def list_relations( return () information_schema: str = self._information_schema(database) query: str = ( - f"SELECT table_name, table_schema, table_type FROM {information_schema}.tables " - "WHERE 1=1" + "SELECT table_name, table_schema, table_type, created, last_altered " + f"FROM {information_schema}.tables WHERE 1=1" ) query += self._build_in_filter(column="table_schema", values=schemas) if names: @@ -1424,10 +1425,20 @@ def list_relations( schema=str(row[1]).lower(), name=str(row[0]).lower(), relation_type=self._normalize_relation_type(str(row[2])), + created_at=relation_age_timestamp_utc(row[3]), + last_altered_at=relation_age_timestamp_utc(row[4]), ) for row in rows ) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 9ef81a54fa..31a20b204c 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -110,6 +110,8 @@ def supports_durable_clone(self) -> bool: return False def supports_relation_age_metadata(self) -> bool: + """Return False because the Postgres catalog records no relation timestamps.""" + return False def supports_table_freshness_metadata(self) -> bool: @@ -307,6 +309,14 @@ def list_relations( for row in cursor.fetchall() ) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index db3d8c030a..58fc8b415e 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -6,7 +6,7 @@ import json import logging from dataclasses import replace -from datetime import UTC, date, datetime +from datetime import date, datetime from decimal import Decimal from pathlib import Path from typing import Any, ClassVar @@ -57,6 +57,7 @@ RetentionScope, TablePromotionMode, ) +from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( render_insert_source_freshness_records_sql, ) @@ -542,14 +543,6 @@ def render_prune_source_freshness_history_sql( def supports_relation_age_metadata(self) -> bool: return True - @staticmethod - def _utc_timestamp(*, value: object) -> datetime | None: - if not isinstance(value, datetime): - return None - if value.tzinfo is None or value.utcoffset() is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) - def supports_table_freshness_metadata(self) -> bool: return True @@ -1831,13 +1824,21 @@ def list_relations( is_transient=( None if row[3] is None else str(row[3]).upper() == TRUE_METADATA_VALUE ), - created_at=self._utc_timestamp(value=row[4]), - last_altered_at=self._utc_timestamp(value=row[5]), + created_at=relation_age_timestamp_utc(row[4]), + last_altered_at=relation_age_timestamp_utc(row[5]), retention_days=None if row[6] is None else int(row[6]), ) for row in rows ) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 8fbffec680..cb9f464a30 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -1674,6 +1674,14 @@ def get_columns( cursor: Any = connection.execute(query) return tuple(ColumnInfo(name=row[0], type=row[1]) for row in cursor.fetchall()) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, @@ -2922,6 +2930,8 @@ def supports_python_functions(self) -> bool: return False def supports_relation_age_metadata(self) -> bool: + """Return False because sys.objects.modify_date ignores DML, so written tables look old.""" + return False def supports_table_functions(self) -> bool: diff --git a/src/sqlbuild/executor/janitor/_helpers/plan.py b/src/sqlbuild/executor/janitor/_helpers/plan.py index 3e8e4c466f..10f484e7f5 100644 --- a/src/sqlbuild/executor/janitor/_helpers/plan.py +++ b/src/sqlbuild/executor/janitor/_helpers/plan.py @@ -135,10 +135,13 @@ def list_target_schema_relations( if not concrete_schemas: concrete_schemas = None relation: RelationInfo - for relation in adapter.list_relations( + for relation in adapter.with_relation_age_metadata( connection=connection, - database=database, - schemas=concrete_schemas, + relations=adapter.list_relations( + connection=connection, + database=database, + schemas=concrete_schemas, + ), ): target_key: tuple[str | None, str | None] | None = target_by_normalized.get( normalized_schema_key((relation.database, relation.schema)) diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 18726d0de0..39275d97d8 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -69,3 +69,10 @@ class RowDiffSampleSqlTestCase: description: str adapter: BaseAdapter expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class AdapterRelationAgeMetadataCapabilityTestCase: + description: str + adapter: StrictAdapter + expected_supported: bool diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index 2003efebbb..de9d0bee25 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -202,3 +202,10 @@ class BigQueryTableFreshnessWildcardTestCase: description: str table_name: str expected_error_fragment: str + + +@dataclass(frozen=True) +class BigQueryRelationAgeMetadataTestCase: + description: str + metadata_rows: tuple[tuple[object, ...], ...] + expected_timestamps: tuple[tuple[str, str], ...] diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py index 0bac9a15f3..5a6bbfdafa 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime, timedelta, timezone import pytest @@ -10,6 +10,7 @@ CursorValue, ExpressionInferenceProfile, QueryResult, + RelationInfo, RowDiffColumnResult, RowDiffResult, RowDiffSampleCell, @@ -43,6 +44,7 @@ BigQueryMergeExclusionTestCase, BigQueryPruneSqlTestCase, BigQueryQueryTestCase, + BigQueryRelationAgeMetadataTestCase, BigQueryRenderCloneTestCase, BigQueryRenderCursorBoundLiteralTestCase, BigQueryRenderDeleteInsertTestCase, @@ -1104,3 +1106,68 @@ def test_given_timestamp_cursor_when_counting_rows_then_bigquery_uses_typed_filt assert result == test_case.expected_count assert executed_sql == [test_case.expected_sql] + + +@pytest.mark.parametrize( + "test_case", + [ + BigQueryRelationAgeMetadataTestCase( + description="dataset metadata supplies UTC creation and last-modified times", + metadata_rows=( + ( + "Old_Orders", + datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + datetime(2026, 8, 2, 10, 30, tzinfo=UTC), + ), + ("old_orders_view", datetime(2026, 8, 3, 4, 0), None), + ), + expected_timestamps=( + ("2026-08-01 10:00:00+00:00", "2026-08-02 10:30:00+00:00"), + ("2026-08-03 04:00:00+00:00", "None"), + ), + ), + BigQueryRelationAgeMetadataTestCase( + description="relations missing from dataset metadata stay unknown", + metadata_rows=(), + expected_timestamps=(("None", "None"), ("None", "None")), + ), + ], + ids=lambda case: case.description, +) +def test_given_listed_relations_when_adding_age_metadata_then_bigquery_reads_each_dataset_once( + test_case: BigQueryRelationAgeMetadataTestCase, +) -> None: + client: FakeBigQueryClient = FakeBigQueryClient( + rows=FakeBigQueryRows( + columns=("table_id", "creation_time", "last_modified_time"), + rows=test_case.metadata_rows, + ) + ) + relations: tuple[RelationInfo, ...] = ( + RelationInfo( + database="example-project", + schema="dev_orders", + name="old_orders", + relation_type="table", + ), + RelationInfo( + database="example-project", + schema="dev_orders", + name="old_orders_view", + relation_type="view", + ), + ) + + enriched: tuple[RelationInfo, ...] = BigQueryAdapter().with_relation_age_metadata( + connection=_BigQueryConnection(client=client, location=None), + relations=relations, + ) + + assert ( + tuple((str(relation.created_at), str(relation.last_altered_at)) for relation in enriched) + == test_case.expected_timestamps + ) + assert tuple(sql for sql, _ in client.queries) == ( + "SELECT table_id, TIMESTAMP_MILLIS(creation_time), " + "TIMESTAMP_MILLIS(last_modified_time) FROM `example-project.dev_orders.__TABLES__`", + ) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index f16b3f2dd6..45a6b4c52d 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -127,3 +127,13 @@ class DatabricksTableFreshnessFallbackTestCase: description: str expected_data_versions: tuple[datetime, ...] expected_query_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DatabricksRelationAgeMetadataTestCase: + description: str + table_type: str + created: datetime | None + last_altered: datetime | None + expected_relation_type: str + expected_timestamps: tuple[str, str] diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index a1765420ab..a32955e32f 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -1,11 +1,13 @@ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timedelta, timezone +from typing import Any, cast import pytest from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, + RelationInfo, TableFreshnessMetadata, TableFreshnessRequest, ) @@ -22,6 +24,7 @@ DatabricksMergeExclusionTestCase, DatabricksPruneSqlTestCase, DatabricksPythonFunctionSupportTestCase, + DatabricksRelationAgeMetadataTestCase, DatabricksRenderCloneTestCase, DatabricksRenderDeleteInsertCursorTestCase, DatabricksRenderDurableCloneTestCase, @@ -575,3 +578,64 @@ def test_given_table_function_when_rendering_then_databricks_returns_expected_dd ) assert statements == test_case.expected_statements + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksRelationAgeMetadataTestCase( + description="managed table offset timestamps are normalized to UTC", + table_type="MANAGED", + created=datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + last_altered=datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), + expected_relation_type="table", + expected_timestamps=("2026-08-01 10:00:00+00:00", "2026-08-02 10:30:00+00:00"), + ), + DatabricksRelationAgeMetadataTestCase( + description="view naive timestamps are treated as UTC", + table_type="VIEW", + created=datetime(2026, 8, 1, 3, 0), + last_altered=datetime(2026, 8, 3, 4, 0), + expected_relation_type="view", + expected_timestamps=("2026-08-01 03:00:00+00:00", "2026-08-03 04:00:00+00:00"), + ), + DatabricksRelationAgeMetadataTestCase( + description="missing timestamps stay unknown", + table_type="MANAGED", + created=None, + last_altered=None, + expected_relation_type="table", + expected_timestamps=("None", "None"), + ), + ], + ids=lambda case: case.description, +) +def test_given_databricks_relation_timestamps_when_listing_then_exposes_utc_age_metadata( + test_case: DatabricksRelationAgeMetadataTestCase, +) -> None: + cursor: FakeDatabricksMetadataCursor = FakeDatabricksMetadataCursor( + rows=[ + ( + "OLD_ORDERS", + "DEV_ORDERS", + test_case.table_type, + test_case.created, + test_case.last_altered, + ) + ] + ) + adapter: DatabricksAdapter = DatabricksAdapter() + + relations: tuple[RelationInfo, ...] = adapter.list_relations( + connection=cast(Any, FakeDatabricksMetadataConnection((cursor,))), + database="analytics", + schemas=("dev_orders",), + ) + + assert "table_type, created, last_altered" in str(cursor.executed_sql) + assert relations[0].name == "old_orders" + assert relations[0].relation_type == test_case.expected_relation_type + assert ( + str(relations[0].created_at), + str(relations[0].last_altered_at), + ) == test_case.expected_timestamps diff --git a/tests/unit/src/sqlbuild/adapters/test_relation_age_metadata_capability.py b/tests/unit/src/sqlbuild/adapters/test_relation_age_metadata_capability.py new file mode 100644 index 0000000000..4a7932ef72 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/test_relation_age_metadata_capability.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from tests.unit.src.sqlbuild.adapters._test_types import ( + AdapterRelationAgeMetadataCapabilityTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + ( + AdapterRelationAgeMetadataCapabilityTestCase( + description="Snowflake reads created and last_altered", + adapter=SnowflakeAdapter(), + expected_supported=True, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="Databricks reads created and last_altered", + adapter=DatabricksAdapter(), + expected_supported=True, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="BigQuery reads creation and last-modified times", + adapter=BigQueryAdapter(), + expected_supported=True, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="SQL Server modify_date ignores data writes", + adapter=SqlServerAdapter(), + expected_supported=False, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="Postgres catalog has no relation timestamps", + adapter=PostgresAdapter(), + expected_supported=False, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="DuckDB catalog has no relation timestamps", + adapter=DuckDbAdapter(), + expected_supported=False, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="MotherDuck catalog has no relation timestamps", + adapter=MotherDuckAdapter(), + expected_supported=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_adapter_when_checking_relation_age_metadata_then_matches_catalog_reliability( + test_case: AdapterRelationAgeMetadataCapabilityTestCase, +) -> None: + assert test_case.adapter.supports_relation_age_metadata() is test_case.expected_supported diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index b8c7607fe2..1248c0579b 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -150,3 +150,11 @@ class JanitorFoldedIdentifierPlanTestCase: expected_skipped_relations: tuple[tuple[str, str], ...] = field(default_factory=tuple) expected_blocked_sources: tuple[str, ...] = field(default_factory=tuple) expected_suppressed_names: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorSeparateAgeMetadataPlanTestCase: + description: str + relation_ages: dict[str, datetime] + expected_candidate_names: tuple[str, ...] + expected_skipped_relations: tuple[tuple[str, str], ...] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 35caabbe48..4c35f24e7e 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -5,6 +5,7 @@ import base64 from collections.abc import Callable from dataclasses import replace +from datetime import datetime from pathlib import Path from typing import Any, ClassVar, cast @@ -498,6 +499,32 @@ def list_relations( ) +class SeparateAgeMetadataJanitorAdapter(FakeJanitorAdapter): + """Warehouse double that lists without timestamps and supplies ages separately.""" + + def __init__( + self, + *, + relation_infos: tuple[RelationInfo, ...], + relation_ages: dict[str, datetime], + ) -> None: + super().__init__(relation_infos=relation_infos) + self.relation_ages: dict[str, datetime] = relation_ages + self.age_metadata_requests: list[tuple[str, ...]] = [] + + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + self.age_metadata_requests.append(tuple(relation.name for relation in relations)) + return tuple( + replace(relation, last_altered_at=self.relation_ages.get(relation.name)) + for relation in relations + ) + + class FailingJanitorEventAdapter(FakeJanitorAdapter): def _execute(self, connection: Any, sql: str) -> Any: is_event_insert: bool = sql.startswith("INSERT INTO") and JANITOR_EVENTS_TABLE_NAME in sql diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py new file mode 100644 index 0000000000..e234cd0adb --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py @@ -0,0 +1,72 @@ +"""Janitor planning against adapters that read relation ages separately from listing.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorSeparateAgeMetadataPlanTestCase, + relation_info, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + SeparateAgeMetadataJanitorAdapter, + build_project, +) + +OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) +RECENT_TIME: datetime = datetime.now(UTC) - timedelta(days=1) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorSeparateAgeMetadataPlanTestCase( + description="separately read ages decide which stale relations are old enough", + relation_ages={"old_orders": OLD_TIME, "recent_orders": RECENT_TIME}, + expected_candidate_names=("old_orders",), + expected_skipped_relations=( + ("analytics.recent_orders", "relation is newer than 7 days"), + ), + ), + JanitorSeparateAgeMetadataPlanTestCase( + description="relations without separately read ages are skipped as unknown", + relation_ages={}, + expected_candidate_names=(), + expected_skipped_relations=( + ("analytics.old_orders", "relation age is unavailable"), + ("analytics.recent_orders", "relation age is unavailable"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_reading_ages_separately_when_planning_janitor_then_uses_those_ages( + test_case: JanitorSeparateAgeMetadataPlanTestCase, +) -> None: + adapter: SeparateAgeMetadataJanitorAdapter = SeparateAgeMetadataJanitorAdapter( + relation_infos=(relation_info("old_orders"), relation_info("recent_orders")), + relation_ages=test_case.relation_ages, + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings(enabled=True), + ) + + assert ( + tuple(candidate.key.name for candidate in plan.candidates) + == test_case.expected_candidate_names + ) + assert ( + tuple((skipped.key.display_name(), skipped.reason) for skipped in plan.skipped_relations) + == test_case.expected_skipped_relations + ) + assert len(adapter.age_metadata_requests) == 1 From 399c702c165e5b8f08f0ea3abb87159f968633cd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:58:50 +0100 Subject: [PATCH 7/8] fix(janitor): read relation ages for candidates only and use Delta write times on Databricks Janitor classification now asks the adapter for age metadata only for relations that survive the desired, tracked, exclude, protection, and addressing filters, in both direct and virtual mode. Databricks information_schema last_altered tracks DDL only, so non-view relations no longer take ages from it; the Databricks hook reads createdAt and lastModified from DESCRIBE DETAIL per candidate and leaves the age unknown when that fails. Views keep their information_schema timestamps. --- .../databricks/classes/databricks_adapter.py | 63 +++++++-- .../janitor/_helpers/classification.py | 15 +- .../executor/janitor/_helpers/plan.py | 9 +- .../janitor/_helpers/schema_planning.py | 11 +- .../janitor/classes/relation_age_reader.py | 30 ++++ src/sqlbuild/executor/janitor/main/plan.py | 8 +- .../adapters/databricks/_test_types.py | 11 ++ .../sqlbuild/adapters/databricks/helpers.py | 9 +- .../adapters/databricks/test_client.py | 129 +++++++++++++++++- .../executor/janitor/main/_test_types.py | 12 ++ .../sqlbuild/executor/janitor/main/helpers.py | 3 +- .../main/test_plan_separate_age_metadata.py | 87 +++++++++++- 12 files changed, 349 insertions(+), 38 deletions(-) create mode 100644 src/sqlbuild/executor/janitor/classes/relation_age_reader.py diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index e10ec8f6ae..e7fb0bc6ef 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -7,6 +7,7 @@ import json import logging import re +from dataclasses import replace from datetime import date, datetime from decimal import Decimal from pathlib import Path @@ -1419,16 +1420,18 @@ def list_relations( rows: list[tuple[Any, ...]] = cursor.fetchall() finally: cursor.close() - return tuple( - RelationInfo( - database=database, - schema=str(row[1]).lower(), - name=str(row[0]).lower(), - relation_type=self._normalize_relation_type(str(row[2])), - created_at=relation_age_timestamp_utc(row[3]), - last_altered_at=relation_age_timestamp_utc(row[4]), - ) - for row in rows + return tuple(self._listed_relation(database=database, row=row) for row in rows) + + def _listed_relation(self, *, database: str, row: tuple[Any, ...]) -> RelationInfo: + relation_type: str = self._normalize_relation_type(str(row[2])) + is_view: bool = relation_type == RelationType.VIEW + return RelationInfo( + database=database, + schema=str(row[1]).lower(), + name=str(row[0]).lower(), + relation_type=relation_type, + created_at=relation_age_timestamp_utc(row[3]) if is_view else None, + last_altered_at=relation_age_timestamp_utc(row[4]) if is_view else None, ) def with_relation_age_metadata( @@ -1437,7 +1440,45 @@ def with_relation_age_metadata( connection: Any, relations: tuple[RelationInfo, ...], ) -> tuple[RelationInfo, ...]: - return relations + """Read data-write ages for non-view relations; information_schema tracks only DDL.""" + + return tuple( + relation + if relation.relation_type == RelationType.VIEW + else self._with_delta_detail_ages(connection=connection, relation=relation) + for relation in relations + ) + + def _with_delta_detail_ages(self, *, connection: Any, relation: RelationInfo) -> RelationInfo: + unknown_age: RelationInfo = replace(relation, created_at=None, last_altered_at=None) + if relation.database is None or relation.schema is None: + return unknown_age + target: str = ".".join( + self.render_identifier(part) + for part in (relation.database, relation.schema, relation.name) + ) + cursor: Any = connection.cursor() + try: + cursor.execute(f"DESCRIBE DETAIL {target}") + row: tuple[Any, ...] | None = cursor.fetchone() + description: tuple[Any, ...] | None = cursor.description + except Exception: + return unknown_age + finally: + cursor.close() + if row is None or description is None: + return unknown_age + detail: dict[str, object] = { + str(column[0]).lower(): value for column, value in zip(description, row, strict=False) + } + last_modified: datetime | None = relation_age_timestamp_utc(detail.get("lastmodified")) + if last_modified is None: + return unknown_age + return replace( + relation, + created_at=relation_age_timestamp_utc(detail.get("createdat")), + last_altered_at=last_modified, + ) def list_functions( self, diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index 78f282f290..a270514f01 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -32,6 +32,7 @@ unaddressable_relation_reason, ) from sqlbuild.executor.janitor._helpers.tracking import collect_tracked_relation_keys +from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader from sqlbuild.executor.janitor.models import ( JanitorDeleteCandidate, JanitorDirectStatePruneCandidate, @@ -172,7 +173,7 @@ def classify_janitor_relations( effective_exclude_patterns: tuple[str, ...], delete_tracked_only: bool, retention_days: int, - age_supported: bool, + age_reader: JanitorRelationAgeReader, now: datetime, direct_mode: bool = False, ) -> JanitorRelationClassification: @@ -180,6 +181,7 @@ def classify_janitor_relations( candidates: list[JanitorDeleteCandidate] = [] skipped_relations: list[JanitorSkippedRelation] = [] + eligible_relations: list[RelationInfo] = [] colliding_names: frozenset[str] = case_colliding_names( relation.name for relation in schema_relations ) @@ -207,7 +209,12 @@ def classify_janitor_relations( JanitorSkippedRelation(key=relation_key, relation=relation, reason=skip_reason) ) continue - age_timestamp: datetime | None = relation_age_timestamp(relation) + eligible_relations.append(relation) + age_supported: bool = age_reader.supported() + aged_relation: RelationInfo + for aged_relation in age_reader.read(tuple(eligible_relations)): + relation_key = build_relation_key(aged_relation) + age_timestamp: datetime | None = relation_age_timestamp(aged_relation) if retention_days > 0: retention_skip_reason: str | None = _retention_skip_reason( age_timestamp=age_timestamp, @@ -219,7 +226,7 @@ def classify_janitor_relations( skipped_relations.append( JanitorSkippedRelation( key=relation_key, - relation=relation, + relation=aged_relation, reason=retention_skip_reason, ) ) @@ -227,7 +234,7 @@ def classify_janitor_relations( candidates.append( JanitorDeleteCandidate( key=relation_key, - relation=relation, + relation=aged_relation, age_timestamp=age_timestamp, ) ) diff --git a/src/sqlbuild/executor/janitor/_helpers/plan.py b/src/sqlbuild/executor/janitor/_helpers/plan.py index 10f484e7f5..3e8e4c466f 100644 --- a/src/sqlbuild/executor/janitor/_helpers/plan.py +++ b/src/sqlbuild/executor/janitor/_helpers/plan.py @@ -135,13 +135,10 @@ def list_target_schema_relations( if not concrete_schemas: concrete_schemas = None relation: RelationInfo - for relation in adapter.with_relation_age_metadata( + for relation in adapter.list_relations( connection=connection, - relations=adapter.list_relations( - connection=connection, - database=database, - schemas=concrete_schemas, - ), + database=database, + schemas=concrete_schemas, ): target_key: tuple[str | None, str | None] | None = target_by_normalized.get( normalized_schema_key((relation.database, relation.schema)) diff --git a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py index 6d84225372..6e72f870b4 100644 --- a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py +++ b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py @@ -10,6 +10,7 @@ blocking_source_names, source_names_for_schema, ) +from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader from sqlbuild.executor.janitor.constants import BUILT_IN_EXCLUDE_PATTERNS from sqlbuild.executor.janitor.models import ( JanitorBlockedSchema, @@ -32,7 +33,7 @@ def classify_target_schemas( exclude_patterns: tuple[str, ...], delete_tracked_only: bool, retention_days: int, - age_supported: bool, + age_reader: JanitorRelationAgeReader, now: datetime, direct_mode: bool, ) -> JanitorSchemaClassification: @@ -61,7 +62,7 @@ def classify_target_schemas( exclude_patterns=exclude_patterns, delete_tracked_only=delete_tracked_only, retention_days=retention_days, - age_supported=age_supported, + age_reader=age_reader, now=now, direct_mode=direct_mode, ) @@ -90,7 +91,7 @@ def classify_target_schemas( exclude_patterns=exclude_patterns, delete_tracked_only=delete_tracked_only, retention_days=retention_days, - age_supported=age_supported, + age_reader=age_reader, now=now, direct_mode=direct_mode, ) @@ -112,7 +113,7 @@ def _classify_schema( exclude_patterns: tuple[str, ...], delete_tracked_only: bool, retention_days: int, - age_supported: bool, + age_reader: JanitorRelationAgeReader, now: datetime, direct_mode: bool, ) -> JanitorRelationClassification: @@ -124,7 +125,7 @@ def _classify_schema( effective_exclude_patterns=BUILT_IN_EXCLUDE_PATTERNS + exclude_patterns, delete_tracked_only=delete_tracked_only, retention_days=retention_days, - age_supported=age_supported, + age_reader=age_reader, now=now, direct_mode=direct_mode, ) diff --git a/src/sqlbuild/executor/janitor/classes/relation_age_reader.py b/src/sqlbuild/executor/janitor/classes/relation_age_reader.py new file mode 100644 index 0000000000..250082de9f --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/relation_age_reader.py @@ -0,0 +1,30 @@ +"""Candidate-scoped relation age metadata reader for janitor planning.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo + + +class JanitorRelationAgeReader: + """Adapter connection used to read age metadata for janitor candidates only.""" + + def __init__(self, *, adapter: BaseAdapter, connection: Any) -> None: + self.adapter: BaseAdapter = adapter + self.connection: Any = connection + + def supported(self) -> bool: + """Return whether the adapter reports reliable relation age metadata.""" + + return self.adapter.supports_relation_age_metadata() + + def read(self, relations: tuple[RelationInfo, ...]) -> tuple[RelationInfo, ...]: + """Return candidate relations with age timestamps filled when supported.""" + + if not relations or not self.supported(): + return relations + return self.adapter.with_relation_age_metadata( + connection=self.connection, relations=relations + ) diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py index 123f0b3b1f..45c88ec42f 100644 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ b/src/sqlbuild/executor/janitor/main/plan.py @@ -16,6 +16,7 @@ ) from sqlbuild.executor.janitor._helpers.plan import collect_scan_schemas, collect_target_schemas from sqlbuild.executor.janitor._helpers.schema_planning import classify_target_schemas +from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader from sqlbuild.executor.janitor.models import ( JanitorArchivePlanning, JanitorDirectModeSettings, @@ -111,8 +112,12 @@ def build_janitor_plan( ) ) inspection.completed(metadata={"item_count": len(target_schemas)}) + age_reader: JanitorRelationAgeReader = JanitorRelationAgeReader( + adapter=adapter, connection=connection + ) age_supported: bool = adapter.supports_relation_age_metadata() schemas: JanitorSchemaClassification = classify_target_schemas( + age_reader=age_reader, target_schemas=target_schemas, managed_target_schemas=managed_target_schemas, facts=facts, @@ -120,7 +125,6 @@ def build_janitor_plan( exclude_patterns=exclude_patterns, delete_tracked_only=delete_tracked_only, retention_days=retention_days, - age_supported=age_supported, now=now, direct_mode=direct.enabled, ) @@ -153,7 +157,7 @@ def build_janitor_plan( virtual_state_prune_candidates=state.virtual_state_prune_candidates, direct_state_prune_candidates=direct_state_prune_candidates, skipped_relations=( - *tuple( + *( skipped for skipped in schemas.skipped_relations if not QueryDiffArtifactLifecycle.is_artifact_name(skipped.key.name) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index 45a6b4c52d..747801d996 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -1,7 +1,9 @@ from dataclasses import dataclass from datetime import datetime +from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.lineage.types import InferredNullability +from tests.unit.src.sqlbuild.adapters.databricks.helpers import FakeDatabricksMetadataCursor @dataclass(frozen=True) @@ -137,3 +139,12 @@ class DatabricksRelationAgeMetadataTestCase: last_altered: datetime | None expected_relation_type: str expected_timestamps: tuple[str, str] + + +@dataclass(frozen=True) +class DatabricksDeltaDetailAgeTestCase: + description: str + relation: RelationInfo + cursors: tuple[FakeDatabricksMetadataCursor, ...] + expected_sql: tuple[str, ...] + expected_timestamps: tuple[str, str] diff --git a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py index 66addaffd7..2f5ddf88fb 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py @@ -20,9 +20,14 @@ def build_retention_request(*, desired_days: int) -> RetentionRequest: class FakeDatabricksMetadataCursor: - def __init__(self, *, rows: list[tuple[object, ...]] | None = None) -> None: + def __init__( + self, + *, + rows: list[tuple[object, ...]] | None = None, + description: tuple[tuple[str], ...] = (("format",), ("properties",)), + ) -> None: self.rows: list[tuple[object, ...]] = rows or [] - self.description: tuple[tuple[str], ...] = (("format",), ("properties",)) + self.description: tuple[tuple[str], ...] = description self.executed_sql: str | None = None self.closed: bool = False diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index a32955e32f..2b624a4b33 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta, timezone from typing import Any, cast import pytest @@ -20,6 +20,7 @@ from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.adapters.databricks._test_types import ( + DatabricksDeltaDetailAgeTestCase, DatabricksExpressionInferenceProfileTestCase, DatabricksMergeExclusionTestCase, DatabricksPruneSqlTestCase, @@ -584,11 +585,19 @@ def test_given_table_function_when_rendering_then_databricks_returns_expected_dd "test_case", [ DatabricksRelationAgeMetadataTestCase( - description="managed table offset timestamps are normalized to UTC", + description="managed table ignores DDL-only information_schema timestamps", table_type="MANAGED", created=datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), last_altered=datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), expected_relation_type="table", + expected_timestamps=("None", "None"), + ), + DatabricksRelationAgeMetadataTestCase( + description="view offset timestamps are normalized to UTC", + table_type="VIEW", + created=datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + last_altered=datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), + expected_relation_type="view", expected_timestamps=("2026-08-01 10:00:00+00:00", "2026-08-02 10:30:00+00:00"), ), DatabricksRelationAgeMetadataTestCase( @@ -600,17 +609,17 @@ def test_given_table_function_when_rendering_then_databricks_returns_expected_dd expected_timestamps=("2026-08-01 03:00:00+00:00", "2026-08-03 04:00:00+00:00"), ), DatabricksRelationAgeMetadataTestCase( - description="missing timestamps stay unknown", - table_type="MANAGED", + description="missing view timestamps stay unknown", + table_type="VIEW", created=None, last_altered=None, - expected_relation_type="table", + expected_relation_type="view", expected_timestamps=("None", "None"), ), ], ids=lambda case: case.description, ) -def test_given_databricks_relation_timestamps_when_listing_then_exposes_utc_age_metadata( +def test_given_databricks_relation_timestamps_when_listing_then_exposes_view_ages_only( test_case: DatabricksRelationAgeMetadataTestCase, ) -> None: cursor: FakeDatabricksMetadataCursor = FakeDatabricksMetadataCursor( @@ -639,3 +648,111 @@ def test_given_databricks_relation_timestamps_when_listing_then_exposes_utc_age_ str(relations[0].created_at), str(relations[0].last_altered_at), ) == test_case.expected_timestamps + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksDeltaDetailAgeTestCase( + description="table age comes from Delta detail lastModified in UTC", + relation=RelationInfo( + database="analytics", schema="dev_orders", name="old_orders", relation_type="table" + ), + cursors=( + FakeDatabricksMetadataCursor( + rows=[ + ( + "delta", + datetime(2025, 1, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), + ) + ], + description=(("format",), ("createdAt",), ("lastModified",)), + ), + ), + expected_sql=("DESCRIBE DETAIL `analytics`.`dev_orders`.`old_orders`",), + expected_timestamps=("2025-01-01 10:00:00+00:00", "2026-08-02 10:30:00+00:00"), + ), + DatabricksDeltaDetailAgeTestCase( + description="backticks in table identifiers are escaped", + relation=RelationInfo( + database="analytics", schema="dev_orders", name="odd`orders", relation_type="table" + ), + cursors=( + FakeDatabricksMetadataCursor( + rows=[("delta", None, datetime(2026, 8, 2, 10, 30))], + description=(("format",), ("createdAt",), ("lastModified",)), + ), + ), + expected_sql=("DESCRIBE DETAIL `analytics`.`dev_orders`.`odd``orders`",), + expected_timestamps=("None", "2026-08-02 10:30:00+00:00"), + ), + DatabricksDeltaDetailAgeTestCase( + description="views keep information_schema ages without Delta detail", + relation=RelationInfo( + database="analytics", + schema="dev_orders", + name="old_orders_view", + relation_type="view", + created_at=datetime(2026, 8, 1, tzinfo=UTC), + last_altered_at=datetime(2026, 8, 3, tzinfo=UTC), + ), + cursors=(), + expected_sql=(), + expected_timestamps=("2026-08-01 00:00:00+00:00", "2026-08-03 00:00:00+00:00"), + ), + DatabricksDeltaDetailAgeTestCase( + description="failed Delta detail leaves table age unknown", + relation=RelationInfo( + database="analytics", + schema="dev_orders", + name="old_orders", + relation_type="table", + created_at=datetime(2025, 1, 1, tzinfo=UTC), + ), + cursors=( + FailingDatabricksMetadataCursor(execute_error=RuntimeError("not a Delta table")), + ), + expected_sql=("DESCRIBE DETAIL `analytics`.`dev_orders`.`old_orders`",), + expected_timestamps=("None", "None"), + ), + DatabricksDeltaDetailAgeTestCase( + description="Delta detail without lastModified leaves table age unknown", + relation=RelationInfo( + database="analytics", schema="dev_orders", name="old_orders", relation_type="table" + ), + cursors=( + FakeDatabricksMetadataCursor( + rows=[("delta", datetime(2025, 1, 1, tzinfo=UTC))], + description=(("format",), ("createdAt",)), + ), + ), + expected_sql=("DESCRIBE DETAIL `analytics`.`dev_orders`.`old_orders`",), + expected_timestamps=("None", "None"), + ), + ], + ids=lambda case: case.description, +) +def test_given_candidate_relations_when_adding_age_metadata_then_databricks_reads_delta_detail( + test_case: DatabricksDeltaDetailAgeTestCase, +) -> None: + connection: FakeDatabricksMetadataConnection = FakeDatabricksMetadataConnection( + test_case.cursors + ) + + enriched: tuple[RelationInfo, ...] = DatabricksAdapter().with_relation_age_metadata( + connection=cast(Any, connection), + relations=(test_case.relation,), + ) + + assert ( + tuple(cursor.executed_sql for cursor in connection.returned_cursors) + == test_case.expected_sql + ) + assert tuple(cursor.closed for cursor in connection.returned_cursors) == tuple( + True for _ in test_case.expected_sql + ) + assert ( + str(enriched[0].created_at), + str(enriched[0].last_altered_at), + ) == test_case.expected_timestamps diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 1248c0579b..8e83003fa5 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -158,3 +158,15 @@ class JanitorSeparateAgeMetadataPlanTestCase: relation_ages: dict[str, datetime] expected_candidate_names: tuple[str, ...] expected_skipped_relations: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class JanitorAgeReadScopeTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + direct_mode: bool + delete_tracked_only: bool + tracked_relations: tuple[tuple[str | None, str | None, str], ...] + exclude_patterns: tuple[str, ...] + protected_relation_keys: frozenset[JanitorRelationKey] + expected_age_requests: tuple[tuple[str, ...], ...] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 4c35f24e7e..9b142958f4 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -507,8 +507,9 @@ def __init__( *, relation_infos: tuple[RelationInfo, ...], relation_ages: dict[str, datetime], + tracked_relations: tuple[tuple[str | None, str | None, str], ...] = (), ) -> None: - super().__init__(relation_infos=relation_infos) + super().__init__(relation_infos=relation_infos, tracked_relations=tracked_relations) self.relation_ages: dict[str, datetime] = relation_ages self.age_metadata_requests: list[tuple[str, ...]] = [] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py index e234cd0adb..ef7fc79439 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py @@ -7,8 +7,14 @@ import pytest from sqlbuild.executor.janitor.main.plan import build_janitor_plan -from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan +from sqlbuild.executor.janitor.models import ( + JanitorDirectModeSettings, + JanitorPlan, + JanitorRelationKey, + JanitorRelationScope, +) from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorAgeReadScopeTestCase, JanitorSeparateAgeMetadataPlanTestCase, relation_info, ) @@ -70,3 +76,82 @@ def test_given_adapter_reading_ages_separately_when_planning_janitor_then_uses_t == test_case.expected_skipped_relations ) assert len(adapter.age_metadata_requests) == 1 + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorAgeReadScopeTestCase( + description="direct mode reads ages only for unfiltered stale relations", + relation_infos=( + relation_info("orders"), + relation_info("old_orders"), + relation_info("stray_orders"), + relation_info("tmp_orders"), + relation_info("_sqlbuild_fingerprints"), + relation_info("Mixed_Orders"), + ), + direct_mode=True, + delete_tracked_only=True, + tracked_relations=( + (None, "analytics", "old_orders"), + (None, "analytics", "tmp_orders"), + (None, "analytics", "Mixed_Orders"), + ), + exclude_patterns=("tmp_*",), + protected_relation_keys=frozenset(), + expected_age_requests=(("old_orders",),), + ), + JanitorAgeReadScopeTestCase( + description="virtual mode reads ages only for unprotected stale relations", + relation_infos=( + relation_info("orders"), + relation_info("orders__v0"), + relation_info("orders__v1"), + relation_info("tmp_orders"), + ), + direct_mode=False, + delete_tracked_only=False, + tracked_relations=(), + exclude_patterns=("tmp_*",), + protected_relation_keys=frozenset( + (JanitorRelationKey(database=None, schema="analytics", name="orders__v1"),) + ), + expected_age_requests=(("orders__v0",),), + ), + JanitorAgeReadScopeTestCase( + description="no age read happens when nothing survives the filters", + relation_infos=(relation_info("orders"), relation_info("stray_orders")), + direct_mode=True, + delete_tracked_only=True, + tracked_relations=(), + exclude_patterns=(), + protected_relation_keys=frozenset(), + expected_age_requests=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_filtered_relations_when_planning_janitor_then_ages_are_read_for_candidates_only( + test_case: JanitorAgeReadScopeTestCase, +) -> None: + adapter: SeparateAgeMetadataJanitorAdapter = SeparateAgeMetadataJanitorAdapter( + relation_infos=test_case.relation_infos, + relation_ages={}, + tracked_relations=test_case.tracked_relations, + ) + + build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=test_case.delete_tracked_only, + exclude_patterns=test_case.exclude_patterns, + relation_scope=JanitorRelationScope( + protected_relation_keys=test_case.protected_relation_keys + ), + direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + ) + + assert tuple(adapter.age_metadata_requests) == test_case.expected_age_requests From 4195902f086d1709145d0a15a65a0042040b1f0d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:06:10 +0100 Subject: [PATCH 8/8] fix(janitor): guard case-variant relations and keep zero-retention cleanup metadata-free Folded relation identity made tracked-only membership case-insensitive, so virtual mode could drop an excluded lowercase table through an untracked quoted twin. Virtual mode now applies the case-collision guard, and exclude patterns match case-insensitively in both modes. BigQuery age metadata is keyed by the exact case-sensitive table ID. Janitor no longer reads relation ages when retention_days is 0. --- .../bigquery/classes/bigquery_adapter.py | 4 +- .../janitor/_helpers/classification.py | 16 ++- .../janitor/_helpers/relation_addressing.py | 8 +- .../sqlbuild/adapters/bigquery/_test_types.py | 1 + .../sqlbuild/adapters/bigquery/test_client.py | 36 ++++-- .../executor/janitor/main/_test_types.py | 12 ++ .../janitor/main/test_plan_case_safety.py | 111 ++++++++++++++++++ .../main/test_plan_separate_age_metadata.py | 24 +++- 8 files changed, 194 insertions(+), 18 deletions(-) create mode 100644 tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 95bfc19c6e..d5081c4a1d 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -2090,7 +2090,7 @@ def _dataset_relation_timestamps( raise return {} return { - (database, schema.lower(), str(row[0]).lower()): ( + (database, schema, str(row[0])): ( relation_age_timestamp_utc(row[1]), relation_age_timestamp_utc(row[2]), ) @@ -2099,7 +2099,7 @@ def _dataset_relation_timestamps( @staticmethod def _age_key(relation: RelationInfo) -> tuple[str | None, str, str]: - return (relation.database, str(relation.schema).lower(), relation.name.lower()) + return (relation.database, str(relation.schema), relation.name) def list_functions( self, diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index a270514f01..eb2aa98a03 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -29,6 +29,7 @@ ) from sqlbuild.executor.janitor._helpers.relation_addressing import ( case_colliding_names, + case_collision_reason, unaddressable_relation_reason, ) from sqlbuild.executor.janitor._helpers.tracking import collect_tracked_relation_keys @@ -204,6 +205,10 @@ def classify_janitor_relations( skip_reason = unaddressable_relation_reason( name=relation_key.name, colliding_names=colliding_names ) + elif skip_reason is None: + skip_reason = case_collision_reason( + name=relation_key.name, colliding_names=colliding_names + ) if skip_reason is not None: skipped_relations.append( JanitorSkippedRelation(key=relation_key, relation=relation, reason=skip_reason) @@ -212,7 +217,8 @@ def classify_janitor_relations( eligible_relations.append(relation) age_supported: bool = age_reader.supported() aged_relation: RelationInfo - for aged_relation in age_reader.read(tuple(eligible_relations)): + eligible: tuple[RelationInfo, ...] = tuple(eligible_relations) + for aged_relation in age_reader.read(eligible) if retention_days > 0 else eligible: relation_key = build_relation_key(aged_relation) age_timestamp: datetime | None = relation_age_timestamp(aged_relation) if retention_days > 0: @@ -296,12 +302,14 @@ def matching_exclude_pattern( key: JanitorRelationKey, patterns: tuple[str, ...], ) -> str | None: - """Return the first exclude pattern matching a relation name or qualified name.""" + """Return the first exclude pattern matching a relation or qualified name, ignoring case.""" - display_name: str = key.display_name() + name: str = key.name.lower() + display_name: str = key.display_name().lower() pattern: str for pattern in patterns: - if fnmatchcase(key.name, pattern) or fnmatchcase(display_name, pattern): + folded_pattern: str = pattern.lower() + if fnmatchcase(name, folded_pattern) or fnmatchcase(display_name, folded_pattern): return pattern return None diff --git a/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py b/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py index 4f499785cd..d59933ae78 100644 --- a/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py +++ b/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py @@ -1,4 +1,4 @@ -"""Direct-mode janitor guard for relation names that are safe to address unquoted.""" +"""Janitor guards for relation names that are safe to address unquoted.""" from __future__ import annotations @@ -26,6 +26,12 @@ def unaddressable_relation_reason(*, name: str, colliding_names: frozenset[str]) if _PLAIN_LOWERCASE_IDENTIFIER_RE.fullmatch(name) is None: return UNQUOTED_ADDRESSING_REASON + return case_collision_reason(name=name, colliding_names=colliding_names) + + +def case_collision_reason(*, name: str, colliding_names: frozenset[str]) -> str | None: + """Return why a relation whose folded name is shared in its listing must not be dropped.""" + if name.lower() in colliding_names: return CASE_COLLISION_REASON return None diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index de9d0bee25..5ed09e990c 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -209,3 +209,4 @@ class BigQueryRelationAgeMetadataTestCase: description: str metadata_rows: tuple[tuple[object, ...], ...] expected_timestamps: tuple[tuple[str, str], ...] + relation_names: tuple[str, ...] = ("old_orders", "old_orders_view") diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py index 5a6bbfdafa..dd9561133e 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py @@ -1115,7 +1115,7 @@ def test_given_timestamp_cursor_when_counting_rows_then_bigquery_uses_typed_filt description="dataset metadata supplies UTC creation and last-modified times", metadata_rows=( ( - "Old_Orders", + "old_orders", datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), datetime(2026, 8, 2, 10, 30, tzinfo=UTC), ), @@ -1131,6 +1131,27 @@ def test_given_timestamp_cursor_when_counting_rows_then_bigquery_uses_typed_filt metadata_rows=(), expected_timestamps=(("None", "None"), ("None", "None")), ), + BigQueryRelationAgeMetadataTestCase( + description="case-distinct tables keep their own timestamps", + metadata_rows=( + ( + "Orders", + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2025, 1, 2, tzinfo=UTC), + ), + ( + "orders", + datetime(2026, 8, 1, tzinfo=UTC), + datetime(2026, 8, 2, tzinfo=UTC), + ), + ), + expected_timestamps=( + ("2026-08-01 00:00:00+00:00", "2026-08-02 00:00:00+00:00"), + ("2025-01-01 00:00:00+00:00", "2025-01-02 00:00:00+00:00"), + ("None", "None"), + ), + relation_names=("orders", "Orders", "ORDERS"), + ), ], ids=lambda case: case.description, ) @@ -1143,19 +1164,14 @@ def test_given_listed_relations_when_adding_age_metadata_then_bigquery_reads_eac rows=test_case.metadata_rows, ) ) - relations: tuple[RelationInfo, ...] = ( + relations: tuple[RelationInfo, ...] = tuple( RelationInfo( database="example-project", schema="dev_orders", - name="old_orders", + name=name, relation_type="table", - ), - RelationInfo( - database="example-project", - schema="dev_orders", - name="old_orders_view", - relation_type="view", - ), + ) + for name in test_case.relation_names ) enriched: tuple[RelationInfo, ...] = BigQueryAdapter().with_relation_age_metadata( diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 8e83003fa5..68a1d40b84 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -170,3 +170,15 @@ class JanitorAgeReadScopeTestCase: exclude_patterns: tuple[str, ...] protected_relation_keys: frozenset[JanitorRelationKey] expected_age_requests: tuple[tuple[str, ...], ...] + retention_days: int = 7 + + +@dataclass(frozen=True) +class JanitorCaseSafetyTestCase: + description: str + relation_names: tuple[str, ...] + direct_mode: bool + tracked_names: tuple[str, ...] + exclude_patterns: tuple[str, ...] + expected_skipped_relations: tuple[tuple[str, str], ...] + expected_dropped_targets: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py new file mode 100644 index 0000000000..f12ffb05f1 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py @@ -0,0 +1,111 @@ +"""Janitor safety for relation names that differ only by case.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.executor.janitor.constants import CASE_COLLISION_REASON +from sqlbuild.executor.janitor.main.execute import execute_janitor_plan +from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorCaseSafetyTestCase, + relation_info, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FakeJanitorAdapter, + build_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorCaseSafetyTestCase( + description="virtual mode never drops an excluded table through a quoted twin", + relation_names=("orders__v0", "Orders__v0"), + direct_mode=False, + tracked_names=("orders__v0",), + exclude_patterns=("orders__v0",), + expected_skipped_relations=( + ("analytics.orders__v0", "relation matches exclude pattern 'orders__v0'"), + ("analytics.Orders__v0", "relation matches exclude pattern 'orders__v0'"), + ), + expected_dropped_targets=(), + ), + JanitorCaseSafetyTestCase( + description="virtual mode skips relations whose folded names collide", + relation_names=("orders__v0", "Orders__v0"), + direct_mode=False, + tracked_names=("orders__v0",), + exclude_patterns=(), + expected_skipped_relations=( + ("analytics.orders__v0", CASE_COLLISION_REASON), + ("analytics.Orders__v0", CASE_COLLISION_REASON), + ), + expected_dropped_targets=(), + ), + JanitorCaseSafetyTestCase( + description="virtual mode still drops a non-colliding tracked relation", + relation_names=("orders__v0",), + direct_mode=False, + tracked_names=("orders__v0",), + exclude_patterns=(), + expected_skipped_relations=(), + expected_dropped_targets=("analytics.orders__v0",), + ), + JanitorCaseSafetyTestCase( + description="virtual exclude patterns match regardless of case", + relation_names=("orders__v0",), + direct_mode=False, + tracked_names=("orders__v0",), + exclude_patterns=("ORDERS__*",), + expected_skipped_relations=( + ("analytics.orders__v0", "relation matches exclude pattern 'ORDERS__*'"), + ), + expected_dropped_targets=(), + ), + JanitorCaseSafetyTestCase( + description="direct exclude patterns match regardless of case", + relation_names=("old_orders",), + direct_mode=True, + tracked_names=("old_orders",), + exclude_patterns=("ANALYTICS.OLD_*",), + expected_skipped_relations=( + ("analytics.old_orders", "relation matches exclude pattern 'ANALYTICS.OLD_*'"), + ), + expected_dropped_targets=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_case_variant_relations_when_running_janitor_then_excluded_tables_are_not_dropped( + test_case: JanitorCaseSafetyTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter( + relation_infos=tuple(relation_info(name) for name in test_case.relation_names), + tracked_relations=tuple((None, "analytics", name) for name in test_case.tracked_names), + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=0, + exclude_patterns=test_case.exclude_patterns, + direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + ) + execute_janitor_plan(plan=plan, adapter=adapter, connection=object()) + + assert ( + tuple( + (skipped.key.display_name(), skipped.reason) + for skipped in filter( + lambda skipped: skipped.key.name != "_sqlbuild_fingerprints", + plan.skipped_relations, + ) + ) + == test_case.expected_skipped_relations + ) + assert tuple(adapter.dropped_targets) == test_case.expected_dropped_targets + assert adapter.renamed_targets == [] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py index ef7fc79439..1c42809a90 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py @@ -129,6 +129,28 @@ def test_given_adapter_reading_ages_separately_when_planning_janitor_then_uses_t protected_relation_keys=frozenset(), expected_age_requests=(), ), + JanitorAgeReadScopeTestCase( + description="zero retention does not read ages", + relation_infos=(relation_info("orders"), relation_info("old_orders")), + direct_mode=True, + delete_tracked_only=False, + tracked_relations=(), + exclude_patterns=(), + protected_relation_keys=frozenset(), + expected_age_requests=(), + retention_days=0, + ), + JanitorAgeReadScopeTestCase( + description="zero retention in virtual mode does not read ages", + relation_infos=(relation_info("orders"), relation_info("orders__v0")), + direct_mode=False, + delete_tracked_only=False, + tracked_relations=(), + exclude_patterns=(), + protected_relation_keys=frozenset(), + expected_age_requests=(), + retention_days=0, + ), ], ids=lambda case: case.description, ) @@ -145,7 +167,7 @@ def test_given_filtered_relations_when_planning_janitor_then_ages_are_read_for_c project=build_project(), adapter=adapter, connection=object(), - retention_days=7, + retention_days=test_case.retention_days, delete_tracked_only=test_case.delete_tracked_only, exclude_patterns=test_case.exclude_patterns, relation_scope=JanitorRelationScope(