diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 174f4b76a0..d2bf04d48a 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -389,6 +389,14 @@ def relation_exists( ) return cursor.fetchone() is not None + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_relations( self, *, @@ -626,6 +634,9 @@ def render_drop_view(self, *, destination: str, if_exists: bool = True) -> tuple def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER TABLE {origin} RENAME TO {destination}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER VIEW {origin} RENAME TO {destination}",) + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: staging: str = self._with_replaced_relation_name( relation=left, name=f"{self._relation_name(left)}__swap_staging" @@ -2342,6 +2353,21 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the janitor audit event table when it is missing.""" + + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 31f77c7009..c87ab8f663 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -97,6 +97,8 @@ def supports_durable_clone(self) -> bool: return False def supports_relation_age_metadata(self) -> bool: + """Return False because the DuckDB and MotherDuck catalogs record no relation timestamps.""" + return False def supports_table_freshness_metadata(self) -> bool: @@ -1029,6 +1031,18 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1311,6 +1325,14 @@ def list_relations( for row in rows ) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, @@ -1626,6 +1648,10 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: unqualified_destination: str = destination.rsplit(".", 1)[-1] return (f"ALTER TABLE {origin} RENAME TO {unqualified_destination}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + unqualified_destination: str = destination.rsplit(".", 1)[-1] + return (f"ALTER VIEW {origin} RENAME TO {unqualified_destination}",) + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: staging: str = self._with_replaced_relation_name( relation=left, name=f"{self._relation_name(left)}__swap_staging" diff --git a/src/sqlbuild/adapter/contract/classes/schema.py b/src/sqlbuild/adapter/contract/classes/schema.py index 980e41a298..501ccffd76 100644 --- a/src/sqlbuild/adapter/contract/classes/schema.py +++ b/src/sqlbuild/adapter/contract/classes/schema.py @@ -23,6 +23,16 @@ def list_relations( """Return all relations in the given database/schema scope.""" ... + @abstractmethod + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + """Return listed relations with age timestamps that listing does not read inline.""" + ... + @abstractmethod def list_functions( self, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index e76502d06d..c436c727c0 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -312,6 +312,11 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: """Render SQL statements that rename a relation.""" ... + @abstractmethod + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + """Render SQL statements that rename a view relation.""" + ... + @abstractmethod def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: """Render SQL statements that swap two relations.""" @@ -875,6 +880,11 @@ def render_create_audit_result_index_sqls( """Render optional audit result table index DDL statements.""" ... + @abstractmethod + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the janitor audit event table when it is missing.""" + ... + @abstractmethod def render_prune_fingerprint_history_sql( self, diff --git a/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py b/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py new file mode 100644 index 0000000000..f920eac27c --- /dev/null +++ b/src/sqlbuild/adapter/relations/_helpers/identifier_fitting.py @@ -0,0 +1,35 @@ +"""Identifier-limit fitting implementations for generated relation names.""" + +from __future__ import annotations + +import hashlib + +from sqlbuild.adapter.relations.constants import SHORTENED_LOGICAL_NAME_HASH_LENGTH +from sqlbuild.errors.contracts.exceptions import SharedInputError + + +def fit_artifact_logical_name_impl( + *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str +) -> str: + """Fit a readable logical component with a deterministic hash suffix.""" + + max_logical_length: int = identifier_limit - len(fixed_prefix) + if max_logical_length < 1: + raise SharedInputError( + f"{artifact_label} prefix '{fixed_prefix}' does not fit within identifier " + f"limit {identifier_limit}" + ) + if len(logical_name) <= max_logical_length: + return logical_name + + suffix_length: int = SHORTENED_LOGICAL_NAME_HASH_LENGTH + 1 + if max_logical_length <= suffix_length: + raise SharedInputError( + f"{artifact_label} name for '{logical_name}' cannot fit within identifier " + f"limit {identifier_limit}" + ) + logical_hash: str = hashlib.sha256(logical_name.encode("utf-8")).hexdigest()[ + :SHORTENED_LOGICAL_NAME_HASH_LENGTH + ] + prefix_length: int = max_logical_length - suffix_length + return f"{logical_name[:prefix_length]}_{logical_hash}" diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 19cede8486..e55b715814 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -1,3 +1,4 @@ """Shared relation metadata limits.""" METADATA_NAME_FILTER_LIMIT: int = 250 +SHORTENED_LOGICAL_NAME_HASH_LENGTH: int = 8 diff --git a/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py b/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py new file mode 100644 index 0000000000..2c601d9173 --- /dev/null +++ b/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name.py @@ -0,0 +1,18 @@ +"""Generated relation name fitting entrypoint.""" + +from __future__ import annotations + +from sqlbuild.adapter.relations._helpers.identifier_fitting import fit_artifact_logical_name_impl + + +def fit_artifact_logical_name( + *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str +) -> str: + """Fit a readable logical component after a fixed prefix within an identifier limit.""" + + return fit_artifact_logical_name_impl( + logical_name=logical_name, + fixed_prefix=fixed_prefix, + identifier_limit=identifier_limit, + artifact_label=artifact_label, + ) diff --git a/src/sqlbuild/adapter/relations/main/relation_age_timestamp.py b/src/sqlbuild/adapter/relations/main/relation_age_timestamp.py new file mode 100644 index 0000000000..77337b3044 --- /dev/null +++ b/src/sqlbuild/adapter/relations/main/relation_age_timestamp.py @@ -0,0 +1,15 @@ +"""Relation catalog timestamp normalization entrypoint.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def relation_age_timestamp_utc(value: object) -> datetime | None: + """Return one catalog timestamp as aware UTC, treating offset-free values as UTC.""" + + if not isinstance(value, datetime): + return None + if value.tzinfo is None or value.utcoffset() is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) diff --git a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py new file mode 100644 index 0000000000..bbf7c9b285 --- /dev/null +++ b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py @@ -0,0 +1,32 @@ +"""Portable append-only state table DDL implementations.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.sql_values.types import StateSqlValueType + + +def render_state_table_create_sql_impl( + *, + qualified_name: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + required_columns: frozenset[str], + render_framework_type: Callable[[FrameworkType], str], + transient: bool, +) -> str: + """Render create-if-missing DDL for string, integer, and text-timestamp state columns.""" + + rendered_types: dict[StateSqlValueType, str] = { + StateSqlValueType.STRING: render_framework_type(FrameworkType.STRING), + StateSqlValueType.INTEGER: render_framework_type(FrameworkType.INTEGER), + StateSqlValueType.TEXT_TIMESTAMP: render_framework_type(FrameworkType.TIMESTAMP), + } + definitions: list[str] = [] + for column in columns: + required: str = " NOT NULL" if column in required_columns else "" + definitions.append(f"{column} {rendered_types[column_types[column]]}{required}") + table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" + return f"CREATE {table_kind} IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" diff --git a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py new file mode 100644 index 0000000000..7d0a48d69a --- /dev/null +++ b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py @@ -0,0 +1,28 @@ +"""Public append-only state table DDL rendering operation.""" + +from collections.abc import Callable, Mapping + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql._helpers.table_ddl import render_state_table_create_sql_impl +from sqlbuild.sql_values.types import StateSqlValueType + + +def render_state_table_create_sql( + *, + qualified_name: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + required_columns: frozenset[str], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + """Render create-if-missing DDL for one append-only state table.""" + + return render_state_table_create_sql_impl( + qualified_name=qualified_name, + columns=columns, + column_types=column_types, + required_columns=required_columns, + render_framework_type=render_framework_type, + transient=transient, + ) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index d37debc4bd..d5081c4a1d 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -4,6 +4,7 @@ import json import logging +from dataclasses import replace from datetime import date, datetime from decimal import Decimal from pathlib import Path @@ -64,6 +65,7 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) +from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( render_insert_source_freshness_records_sql, ) @@ -282,6 +284,18 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -350,7 +364,7 @@ def render_prune_source_freshness_history_sql( ) def supports_relation_age_metadata(self) -> bool: - return False + return True def supports_table_freshness_metadata(self) -> bool: return True @@ -1825,6 +1839,10 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: destination_name: str = self._strip_identifier_quotes(destination).split(".")[-1] return (f"ALTER TABLE {self._quote_identifier_path(origin)} RENAME TO {destination_name}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + del origin, destination + raise AdapterUserError(message="BigQuery does not support renaming views") + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: raise AdapterUserError(message="BigQuery does not support atomic table swap") @@ -2030,6 +2048,59 @@ def list_relations( raise return tuple(relations) + def with_relation_age_metadata( + self, + *, + connection: _BigQueryConnection, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + timestamps: dict[tuple[str | None, str, str], tuple[datetime | None, datetime | None]] = {} + dataset_key: tuple[str | None, str] + for dataset_key in sorted( + {(relation.database, str(relation.schema)) for relation in relations}, + key=lambda key: (key[0] or "", key[1]), + ): + timestamps.update( + self._dataset_relation_timestamps( + connection=connection, database=dataset_key[0], schema=dataset_key[1] + ) + ) + return tuple( + replace( + relation, + created_at=timestamps.get(self._age_key(relation), (None, None))[0], + last_altered_at=timestamps.get(self._age_key(relation), (None, None))[1], + ) + for relation in relations + ) + + def _dataset_relation_timestamps( + self, *, connection: _BigQueryConnection, database: str | None, schema: str + ) -> dict[tuple[str | None, str, str], tuple[datetime | None, datetime | None]]: + dataset_id: str = self._build_dataset_id(database=database, schema=schema) + try: + cursor: _BigQueryCursor = self.execute( + connection=connection, + sql="SELECT table_id, TIMESTAMP_MILLIS(creation_time), " + f"TIMESTAMP_MILLIS(last_modified_time) FROM `{dataset_id}.__TABLES__`", + ) + rows: list[tuple[Any, ...]] = cursor.fetchall() + except Exception as error: + if not self._is_google_not_found(error): + raise + return {} + return { + (database, schema, str(row[0])): ( + relation_age_timestamp_utc(row[1]), + relation_age_timestamp_utc(row[2]), + ) + for row in rows + } + + @staticmethod + def _age_key(relation: RelationInfo) -> tuple[str | None, str, str]: + return (relation.database, str(relation.schema), relation.name) + def list_functions( self, *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 743883bbc2..e7fb0bc6ef 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -7,6 +7,7 @@ import json import logging import re +from dataclasses import replace from datetime import date, datetime from decimal import Decimal from pathlib import Path @@ -63,6 +64,7 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) +from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( render_insert_source_freshness_records_sql, ) @@ -376,6 +378,18 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1257,7 +1271,7 @@ def supports_durable_clone(self) -> bool: return True def supports_relation_age_metadata(self) -> bool: - return False + return True def recommended_max_sql_length(self) -> int | None: return 256_000 @@ -1394,8 +1408,8 @@ def list_relations( return () information_schema: str = self._information_schema(database) query: str = ( - f"SELECT table_name, table_schema, table_type FROM {information_schema}.tables " - "WHERE 1=1" + "SELECT table_name, table_schema, table_type, created, last_altered " + f"FROM {information_schema}.tables WHERE 1=1" ) query += self._build_in_filter(column="table_schema", values=schemas) if names: @@ -1406,14 +1420,64 @@ def list_relations( rows: list[tuple[Any, ...]] = cursor.fetchall() finally: cursor.close() + return tuple(self._listed_relation(database=database, row=row) for row in rows) + + def _listed_relation(self, *, database: str, row: tuple[Any, ...]) -> RelationInfo: + relation_type: str = self._normalize_relation_type(str(row[2])) + is_view: bool = relation_type == RelationType.VIEW + return RelationInfo( + database=database, + schema=str(row[1]).lower(), + name=str(row[0]).lower(), + relation_type=relation_type, + created_at=relation_age_timestamp_utc(row[3]) if is_view else None, + last_altered_at=relation_age_timestamp_utc(row[4]) if is_view else None, + ) + + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + """Read data-write ages for non-view relations; information_schema tracks only DDL.""" + return tuple( - RelationInfo( - database=database, - schema=str(row[1]).lower(), - name=str(row[0]).lower(), - relation_type=self._normalize_relation_type(str(row[2])), - ) - for row in rows + relation + if relation.relation_type == RelationType.VIEW + else self._with_delta_detail_ages(connection=connection, relation=relation) + for relation in relations + ) + + def _with_delta_detail_ages(self, *, connection: Any, relation: RelationInfo) -> RelationInfo: + unknown_age: RelationInfo = replace(relation, created_at=None, last_altered_at=None) + if relation.database is None or relation.schema is None: + return unknown_age + target: str = ".".join( + self.render_identifier(part) + for part in (relation.database, relation.schema, relation.name) + ) + cursor: Any = connection.cursor() + try: + cursor.execute(f"DESCRIBE DETAIL {target}") + row: tuple[Any, ...] | None = cursor.fetchone() + description: tuple[Any, ...] | None = cursor.description + except Exception: + return unknown_age + finally: + cursor.close() + if row is None or description is None: + return unknown_age + detail: dict[str, object] = { + str(column[0]).lower(): value for column, value in zip(description, row, strict=False) + } + last_modified: datetime | None = relation_age_timestamp_utc(detail.get("lastmodified")) + if last_modified is None: + return unknown_age + return replace( + relation, + created_at=relation_age_timestamp_utc(detail.get("createdat")), + last_altered_at=last_modified, ) def list_functions( @@ -1833,6 +1897,9 @@ def render_delete_insert_cursor( def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER TABLE {origin} RENAME TO {destination}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER VIEW {origin} RENAME TO {destination}",) + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: raise AdapterUserError(message="Databricks does not support atomic table swap") diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 6b123118f8..31a20b204c 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -110,6 +110,8 @@ def supports_durable_clone(self) -> bool: return False def supports_relation_age_metadata(self) -> bool: + """Return False because the Postgres catalog records no relation timestamps.""" + return False def supports_table_freshness_metadata(self) -> bool: @@ -307,6 +309,14 @@ def list_relations( for row in cursor.fetchall() ) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, @@ -1582,6 +1592,18 @@ def render_create_audit_result_index_sqls( f"ON {table_name} (run_id, invocation_id, result_id)", ) + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_read_latest_source_freshness_sql( self, *, @@ -1881,6 +1903,10 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: destination_name: str = destination.split(".")[-1] return (f"ALTER TABLE {origin} RENAME TO {destination_name}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + destination_name: str = destination.split(".")[-1] + return (f"ALTER VIEW {origin} RENAME TO {destination_name}",) + def load_seed( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index cbf3550519..58fc8b415e 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -57,6 +57,7 @@ RetentionScope, TablePromotionMode, ) +from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( render_insert_source_freshness_records_sql, ) @@ -458,6 +459,19 @@ def render_create_audit_result_index_sqls( del database, schema return () + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -527,7 +541,7 @@ def render_prune_source_freshness_history_sql( ) def supports_relation_age_metadata(self) -> bool: - return False + return True def supports_table_freshness_metadata(self) -> bool: return True @@ -1810,13 +1824,21 @@ def list_relations( is_transient=( None if row[3] is None else str(row[3]).upper() == TRUE_METADATA_VALUE ), - created_at=None if row[4] is None else row[4], - last_altered_at=None if row[5] is None else row[5], + created_at=relation_age_timestamp_utc(row[4]), + last_altered_at=relation_age_timestamp_utc(row[5]), retention_days=None if row[6] is None else int(row[6]), ) for row in rows ) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, @@ -2340,6 +2362,9 @@ def render_drop(self, *, destination: str, if_exists: bool = True) -> tuple[str, def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER TABLE {origin} RENAME TO {destination}",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER VIEW {origin} RENAME TO {destination}",) + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: return (f"ALTER TABLE {left} SWAP WITH {right}",) diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 4e60c013e1..cb9f464a30 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -504,16 +504,12 @@ def render_create_fingerprint_table_sql( f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL" f")" ) - escaped_schema: str = schema.replace("'", "''") - escaped_table: str = FINGERPRINT_TABLE_NAME.replace("'", "''") - exists_sql: str = ( - "SELECT 1 FROM information_schema.tables " - f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=FINGERPRINT_TABLE_NAME, ) - if database is not None: - escaped_database: str = database.replace("'", "''") - exists_sql += f" AND table_catalog = '{escaped_database}'" - return f"IF NOT EXISTS ({exists_sql}) {create_sql}" def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: from sqlbuild.microbatches.constants import MICROBATCH_TABLE_NAME @@ -525,16 +521,12 @@ def render_create_microbatch_state_table_sql(self, *, database: str | None, sche render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, ).replace("CREATE TABLE IF NOT EXISTS ", "CREATE TABLE ", 1) - escaped_schema: str = schema.replace("'", "''") - escaped_table: str = MICROBATCH_TABLE_NAME.replace("'", "''") - exists_sql: str = ( - "SELECT 1 FROM information_schema.tables " - f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=MICROBATCH_TABLE_NAME, ) - if database is not None: - escaped_database: str = database.replace("'", "''") - exists_sql += f" AND table_catalog = '{escaped_database}'" - return f"IF NOT EXISTS ({exists_sql}) {create_sql}" def render_create_fingerprint_index_sqls( self, @@ -676,16 +668,12 @@ def render_create_node_result_table_sql( "status", ): create_sql = create_sql.replace(f"{column} NVARCHAR(MAX)", f"{column} NVARCHAR(450)") - escaped_schema: str = schema.replace("'", "''") - escaped_table: str = NODE_RESULTS_TABLE_NAME.replace("'", "''") - exists_sql: str = ( - "SELECT 1 FROM information_schema.tables " - f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=NODE_RESULTS_TABLE_NAME, ) - if database is not None: - escaped_database: str = database.replace("'", "''") - exists_sql += f" AND table_catalog = '{escaped_database}'" - return f"IF NOT EXISTS ({exists_sql}) {create_sql}" def render_create_node_result_index_sqls( self, @@ -752,16 +740,12 @@ def render_create_audit_result_table_sql(self, *, database: str | None, schema: "outcome", ): create_sql = create_sql.replace(f"{column} NVARCHAR(MAX)", f"{column} NVARCHAR(450)") - escaped_schema: str = schema.replace("'", "''") - escaped_table: str = AUDIT_RESULTS_TABLE_NAME.replace("'", "''") - exists_sql: str = ( - "SELECT 1 FROM information_schema.tables " - f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=AUDIT_RESULTS_TABLE_NAME, ) - if database is not None: - escaped_database: str = database.replace("'", "''") - exists_sql += f" AND table_catalog = '{escaped_database}'" - return f"IF NOT EXISTS ({exists_sql}) {create_sql}" def render_create_audit_result_index_sqls( self, *, database: str | None, schema: str @@ -792,6 +776,25 @@ def render_create_audit_result_index_sqls( "(run_id, invocation_id, result_id)", ) + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + create_sql: str = build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ).replace("CREATE TABLE IF NOT EXISTS", "CREATE TABLE", 1) + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=JANITOR_EVENTS_TABLE_NAME, + ) + def render_read_latest_source_freshness_sql( self, *, @@ -892,6 +895,23 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: destination_name: str = self._unquote_relation_part(destination.split(".")[-1]) return (f"EXEC sp_rename '{origin_name}', '{destination_name}'",) + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return self.render_rename(origin=origin, destination=destination) + + def _create_table_if_missing_sql( + self, *, create_sql: str, database: str | None, schema: str, table_name: str + ) -> str: + escaped_schema: str = schema.replace("'", "''") + escaped_table: str = table_name.replace("'", "''") + exists_sql: str = ( + "SELECT 1 FROM information_schema.tables " + f"WHERE table_schema = '{escaped_schema}' AND table_name = '{escaped_table}'" + ) + if database is not None: + escaped_database: str = database.replace("'", "''") + exists_sql += f" AND table_catalog = '{escaped_database}'" + return f"IF NOT EXISTS ({exists_sql}) {create_sql}" + def render_add_columns( self, *, destination: str, columns: tuple[ColumnInfo, ...] ) -> tuple[str, ...]: @@ -1654,6 +1674,14 @@ def get_columns( cursor: Any = connection.execute(query) return tuple(ColumnInfo(name=row[0], type=row[1]) for row in cursor.fetchall()) + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + def list_functions( self, *, @@ -2902,6 +2930,8 @@ def supports_python_functions(self) -> bool: return False def supports_relation_age_metadata(self) -> bool: + """Return False because sys.objects.modify_date ignores DML, so written tables look old.""" + return False def supports_table_functions(self) -> bool: diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py index 9f479ef0b7..a516b1629c 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py @@ -2,9 +2,12 @@ from __future__ import annotations +from datetime import datetime from typing import TextIO from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, JanitorCheckpointCandidate, JanitorDeleteCandidate, JanitorDetachedVirtualEnvironmentCandidate, @@ -24,15 +27,15 @@ def write_disabled(*, stream: TextIO, use_color: bool = False) -> None: style: CliStyle = CliStyle(use_color=use_color) disabled_detail: str = ( - "Janitor is opt-in. It previews stale warehouse objects and asks before deleting anything." + "Janitor is opt-in. It previews stale warehouse objects and asks before archiving or " + "deleting anything." ) stream.write( f"{style.warning_strong('Janitor is disabled for this project.')}\n\n" f"{style.warning(disabled_detail)}\n\n" f"Add this block to {style.object_name('sqlbuild_project.toml')}:\n\n" - "janitor:\n" - " enabled: true\n" - " retention_days: 30\n\n" + "[janitor]\n" + "enabled = true\n\n" "After enabling, run janitor again to preview cleanup:\n" f" {style.accent('sqb janitor')}\n" ) @@ -46,14 +49,34 @@ def _write_plan_summary(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) - stream.write(f" {'retention':<22} {style.accent(f'{plan.retention_days} days')}\n") if not plan.age_metadata_supported: stream.write(f" {'age metadata':<22} {style.accent('unavailable')}\n") + if plan.direct_mode: + archive_retention: str = f"{plan.archive_retention_days} days" + stream.write(f" {'archive retention':<22} {style.accent(archive_retention)}\n") stream.write(f" {'schemas scanned':<22} {style.accent(str(plan.scanned_schema_count))}\n") stream.write(f" {'schemas skipped':<22} {style.accent(str(len(plan.skipped_schemas)))}\n") - _write_count_row( - stream=stream, - style=style, - label="reported objects" if plan.direct_mode else "eligible for deletion", - items=plan.candidates, - ) + if plan.direct_mode: + _write_count_row( + stream=stream, + style=style, + label="relations to archive", + items=plan.archive_candidates, + ) + _write_count_row( + stream=stream, + style=style, + label="archives to delete", + items=plan.archive_deletion_candidates, + ) + stream.write( + f" {'archives retained':<22} {style.accent(str(len(plan.retained_archives)))}\n" + ) + else: + _write_count_row( + stream=stream, + style=style, + label="eligible for deletion", + items=plan.candidates, + ) _write_count_row( stream=stream, style=style, @@ -123,6 +146,77 @@ def _write_query_artifact_candidates(*, plan: JanitorPlan, stream: TextIO, style stream.write(f" {style.object_name(query_artifact.key.display_name())}\n") +def _write_archive_sections(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: + if plan.archive_candidates: + stream.write(f"\n{style.success('Relations to archive')}\n") + archive_candidate: JanitorArchiveCandidate + for archive_candidate in plan.archive_candidates: + expiry: str = ( + "deleted in this run" + if archive_candidate.expires_at <= archive_candidate.archived_at + else f"delete after {_utc(archive_candidate.expires_at)}" + ) + detail: str = f"{_age(value=archive_candidate.age_timestamp, plan=plan)}, {expiry}" + stream.write( + f" {style.object_name(archive_candidate.key.display_name())} -> " + f"{style.object_name(archive_candidate.archive_key.display_name())} " + f"{style.muted(detail)}\n" + ) + if plan.archive_deletion_candidates: + stream.write(f"\n{style.success('Archives to delete')}\n") + deletion: JanitorArchivedRelation + for deletion in plan.archive_deletion_candidates: + detail = ( + "archived in this run" + if deletion.original_key is not None + else f"archived {_utc(deletion.archived_at)}, " + f"{_age(value=deletion.archived_at, plan=plan)}, " + f"expired {_utc(deletion.expires_at)}" + ) + stream.write( + f" {style.object_name(deletion.key.display_name())} {style.muted(detail)}\n" + ) + if plan.retained_archives: + stream.write(f"\n{style.success('Retained archives')}\n") + retained: JanitorArchivedRelation + for retained in plan.retained_archives: + detail = f"archived {_utc(retained.archived_at)}, expires {_utc(retained.expires_at)}" + stream.write( + f" {style.object_name(retained.key.display_name())} {style.muted(detail)}\n" + ) + + +def _utc(value: datetime) -> str: + return f"{value:%Y-%m-%d %H:%M:%S} UTC" + + +def _age(*, value: datetime | None, plan: JanitorPlan) -> str: + if value is None or plan.planned_at is None: + return "age unknown" + return f"age {max((plan.planned_at - value).days, 0)}d" + + +def _write_blocked_schemas(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) -> None: + if not plan.blocked_schemas: + return + stream.write(f"\n{style.error_strong('Janitor blocked')}\n") + stream.write(f" {style.error('Managed target schemas contain active configured sources.')}\n") + suppressed_action: str = "archive" if plan.direct_mode else "deletion" + for blocked_schema in plan.blocked_schemas: + sources: str = ", ".join(blocked_schema.source_names) + stream.write( + f" {style.object_name(blocked_schema.display_name())} " + f"{style.error_muted('active sources: ' + sources)}\n" + ) + for candidate in blocked_schema.suppressed_candidates: + suppressed: str = f"suppressed {suppressed_action}: {candidate.key.display_name()}" + stream.write(f" {style.error_muted(suppressed)}\n") + for archived in blocked_schema.suppressed_archive_deletions: + suppressed = f"suppressed archive deletion: {archived.key.display_name()}" + stream.write(f" {style.error_muted(suppressed)}\n") + stream.write(f" {style.error('No janitor actions will be performed.')}\n") + + def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> None: """Write a janitor preview.""" @@ -132,21 +226,7 @@ def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> stream.write(f"{style.title('Janitor preview')} {rendered_env}\n\n") _write_plan_summary(plan=plan, stream=stream, style=style) - if plan.blocked_schemas: - stream.write(f"\n{style.error_strong('Janitor blocked')}\n") - stream.write( - f" {style.error('Managed target schemas contain active configured sources.')}\n" - ) - for blocked_schema in plan.blocked_schemas: - sources: str = ", ".join(blocked_schema.source_names) - stream.write( - f" {style.object_name(blocked_schema.display_name())} " - f"{style.error_muted('active sources: ' + sources)}\n" - ) - for candidate in blocked_schema.suppressed_candidates: - suppressed: str = f"suppressed deletion: {candidate.key.display_name()}" - stream.write(f" {style.error_muted(suppressed)}\n") - stream.write(f" {style.error('No janitor actions will be performed.')}\n") + _write_blocked_schemas(plan=plan, stream=stream, style=style) if plan.skipped_schemas: stream.write(f"\n{style.success('Skipped schemas')}\n") @@ -158,15 +238,13 @@ def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> f"{style.muted('contains active source ' + sources)}\n" ) - if plan.candidates: - heading: str = "Report-only objects" if plan.direct_mode else "Eligible objects" - stream.write(f"\n{style.success(heading)}\n") + if plan.candidates and not plan.direct_mode: + stream.write(f"\n{style.success('Eligible objects')}\n") candidate: JanitorDeleteCandidate for candidate in plan.candidates: - reason: str = " direct mode does not delete relations" if plan.direct_mode else "" - stream.write( - f" {style.object_name(candidate.key.display_name())}{style.muted(reason)}\n" - ) + stream.write(f" {style.object_name(candidate.key.display_name())}\n") + + _write_archive_sections(plan=plan, stream=stream, style=style) _write_query_artifact_candidates(plan=plan, stream=stream, style=style) @@ -255,16 +333,26 @@ def confirmation_text(plan: JanitorPlan) -> str: + len(plan.direct_state_prune_candidates) + len(plan.virtual_state_prune_candidates) ) + archive_prefix: str = ( + f"archive {len(plan.archive_candidates)} and " if plan.archive_candidates else "" + ) + physical_deletion_count: int = physical_janitor_deletion_count(plan) if state_candidate_count == 0: - deletion_count: int = (0 if plan.direct_mode else len(plan.candidates)) + len( - plan.query_diff_artifact_candidates + return ( + f"{archive_prefix}delete {physical_deletion_count} objects " + f"from {environment_label(plan)}" ) - return f"delete {deletion_count} objects from {environment_label(plan)}" - physical_deletion_count: int = (0 if plan.direct_mode else len(plan.candidates)) + len( - plan.query_diff_artifact_candidates - ) deletion_count: int = physical_deletion_count + state_candidate_count - return f"delete {deletion_count} items from {environment_label(plan)}" + return f"{archive_prefix}delete {deletion_count} items from {environment_label(plan)}" + + +def physical_janitor_deletion_count(plan: JanitorPlan) -> int: + """Count warehouse relations the janitor plan will drop.""" + + relation_count: int = ( + len(plan.archive_deletion_candidates) if plan.direct_mode else len(plan.candidates) + ) + return relation_count + len(plan.query_diff_artifact_candidates) def environment_label(plan: JanitorPlan) -> str: diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py index 8aaedb868b..af1b6480bf 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py @@ -7,6 +7,7 @@ from sqlbuild.cli.commands._helpers.janitor_output.output import ( confirmation_text, environment_label, + physical_janitor_deletion_count, write_disabled, write_plan, ) @@ -38,6 +39,8 @@ def janitor_plan_has_work(planning_result: JanitorPlanningResult) -> bool: plan: JanitorPlan = planning_result.plan return bool( (plan.candidates and not plan.direct_mode) + or plan.archive_candidates + or plan.archive_deletion_candidates or plan.query_diff_artifact_candidates or plan.checkpoint_candidates or plan.detached_virtual_environment_candidates @@ -65,18 +68,24 @@ def confirm_janitor_plan(*, planning_result: JanitorPlanningResult) -> bool: prune_count: int = len(plan.direct_state_prune_candidates) + len( plan.virtual_state_prune_candidates ) + archive_prefix: str = ( + f"archive {len(plan.archive_candidates)} and " if plan.archive_candidates else "" + ) if state_candidate_count or prune_count: - physical_deletion_count: int = (0 if plan.direct_mode else len(plan.candidates)) + len( - plan.query_diff_artifact_candidates - ) + physical_deletion_count: int = physical_janitor_deletion_count(plan) deletion_count: int = physical_deletion_count + state_candidate_count + prune_count sys.stdout.write( - f"Janitor will delete {deletion_count} items from {environment_label(plan)}.\n" + f"Janitor will {archive_prefix}delete {deletion_count} items " + f"from {environment_label(plan)}.\n" ) else: + object_count: int = ( + physical_janitor_deletion_count(plan) + if plan.direct_mode + else len(plan.candidates) + len(plan.query_diff_artifact_candidates) + ) sys.stdout.write( - "Janitor will delete " - f"{len(plan.candidates) + len(plan.query_diff_artifact_candidates)} objects " + f"Janitor will {archive_prefix}delete {object_count} objects " f"from {environment_label(plan)}.\n" ) if plan.retention_days == 0: @@ -84,6 +93,8 @@ def confirm_janitor_plan(*, planning_result: JanitorPlanningResult) -> bool: sys.stdout.write("Age metadata will not be checked.\n") else: sys.stdout.write(f"Retention: {plan.retention_days} days\n") + if plan.direct_mode: + sys.stdout.write(f"Archive retention: {plan.archive_retention_days} days\n") sys.stdout.write(f"\nType `{expected}` to continue: ") sys.stdout.flush() try: @@ -110,6 +121,11 @@ def write_janitor_completion( def _deleted_message(*, result: JanitorExecutionResult) -> str: + archived_prefix: str = f"Archived {len(result.archived)} relations. " if result.archived else "" + return archived_prefix + _deletion_summary(result=result) + + +def _deletion_summary(*, result: JanitorExecutionResult) -> str: deleted_state_count: int = ( len(result.deleted_checkpoints) + len(result.deleted_detached_virtual_environments) @@ -117,7 +133,11 @@ def _deleted_message(*, result: JanitorExecutionResult) -> str: + len(result.deleted_state_backups) + len(result.deleted_expired_locks) ) - deleted_object_count: int = len(result.deleted) + len(result.deleted_query_diff_artifacts) + deleted_object_count: int = ( + len(result.deleted) + + len(result.deleted_archives) + + len(result.deleted_query_diff_artifacts) + ) pruned_state_count: int = len(result.pruned_direct_state) + len(result.pruned_virtual_state) non_checkpoint_state_count: int = deleted_state_count - len(result.deleted_checkpoints) if non_checkpoint_state_count or pruned_state_count: diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py index 7f2bf8288d..53e5bee982 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py @@ -12,6 +12,7 @@ ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days from sqlbuild.presentation.main.supports_color import supports_color @@ -36,10 +37,10 @@ def resolve_janitor_settings( ) -> JanitorSettings: """Resolve and validate effective janitor settings.""" - retention_days: int = ( - request.retention_days - if request.retention_days is not None - else invocation.discovered_inputs.project_config.janitor.retention_days + retention_days: int = resolve_janitor_retention_days( + override=request.retention_days, + configured=invocation.discovered_inputs.project_config.janitor.retention_days, + virtual_environments=invocation.discovered_inputs.project_config.settings.virtual_environments, ) if retention_days < 0: raise CliUserError("janitor --retention-days must be >= 0", code="C501") @@ -53,4 +54,7 @@ def resolve_janitor_settings( return JanitorSettings( retention_days=retention_days, direct_state_history_versions=direct_state_history_versions, + archive_retention_days=( + invocation.discovered_inputs.project_config.janitor.archive_retention_days + ), ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py index 4b0249cadb..fa126f10e7 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py @@ -191,6 +191,7 @@ def build_janitor_execution_plan( not invocation.discovered_inputs.project_config.settings.virtual_environments ), state_history_versions=settings.direct_state_history_versions, + archive_retention_days=settings.archive_retention_days, ), ) lifecycle.completed(metadata={"item_count": _janitor_candidate_count(plan)}) @@ -206,6 +207,7 @@ def _janitor_candidate_count(plan: JanitorPlan) -> int: len(candidates) for candidates in ( plan.candidates, + plan.archive_deletion_candidates, plan.query_diff_artifact_candidates, plan.checkpoint_candidates, plan.detached_virtual_environment_candidates, diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 98f8c3ec2d..5d8dabd540 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -784,6 +784,7 @@ class JanitorSettings: retention_days: int direct_state_history_versions: int + archive_retention_days: int = 14 @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index 06869d331a..0662eecb27 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -1427,7 +1427,10 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: payload=payload, label="janitor", file_path=file_path ) enabled: bool = _optional_bool(mapping=mapping, key="enabled", default=False) - retention_days: int = _optional_int(mapping=mapping, key="retention_days", default=30) + retention_days: int | None = _optional_nullable_int(mapping=mapping, key="retention_days") + archive_retention_days: int = _optional_int( + mapping=mapping, key="archive_retention_days", default=14 + ) max_checkpoints: int = _optional_int(mapping=mapping, key="max_checkpoints", default=20) direct_state_history_versions: int = _optional_int( mapping=mapping, @@ -1446,8 +1449,10 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: file_path=file_path, ) ) - if retention_days < 0: + if retention_days is not None and retention_days < 0: raise ProjectConfigError(f"{file_path} janitor.retention_days must be >= 0") + if archive_retention_days < 0: + raise ProjectConfigError(f"{file_path} janitor.archive_retention_days must be >= 0") if max_checkpoints < 1: raise ProjectConfigError(f"{file_path} janitor.max_checkpoints must be >= 1") if direct_state_history_versions < 0: @@ -1455,6 +1460,7 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: return JanitorConfig( enabled=enabled, retention_days=retention_days, + archive_retention_days=archive_retention_days, max_checkpoints=max_checkpoints, direct_state_history_versions=direct_state_history_versions, delete_tracked_only=delete_tracked_only, diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py b/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py index e3ad2d2783..1de215dfe6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/artifact_names.py @@ -2,17 +2,15 @@ from __future__ import annotations -import hashlib import re +from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name from sqlbuild.compiler.planner.constants import ( SCENARIO_ARTIFACT_KINDS, SCENARIO_ARTIFACT_PREFIX, SCENARIO_HASH_PREFIX_LENGTH, - SCENARIO_SHORTENED_LOGICAL_HASH_LENGTH, ) from sqlbuild.compiler.planner.models import ParsedScenarioArtifactName -from sqlbuild.errors.contracts.exceptions import SharedInputError _SCENARIO_ARTIFACT_NAME_RE: re.Pattern[str] = re.compile( rf"^{re.escape(SCENARIO_ARTIFACT_PREFIX)}" @@ -62,30 +60,3 @@ def fit_scenario_artifact_logical_name( identifier_limit=identifier_limit, artifact_label="Scenario artifact", ) - - -def fit_artifact_logical_name( - *, logical_name: str, fixed_prefix: str, identifier_limit: int, artifact_label: str -) -> str: - """Fit a readable logical component with a deterministic hash suffix.""" - - max_logical_length: int = identifier_limit - len(fixed_prefix) - if max_logical_length < 1: - raise SharedInputError( - f"{artifact_label} prefix '{fixed_prefix}' does not fit within identifier " - f"limit {identifier_limit}" - ) - if len(logical_name) <= max_logical_length: - return logical_name - - suffix_length: int = SCENARIO_SHORTENED_LOGICAL_HASH_LENGTH + 1 - if max_logical_length <= suffix_length: - raise SharedInputError( - f"{artifact_label} name for '{logical_name}' cannot fit within identifier " - f"limit {identifier_limit}" - ) - logical_hash: str = hashlib.sha256(logical_name.encode("utf-8")).hexdigest()[ - :SCENARIO_SHORTENED_LOGICAL_HASH_LENGTH - ] - prefix_length: int = max_logical_length - suffix_length - return f"{logical_name[:prefix_length]}_{logical_hash}" diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 5f059d15dd..dc7fa0bb6a 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -43,7 +43,6 @@ METADATA_NAME_FILTER_LIMIT: int = 250 SCENARIO_ARTIFACT_PREFIX: str = "__sqb_" SCENARIO_HASH_PREFIX_LENGTH: int = 12 -SCENARIO_SHORTENED_LOGICAL_HASH_LENGTH: int = 8 SCENARIO_ARTIFACT_KINDS: tuple[str, ...] = ("source", "ref", "seed", "dbt_ref", "model") SCENARIO_DEFAULT_IDENTIFIER_LIMIT: int = 63 SCENARIO_PLAN_INVALID_HASH_PREFIX: str = "S501" diff --git a/src/sqlbuild/executor/audit_results/_helpers/sql.py b/src/sqlbuild/executor/audit_results/_helpers/sql.py index 1aba90f906..6944e12702 100644 --- a/src/sqlbuild/executor/audit_results/_helpers/sql.py +++ b/src/sqlbuild/executor/audit_results/_helpers/sql.py @@ -5,6 +5,9 @@ from collections.abc import Callable, Sequence from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.audit_results.constants import ( AUDIT_RESULT_COLUMN_TYPES, @@ -52,7 +55,6 @@ ) from sqlbuild.executor.audit_results.models import AuditResultRecord from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal -from sqlbuild.sql_values.types import StateSqlValueType _REQUIRED_COLUMNS: frozenset[str] = frozenset( { @@ -95,21 +97,16 @@ def build_create_table_sql( render_framework_type: Callable[[FrameworkType], str], transient: bool = False, ) -> str: - qualified_name: str = build_qualified_table_name( - database=database, schema=schema, render_qualified_name=render_qualified_name + return render_state_table_create_sql( + qualified_name=build_qualified_table_name( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=AUDIT_RESULT_COLUMNS, + column_types=AUDIT_RESULT_COLUMN_TYPES, + required_columns=_REQUIRED_COLUMNS, + render_framework_type=render_framework_type, + transient=transient, ) - rendered_types: dict[StateSqlValueType, str] = { - StateSqlValueType.STRING: render_framework_type(FrameworkType.STRING), - StateSqlValueType.INTEGER: render_framework_type(FrameworkType.INTEGER), - StateSqlValueType.TEXT_TIMESTAMP: render_framework_type(FrameworkType.TIMESTAMP), - } - definitions: list[str] = [] - for column in AUDIT_RESULT_COLUMNS: - required: str = " NOT NULL" if column in _REQUIRED_COLUMNS else "" - column_type: str = rendered_types[AUDIT_RESULT_COLUMN_TYPES[column]] - definitions.append(f"{column} {column_type}{required}") - table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" - return f"CREATE {table_kind} IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" def build_insert_sql( diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_execution.py b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py new file mode 100644 index 0000000000..14b48201e4 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/archive_execution.py @@ -0,0 +1,222 @@ +"""Direct-mode janitor archive renames, archive drops, and audit events.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import RelationType +from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( + resolve_qualified_name_parts, +) +from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, + JanitorPlan, + JanitorRelationKey, +) +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENT_SCHEMA_VERSION +from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id +from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType +from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle +from sqlbuild.runtime.observability.main.current_execution_identity import ( + current_execution_identity, +) +from sqlbuild.runtime.observability.models import ExecutionIdentity + + +def janitor_run_id() -> str: + """Return the invocation identity used to group janitor audit events.""" + + identity: ExecutionIdentity | None = current_execution_identity() + return identity.invocation_id if identity is not None else uuid4().hex + + +def apply_direct_archives( + *, + plan: JanitorPlan, + adapter: BaseAdapter, + connection: Any, + recorder: StatementRecorder, + run_id: str, +) -> tuple[tuple[JanitorArchiveCandidate, ...], tuple[JanitorArchivedRelation, ...]]: + """Rename stale relations to archives, then drop expired archives by name.""" + + _create_event_tables(adapter=adapter, connection=connection, plan=plan) + archived: list[JanitorArchiveCandidate] = [] + candidate: JanitorArchiveCandidate + for candidate in plan.archive_candidates: + with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): + _rename_relation( + adapter=adapter, + connection=connection, + candidate=candidate, + recorder=recorder, + ) + archived.append(candidate) + _write_event( + adapter=adapter, + connection=connection, + record=_event_record( + adapter=adapter, + event_type=JanitorEventType.ARCHIVE, + run_id=run_id, + archive=JanitorArchivedRelation( + key=candidate.archive_key, + relation_type=candidate.relation.relation_type, + archived_at=candidate.archived_at, + expires_at=candidate.expires_at, + original_key=candidate.key, + ), + ), + ) + deleted: list[JanitorArchivedRelation] = [] + archive: JanitorArchivedRelation + for archive in plan.archive_deletion_candidates: + with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): + _drop_archive( + adapter=adapter, connection=connection, archive=archive, recorder=recorder + ) + deleted.append(archive) + _write_event( + adapter=adapter, + connection=connection, + record=_event_record( + adapter=adapter, + event_type=JanitorEventType.DELETE, + run_id=run_id, + archive=archive, + ), + ) + return tuple(archived), tuple(deleted) + + +def _required_schema(key: JanitorRelationKey) -> str: + if key.schema is None: + raise ExecutorInputError(f"janitor archive action requires a schema: {key.display_name()}") + return key.schema + + +def _is_view(relation_type: str | None) -> bool: + return relation_type is not None and normalize_relation_type(relation_type) == RelationType.VIEW + + +def _qualified(*, adapter: BaseAdapter, key: JanitorRelationKey) -> str: + return resolve_qualified_name_parts( + adapter=adapter, database=key.database, schema=key.schema, name=key.name + ) + + +def _rename_relation( + *, + adapter: BaseAdapter, + connection: Any, + candidate: JanitorArchiveCandidate, + recorder: StatementRecorder, +) -> None: + origin: str = _qualified(adapter=adapter, key=candidate.key) + destination: str = _qualified(adapter=adapter, key=candidate.archive_key) + if not _is_view(candidate.relation.relation_type): + adapter.rename( + connection=connection, + origin=origin, + destination=destination, + statement_recorder=recorder, + ) + return + statements: tuple[str, ...] = adapter.render_rename_view(origin=origin, destination=destination) + recorder.record_many(statements) + statement: str + for statement in statements: + _ = adapter.execute(connection=connection, sql=statement) + + +def _drop_archive( + *, + adapter: BaseAdapter, + connection: Any, + archive: JanitorArchivedRelation, + recorder: StatementRecorder, +) -> None: + destination: str = _qualified(adapter=adapter, key=archive.key) + if _is_view(archive.relation_type): + adapter.drop_view( + connection=connection, + destination=destination, + if_exists=True, + statement_recorder=recorder, + ) + return + adapter.drop( + connection=connection, + destination=destination, + if_exists=True, + statement_recorder=recorder, + ) + + +def _create_event_tables(*, adapter: BaseAdapter, connection: Any, plan: JanitorPlan) -> None: + keys: tuple[JanitorRelationKey, ...] = ( + *(candidate.key for candidate in plan.archive_candidates), + *(archive.key for archive in plan.archive_deletion_candidates), + ) + schemas: dict[tuple[str | None, str], None] = dict.fromkeys( + (key.database, _required_schema(key)) for key in keys + ) + database: str | None + schema: str + for database, schema in schemas: + _ = adapter.execute( + connection=connection, + sql=adapter.render_create_janitor_event_table_sql(database=database, schema=schema), + ) + + +def _event_record( + *, + adapter: BaseAdapter, + event_type: JanitorEventType, + run_id: str, + archive: JanitorArchivedRelation, +) -> JanitorEventRecord: + schema: str = _required_schema(archive.key) + original: JanitorRelationKey | None = archive.original_key + return JanitorEventRecord( + event_id=build_janitor_event_id( + event_type=event_type, + run_id=run_id, + relation_database=archive.key.database, + relation_schema=schema, + archive_name=archive.key.name, + ), + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=event_type, + occurred_at=datetime.now(UTC), + run_id=run_id, + relation_database=archive.key.database, + relation_schema=schema, + relation_type=archive.relation_type, + original_name=None if original is None else original.name, + original_qualified_name=( + None if original is None else _qualified(adapter=adapter, key=original) + ), + archive_name=archive.key.name, + archive_qualified_name=_qualified(adapter=adapter, key=archive.key), + archived_at=archive.archived_at, + ) + + +def _write_event(*, adapter: BaseAdapter, connection: Any, record: JanitorEventRecord) -> None: + _ = write_janitor_event_record( + connection=connection, + execute=adapter.execute, + record=record, + render_qualified_name=adapter.render_qualified_name, + ) diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_names.py b/src/sqlbuild/executor/janitor/_helpers/archive_names.py new file mode 100644 index 0000000000..b9382bfa27 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/archive_names.py @@ -0,0 +1,72 @@ +"""Strict janitor archive relation name grammar.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name +from sqlbuild.executor.janitor.constants import ( + ARCHIVE_ARTIFACT_LABEL, + ARCHIVE_LOOKALIKE_PREFIX, + ARCHIVE_NAME_PREFIX, + ARCHIVE_NAME_SEPARATOR, + ARCHIVE_TIMESTAMP_FORMAT, +) +from sqlbuild.executor.janitor.models import JanitorParsedArchiveName + +_ARCHIVE_NAME_RE: re.Pattern[str] = re.compile( + rf"^{re.escape(ARCHIVE_NAME_PREFIX)}" + r"(?P[0-9]{8}T[0-9]{6}Z)" + rf"{re.escape(ARCHIVE_NAME_SEPARATOR)}" + r"(?P.+)$", + re.IGNORECASE, +) + + +def archive_timestamp(value: datetime) -> datetime: + """Normalize a timestamp to the UTC second resolution embedded in archive names.""" + + aware: datetime = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + return aware.replace(microsecond=0) + + +def build_archive_name(*, original_name: str, archived_at: datetime, identifier_limit: int) -> str: + """Build the fitted, case-folded archive name, keeping the prefix and timestamp intact.""" + + fixed_prefix: str = ( + f"{ARCHIVE_NAME_PREFIX}" + f"{archive_timestamp(archived_at).strftime(ARCHIVE_TIMESTAMP_FORMAT)}" + f"{ARCHIVE_NAME_SEPARATOR}" + ) + logical_part: str = fit_artifact_logical_name( + logical_name=original_name, + fixed_prefix=fixed_prefix, + identifier_limit=identifier_limit, + artifact_label=ARCHIVE_ARTIFACT_LABEL, + ) + return f"{fixed_prefix}{logical_part}".lower() + + +def parse_archive_name(name: str) -> JanitorParsedArchiveName | None: + """Parse a strict archive name, returning None for anything else.""" + + match: re.Match[str] | None = _ARCHIVE_NAME_RE.fullmatch(name) + if match is None: + return None + try: + archived_at: datetime = datetime.strptime( + match.group("timestamp").upper(), ARCHIVE_TIMESTAMP_FORMAT + ).replace(tzinfo=UTC) + except ValueError: + return None + return JanitorParsedArchiveName( + archived_at=archived_at, + logical_name=match.group("logical_name"), + ) + + +def is_archive_lookalike_name(name: str) -> bool: + """Return whether a relation name is reserved for janitor archive handling.""" + + return name.lower().startswith(ARCHIVE_LOOKALIKE_PREFIX) diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_planning.py b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py new file mode 100644 index 0000000000..d0749b5dbd --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py @@ -0,0 +1,208 @@ +"""Direct-mode janitor archive and archive-expiry planning.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timedelta + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.executor.janitor._helpers.archive_names import ( + archive_timestamp, + build_archive_name, + is_archive_lookalike_name, + parse_archive_name, +) +from sqlbuild.executor.janitor._helpers.classification import matching_exclude_pattern +from sqlbuild.executor.janitor._helpers.plan import relation_key as build_relation_key +from sqlbuild.executor.janitor._helpers.relation_addressing import ( + case_colliding_names, + unaddressable_relation_reason, +) +from sqlbuild.executor.janitor.constants import BUILT_IN_EXCLUDE_PATTERNS, MALFORMED_ARCHIVE_REASON +from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, + JanitorArchivePlanning, + JanitorBlockedSchema, + JanitorDeleteCandidate, + JanitorParsedArchiveName, + JanitorRelationKey, + JanitorSchemaClassification, + JanitorSkippedRelation, + JanitorWarehouseFacts, +) + + +def plan_janitor_archives( + *, + direct_mode: bool, + adapter: BaseAdapter, + schemas: JanitorSchemaClassification, + facts: JanitorWarehouseFacts, + managed_target_schemas: set[tuple[str | None, str | None]], + exclude_patterns: tuple[str, ...], + archive_retention_days: int, + now: datetime, +) -> JanitorArchivePlanning: + """Plan direct-mode archives; virtual mode never archives or expires archives.""" + + if not direct_mode: + return JanitorArchivePlanning( + archive_candidates=(), + archive_deletion_candidates=(), + retained_archives=(), + skipped_relations=(), + blocked_schemas=schemas.blocked_schemas, + ) + return _plan_direct_archives( + candidates=schemas.candidates, + facts=facts, + managed_target_schemas=managed_target_schemas, + blocked_schemas=schemas.blocked_schemas, + exclude_patterns=exclude_patterns, + archive_retention_days=archive_retention_days, + identifier_limit=adapter.maximum_identifier_length(), + now=now, + ) + + +def _plan_direct_archives( + *, + candidates: tuple[JanitorDeleteCandidate, ...], + facts: JanitorWarehouseFacts, + managed_target_schemas: set[tuple[str | None, str | None]], + blocked_schemas: tuple[JanitorBlockedSchema, ...], + exclude_patterns: tuple[str, ...], + archive_retention_days: int, + identifier_limit: int, + now: datetime, +) -> JanitorArchivePlanning: + """Plan archive renames and name-based expiry of existing archives.""" + + archived_at: datetime = archive_timestamp(now) + retention: timedelta = timedelta(days=archive_retention_days) + archive_candidates: tuple[JanitorArchiveCandidate, ...] = tuple( + _archive_candidate( + candidate=candidate, + archived_at=archived_at, + retention=retention, + identifier_limit=identifier_limit, + ) + for candidate in candidates + ) + blocked_schema_keys: frozenset[tuple[str | None, str | None]] = frozenset( + (blocked.database, blocked.schema) for blocked in blocked_schemas + ) + deletions: list[JanitorArchivedRelation] = [] + retained: list[JanitorArchivedRelation] = [] + skipped: list[JanitorSkippedRelation] = [] + suppressed_by_schema: dict[tuple[str | None, str | None], list[JanitorArchivedRelation]] = {} + effective_patterns: tuple[str, ...] = BUILT_IN_EXCLUDE_PATTERNS + exclude_patterns + schema_key: tuple[str | None, str | None] + for schema_key in sorted(managed_target_schemas, key=lambda key: (key[0] or "", key[1] or "")): + schema_relations: tuple[RelationInfo, ...] = facts.relations_by_schema.get(schema_key, ()) + colliding_names: frozenset[str] = case_colliding_names( + relation.name for relation in schema_relations + ) + relation: RelationInfo + for relation in schema_relations: + key: JanitorRelationKey = build_relation_key(relation) + if key in facts.desired_keys or not is_archive_lookalike_name(key.name): + continue + parsed: JanitorParsedArchiveName | None = parse_archive_name(key.name) + if parsed is None: + skipped.append( + JanitorSkippedRelation( + key=key, relation=relation, reason=MALFORMED_ARCHIVE_REASON + ) + ) + continue + skip_reason: str | None = _archive_skip_reason( + key=key, colliding_names=colliding_names, effective_patterns=effective_patterns + ) + if skip_reason is not None: + skipped.append( + JanitorSkippedRelation(key=key, relation=relation, reason=skip_reason) + ) + continue + archived: JanitorArchivedRelation = JanitorArchivedRelation( + key=key, + relation_type=relation.relation_type, + archived_at=parsed.archived_at, + expires_at=parsed.archived_at + retention, + ) + if archived.expires_at > now: + retained.append(archived) + elif schema_key in blocked_schema_keys: + suppressed_by_schema.setdefault(schema_key, []).append(archived) + else: + deletions.append(archived) + same_run_deletions: tuple[JanitorArchivedRelation, ...] = tuple( + JanitorArchivedRelation( + key=candidate.archive_key, + relation_type=candidate.relation.relation_type, + archived_at=candidate.archived_at, + expires_at=candidate.expires_at, + original_key=candidate.key, + ) + for candidate in archive_candidates + if candidate.expires_at <= archived_at + ) + return JanitorArchivePlanning( + archive_candidates=archive_candidates, + archive_deletion_candidates=(*deletions, *same_run_deletions), + retained_archives=tuple(retained), + skipped_relations=tuple(skipped), + blocked_schemas=tuple( + replace( + blocked, + suppressed_archive_deletions=tuple( + suppressed_by_schema.get((blocked.database, blocked.schema), ()) + ), + ) + for blocked in blocked_schemas + ), + ) + + +def _archive_skip_reason( + *, + key: JanitorRelationKey, + colliding_names: frozenset[str], + effective_patterns: tuple[str, ...], +) -> str | None: + addressing_reason: str | None = unaddressable_relation_reason( + name=key.name, colliding_names=colliding_names + ) + if addressing_reason is not None: + return addressing_reason + exclude_pattern: str | None = matching_exclude_pattern(key=key, patterns=effective_patterns) + if exclude_pattern is not None: + return f"relation matches exclude pattern {exclude_pattern!r}" + return None + + +def _archive_candidate( + *, + candidate: JanitorDeleteCandidate, + archived_at: datetime, + retention: timedelta, + identifier_limit: int, +) -> JanitorArchiveCandidate: + return JanitorArchiveCandidate( + key=candidate.key, + relation=candidate.relation, + age_timestamp=candidate.age_timestamp, + archive_key=JanitorRelationKey( + database=candidate.key.database, + schema=candidate.key.schema, + name=build_archive_name( + original_name=candidate.key.name, + archived_at=archived_at, + identifier_limit=identifier_limit, + ), + ), + archived_at=archived_at, + expires_at=archived_at + retention, + ) diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index 1675314741..eb2aa98a03 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -17,6 +17,7 @@ from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle from sqlbuild.executor.diff.models import QueryDiffArtifactInspection +from sqlbuild.executor.janitor._helpers.archive_names import is_archive_lookalike_name from sqlbuild.executor.janitor._helpers.plan import ( collect_desired_keys, collect_source_schemas, @@ -26,7 +27,13 @@ from sqlbuild.executor.janitor._helpers.plan import ( relation_key as build_relation_key, ) +from sqlbuild.executor.janitor._helpers.relation_addressing import ( + case_colliding_names, + case_collision_reason, + unaddressable_relation_reason, +) from sqlbuild.executor.janitor._helpers.tracking import collect_tracked_relation_keys +from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader from sqlbuild.executor.janitor.models import ( JanitorDeleteCandidate, JanitorDirectStatePruneCandidate, @@ -167,18 +174,25 @@ def classify_janitor_relations( effective_exclude_patterns: tuple[str, ...], delete_tracked_only: bool, retention_days: int, - age_supported: bool, + age_reader: JanitorRelationAgeReader, now: datetime, + direct_mode: bool = False, ) -> JanitorRelationClassification: """Split one schema's relations into delete candidates and skipped relations.""" candidates: list[JanitorDeleteCandidate] = [] skipped_relations: list[JanitorSkippedRelation] = [] + eligible_relations: list[RelationInfo] = [] + colliding_names: frozenset[str] = case_colliding_names( + relation.name for relation in schema_relations + ) relation: RelationInfo for relation in schema_relations: relation_key: JanitorRelationKey = build_relation_key(relation) if relation_key in facts.desired_keys: continue + if direct_mode and is_archive_lookalike_name(relation_key.name): + continue skip_reason: str | None = _relation_skip_reason( relation_key=relation_key, facts=facts, @@ -187,12 +201,26 @@ def classify_janitor_relations( effective_exclude_patterns=effective_exclude_patterns, delete_tracked_only=delete_tracked_only, ) + if skip_reason is None and direct_mode: + skip_reason = unaddressable_relation_reason( + name=relation_key.name, colliding_names=colliding_names + ) + elif skip_reason is None: + skip_reason = case_collision_reason( + name=relation_key.name, colliding_names=colliding_names + ) if skip_reason is not None: skipped_relations.append( JanitorSkippedRelation(key=relation_key, relation=relation, reason=skip_reason) ) continue - age_timestamp: datetime | None = relation_age_timestamp(relation) + eligible_relations.append(relation) + age_supported: bool = age_reader.supported() + aged_relation: RelationInfo + eligible: tuple[RelationInfo, ...] = tuple(eligible_relations) + for aged_relation in age_reader.read(eligible) if retention_days > 0 else eligible: + relation_key = build_relation_key(aged_relation) + age_timestamp: datetime | None = relation_age_timestamp(aged_relation) if retention_days > 0: retention_skip_reason: str | None = _retention_skip_reason( age_timestamp=age_timestamp, @@ -204,7 +232,7 @@ def classify_janitor_relations( skipped_relations.append( JanitorSkippedRelation( key=relation_key, - relation=relation, + relation=aged_relation, reason=retention_skip_reason, ) ) @@ -212,7 +240,7 @@ def classify_janitor_relations( candidates.append( JanitorDeleteCandidate( key=relation_key, - relation=relation, + relation=aged_relation, age_timestamp=age_timestamp, ) ) @@ -238,7 +266,7 @@ def _relation_skip_reason( relation_key, "relation is referenced by a retained virtual checkpoint", ) - exclude_pattern: str | None = _matching_exclude_pattern( + exclude_pattern: str | None = matching_exclude_pattern( key=relation_key, patterns=effective_exclude_patterns, ) @@ -269,15 +297,19 @@ def _retention_skip_reason( return None -def _matching_exclude_pattern( +def matching_exclude_pattern( *, key: JanitorRelationKey, patterns: tuple[str, ...], ) -> str | None: - display_name: str = key.display_name() + """Return the first exclude pattern matching a relation or qualified name, ignoring case.""" + + name: str = key.name.lower() + display_name: str = key.display_name().lower() pattern: str for pattern in patterns: - if fnmatchcase(key.name, pattern) or fnmatchcase(display_name, pattern): + folded_pattern: str = pattern.lower() + if fnmatchcase(name, folded_pattern) or fnmatchcase(display_name, folded_pattern): return pattern return None diff --git a/src/sqlbuild/executor/janitor/_helpers/plan.py b/src/sqlbuild/executor/janitor/_helpers/plan.py index 763cf41591..3e8e4c466f 100644 --- a/src/sqlbuild/executor/janitor/_helpers/plan.py +++ b/src/sqlbuild/executor/janitor/_helpers/plan.py @@ -3,10 +3,11 @@ from __future__ import annotations from collections import defaultdict +from dataclasses import replace from datetime import UTC, datetime from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledProject, @@ -42,16 +43,45 @@ def collect_desired_keys(project: CompiledProject) -> set[JanitorRelationKey]: def collect_target_schemas(project: CompiledProject) -> set[tuple[str | None, str | None]]: - """Collect schemas where the compiled project writes relations.""" + """Collect schemas where the compiled project writes relations, one spelling per schema.""" - schemas: set[tuple[str | None, str | None]] = set() + schemas: list[tuple[str | None, str | None]] = [] model: CompiledModel for model in project.models: - schemas.add((model.destination.database, model.destination.schema)) + schemas.append((model.destination.database, model.destination.schema)) seed: CompiledSeed for seed in project.seeds: - schemas.add((seed.destination.database, seed.destination.schema)) - return schemas + schemas.append((seed.destination.database, seed.destination.schema)) + return _dedupe_schema_keys(schemas) + + +def collect_scan_schemas( + *, + managed_target_schemas: set[tuple[str | None, str | None]], + relation_keys: frozenset[JanitorRelationKey], +) -> set[tuple[str | None, str | None]]: + """Add schemas of scanned or protected relations without duplicating folded spellings.""" + + return _dedupe_schema_keys( + [ + *sorted(managed_target_schemas, key=_schema_sort_key), + *sorted(((key.database, key.schema) for key in relation_keys), key=_schema_sort_key), + ] + ) + + +def _dedupe_schema_keys( + schema_keys: list[tuple[str | None, str | None]], +) -> set[tuple[str | None, str | None]]: + kept: dict[tuple[str | None, str | None], tuple[str | None, str | None]] = {} + schema_key: tuple[str | None, str | None] + for schema_key in schema_keys: + kept.setdefault(normalized_schema_key(schema_key), schema_key) + return set(kept.values()) + + +def _schema_sort_key(schema_key: tuple[str | None, str | None]) -> tuple[str, str]: + return (schema_key[0] or "", schema_key[1] or "") def collect_source_schemas( @@ -85,8 +115,11 @@ def list_target_schema_relations( connection: object, target_schemas: set[tuple[str | None, str | None]], ) -> dict[tuple[str | None, str | None], tuple[RelationInfo, ...]]: - """List warehouse relations for the target schemas.""" + """List relations matched case-insensitively and respelled as their target schema.""" + target_by_normalized: dict[tuple[str | None, str | None], tuple[str | None, str | None]] = { + normalized_schema_key(schema_key): schema_key for schema_key in target_schemas + } by_database: dict[str | None, set[str | None]] = defaultdict(set) schema_key: tuple[str | None, str | None] for schema_key in target_schemas: @@ -107,13 +140,27 @@ def list_target_schema_relations( database=database, schemas=concrete_schemas, ): - key: tuple[str | None, str | None] = (relation.database, relation.schema) - if key in target_schemas: - result[key].append(relation) + target_key: tuple[str | None, str | None] | None = target_by_normalized.get( + normalized_schema_key((relation.database, relation.schema)) + ) + if target_key is None: + continue + result[target_key].append( + replace(relation, database=target_key[0], schema=target_key[1]) + ) return {key: tuple(value) for key, value in result.items()} +def normalized_schema_key( + schema_key: tuple[str | None, str | None], +) -> tuple[str | None, str | None]: + """Return the case-insensitive comparison form of one database and schema pair.""" + + database, schema, _ = RelationLookup.key(database=schema_key[0], schema=schema_key[1], name="") + return (database, schema) + + def relation_key(relation: RelationInfo) -> JanitorRelationKey: """Build a janitor relation key from adapter relation metadata.""" @@ -140,4 +187,4 @@ def relation_age_timestamp(relation: RelationInfo) -> datetime | None: def _ensure_aware(value: datetime) -> datetime: if value.tzinfo is None: return value.replace(tzinfo=UTC) - return value + return value.astimezone(UTC) diff --git a/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py b/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py new file mode 100644 index 0000000000..d59933ae78 --- /dev/null +++ b/src/sqlbuild/executor/janitor/_helpers/relation_addressing.py @@ -0,0 +1,37 @@ +"""Janitor guards for relation names that are safe to address unquoted.""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Iterable + +from sqlbuild.executor.janitor.constants import ( + CASE_COLLISION_REASON, + UNQUOTED_ADDRESSING_REASON, +) + +_PLAIN_LOWERCASE_IDENTIFIER_RE: re.Pattern[str] = re.compile(r"^[a-z_][a-z0-9_$]*$") + + +def case_colliding_names(names: Iterable[str]) -> frozenset[str]: + """Return case-folded names shared by more than one relation in a listing.""" + + counts: Counter[str] = Counter(name.lower() for name in names) + return frozenset(name for name, count in counts.items() if count > 1) + + +def unaddressable_relation_reason(*, name: str, colliding_names: frozenset[str]) -> str | None: + """Return why the janitor must not rename or drop a relation by its catalog name.""" + + if _PLAIN_LOWERCASE_IDENTIFIER_RE.fullmatch(name) is None: + return UNQUOTED_ADDRESSING_REASON + return case_collision_reason(name=name, colliding_names=colliding_names) + + +def case_collision_reason(*, name: str, colliding_names: frozenset[str]) -> str | None: + """Return why a relation whose folded name is shared in its listing must not be dropped.""" + + if name.lower() in colliding_names: + return CASE_COLLISION_REASON + return None diff --git a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py index ba5ef885e0..6e72f870b4 100644 --- a/src/sqlbuild/executor/janitor/_helpers/schema_planning.py +++ b/src/sqlbuild/executor/janitor/_helpers/schema_planning.py @@ -6,7 +6,11 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.executor.janitor._helpers.classification import classify_janitor_relations -from sqlbuild.executor.janitor._helpers.source_safety import blocking_source_names +from sqlbuild.executor.janitor._helpers.source_safety import ( + blocking_source_names, + source_names_for_schema, +) +from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader from sqlbuild.executor.janitor.constants import BUILT_IN_EXCLUDE_PATTERNS from sqlbuild.executor.janitor.models import ( JanitorBlockedSchema, @@ -29,7 +33,7 @@ def classify_target_schemas( exclude_patterns: tuple[str, ...], delete_tracked_only: bool, retention_days: int, - age_supported: bool, + age_reader: JanitorRelationAgeReader, now: datetime, direct_mode: bool, ) -> JanitorSchemaClassification: @@ -41,7 +45,9 @@ def classify_target_schemas( blocked_schemas: list[JanitorBlockedSchema] = [] for schema_key in sorted(target_schemas, key=lambda key: (key[0] or "", key[1] or "")): schema_relations: tuple[RelationInfo, ...] = facts.relations_by_schema.get(schema_key, ()) - source_names: set[str] | None = facts.source_schema_names.get(schema_key) + source_names: set[str] = source_names_for_schema( + schema_key=schema_key, source_schema_names=facts.source_schema_names + ) blocking_sources: tuple[str, ...] = blocking_source_names( schema_key=schema_key, managed_schema_keys=managed_target_schemas, @@ -56,8 +62,9 @@ def classify_target_schemas( exclude_patterns=exclude_patterns, delete_tracked_only=delete_tracked_only, retention_days=retention_days, - age_supported=age_supported, + age_reader=age_reader, now=now, + direct_mode=direct_mode, ) blocked_schemas.append( JanitorBlockedSchema( @@ -84,8 +91,9 @@ def classify_target_schemas( exclude_patterns=exclude_patterns, delete_tracked_only=delete_tracked_only, retention_days=retention_days, - age_supported=age_supported, + age_reader=age_reader, now=now, + direct_mode=direct_mode, ) candidates.extend(classification.candidates) skipped_relations.extend(classification.skipped_relations) @@ -105,8 +113,9 @@ def _classify_schema( exclude_patterns: tuple[str, ...], delete_tracked_only: bool, retention_days: int, - age_supported: bool, + age_reader: JanitorRelationAgeReader, now: datetime, + direct_mode: bool, ) -> JanitorRelationClassification: return classify_janitor_relations( schema_relations=schema_relations, @@ -116,6 +125,7 @@ def _classify_schema( effective_exclude_patterns=BUILT_IN_EXCLUDE_PATTERNS + exclude_patterns, delete_tracked_only=delete_tracked_only, retention_days=retention_days, - age_supported=age_supported, + age_reader=age_reader, now=now, + direct_mode=direct_mode, ) diff --git a/src/sqlbuild/executor/janitor/_helpers/source_safety.py b/src/sqlbuild/executor/janitor/_helpers/source_safety.py index eaa1beb42e..695061f74c 100644 --- a/src/sqlbuild/executor/janitor/_helpers/source_safety.py +++ b/src/sqlbuild/executor/janitor/_helpers/source_safety.py @@ -2,6 +2,8 @@ from __future__ import annotations +from sqlbuild.executor.janitor._helpers.plan import normalized_schema_key + def blocking_source_names( *, @@ -11,22 +13,26 @@ def blocking_source_names( ) -> tuple[str, ...]: """Return active sources when one managed schema has mixed ownership.""" - normalized: tuple[str | None, str | None] = _normalized_schema_key(schema_key) - if not any( - _normalized_schema_key(candidate) == normalized for candidate in managed_schema_keys - ): + normalized: tuple[str | None, str | None] = normalized_schema_key(schema_key) + if not any(normalized_schema_key(candidate) == normalized for candidate in managed_schema_keys): return () - for candidate, source_names in source_schema_names.items(): - if _normalized_schema_key(candidate) == normalized: - return tuple(sorted(source_names)) - return () + return tuple( + sorted( + source_names_for_schema(schema_key=schema_key, source_schema_names=source_schema_names) + ) + ) -def _normalized_schema_key( +def source_names_for_schema( + *, schema_key: tuple[str | None, str | None], -) -> tuple[str | None, str | None]: - database, schema = schema_key - return ( - None if database is None else database.lower(), - None if schema is None else schema.lower(), - ) + source_schema_names: dict[tuple[str | None, str | None], set[str]], +) -> set[str]: + """Return active source names configured in one schema, compared case-insensitively.""" + + normalized: tuple[str | None, str | None] = normalized_schema_key(schema_key) + names: set[str] = set() + for candidate, source_names in source_schema_names.items(): + if normalized_schema_key(candidate) == normalized: + names.update(source_names) + return names diff --git a/src/sqlbuild/executor/janitor/classes/relation_age_reader.py b/src/sqlbuild/executor/janitor/classes/relation_age_reader.py new file mode 100644 index 0000000000..250082de9f --- /dev/null +++ b/src/sqlbuild/executor/janitor/classes/relation_age_reader.py @@ -0,0 +1,30 @@ +"""Candidate-scoped relation age metadata reader for janitor planning.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo + + +class JanitorRelationAgeReader: + """Adapter connection used to read age metadata for janitor candidates only.""" + + def __init__(self, *, adapter: BaseAdapter, connection: Any) -> None: + self.adapter: BaseAdapter = adapter + self.connection: Any = connection + + def supported(self) -> bool: + """Return whether the adapter reports reliable relation age metadata.""" + + return self.adapter.supports_relation_age_metadata() + + def read(self, relations: tuple[RelationInfo, ...]) -> tuple[RelationInfo, ...]: + """Return candidate relations with age timestamps filled when supported.""" + + if not relations or not self.supported(): + return relations + return self.adapter.with_relation_age_metadata( + connection=self.connection, relations=relations + ) diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 520004962a..5cd2663418 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -4,10 +4,35 @@ from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME +from sqlbuild.executor.audit_results.constants import AUDIT_RESULTS_TABLE_NAME +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME +from sqlbuild.executor.node_results.constants import NODE_RESULTS_TABLE_NAME from sqlbuild.microbatches.constants import MICROBATCH_TABLE_NAME BUILT_IN_EXCLUDE_PATTERNS: tuple[str, ...] = ( FINGERPRINT_TABLE_NAME, MICROBATCH_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, + AUDIT_RESULTS_TABLE_NAME, + JANITOR_EVENTS_TABLE_NAME, +) + +DIRECT_DEFAULT_RETENTION_DAYS: int = 14 +VIRTUAL_DEFAULT_RETENTION_DAYS: int = 30 + +ARCHIVE_NAME_PREFIX: str = "_SQB_ARCHIVE__" +ARCHIVE_NAME_SEPARATOR: str = "__" +ARCHIVE_TIMESTAMP_FORMAT: str = "%Y%m%dT%H%M%SZ" +ARCHIVE_LOOKALIKE_PREFIX: str = "_sqb_archive" +ARCHIVE_ARTIFACT_LABEL: str = "Janitor archive" +UNQUOTED_ADDRESSING_REASON: str = ( + "relation name is not a plain lowercase identifier and may require quoting; " + "janitor does not act on it" +) +CASE_COLLISION_REASON: str = ( + "relation name collides case-insensitively with another relation; janitor does not act on it" +) +MALFORMED_ARCHIVE_REASON: str = ( + "name resembles a janitor archive but does not match the strict archive grammar" ) diff --git a/src/sqlbuild/executor/janitor/main/execute.py b/src/sqlbuild/executor/janitor/main/execute.py index e603f5094a..58ad35c963 100644 --- a/src/sqlbuild/executor/janitor/main/execute.py +++ b/src/sqlbuild/executor/janitor/main/execute.py @@ -7,8 +7,14 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.executor.janitor._helpers.archive_execution import ( + apply_direct_archives, + janitor_run_id, +) from sqlbuild.executor.janitor._helpers.deletion import apply_janitor_deletions from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, JanitorCheckpointCandidate, JanitorDeleteCandidate, JanitorDetachedVirtualEnvironmentCandidate, @@ -87,6 +93,19 @@ def _execute_janitor_plan( if_exists=True, statement_recorder=recorder, ) + archived: tuple[JanitorArchiveCandidate, ...] + deleted_archives: tuple[JanitorArchivedRelation, ...] + archived, deleted_archives = ( + apply_direct_archives( + plan=plan, + adapter=adapter, + connection=connection, + recorder=recorder, + run_id=janitor_run_id(), + ) + if plan.direct_mode + else ((), ()) + ) query_artifact_candidate: JanitorQueryDiffArtifactCandidate for query_artifact_candidate in plan.query_diff_artifact_candidates: with OperationLifecycle(operation_kind="janitor", operation_name="janitor_cleanup_action"): @@ -137,6 +156,8 @@ def _execute_janitor_plan( ) return JanitorExecutionResult( deleted=() if plan.direct_mode else plan.candidates, + archived=archived, + deleted_archives=deleted_archives, deleted_query_diff_artifacts=plan.query_diff_artifact_candidates, deleted_checkpoints=deleted_checkpoints, deleted_detached_virtual_environments=deleted_detached_virtual_environments, @@ -151,6 +172,8 @@ def _execute_janitor_plan( def _janitor_action_count(plan: JanitorPlan) -> int: relation_count: int = 0 if plan.direct_mode else len(plan.candidates) relation_count += len(plan.query_diff_artifact_candidates) + if plan.direct_mode: + relation_count += len(plan.archive_candidates) + len(plan.archive_deletion_candidates) return relation_count + sum( len(candidates) for candidates in ( diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py index ae7f92df3e..45c88ec42f 100644 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ b/src/sqlbuild/executor/janitor/main/plan.py @@ -8,14 +8,17 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle +from sqlbuild.executor.janitor._helpers.archive_planning import plan_janitor_archives from sqlbuild.executor.janitor._helpers.classification import ( collect_direct_state_prune_candidates, collect_query_diff_artifact_candidates, gather_janitor_warehouse_facts, ) -from sqlbuild.executor.janitor._helpers.plan import collect_target_schemas +from sqlbuild.executor.janitor._helpers.plan import collect_scan_schemas, collect_target_schemas from sqlbuild.executor.janitor._helpers.schema_planning import classify_target_schemas +from sqlbuild.executor.janitor.classes.relation_age_reader import JanitorRelationAgeReader from sqlbuild.executor.janitor.models import ( + JanitorArchivePlanning, JanitorDirectModeSettings, JanitorDirectStatePruneCandidate, JanitorPlan, @@ -49,9 +52,10 @@ def build_janitor_plan( ) direct: JanitorDirectModeSettings = direct_settings or JanitorDirectModeSettings() managed_target_schemas: set[tuple[str | None, str | None]] = collect_target_schemas(project) - target_schemas: set[tuple[str | None, str | None]] = set(managed_target_schemas) - target_schemas.update((key.database, key.schema) for key in scope.protected_relation_keys) - target_schemas.update((key.database, key.schema) for key in scope.scan_relation_keys) + target_schemas: set[tuple[str | None, str | None]] = collect_scan_schemas( + managed_target_schemas=managed_target_schemas, + relation_keys=scope.protected_relation_keys | scope.scan_relation_keys, + ) query_artifact_schemas: set[tuple[str | None, str]] = { (database, schema) for database, schema in target_schemas if schema is not None } @@ -74,6 +78,7 @@ def build_janitor_plan( target_name=project.effective_target_name, retention_days=retention_days, direct_mode=direct.enabled, + archive_retention_days=direct.archive_retention_days, query_diff_artifact_candidates=query_diff_artifact_candidates, checkpoint_candidates=state.checkpoint_candidates, detached_virtual_environment_candidates=(state.detached_virtual_environment_candidates), @@ -85,6 +90,7 @@ def build_janitor_plan( skipped_relations=query_diff_artifact_skipped, scanned_schema_count=len(query_artifact_schemas), age_metadata_supported=adapter.supports_relation_age_metadata(), + planned_at=now, ) with OperationLifecycle( @@ -106,8 +112,12 @@ def build_janitor_plan( ) ) inspection.completed(metadata={"item_count": len(target_schemas)}) + age_reader: JanitorRelationAgeReader = JanitorRelationAgeReader( + adapter=adapter, connection=connection + ) age_supported: bool = adapter.supports_relation_age_metadata() schemas: JanitorSchemaClassification = classify_target_schemas( + age_reader=age_reader, target_schemas=target_schemas, managed_target_schemas=managed_target_schemas, facts=facts, @@ -115,16 +125,29 @@ def build_janitor_plan( exclude_patterns=exclude_patterns, delete_tracked_only=delete_tracked_only, retention_days=retention_days, - age_supported=age_supported, now=now, direct_mode=direct.enabled, ) + archives: JanitorArchivePlanning = plan_janitor_archives( + direct_mode=direct.enabled, + adapter=adapter, + schemas=schemas, + facts=facts, + managed_target_schemas=managed_target_schemas, + exclude_patterns=exclude_patterns, + archive_retention_days=direct.archive_retention_days, + now=now, + ) return JanitorPlan( target_name=project.effective_target_name, retention_days=retention_days, direct_mode=direct.enabled, + archive_retention_days=direct.archive_retention_days, candidates=schemas.candidates, + archive_candidates=archives.archive_candidates, + archive_deletion_candidates=archives.archive_deletion_candidates, + retained_archives=archives.retained_archives, query_diff_artifact_candidates=query_diff_artifact_candidates, checkpoint_candidates=state.checkpoint_candidates, detached_virtual_environment_candidates=state.detached_virtual_environment_candidates, @@ -134,15 +157,17 @@ def build_janitor_plan( virtual_state_prune_candidates=state.virtual_state_prune_candidates, direct_state_prune_candidates=direct_state_prune_candidates, skipped_relations=( - *tuple( + *( skipped for skipped in schemas.skipped_relations if not QueryDiffArtifactLifecycle.is_artifact_name(skipped.key.name) ), + *archives.skipped_relations, *query_diff_artifact_skipped, ), skipped_schemas=schemas.skipped_schemas, - blocked_schemas=schemas.blocked_schemas, + blocked_schemas=archives.blocked_schemas, scanned_schema_count=len(target_schemas | set(query_artifact_schemas)), age_metadata_supported=age_supported, + planned_at=now, ) diff --git a/src/sqlbuild/executor/janitor/main/resolve_retention_days.py b/src/sqlbuild/executor/janitor/main/resolve_retention_days.py new file mode 100644 index 0000000000..19e6372361 --- /dev/null +++ b/src/sqlbuild/executor/janitor/main/resolve_retention_days.py @@ -0,0 +1,20 @@ +"""Janitor relation retention resolution entrypoint.""" + +from __future__ import annotations + +from sqlbuild.executor.janitor.constants import ( + DIRECT_DEFAULT_RETENTION_DAYS, + VIRTUAL_DEFAULT_RETENTION_DAYS, +) + + +def resolve_janitor_retention_days( + *, override: int | None, configured: int | None, virtual_environments: bool +) -> int: + """Resolve CLI override, explicit config, then the mode-specific default.""" + + if override is not None: + return override + if configured is not None: + return configured + return VIRTUAL_DEFAULT_RETENTION_DAYS if virtual_environments else DIRECT_DEFAULT_RETENTION_DAYS diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index ebbcdebae1..5601dfcc81 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -5,17 +5,30 @@ from dataclasses import dataclass, field from datetime import datetime -from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -@dataclass(frozen=True) +@dataclass(frozen=True, eq=False) class JanitorRelationKey: - """Physical identity for a warehouse relation considered by janitor.""" + """Physical relation identity compared case-insensitively but rendered as spelled.""" database: str | None schema: str | None name: str + def identity(self) -> tuple[str | None, str | None, str]: + """Return the case-insensitive comparison identity.""" + + return RelationLookup.key(database=self.database, schema=self.schema, name=self.name) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, JanitorRelationKey): + return NotImplemented + return self.identity() == other.identity() + + def __hash__(self) -> int: + return hash(self.identity()) + def display_name(self) -> str: """Render a qualified display name.""" @@ -37,6 +50,37 @@ class JanitorDeleteCandidate: age_timestamp: datetime | None +@dataclass(frozen=True) +class JanitorParsedArchiveName: + """Structured parts of one strict janitor archive relation name.""" + + archived_at: datetime + logical_name: str + + +@dataclass(frozen=True) +class JanitorArchiveCandidate: + """One stale direct-mode relation that will be renamed to an archive name.""" + + key: JanitorRelationKey + relation: RelationInfo + age_timestamp: datetime | None + archive_key: JanitorRelationKey + archived_at: datetime + expires_at: datetime + + +@dataclass(frozen=True) +class JanitorArchivedRelation: + """One relation whose name strictly matches the janitor archive grammar.""" + + key: JanitorRelationKey + relation_type: str | None + archived_at: datetime + expires_at: datetime + original_key: JanitorRelationKey | None = None + + @dataclass(frozen=True) class JanitorQueryDiffArtifactCandidate: """One expired, fingerprint-owned query-diff artifact eligible for deletion.""" @@ -156,6 +200,7 @@ class JanitorBlockedSchema: schema: str | None source_names: tuple[str, ...] suppressed_candidates: tuple[JanitorDeleteCandidate, ...] = field(default_factory=tuple) + suppressed_archive_deletions: tuple[JanitorArchivedRelation, ...] = field(default_factory=tuple) def display_name(self) -> str: """Render a schema display name.""" @@ -197,6 +242,17 @@ class JanitorSchemaClassification: blocked_schemas: tuple[JanitorBlockedSchema, ...] +@dataclass(frozen=True) +class JanitorArchivePlanning: + """Direct-mode archive and archive-expiry decisions for one janitor plan.""" + + archive_candidates: tuple[JanitorArchiveCandidate, ...] + archive_deletion_candidates: tuple[JanitorArchivedRelation, ...] + retained_archives: tuple[JanitorArchivedRelation, ...] + skipped_relations: tuple[JanitorSkippedRelation, ...] + blocked_schemas: tuple[JanitorBlockedSchema, ...] + + @dataclass(frozen=True) class JanitorStateCandidates: """Precomputed state-side cleanup candidates for one janitor plan.""" @@ -228,6 +284,7 @@ class JanitorDirectModeSettings: enabled: bool = False state_history_versions: int = 20 + archive_retention_days: int = 14 @dataclass(frozen=True) @@ -237,7 +294,11 @@ class JanitorPlan: target_name: str | None retention_days: int direct_mode: bool = False + archive_retention_days: int = 14 candidates: tuple[JanitorDeleteCandidate, ...] = field(default_factory=tuple) + archive_candidates: tuple[JanitorArchiveCandidate, ...] = field(default_factory=tuple) + archive_deletion_candidates: tuple[JanitorArchivedRelation, ...] = field(default_factory=tuple) + retained_archives: tuple[JanitorArchivedRelation, ...] = field(default_factory=tuple) query_diff_artifact_candidates: tuple[JanitorQueryDiffArtifactCandidate, ...] = field( default_factory=tuple ) @@ -261,6 +322,7 @@ class JanitorPlan: blocked_schemas: tuple[JanitorBlockedSchema, ...] = field(default_factory=tuple) scanned_schema_count: int = 0 age_metadata_supported: bool = False + planned_at: datetime | None = None @dataclass(frozen=True) @@ -268,6 +330,8 @@ class JanitorExecutionResult: """Result from deleting janitor candidates.""" deleted: tuple[JanitorDeleteCandidate, ...] = field(default_factory=tuple) + archived: tuple[JanitorArchiveCandidate, ...] = field(default_factory=tuple) + deleted_archives: tuple[JanitorArchivedRelation, ...] = field(default_factory=tuple) deleted_query_diff_artifacts: tuple[JanitorQueryDiffArtifactCandidate, ...] = field( default_factory=tuple ) diff --git a/src/sqlbuild/executor/janitor_events/__init__.py b/src/sqlbuild/executor/janitor_events/__init__.py new file mode 100644 index 0000000000..58de8b3059 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/__init__.py @@ -0,0 +1 @@ +"""Append-only direct-mode janitor audit events.""" diff --git a/src/sqlbuild/executor/janitor_events/_helpers/__init__.py b/src/sqlbuild/executor/janitor_events/_helpers/__init__.py new file mode 100644 index 0000000000..ca84bc0c46 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/_helpers/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event persistence helpers.""" diff --git a/src/sqlbuild/executor/janitor_events/_helpers/identity.py b/src/sqlbuild/executor/janitor_events/_helpers/identity.py new file mode 100644 index 0000000000..f0cbb41021 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/_helpers/identity.py @@ -0,0 +1,29 @@ +"""Deterministic janitor audit event identity helpers.""" + +from __future__ import annotations + +import hashlib +import json + +from sqlbuild.executor.janitor_events.types import JanitorEventType + + +def build_janitor_event_id_impl( + *, + event_type: JanitorEventType, + run_id: str, + relation_database: str | None, + relation_schema: str, + archive_name: str, +) -> str: + """Return the content-addressed ID for one janitor action within one run.""" + + identity: tuple[object, ...] = ( + event_type.value, + run_id, + relation_database, + relation_schema, + archive_name, + ) + encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py new file mode 100644 index 0000000000..8f9bf5e1da --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -0,0 +1,112 @@ +"""Portable SQL builders for append-only janitor audit events.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor_events.constants import ( + COLUMN_ARCHIVE_NAME, + COLUMN_ARCHIVE_QUALIFIED_NAME, + COLUMN_ARCHIVED_AT, + COLUMN_EVENT_ID, + COLUMN_EVENT_TYPE, + COLUMN_OCCURRED_AT, + COLUMN_ORIGINAL_NAME, + COLUMN_ORIGINAL_QUALIFIED_NAME, + COLUMN_RELATION_DATABASE, + COLUMN_RELATION_SCHEMA, + COLUMN_RELATION_TYPE, + COLUMN_RUN_ID, + COLUMN_SCHEMA_VERSION, + JANITOR_EVENT_COLUMN_TYPES, + JANITOR_EVENT_COLUMNS, + JANITOR_EVENT_REQUIRED_COLUMNS, + JANITOR_EVENTS_TABLE_NAME, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal + + +def build_qualified_table_name( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: + qualified_name: str | None = render_qualified_name( + database=database, schema=schema, name=JANITOR_EVENTS_TABLE_NAME + ) + if qualified_name is None: + raise ExecutorInputError("janitor event table requires a target schema") + return qualified_name + + +def build_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + return render_state_table_create_sql( + qualified_name=build_qualified_table_name( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=JANITOR_EVENT_COLUMNS, + column_types=JANITOR_EVENT_COLUMN_TYPES, + required_columns=JANITOR_EVENT_REQUIRED_COLUMNS, + render_framework_type=render_framework_type, + transient=transient, + ) + + +def build_existing_event_sql( + *, + record: JanitorEventRecord, + render_qualified_name: Callable[..., str | None], +) -> str: + qualified_name: str = build_qualified_table_name( + database=record.relation_database, + schema=record.relation_schema, + render_qualified_name=render_qualified_name, + ) + event_id: str = _column_literal(column=COLUMN_EVENT_ID, value=record.event_id) + return f"SELECT {COLUMN_EVENT_ID} FROM {qualified_name} WHERE {COLUMN_EVENT_ID} = {event_id}" + + +def build_insert_sql( + *, + record: JanitorEventRecord, + render_qualified_name: Callable[..., str | None], +) -> str: + qualified_name: str = build_qualified_table_name( + database=record.relation_database, + schema=record.relation_schema, + render_qualified_name=render_qualified_name, + ) + values: dict[str, object | None] = { + COLUMN_EVENT_ID: record.event_id, + COLUMN_SCHEMA_VERSION: record.schema_version, + COLUMN_EVENT_TYPE: record.event_type.value, + COLUMN_OCCURRED_AT: record.occurred_at, + COLUMN_RUN_ID: record.run_id, + COLUMN_RELATION_DATABASE: record.relation_database, + COLUMN_RELATION_SCHEMA: record.relation_schema, + COLUMN_RELATION_TYPE: record.relation_type, + COLUMN_ORIGINAL_NAME: record.original_name, + COLUMN_ORIGINAL_QUALIFIED_NAME: record.original_qualified_name, + COLUMN_ARCHIVE_NAME: record.archive_name, + COLUMN_ARCHIVE_QUALIFIED_NAME: record.archive_qualified_name, + COLUMN_ARCHIVED_AT: record.archived_at, + } + literals: str = ", ".join( + _column_literal(column=column, value=values[column]) for column in JANITOR_EVENT_COLUMNS + ) + return f"INSERT INTO {qualified_name} ({', '.join(JANITOR_EVENT_COLUMNS)}) VALUES ({literals})" + + +def _column_literal(*, column: str, value: object | None) -> str: + return render_state_sql_literal(value=value, declared_type=JANITOR_EVENT_COLUMN_TYPES[column]) diff --git a/src/sqlbuild/executor/janitor_events/constants.py b/src/sqlbuild/executor/janitor_events/constants.py new file mode 100644 index 0000000000..721f93d887 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/constants.py @@ -0,0 +1,57 @@ +"""Janitor audit event warehouse storage constants.""" + +from sqlbuild.sql_values.types import StateSqlValueType + +JANITOR_EVENTS_TABLE_NAME: str = "_sqlbuild_janitor_events" +JANITOR_EVENT_SCHEMA_VERSION: int = 1 + +COLUMN_EVENT_ID: str = "event_id" +COLUMN_SCHEMA_VERSION: str = "schema_version" +COLUMN_EVENT_TYPE: str = "event_type" +COLUMN_OCCURRED_AT: str = "occurred_at" +COLUMN_RUN_ID: str = "run_id" +COLUMN_RELATION_DATABASE: str = "relation_database" +COLUMN_RELATION_SCHEMA: str = "relation_schema" +COLUMN_RELATION_TYPE: str = "relation_type" +COLUMN_ORIGINAL_NAME: str = "original_name" +COLUMN_ORIGINAL_QUALIFIED_NAME: str = "original_qualified_name" +COLUMN_ARCHIVE_NAME: str = "archive_name" +COLUMN_ARCHIVE_QUALIFIED_NAME: str = "archive_qualified_name" +COLUMN_ARCHIVED_AT: str = "archived_at" + +JANITOR_EVENT_COLUMNS: tuple[str, ...] = ( + COLUMN_EVENT_ID, + COLUMN_SCHEMA_VERSION, + COLUMN_EVENT_TYPE, + COLUMN_OCCURRED_AT, + COLUMN_RUN_ID, + COLUMN_RELATION_DATABASE, + COLUMN_RELATION_SCHEMA, + COLUMN_RELATION_TYPE, + COLUMN_ORIGINAL_NAME, + COLUMN_ORIGINAL_QUALIFIED_NAME, + COLUMN_ARCHIVE_NAME, + COLUMN_ARCHIVE_QUALIFIED_NAME, + COLUMN_ARCHIVED_AT, +) + +JANITOR_EVENT_REQUIRED_COLUMNS: frozenset[str] = frozenset( + { + COLUMN_EVENT_ID, + COLUMN_SCHEMA_VERSION, + COLUMN_EVENT_TYPE, + COLUMN_OCCURRED_AT, + COLUMN_RUN_ID, + COLUMN_RELATION_SCHEMA, + COLUMN_ARCHIVE_NAME, + COLUMN_ARCHIVE_QUALIFIED_NAME, + COLUMN_ARCHIVED_AT, + } +) + +JANITOR_EVENT_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in JANITOR_EVENT_COLUMNS}, + COLUMN_SCHEMA_VERSION: StateSqlValueType.INTEGER, + COLUMN_OCCURRED_AT: StateSqlValueType.TEXT_TIMESTAMP, + COLUMN_ARCHIVED_AT: StateSqlValueType.TEXT_TIMESTAMP, +} diff --git a/src/sqlbuild/executor/janitor_events/main/__init__.py b/src/sqlbuild/executor/janitor_events/main/__init__.py new file mode 100644 index 0000000000..92857c3ea7 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/main/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event persistence entrypoints.""" diff --git a/src/sqlbuild/executor/janitor_events/main/create_table_sql.py b/src/sqlbuild/executor/janitor_events/main/create_table_sql.py new file mode 100644 index 0000000000..4b694023e6 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/main/create_table_sql.py @@ -0,0 +1,27 @@ +"""Janitor audit event table SQL entrypoint.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.executor.janitor_events._helpers.sql import build_create_table_sql + + +def build_janitor_events_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + """Build DDL that creates the janitor audit event table when it is missing.""" + + return build_create_table_sql( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + render_framework_type=render_framework_type, + transient=transient, + ) diff --git a/src/sqlbuild/executor/janitor_events/main/janitor_event_id.py b/src/sqlbuild/executor/janitor_events/main/janitor_event_id.py new file mode 100644 index 0000000000..997fe7811b --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/main/janitor_event_id.py @@ -0,0 +1,25 @@ +"""Janitor audit event identity entrypoint.""" + +from __future__ import annotations + +from sqlbuild.executor.janitor_events._helpers.identity import build_janitor_event_id_impl +from sqlbuild.executor.janitor_events.types import JanitorEventType + + +def build_janitor_event_id( + *, + event_type: JanitorEventType, + run_id: str, + relation_database: str | None, + relation_schema: str, + archive_name: str, +) -> str: + """Return the deterministic ID for one janitor action within one run.""" + + return build_janitor_event_id_impl( + event_type=event_type, + run_id=run_id, + relation_database=relation_database, + relation_schema=relation_schema, + archive_name=archive_name, + ) diff --git a/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py b/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py new file mode 100644 index 0000000000..3dc22a22e4 --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/main/write_janitor_event.py @@ -0,0 +1,35 @@ +"""Janitor audit event warehouse write operation.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.executor.janitor_events._helpers.sql import ( + build_existing_event_sql, + build_insert_sql, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord + + +def write_janitor_event_record( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + record: JanitorEventRecord, + render_qualified_name: Callable[..., str | None], +) -> bool: + """Append one janitor event unless its deterministic ID was already written.""" + + cursor: Any = execute( + connection=connection, + sql=build_existing_event_sql(record=record, render_qualified_name=render_qualified_name), + ) + if cursor.fetchall(): + return False + _ = execute( + connection=connection, + sql=build_insert_sql(record=record, render_qualified_name=render_qualified_name), + ) + return True diff --git a/src/sqlbuild/executor/janitor_events/models.py b/src/sqlbuild/executor/janitor_events/models.py new file mode 100644 index 0000000000..2e99b267ce --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/models.py @@ -0,0 +1,27 @@ +"""Immutable records persisted to the janitor audit table.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from sqlbuild.executor.janitor_events.types import JanitorEventType + + +@dataclass(frozen=True) +class JanitorEventRecord: + """One immutable janitor archive or delete fact.""" + + event_id: str + schema_version: int + event_type: JanitorEventType + occurred_at: datetime + run_id: str + relation_database: str | None + relation_schema: str + relation_type: str | None + original_name: str | None + original_qualified_name: str | None + archive_name: str + archive_qualified_name: str + archived_at: datetime diff --git a/src/sqlbuild/executor/janitor_events/types.py b/src/sqlbuild/executor/janitor_events/types.py new file mode 100644 index 0000000000..2223996f3c --- /dev/null +++ b/src/sqlbuild/executor/janitor_events/types.py @@ -0,0 +1,12 @@ +"""Janitor audit event type declarations.""" + +from __future__ import annotations + +from enum import StrEnum + + +class JanitorEventType(StrEnum): + """Physical janitor actions recorded in the audit table.""" + + ARCHIVE = "archive" + DELETE = "delete" diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index ae9e9d557d..8d92ed41af 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -313,7 +313,8 @@ class JanitorConfig: """Janitor command defaults.""" enabled: bool = False - retention_days: int = 30 + retention_days: int | None = None + archive_retention_days: int = 14 max_checkpoints: int = 20 direct_state_history_versions: int = 20 delete_tracked_only: bool = True diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index 0ec40455a1..d12e339fac 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -26,6 +26,7 @@ class JanitorCleanupE2ETestCase: expected_stdout_fragments: tuple[str, ...] expected_existing_tables: tuple[str, ...] expected_missing_tables: tuple[str, ...] + expected_archived_original_names: tuple[str, ...] = field(default_factory=tuple) @dataclass(frozen=True) @@ -152,3 +153,79 @@ class JanitorDirectStatePruningE2ETestCase: expected_source_freshness_count_after: int expected_fingerprint_run_ids_after: tuple[str, ...] expected_source_freshness_run_ids_after: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorArchiveRetentionE2ETestCase: + """Test case for direct-mode archival after the relation retention period.""" + + description: str + janitor_command: tuple[str, ...] + retired_model_ages_days: dict[str, int] + expected_archived_names: tuple[str, ...] + expected_retained_names: tuple[str, ...] + expected_stdout_fragments: tuple[str, ...] + expected_second_run_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorArchiveExpiryE2ETestCase: + """Test case for name-based deletion of expired archives.""" + + description: str + janitor_command: tuple[str, ...] + expected_deleted_names: tuple[str, ...] + expected_kept_names: tuple[str, ...] + expected_kept_other_schema_names: tuple[str, ...] + expected_stdout_fragments: tuple[str, ...] + expected_delete_event_count: int + expected_total_event_count: int + + +@dataclass(frozen=True) +class JanitorZeroRetentionE2ETestCase: + """Test case for archive and delete in one run.""" + + description: str + janitor_command: tuple[str, ...] + stale_tables: tuple[str, ...] + stale_views: tuple[str, ...] + expected_stdout_fragments: tuple[str, ...] + expected_event_types: tuple[tuple[str, str], ...] + expected_fingerprint_count_after: int + expected_source_freshness_count_after: int + + +@dataclass(frozen=True) +class JanitorArchiveNameFittingE2ETestCase: + """Test case for identifier-fitted archive names.""" + + description: str + janitor_command: tuple[str, ...] + long_relation_name: str + identifier_limit: int + expected_archive_prefix_length: int + + +@dataclass(frozen=True) +class JanitorArchiveInterruptionE2ETestCase: + """Test case for an audit write failure after a rename.""" + + description: str + janitor_command: tuple[str, ...] + stale_table: str + expected_first_exit_code: int + expected_first_output_fragments: tuple[str, ...] + expected_second_exit_code: int + expected_second_stdout_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorUnaddressableRelationE2ETestCase: + """Test case for relations the janitor cannot safely address unquoted.""" + + description: str + janitor_command: tuple[str, ...] + plain_relation: str + quoted_relation: str + expected_stdout_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 6b79ef6612..ca85dbc78a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -2,9 +2,11 @@ from __future__ import annotations +from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path from textwrap import dedent +from types import MappingProxyType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.fingerprints.main.write import write_fingerprint @@ -14,7 +16,10 @@ SourceFreshnessRecord, SourceFreshnessRenderers, ) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, +) def prepare_janitor_project( @@ -194,3 +199,164 @@ def create_direct_state_history(*, db_path: Path) -> None: ) finally: connection.close() + + +AGED_JANITOR_ADAPTER_NAME: str = "aged_janitor_duckdb" +JANITOR_TEST_SETTINGS_FILENAME: str = "janitor_test_settings.json" +AGED_JANITOR_ADAPTER_SOURCE: str = """ +import json +from dataclasses import replace +from datetime import datetime +from pathlib import Path +from typing import Any + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter + +_SETTINGS_PATH = Path(__file__).resolve().parent.parent / "janitor_test_settings.json" + + +def _settings() -> dict[str, Any]: + if not _SETTINGS_PATH.exists(): + return {} + return json.loads(_SETTINGS_PATH.read_text(encoding="utf-8")) + + +class AgedJanitorDuckDbAdapter(DuckDbAdapter): + adapter_name = "aged_janitor_duckdb" + + def supports_relation_age_metadata(self) -> bool: + return True + + def maximum_identifier_length(self) -> int: + return int(_settings().get("identifier_limit", super().maximum_identifier_length())) + + def list_relations( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[RelationInfo, ...]: + created_at: dict[str, str] = _settings().get("created_at", {}) + return tuple( + replace(relation, created_at=datetime.fromisoformat(created_at[relation.name])) + if relation.name in created_at + else relation + for relation in super().list_relations( + connection=connection, database=database, schemas=schemas, names=names + ) + ) + + def _execute(self, *, connection: Any, sql: str) -> Any: + if ( + _settings().get("fail_janitor_event_insert") + and sql.startswith("INSERT INTO") + and "_sqlbuild_janitor_events" in sql + ): + raise AdapterUserError("simulated janitor event write failure") + return super()._execute(connection=connection, sql=sql) +""" + + +def prepare_archive_janitor_project( + *, + tmp_path: Path, + project_name: str, + janitor_config: str, + model_names: tuple[str, ...] = ("orders",), + use_aged_adapter: bool = False, +) -> Path: + """Create a DuckDB project whose janitor archives stale relations.""" + + repo_files: dict[str, str] = { + "sqlbuild_project.toml": ( + f'name = "{project_name}"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + f'database = "{(tmp_path / project_name / "janitor.duckdb").as_posix()}"\n\n' + f"[janitor]\n{dedent(janitor_config).strip()}\n\n" + "[defaults]\n" + 'materialized = "table"\n' + ), + **{ + f"models/{model_name}.sql": f"MODEL ();\n\nSELECT 1 AS {model_name}_id\n" + for model_name in model_names + }, + f"adapters/{AGED_JANITOR_ADAPTER_NAME}.py": AGED_JANITOR_ADAPTER_SOURCE, + **{ + False: {}, + True: {"sqlbuild_local.toml": f'adapter = "{AGED_JANITOR_ADAPTER_NAME}"\n'}, + }[use_aged_adapter], + } + return prepare_inline_project( + tmp_path=tmp_path, + project_name=project_name, + repo_files=repo_files, + ) + + +def write_janitor_test_settings( + *, + project_dir: Path, + created_at: Mapping[str, datetime] = MappingProxyType({}), + identifier_limit: int = 255, + fail_janitor_event_insert: bool = False, +) -> None: + """Write relation ages and fault switches consumed by the aged janitor adapter.""" + + import json + + payload: dict[str, object] = { + "created_at": {name: value.isoformat() for name, value in created_at.items()}, + "identifier_limit": identifier_limit, + "fail_janitor_event_insert": fail_janitor_event_insert, + } + (project_dir / JANITOR_TEST_SETTINGS_FILENAME).write_text(json.dumps(payload), encoding="utf-8") + + +def archive_timestamp_text(value: datetime) -> str: + """Render the UTC timestamp component embedded in janitor archive names.""" + + return value.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ").lower() + + +def list_archive_names(*, db_path: Path, schema: str = "main") -> tuple[str, ...]: + """Return archive-prefixed relation names in one schema.""" + + rows: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' AND starts_with(table_name, '_sqb_archive__') " + "ORDER BY table_name" + ), + ) + return tuple(str(row[0]) for row in rows) + + +def read_janitor_events(*, db_path: Path) -> list[tuple[object, ...]]: + """Return janitor audit rows in a stable order.""" + + return query_duckdb( + db_path=db_path, + sql=( + "SELECT event_type, original_name, original_qualified_name, archive_name, run_id " + "FROM main._sqlbuild_janitor_events ORDER BY occurred_at, event_type, archive_name" + ), + ) + + +def read_current_janitor_events(*, db_path: Path, run_id: str) -> list[tuple[object, ...]]: + """Return janitor audit rows excluding one seeded historical run.""" + + return query_duckdb( + db_path=db_path, + sql=( + "SELECT event_type, original_name, original_qualified_name, archive_name, run_id " + f"FROM main._sqlbuild_janitor_events WHERE run_id <> '{run_id}' " + "ORDER BY archive_name" + ), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 701ef7e277..d59b115508 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -51,7 +51,7 @@ expected_exit_code=0, expected_stdout_fragments=( "Janitor is disabled for this project.", - "enabled: true", + "enabled = true", ), ) ], @@ -81,19 +81,20 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( "test_case", [ JanitorCleanupE2ETestCase( - description="tracked-only direct janitor reports stale relations without deleting", + description="tracked-only direct janitor archives only tracked stale relations", build_command=("--no-color", "build", "--full-refresh"), janitor_command=("janitor", "--auto-approve"), expected_exit_code=0, expected_stdout_fragments=( - "reported objects 1", + "relations to archive 1", + "archives to delete 0", "objects skipped 4", - "main.janitor_tracked_extra", + "main.janitor_tracked_extra -> main._sqb_archive__", "main.janitor_untracked_extra relation is not tracked by SQLBuild", "main.partition_state relation matches exclude pattern 'partition_*'", "main._sqlbuild_fingerprints relation matches exclude pattern", "main._sqlbuild_microbatches relation matches exclude pattern", - "direct mode does not delete relations", + "Archived 1 relations.", ), expected_existing_tables=( "orders", @@ -101,14 +102,15 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( "partition_state", "_sqlbuild_fingerprints", "_sqlbuild_microbatches", - "janitor_tracked_extra", + "_sqlbuild_janitor_events", ), - expected_missing_tables=(), + expected_missing_tables=("janitor_tracked_extra",), + expected_archived_original_names=("janitor_tracked_extra",), ) ], ids=lambda case: case.description, ) -def test_given_stale_relations_when_running_direct_janitor_then_it_reports_without_deleting( +def test_given_stale_relations_when_running_direct_janitor_then_it_archives_tracked_relations( test_case: JanitorCleanupE2ETestCase, tmp_path: Path, ) -> None: @@ -150,6 +152,14 @@ def test_given_stale_relations_when_running_direct_janitor_then_it_reports_witho assert table_exists(db_path=db_path, table_name=table_name) for table_name in test_case.expected_missing_tables: assert not table_exists(db_path=db_path, table_name=table_name) + for original_name in test_case.expected_archived_original_names: + assert query_duckdb( + db_path=db_path, + sql=( + "SELECT COUNT(*) FROM information_schema.tables " + f"WHERE table_name LIKE '_sqb_archive__%__{original_name}'" + ), + ) == [(1,)] assert "Eligible expired VDEs" not in janitor_result.stdout assert "Eligible state backups" not in janitor_result.stdout assert "Eligible expired locks" not in janitor_result.stdout @@ -166,7 +176,7 @@ def test_given_stale_relations_when_running_direct_janitor_then_it_reports_witho "Janitor blocked", "Managed target schemas contain active configured sources.", "main active sources: raw_events", - "suppressed deletion: main.stale_main", + "suppressed archive: main.stale_main", "No janitor actions will be performed.", ), expected_existing_relations=( @@ -355,30 +365,30 @@ def test_given_virtual_microbatch_history_when_running_janitor_then_events_remai "test_case", [ JanitorCleanupE2ETestCase( - description="tracked-only direct janitor reports strict scenario artifacts", + description="tracked-only direct janitor archives strict scenario artifacts", build_command=("--no-color", "build", "--full-refresh"), janitor_command=("janitor", "--auto-approve"), expected_exit_code=0, expected_stdout_fragments=( - "reported objects 2", + "relations to archive 2", "objects skipped 2", - "main.__sqb_a13f09c2e7b8__model__daily_revenue", - "main.__sqb_a13f09c2e7b8__source__raw_orders", + "main.__sqb_a13f09c2e7b8__model__daily_revenue -> main._sqb_archive__", + "main.__sqb_a13f09c2e7b8__source__raw_orders -> main._sqb_archive__", "main.__sqb_a13f09c2e7b__model__daily_revenue relation is not tracked by SQLBuild", - "direct mode does not delete relations", ), expected_existing_tables=( "orders", "__sqb_a13f09c2e7b__model__daily_revenue", + ), + expected_missing_tables=( "__sqb_a13f09c2e7b8__source__raw_orders", "__sqb_a13f09c2e7b8__model__daily_revenue", ), - expected_missing_tables=(), ) ], ids=lambda case: case.description, ) -def test_given_scenario_artifacts_when_running_direct_janitor_then_it_reports_without_deleting( +def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_them( test_case: JanitorCleanupE2ETestCase, tmp_path: Path, ) -> None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py new file mode 100644 index 0000000000..193fd3d19a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_archive.py @@ -0,0 +1,535 @@ +"""E2E tests for direct-mode janitor archival and archive expiry.""" + +from __future__ import annotations + +import re +import subprocess +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorArchiveExpiryE2ETestCase, + JanitorArchiveInterruptionE2ETestCase, + JanitorArchiveNameFittingE2ETestCase, + JanitorArchiveRetentionE2ETestCase, + JanitorUnaddressableRelationE2ETestCase, + JanitorZeroRetentionE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( + archive_timestamp_text, + create_direct_state_history, + list_archive_names, + prepare_archive_janitor_project, + read_current_janitor_events, + read_janitor_events, + write_janitor_test_settings, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + query_duckdb, + run_sqb, + table_exists, +) + +ARCHIVE_NAME_PATTERN: re.Pattern[str] = re.compile( + r"^_sqb_archive__(?P[0-9]{8}t[0-9]{6}z)__(?P.+)$" +) +RECENT_ARCHIVE_NAME: str = ( + f"_sqb_archive__{archive_timestamp_text(datetime.now(UTC) - timedelta(days=1))}__recent_orders" +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveRetentionE2ETestCase( + description="retired tracked relation is archived after retention and then retained", + janitor_command=("--no-color", "janitor", "--auto-approve"), + retired_model_ages_days={"customers": 20, "products": 3}, + expected_archived_names=("customers",), + expected_retained_names=("products",), + expected_stdout_fragments=( + "retention 14 days", + "archive retention 14 days", + "relations to archive 1", + "Relations to archive", + "main.customers -> main._sqb_archive__", + "age 20d, delete after", + "main.products relation is newer than 14 days", + "Archived 1 relations.", + ), + expected_second_run_fragments=( + "relations to archive 0", + "archives to delete 0", + "archives retained 1", + "Retained archives", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_retired_tracked_relation_past_retention_when_running_janitor_then_archives_it( + test_case: JanitorArchiveRetentionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_archive_retention", + janitor_config="enabled = true\n", + model_names=("orders", *test_case.retired_model_ages_days), + use_aged_adapter=True, + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + now: datetime = datetime.now(UTC) + write_janitor_test_settings( + project_dir=project_dir, + created_at={ + name: now - timedelta(days=age_days) + for name, age_days in test_case.retired_model_ages_days.items() + }, + ) + for model_name in test_case.retired_model_ages_days: + (project_dir / "models" / f"{model_name}.sql").unlink() + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in janitor_result.stdout + archive_names: tuple[str, ...] = list_archive_names(db_path=db_path) + assert tuple(ARCHIVE_NAME_PATTERN.sub(r"\g", name) for name in archive_names) == ( + test_case.expected_archived_names + ) + assert all(ARCHIVE_NAME_PATTERN.fullmatch(name) for name in archive_names) + for name in test_case.expected_archived_names: + assert not table_exists(db_path=db_path, table_name=name) + for name in test_case.expected_retained_names: + assert table_exists(db_path=db_path, table_name=name) + events: list[tuple[object, ...]] = read_janitor_events(db_path=db_path) + assert [(event[0], event[1], event[2], event[3]) for event in events] == [ + ("archive", name, f"main.{name}", archive_name) + for name, archive_name in zip(test_case.expected_archived_names, archive_names, strict=True) + ] + assert all(str(event[4]) for event in events) + + second_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert second_result.returncode == 0, second_result.stdout + second_result.stderr + for fragment in test_case.expected_second_run_fragments: + assert fragment in second_result.stdout + assert list_archive_names(db_path=db_path) == archive_names + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveExpiryE2ETestCase( + description="only expired strict archives in managed schemas are deleted", + janitor_command=("--no-color", "janitor", "--auto-approve"), + expected_deleted_names=( + "_sqb_archive__20200101t000000z__old_customers_view", + "_sqb_archive__20200101t000000z__old_products", + ), + expected_kept_names=( + RECENT_ARCHIVE_NAME, + "_sqb_archive__20201399t000000z__bad_month", + "_SQB_ARCHIVE_20200101T000000Z__one_separator", + "_sqb_archive_notes", + "_SQB_ARCHIVE__20200101T000000Z__legacy_orders", + ), + expected_kept_other_schema_names=("_sqb_archive__20200101t000000z__orders",), + expected_stdout_fragments=( + "archives to delete 2", + "archives retained 1", + "Archives to delete", + "main._sqb_archive__20200101t000000z__old_products archived 2020-01-01 00:00:00", + "expired 2020-01-15 00:00:00 UTC", + "Retained archives", + f"main.{RECENT_ARCHIVE_NAME} archived", + "main._sqb_archive__20201399t000000z__bad_month name resembles a janitor archive " + "but does not match the strict archive grammar", + "main._SQB_ARCHIVE_20200101T000000Z__one_separator name resembles a janitor archive", + "main._sqb_archive_notes name resembles a janitor archive", + "main._SQB_ARCHIVE__20200101T000000Z__legacy_orders relation name is not a plain " + "lowercase identifier and may require quoting; janitor does not act on it", + "Deleted 2 objects", + ), + expected_delete_event_count=2, + expected_total_event_count=5, + ) + ], + ids=lambda case: case.description, +) +def test_given_archive_named_relations_when_running_janitor_then_deletes_only_expired_strict_ones( + test_case: JanitorArchiveExpiryE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_archive_expiry", + janitor_config="enabled = true\narchive_retention_days = 14\n", + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + execute_duckdb( + db_path=db_path, + sql=( + 'CREATE TABLE main."_sqb_archive__20200101t000000z__old_products" AS SELECT 1 AS id; ' + 'CREATE VIEW main."_sqb_archive__20200101t000000z__old_customers_view" AS ' + "SELECT 1 AS id; " + f'CREATE TABLE main."{RECENT_ARCHIVE_NAME}" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_sqb_archive__20201399t000000z__bad_month" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_SQB_ARCHIVE_20200101T000000Z__one_separator" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_sqb_archive_notes" AS SELECT 1 AS id; ' + 'CREATE TABLE main."_SQB_ARCHIVE__20200101T000000Z__legacy_orders" AS SELECT 1 AS id; ' + "CREATE SCHEMA other; " + 'CREATE TABLE other."_sqb_archive__20200101t000000z__orders" AS SELECT 1 AS id; ' + + DuckDbAdapter().render_create_janitor_event_table_sql(database=None, schema="main") + + "; INSERT INTO main._sqlbuild_janitor_events " + "(event_id, schema_version, event_type, occurred_at, run_id, relation_schema, " + "original_name, archive_name, archive_qualified_name, archived_at) VALUES " + "('misleading_delete', 1, 'delete', TIMESTAMP '2020-02-01 00:00:00', 'run_old', " + "'main', 'old_products', '_sqb_archive__20200101t000000z__old_products', " + "'main._sqb_archive__20200101t000000z__old_products', " + "TIMESTAMP '2020-01-01 00:00:00'), " + "('misleading_archive', 1, 'archive', TIMESTAMP '2020-01-01 00:00:00', 'run_old', " + f"'main', 'recent_orders', '{RECENT_ARCHIVE_NAME}', 'main.{RECENT_ARCHIVE_NAME}', " + "TIMESTAMP '2020-01-01 00:00:00'), " + "('misleading_malformed', 1, 'archive', TIMESTAMP '2020-01-01 00:00:00', 'run_old', " + "'main', 'bad_month', '_sqb_archive__20201399t000000z__bad_month', " + "'main._sqb_archive__20201399t000000z__bad_month', TIMESTAMP '2020-01-01 00:00:00')" + ), + ) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in janitor_result.stdout + for name in test_case.expected_deleted_names: + assert not table_exists(db_path=db_path, table_name=name) + for name in test_case.expected_kept_names: + assert table_exists(db_path=db_path, table_name=name) + for name in test_case.expected_kept_other_schema_names: + assert table_exists(db_path=db_path, schema="other", table_name=name) + events: list[tuple[object, ...]] = read_janitor_events(db_path=db_path) + current_events: list[tuple[object, ...]] = read_current_janitor_events( + db_path=db_path, run_id="run_old" + ) + assert len(events) == test_case.expected_total_event_count + assert len(current_events) == test_case.expected_delete_event_count + assert [(event[0], event[1], event[3]) for event in current_events] == [ + ("delete", None, name) for name in sorted(test_case.expected_deleted_names) + ] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorZeroRetentionE2ETestCase( + description="zero retention archives and deletes in one run with unchanged pruning", + janitor_command=( + "--no-color", + "janitor", + "--auto-approve", + "--direct-state-history-versions", + "2", + ), + stale_tables=("stale_orders",), + stale_views=("stale_customers_view",), + expected_stdout_fragments=( + "retention disabled (0 days)", + "archive retention 0 days", + "relations to archive 2", + "archives to delete 2", + "deleted in this run", + "archived in this run", + "Eligible direct state pruning", + "main._sqlbuild_fingerprints keep latest 2", + "Archived 2 relations. Deleted 2 objects", + ), + expected_event_types=( + ("archive", "stale_customers_view"), + ("archive", "stale_orders"), + ("delete", "stale_customers_view"), + ("delete", "stale_orders"), + ), + expected_fingerprint_count_after=3, + expected_source_freshness_count_after=2, + ) + ], + ids=lambda case: case.description, +) +def test_given_zero_retention_settings_when_running_janitor_then_archives_and_deletes_in_one_run( + test_case: JanitorZeroRetentionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_zero_retention", + janitor_config=( + "enabled = true\n" + "retention_days = 0\n" + "archive_retention_days = 0\n" + "delete_tracked_only = false\n" + ), + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + create_direct_state_history(db_path=db_path) + execute_duckdb( + db_path=db_path, + sql="; ".join( + ( + *(f"CREATE TABLE main.{name} AS SELECT 1 AS id" for name in test_case.stale_tables), + *(f"CREATE VIEW main.{name} AS SELECT 1 AS id" for name in test_case.stale_views), + ) + ), + ) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in janitor_result.stdout + for name in (*test_case.stale_tables, *test_case.stale_views): + assert not table_exists(db_path=db_path, table_name=name) + assert list_archive_names(db_path=db_path) == () + events: list[tuple[object, ...]] = read_janitor_events(db_path=db_path) + assert tuple(sorted((str(event[0]), str(event[1])) for event in events)) == ( + test_case.expected_event_types + ) + assert len({event[4] for event in events}) == 1 + assert query_duckdb( + db_path=db_path, sql="SELECT COUNT(*) FROM main._sqlbuild_fingerprints" + ) == [(test_case.expected_fingerprint_count_after,)] + assert query_duckdb( + db_path=db_path, sql="SELECT COUNT(*) FROM main._sqlbuild_source_freshness" + ) == [(test_case.expected_source_freshness_count_after,)] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveNameFittingE2ETestCase( + description="long relation name is fitted to the adapter limit with timestamp intact", + janitor_command=("--no-color", "janitor", "--auto-approve"), + long_relation_name="customer_order_fulfillment_history_snapshot_backup", + identifier_limit=48, + expected_archive_prefix_length=len("_SQB_ARCHIVE__20260101T000000Z__"), + ) + ], + ids=lambda case: case.description, +) +def test_given_small_identifier_limit_when_archiving_long_name_then_fits_name_and_keeps_timestamp( + test_case: JanitorArchiveNameFittingE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_archive_name_fitting", + janitor_config="enabled = true\nretention_days = 0\ndelete_tracked_only = false\n", + use_aged_adapter=True, + ) + db_path: Path = project_dir / "janitor.duckdb" + write_janitor_test_settings( + project_dir=project_dir, identifier_limit=test_case.identifier_limit + ) + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + execute_duckdb( + db_path=db_path, + sql=f"CREATE TABLE main.{test_case.long_relation_name} AS SELECT 1 AS id", + ) + before: datetime = datetime.now(UTC).replace(microsecond=0) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + after: datetime = datetime.now(UTC) + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + archive_names: tuple[str, ...] = list_archive_names(db_path=db_path) + assert len(archive_names) == 1 + archive_name: str = archive_names[0] + match: re.Match[str] | None = ARCHIVE_NAME_PATTERN.match(archive_name) + assert match is not None + archived_at: datetime = datetime.strptime( + match.group("timestamp").upper(), "%Y%m%dT%H%M%SZ" + ).replace(tzinfo=UTC) + assert before <= archived_at <= after + logical_name: str = match.group("logical_name") + assert len(archive_name) == test_case.identifier_limit + assert len(archive_name) - len(logical_name) == test_case.expected_archive_prefix_length + assert logical_name != test_case.long_relation_name + assert test_case.long_relation_name.startswith(logical_name.rsplit("_", 1)[0]) + events: list[tuple[object, ...]] = read_janitor_events(db_path=db_path) + assert [(event[0], event[1], event[3]) for event in events] == [ + ("archive", test_case.long_relation_name, archive_name) + ] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveInterruptionE2ETestCase( + description="archive without an audit row is still deleted later by name and age", + janitor_command=("--no-color", "janitor", "--auto-approve"), + stale_table="stale_orders", + expected_first_exit_code=1, + expected_first_output_fragments=("simulated janitor event write failure",), + expected_second_exit_code=0, + expected_second_stdout_fragments=( + "archives to delete 1", + "Archives to delete", + "Deleted 1 objects", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_audit_write_interrupted_after_rename_when_janitor_reruns_then_archive_is_deleted( + test_case: JanitorArchiveInterruptionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_archive_interruption", + janitor_config=( + "enabled = true\n" + "retention_days = 0\n" + "archive_retention_days = 14\n" + "delete_tracked_only = false\n" + ), + use_aged_adapter=True, + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + execute_duckdb( + db_path=db_path, sql=f"CREATE TABLE main.{test_case.stale_table} AS SELECT 1 AS id" + ) + write_janitor_test_settings(project_dir=project_dir, fail_janitor_event_insert=True) + + first_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert first_result.returncode == test_case.expected_first_exit_code, ( + first_result.stdout + first_result.stderr + ) + for fragment in test_case.expected_first_output_fragments: + assert fragment in first_result.stdout + first_result.stderr + stranded_archives: tuple[str, ...] = list_archive_names(db_path=db_path) + assert len(stranded_archives) == 1 + assert not table_exists(db_path=db_path, table_name=test_case.stale_table) + assert read_janitor_events(db_path=db_path) == [] + + write_janitor_test_settings(project_dir=project_dir) + project_file: Path = project_dir / "sqlbuild_project.toml" + project_file.write_text( + project_file.read_text(encoding="utf-8").replace( + "archive_retention_days = 14", "archive_retention_days = 0" + ), + encoding="utf-8", + ) + second_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert second_result.returncode == test_case.expected_second_exit_code, ( + second_result.stdout + second_result.stderr + ) + for fragment in test_case.expected_second_stdout_fragments: + assert fragment in second_result.stdout + assert list_archive_names(db_path=db_path) == () + assert [(event[0], event[1], event[3]) for event in read_janitor_events(db_path=db_path)] == [ + ("delete", None, stranded_archives[0]) + ] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorUnaddressableRelationE2ETestCase( + description="quoted mixed-case relation is reported and left while plain one archives", + janitor_command=("--no-color", "janitor", "--auto-approve"), + plain_relation="old_orders", + quoted_relation="LegacyCustomers", + expected_stdout_fragments=( + "relations to archive 1", + "main.old_orders -> main._sqb_archive__", + "main.LegacyCustomers relation name is not a plain lowercase identifier and may " + "require quoting; janitor does not act on it", + "Archived 1 relations.", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_quoted_mixed_case_relation_when_running_janitor_then_it_is_reported_and_untouched( + test_case: JanitorUnaddressableRelationE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_archive_janitor_project( + tmp_path=tmp_path, + project_name="janitor_unaddressable_relation", + janitor_config="enabled = true\nretention_days = 0\ndelete_tracked_only = false\n", + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + execute_duckdb( + db_path=db_path, + sql=( + f"CREATE TABLE main.{test_case.plain_relation} AS SELECT 1 AS id; " + f'CREATE TABLE main."{test_case.quoted_relation}" AS SELECT 2 AS id' + ), + ) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, project_dir=project_dir + ) + + assert janitor_result.returncode == 0, janitor_result.stdout + janitor_result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in janitor_result.stdout + assert not table_exists(db_path=db_path, table_name=test_case.plain_relation) + assert table_exists(db_path=db_path, table_name=test_case.quoted_relation) + assert query_duckdb( + db_path=db_path, sql=f'SELECT id FROM main."{test_case.quoted_relation}"' + ) == [(2,)] + archive_names: tuple[str, ...] = list_archive_names(db_path=db_path) + assert tuple(ARCHIVE_NAME_PATTERN.sub(r"\g", name) for name in archive_names) == ( + test_case.plain_relation, + ) + assert [(event[0], event[1]) for event in read_janitor_events(db_path=db_path)] == [ + ("archive", test_case.plain_relation) + ] diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/__init__.py b/tests/integration/src/sqlbuild/executor/janitor_events/__init__.py new file mode 100644 index 0000000000..3db13f5218 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py new file mode 100644 index 0000000000..65db89d85b --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/main/__init__.py @@ -0,0 +1 @@ +"""Janitor audit event write integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py new file mode 100644 index 0000000000..bf162f5406 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/main/_test_types.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class JanitorEventWriteTestCase: + description: str + write_attempts: int + expected_row_count: int + expected_write_results: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py b/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py new file mode 100644 index 0000000000..c1ff5d333d --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/janitor_events/main/test_write_janitor_event.py @@ -0,0 +1,103 @@ +"""DuckDB integration tests for append-only janitor audit events.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.janitor_events.constants import ( + JANITOR_EVENT_COLUMNS, + JANITOR_EVENT_SCHEMA_VERSION, +) +from sqlbuild.executor.janitor_events.main.janitor_event_id import build_janitor_event_id +from sqlbuild.executor.janitor_events.main.write_janitor_event import write_janitor_event_record +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType +from tests.integration.src.sqlbuild.executor.janitor_events.main._test_types import ( + JanitorEventWriteTestCase, +) + +ARCHIVE_NAME: str = "_SQB_ARCHIVE__20260924T101500Z__orders" + + +@pytest.mark.parametrize( + "test_case", + ( + JanitorEventWriteTestCase( + description="repeated write of one deterministic event appends a single row", + write_attempts=2, + expected_row_count=1, + expected_write_results=(True, False), + ), + ), + ids=lambda case: case.description, +) +def test_given_repeated_janitor_event_when_writing_then_insert_is_idempotent( + test_case: JanitorEventWriteTestCase, + tmp_path: Path, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": str(tmp_path / "janitor_events.duckdb")}) + archived_at: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) + record: JanitorEventRecord = JanitorEventRecord( + event_id=build_janitor_event_id( + event_type=JanitorEventType.ARCHIVE, + run_id="run_orders_cleanup", + relation_database=None, + relation_schema="main", + archive_name=ARCHIVE_NAME, + ), + schema_version=JANITOR_EVENT_SCHEMA_VERSION, + event_type=JanitorEventType.ARCHIVE, + occurred_at=archived_at, + run_id="run_orders_cleanup", + relation_database=None, + relation_schema="main", + relation_type="BASE TABLE", + original_name="orders", + original_qualified_name="main.orders", + archive_name=ARCHIVE_NAME, + archive_qualified_name=f"main.{ARCHIVE_NAME}", + archived_at=archived_at, + ) + try: + adapter.execute( + connection=connection, + sql=adapter.render_create_janitor_event_table_sql(database=None, schema="main"), + ) + write_results: tuple[bool, ...] = tuple( + write_janitor_event_record( + connection=connection, + execute=adapter.execute, + record=record, + render_qualified_name=adapter.render_qualified_name, + ) + for _ in range(test_case.write_attempts) + ) + rows: list[tuple[object, ...]] = connection.execute( + f"SELECT {', '.join(JANITOR_EVENT_COLUMNS)} FROM main._sqlbuild_janitor_events" + ).fetchall() + finally: + adapter.close(connection) + + assert write_results == test_case.expected_write_results + assert len(rows) == test_case.expected_row_count + assert rows[0] == ( + record.event_id, + JANITOR_EVENT_SCHEMA_VERSION, + "archive", + archived_at.replace(tzinfo=None), + "run_orders_cleanup", + None, + "main", + "BASE TABLE", + "orders", + "main.orders", + ARCHIVE_NAME, + f"main.{ARCHIVE_NAME}", + archived_at.replace(tzinfo=None), + ) diff --git a/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/_test_types.py b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/_test_types.py new file mode 100644 index 0000000000..f14ad41378 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/_test_types.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class FitArtifactLogicalNameTestCase: + description: str + logical_name: str + fixed_prefix: str + identifier_limit: int + expected_name: str + + +@dataclass(frozen=True) +class FitArtifactLogicalNameErrorTestCase: + description: str + logical_name: str + fixed_prefix: str + identifier_limit: int + expected_error_fragment: str diff --git a/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py new file mode 100644 index 0000000000..ffc2743ab3 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/relations/main/fit_artifact_logical_name/test_fit_artifact_logical_name.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import hashlib + +import pytest + +from sqlbuild.adapter.relations.main.fit_artifact_logical_name import fit_artifact_logical_name +from sqlbuild.errors.contracts.exceptions import SharedInputError +from tests.unit.src.sqlbuild.adapter.relations.main.fit_artifact_logical_name._test_types import ( + FitArtifactLogicalNameErrorTestCase, + FitArtifactLogicalNameTestCase, +) + +LONG_NAME: str = "customer_orders_" + "x" * 40 +LONG_NAME_HASH: str = hashlib.sha256(LONG_NAME.encode("utf-8")).hexdigest()[:8] + + +@pytest.mark.parametrize( + "test_case", + [ + FitArtifactLogicalNameTestCase( + description="short name is kept unchanged", + logical_name="orders", + fixed_prefix="__prefix__", + identifier_limit=63, + expected_name="orders", + ), + FitArtifactLogicalNameTestCase( + description="long name is shortened with a deterministic hash suffix", + logical_name=LONG_NAME, + fixed_prefix="__prefix__", + identifier_limit=40, + expected_name=f"{LONG_NAME[:21]}_{LONG_NAME_HASH}", + ), + ], + ids=lambda case: case.description, +) +def test_given_logical_name_when_fitting_artifact_name_then_returns_expected_name( + test_case: FitArtifactLogicalNameTestCase, +) -> None: + fitted: str = fit_artifact_logical_name( + logical_name=test_case.logical_name, + fixed_prefix=test_case.fixed_prefix, + identifier_limit=test_case.identifier_limit, + artifact_label="Test artifact", + ) + + assert fitted == test_case.expected_name + assert len(test_case.fixed_prefix + fitted) <= test_case.identifier_limit + + +@pytest.mark.parametrize( + "test_case", + [ + FitArtifactLogicalNameErrorTestCase( + description="prefix longer than the limit is rejected", + logical_name="orders", + fixed_prefix="__a_prefix_that_is_too_long__", + identifier_limit=10, + expected_error_fragment="does not fit within identifier limit 10", + ), + FitArtifactLogicalNameErrorTestCase( + description="room too small for a hash suffix is rejected", + logical_name=LONG_NAME, + fixed_prefix="__prefix__", + identifier_limit=15, + expected_error_fragment="cannot fit within identifier limit 15", + ), + ], + ids=lambda case: case.description, +) +def test_given_unfittable_name_when_fitting_artifact_name_then_raises_input_error( + test_case: FitArtifactLogicalNameErrorTestCase, +) -> None: + with pytest.raises(SharedInputError) as exc_info: + fit_artifact_logical_name( + logical_name=test_case.logical_name, + fixed_prefix=test_case.fixed_prefix, + identifier_limit=test_case.identifier_limit, + artifact_label="Test artifact", + ) + + assert test_case.expected_error_fragment in str(exc_info.value) diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 18726d0de0..39275d97d8 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -69,3 +69,10 @@ class RowDiffSampleSqlTestCase: description: str adapter: BaseAdapter expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class AdapterRelationAgeMetadataCapabilityTestCase: + description: str + adapter: StrictAdapter + expected_supported: bool diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index 2003efebbb..5ed09e990c 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -202,3 +202,11 @@ class BigQueryTableFreshnessWildcardTestCase: description: str table_name: str expected_error_fragment: str + + +@dataclass(frozen=True) +class BigQueryRelationAgeMetadataTestCase: + description: str + metadata_rows: tuple[tuple[object, ...], ...] + expected_timestamps: tuple[tuple[str, str], ...] + relation_names: tuple[str, ...] = ("old_orders", "old_orders_view") diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py index 0bac9a15f3..dd9561133e 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime, timedelta, timezone import pytest @@ -10,6 +10,7 @@ CursorValue, ExpressionInferenceProfile, QueryResult, + RelationInfo, RowDiffColumnResult, RowDiffResult, RowDiffSampleCell, @@ -43,6 +44,7 @@ BigQueryMergeExclusionTestCase, BigQueryPruneSqlTestCase, BigQueryQueryTestCase, + BigQueryRelationAgeMetadataTestCase, BigQueryRenderCloneTestCase, BigQueryRenderCursorBoundLiteralTestCase, BigQueryRenderDeleteInsertTestCase, @@ -1104,3 +1106,84 @@ def test_given_timestamp_cursor_when_counting_rows_then_bigquery_uses_typed_filt assert result == test_case.expected_count assert executed_sql == [test_case.expected_sql] + + +@pytest.mark.parametrize( + "test_case", + [ + BigQueryRelationAgeMetadataTestCase( + description="dataset metadata supplies UTC creation and last-modified times", + metadata_rows=( + ( + "old_orders", + datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + datetime(2026, 8, 2, 10, 30, tzinfo=UTC), + ), + ("old_orders_view", datetime(2026, 8, 3, 4, 0), None), + ), + expected_timestamps=( + ("2026-08-01 10:00:00+00:00", "2026-08-02 10:30:00+00:00"), + ("2026-08-03 04:00:00+00:00", "None"), + ), + ), + BigQueryRelationAgeMetadataTestCase( + description="relations missing from dataset metadata stay unknown", + metadata_rows=(), + expected_timestamps=(("None", "None"), ("None", "None")), + ), + BigQueryRelationAgeMetadataTestCase( + description="case-distinct tables keep their own timestamps", + metadata_rows=( + ( + "Orders", + datetime(2025, 1, 1, tzinfo=UTC), + datetime(2025, 1, 2, tzinfo=UTC), + ), + ( + "orders", + datetime(2026, 8, 1, tzinfo=UTC), + datetime(2026, 8, 2, tzinfo=UTC), + ), + ), + expected_timestamps=( + ("2026-08-01 00:00:00+00:00", "2026-08-02 00:00:00+00:00"), + ("2025-01-01 00:00:00+00:00", "2025-01-02 00:00:00+00:00"), + ("None", "None"), + ), + relation_names=("orders", "Orders", "ORDERS"), + ), + ], + ids=lambda case: case.description, +) +def test_given_listed_relations_when_adding_age_metadata_then_bigquery_reads_each_dataset_once( + test_case: BigQueryRelationAgeMetadataTestCase, +) -> None: + client: FakeBigQueryClient = FakeBigQueryClient( + rows=FakeBigQueryRows( + columns=("table_id", "creation_time", "last_modified_time"), + rows=test_case.metadata_rows, + ) + ) + relations: tuple[RelationInfo, ...] = tuple( + RelationInfo( + database="example-project", + schema="dev_orders", + name=name, + relation_type="table", + ) + for name in test_case.relation_names + ) + + enriched: tuple[RelationInfo, ...] = BigQueryAdapter().with_relation_age_metadata( + connection=_BigQueryConnection(client=client, location=None), + relations=relations, + ) + + assert ( + tuple((str(relation.created_at), str(relation.last_altered_at)) for relation in enriched) + == test_case.expected_timestamps + ) + assert tuple(sql for sql, _ in client.queries) == ( + "SELECT table_id, TIMESTAMP_MILLIS(creation_time), " + "TIMESTAMP_MILLIS(last_modified_time) FROM `example-project.dev_orders.__TABLES__`", + ) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index f16b3f2dd6..747801d996 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -1,7 +1,9 @@ from dataclasses import dataclass from datetime import datetime +from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.lineage.types import InferredNullability +from tests.unit.src.sqlbuild.adapters.databricks.helpers import FakeDatabricksMetadataCursor @dataclass(frozen=True) @@ -127,3 +129,22 @@ class DatabricksTableFreshnessFallbackTestCase: description: str expected_data_versions: tuple[datetime, ...] expected_query_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DatabricksRelationAgeMetadataTestCase: + description: str + table_type: str + created: datetime | None + last_altered: datetime | None + expected_relation_type: str + expected_timestamps: tuple[str, str] + + +@dataclass(frozen=True) +class DatabricksDeltaDetailAgeTestCase: + description: str + relation: RelationInfo + cursors: tuple[FakeDatabricksMetadataCursor, ...] + expected_sql: tuple[str, ...] + expected_timestamps: tuple[str, str] diff --git a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py index 66addaffd7..2f5ddf88fb 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py @@ -20,9 +20,14 @@ def build_retention_request(*, desired_days: int) -> RetentionRequest: class FakeDatabricksMetadataCursor: - def __init__(self, *, rows: list[tuple[object, ...]] | None = None) -> None: + def __init__( + self, + *, + rows: list[tuple[object, ...]] | None = None, + description: tuple[tuple[str], ...] = (("format",), ("properties",)), + ) -> None: self.rows: list[tuple[object, ...]] = rows or [] - self.description: tuple[tuple[str], ...] = (("format",), ("properties",)) + self.description: tuple[tuple[str], ...] = description self.executed_sql: str | None = None self.closed: bool = False diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index a1765420ab..2b624a4b33 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -1,11 +1,13 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime, timedelta, timezone +from typing import Any, cast import pytest from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, + RelationInfo, TableFreshnessMetadata, TableFreshnessRequest, ) @@ -18,10 +20,12 @@ from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.adapters.databricks._test_types import ( + DatabricksDeltaDetailAgeTestCase, DatabricksExpressionInferenceProfileTestCase, DatabricksMergeExclusionTestCase, DatabricksPruneSqlTestCase, DatabricksPythonFunctionSupportTestCase, + DatabricksRelationAgeMetadataTestCase, DatabricksRenderCloneTestCase, DatabricksRenderDeleteInsertCursorTestCase, DatabricksRenderDurableCloneTestCase, @@ -575,3 +579,180 @@ def test_given_table_function_when_rendering_then_databricks_returns_expected_dd ) assert statements == test_case.expected_statements + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksRelationAgeMetadataTestCase( + description="managed table ignores DDL-only information_schema timestamps", + table_type="MANAGED", + created=datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + last_altered=datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), + expected_relation_type="table", + expected_timestamps=("None", "None"), + ), + DatabricksRelationAgeMetadataTestCase( + description="view offset timestamps are normalized to UTC", + table_type="VIEW", + created=datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + last_altered=datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), + expected_relation_type="view", + expected_timestamps=("2026-08-01 10:00:00+00:00", "2026-08-02 10:30:00+00:00"), + ), + DatabricksRelationAgeMetadataTestCase( + description="view naive timestamps are treated as UTC", + table_type="VIEW", + created=datetime(2026, 8, 1, 3, 0), + last_altered=datetime(2026, 8, 3, 4, 0), + expected_relation_type="view", + expected_timestamps=("2026-08-01 03:00:00+00:00", "2026-08-03 04:00:00+00:00"), + ), + DatabricksRelationAgeMetadataTestCase( + description="missing view timestamps stay unknown", + table_type="VIEW", + created=None, + last_altered=None, + expected_relation_type="view", + expected_timestamps=("None", "None"), + ), + ], + ids=lambda case: case.description, +) +def test_given_databricks_relation_timestamps_when_listing_then_exposes_view_ages_only( + test_case: DatabricksRelationAgeMetadataTestCase, +) -> None: + cursor: FakeDatabricksMetadataCursor = FakeDatabricksMetadataCursor( + rows=[ + ( + "OLD_ORDERS", + "DEV_ORDERS", + test_case.table_type, + test_case.created, + test_case.last_altered, + ) + ] + ) + adapter: DatabricksAdapter = DatabricksAdapter() + + relations: tuple[RelationInfo, ...] = adapter.list_relations( + connection=cast(Any, FakeDatabricksMetadataConnection((cursor,))), + database="analytics", + schemas=("dev_orders",), + ) + + assert "table_type, created, last_altered" in str(cursor.executed_sql) + assert relations[0].name == "old_orders" + assert relations[0].relation_type == test_case.expected_relation_type + assert ( + str(relations[0].created_at), + str(relations[0].last_altered_at), + ) == test_case.expected_timestamps + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksDeltaDetailAgeTestCase( + description="table age comes from Delta detail lastModified in UTC", + relation=RelationInfo( + database="analytics", schema="dev_orders", name="old_orders", relation_type="table" + ), + cursors=( + FakeDatabricksMetadataCursor( + rows=[ + ( + "delta", + datetime(2025, 1, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), + ) + ], + description=(("format",), ("createdAt",), ("lastModified",)), + ), + ), + expected_sql=("DESCRIBE DETAIL `analytics`.`dev_orders`.`old_orders`",), + expected_timestamps=("2025-01-01 10:00:00+00:00", "2026-08-02 10:30:00+00:00"), + ), + DatabricksDeltaDetailAgeTestCase( + description="backticks in table identifiers are escaped", + relation=RelationInfo( + database="analytics", schema="dev_orders", name="odd`orders", relation_type="table" + ), + cursors=( + FakeDatabricksMetadataCursor( + rows=[("delta", None, datetime(2026, 8, 2, 10, 30))], + description=(("format",), ("createdAt",), ("lastModified",)), + ), + ), + expected_sql=("DESCRIBE DETAIL `analytics`.`dev_orders`.`odd``orders`",), + expected_timestamps=("None", "2026-08-02 10:30:00+00:00"), + ), + DatabricksDeltaDetailAgeTestCase( + description="views keep information_schema ages without Delta detail", + relation=RelationInfo( + database="analytics", + schema="dev_orders", + name="old_orders_view", + relation_type="view", + created_at=datetime(2026, 8, 1, tzinfo=UTC), + last_altered_at=datetime(2026, 8, 3, tzinfo=UTC), + ), + cursors=(), + expected_sql=(), + expected_timestamps=("2026-08-01 00:00:00+00:00", "2026-08-03 00:00:00+00:00"), + ), + DatabricksDeltaDetailAgeTestCase( + description="failed Delta detail leaves table age unknown", + relation=RelationInfo( + database="analytics", + schema="dev_orders", + name="old_orders", + relation_type="table", + created_at=datetime(2025, 1, 1, tzinfo=UTC), + ), + cursors=( + FailingDatabricksMetadataCursor(execute_error=RuntimeError("not a Delta table")), + ), + expected_sql=("DESCRIBE DETAIL `analytics`.`dev_orders`.`old_orders`",), + expected_timestamps=("None", "None"), + ), + DatabricksDeltaDetailAgeTestCase( + description="Delta detail without lastModified leaves table age unknown", + relation=RelationInfo( + database="analytics", schema="dev_orders", name="old_orders", relation_type="table" + ), + cursors=( + FakeDatabricksMetadataCursor( + rows=[("delta", datetime(2025, 1, 1, tzinfo=UTC))], + description=(("format",), ("createdAt",)), + ), + ), + expected_sql=("DESCRIBE DETAIL `analytics`.`dev_orders`.`old_orders`",), + expected_timestamps=("None", "None"), + ), + ], + ids=lambda case: case.description, +) +def test_given_candidate_relations_when_adding_age_metadata_then_databricks_reads_delta_detail( + test_case: DatabricksDeltaDetailAgeTestCase, +) -> None: + connection: FakeDatabricksMetadataConnection = FakeDatabricksMetadataConnection( + test_case.cursors + ) + + enriched: tuple[RelationInfo, ...] = DatabricksAdapter().with_relation_age_metadata( + connection=cast(Any, connection), + relations=(test_case.relation,), + ) + + assert ( + tuple(cursor.executed_sql for cursor in connection.returned_cursors) + == test_case.expected_sql + ) + assert tuple(cursor.closed for cursor in connection.returned_cursors) == tuple( + True for _ in test_case.expected_sql + ) + assert ( + str(enriched[0].created_at), + str(enriched[0].last_altered_at), + ) == test_case.expected_timestamps diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 103623818c..48189e6048 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -194,3 +194,13 @@ class SnowflakeBatchedRetentionErrorTestCase: requests: tuple[tuple[str, str], ...] rows: list[tuple[object, ...]] expected_error: str + + +@dataclass(frozen=True) +class SnowflakeRelationAgeMetadataTestCase: + description: str + relation_type: str + created: datetime | None + last_altered: datetime | None + expected_created_at: datetime | None + expected_last_altered_at: datetime | None diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 54bcbf4ef5..d4b7cd9a7e 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta, timezone from pathlib import Path from typing import Any, cast from unittest.mock import patch @@ -36,6 +36,7 @@ SnowflakePruneSqlTestCase, SnowflakeQualifiedColumnInspectionTestCase, SnowflakeQueryColumnNamesTestCase, + SnowflakeRelationAgeMetadataTestCase, SnowflakeRenderCloneTestCase, SnowflakeRenderCursorBoundLiteralTestCase, SnowflakeRenderIdentifierTestCase, @@ -1200,3 +1201,66 @@ def test_given_default_seed_csv_settings_when_loading_seed_then_uses_python_csv_ ) assert cursor.executemany_rows == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeRelationAgeMetadataTestCase( + description="local-offset table timestamps are normalized to UTC", + relation_type="BASE TABLE", + created=datetime(2026, 8, 1, 3, 0, tzinfo=timezone(timedelta(hours=-7))), + last_altered=datetime(2026, 8, 2, 12, 30, tzinfo=timezone(timedelta(hours=2))), + expected_created_at=datetime(2026, 8, 1, 10, 0, tzinfo=UTC), + expected_last_altered_at=datetime(2026, 8, 2, 10, 30, tzinfo=UTC), + ), + SnowflakeRelationAgeMetadataTestCase( + description="view timestamps are read and naive values are treated as UTC", + relation_type="VIEW", + created=datetime(2026, 8, 1, 3, 0), + last_altered=datetime(2026, 8, 3, 4, 0), + expected_created_at=datetime(2026, 8, 1, 3, 0, tzinfo=UTC), + expected_last_altered_at=datetime(2026, 8, 3, 4, 0, tzinfo=UTC), + ), + SnowflakeRelationAgeMetadataTestCase( + description="missing timestamps stay unknown", + relation_type="VIEW", + created=None, + last_altered=None, + expected_created_at=None, + expected_last_altered_at=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_relation_timestamps_when_listing_then_exposes_utc_age_metadata( + test_case: SnowflakeRelationAgeMetadataTestCase, +) -> None: + cursor: FakeSnowflakeMetadataCursor = FakeSnowflakeMetadataCursor( + rows=[ + ( + "OLD_ORDERS", + "DEV_ORDERS", + test_case.relation_type, + "NO", + test_case.created, + test_case.last_altered, + None, + ) + ] + ) + adapter: SnowflakeAdapter = SnowflakeAdapter() + + relations: tuple[Any, ...] = adapter.list_relations( + connection=cast(Any, FakeSnowflakeMetadataConnection(cursor)), + database="ANALYTICS", + schemas=("DEV_ORDERS",), + ) + + assert adapter.supports_relation_age_metadata() is True + assert relations[0].created_at == test_case.expected_created_at + assert relations[0].last_altered_at == test_case.expected_last_altered_at + assert (str(relations[0].created_at), str(relations[0].last_altered_at)) == ( + str(test_case.expected_created_at), + str(test_case.expected_last_altered_at), + ) diff --git a/tests/unit/src/sqlbuild/adapters/test_relation_age_metadata_capability.py b/tests/unit/src/sqlbuild/adapters/test_relation_age_metadata_capability.py new file mode 100644 index 0000000000..4a7932ef72 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/test_relation_age_metadata_capability.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from tests.unit.src.sqlbuild.adapters._test_types import ( + AdapterRelationAgeMetadataCapabilityTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + ( + AdapterRelationAgeMetadataCapabilityTestCase( + description="Snowflake reads created and last_altered", + adapter=SnowflakeAdapter(), + expected_supported=True, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="Databricks reads created and last_altered", + adapter=DatabricksAdapter(), + expected_supported=True, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="BigQuery reads creation and last-modified times", + adapter=BigQueryAdapter(), + expected_supported=True, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="SQL Server modify_date ignores data writes", + adapter=SqlServerAdapter(), + expected_supported=False, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="Postgres catalog has no relation timestamps", + adapter=PostgresAdapter(), + expected_supported=False, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="DuckDB catalog has no relation timestamps", + adapter=DuckDbAdapter(), + expected_supported=False, + ), + AdapterRelationAgeMetadataCapabilityTestCase( + description="MotherDuck catalog has no relation timestamps", + adapter=MotherDuckAdapter(), + expected_supported=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_adapter_when_checking_relation_age_metadata_then_matches_catalog_reliability( + test_case: AdapterRelationAgeMetadataCapabilityTestCase, +) -> None: + assert test_case.adapter.supports_relation_age_metadata() is test_case.expected_supported diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py index dfde23266d..e87261302d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -23,3 +23,20 @@ class JanitorPlanOutputTestCase: use_color: bool expected_output_fragments: tuple[str, ...] unexpected_output_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class JanitorDirectArchiveConfirmationTestCase: + description: str + typed_response: str + expected_result: bool + expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorSettingsResolutionTestCase: + description: str + project_toml: str + cli_retention_days: int | None + expected_retention_days: int + expected_archive_retention_days: int diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py index aa9bbd0754..058912dca2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -1,9 +1,11 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime, timedelta from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.executor.janitor.models import ( + JanitorArchiveCandidate, + JanitorArchivedRelation, JanitorBlockedSchema, JanitorCheckpointCandidate, JanitorDeleteCandidate, @@ -109,3 +111,53 @@ def build_janitor_plan() -> JanitorPlan: scanned_schema_count=2, age_metadata_supported=True, ) + + +def build_direct_archive_plan() -> JanitorPlan: + archived_at: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) + relation: RelationInfo = RelationInfo( + database=None, + schema="dev", + name="old_orders", + relation_type="table", + ) + original_key: JanitorRelationKey = JanitorRelationKey( + database=None, schema="dev", name="old_orders" + ) + return JanitorPlan( + target_name="dev", + retention_days=14, + direct_mode=True, + archive_retention_days=14, + candidates=( + JanitorDeleteCandidate(key=original_key, relation=relation, age_timestamp=None), + ), + archive_candidates=( + JanitorArchiveCandidate( + key=original_key, + relation=relation, + age_timestamp=archived_at - timedelta(days=20), + archive_key=JanitorRelationKey( + database=None, + schema="dev", + name="_sqb_archive__20260924t101500z__old_orders", + ), + archived_at=archived_at, + expires_at=archived_at + timedelta(days=14), + ), + ), + archive_deletion_candidates=( + JanitorArchivedRelation( + key=JanitorRelationKey( + database=None, + schema="dev", + name="_sqb_archive__20260101t000000z__old_products", + ), + relation_type="table", + archived_at=datetime(2026, 1, 1, tzinfo=UTC), + expires_at=datetime(2026, 1, 15, tzinfo=UTC), + ), + ), + age_metadata_supported=True, + planned_at=archived_at, + ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index a86674f25a..259b2a7962 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -18,10 +18,14 @@ ) from tests.unit.src.sqlbuild.cli.commands.main.janitor._test_types import ( JanitorConfirmationInterruptTestCase, + JanitorDirectArchiveConfirmationTestCase, JanitorDisabledOutputTestCase, JanitorPlanOutputTestCase, ) -from tests.unit.src.sqlbuild.cli.commands.main.janitor.helpers import build_janitor_plan +from tests.unit.src.sqlbuild.cli.commands.main.janitor.helpers import ( + build_direct_archive_plan, + build_janitor_plan, +) @pytest.mark.parametrize( @@ -84,11 +88,10 @@ def interrupting_input() -> str: use_color=False, expected_output="Janitor is disabled for this project.\n\n" "Janitor is opt-in. It previews stale warehouse objects and asks before " - "deleting anything.\n\n" + "archiving or deleting anything.\n\n" "Add this block to sqlbuild_project.toml:\n\n" - "janitor:\n" - " enabled: true\n" - " retention_days: 30\n\n" + "[janitor]\n" + "enabled = true\n\n" "After enabling, run janitor again to preview cleanup:\n" " sqb janitor\n", ) @@ -127,7 +130,7 @@ def test_given_disabled_janitor_when_writing_with_color_then_styles_title( assert stream.getvalue().startswith(test_case.expected_prefix) assert ( "\033[33mJanitor is opt-in. It previews stale warehouse objects and asks before " - "deleting anything.\033[0m" + "archiving or deleting anything.\033[0m" ) in stream.getvalue() assert "\033[34msqb janitor\033[0m" in stream.getvalue() @@ -206,3 +209,80 @@ def test_given_janitor_plan_when_writing_with_color_then_uses_semantic_colors( output: str = stream.getvalue() for fragment in test_case.expected_output_fragments: assert fragment in output + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorDirectArchiveConfirmationTestCase( + description="direct archive plan requires typed archive and delete counts", + typed_response="archive 1 and delete 1 objects from dev", + expected_result=True, + expected_output_fragments=( + "Janitor will archive 1 and delete 1 objects from dev.", + "Retention: 14 days", + "Archive retention: 14 days", + "Type `archive 1 and delete 1 objects from dev` to continue: ", + ), + ), + JanitorDirectArchiveConfirmationTestCase( + description="direct archive plan rejects the virtual-mode phrase", + typed_response="delete 1 objects from dev", + expected_result=False, + expected_output_fragments=("Janitor will archive 1 and delete 1 objects from dev.",), + ), + ], + ids=lambda case: case.description, +) +def test_given_direct_archive_plan_when_confirming_then_requires_exact_archive_phrase( + test_case: JanitorDirectArchiveConfirmationTestCase, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(builtins, "input", lambda: test_case.typed_response) + + result: bool = confirm_janitor_plan( + planning_result=JanitorPlanningResult(plan=build_direct_archive_plan()) + ) + + output: str = capsys.readouterr().out + assert result is test_case.expected_result + for fragment in test_case.expected_output_fragments: + assert fragment in output + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorPlanOutputTestCase( + description="direct preview lists exact archive names, ages, and expiry", + use_color=False, + expected_output_fragments=( + " retention 14 days", + " archive retention 14 days", + " relations to archive 1", + " archives to delete 1", + " archives retained 0", + "Relations to archive\n dev.old_orders -> " + "dev._sqb_archive__20260924t101500z__old_orders " + "age 20d, delete after 2026-10-08 10:15:00 UTC", + "Archives to delete\n dev._sqb_archive__20260101t000000z__old_products " + "archived 2026-01-01 00:00:00 UTC, age 266d, expired 2026-01-15 00:00:00 UTC", + ), + unexpected_output_fragments=("eligible for deletion", "Eligible objects"), + ) + ], + ids=lambda case: case.description, +) +def test_given_direct_archive_plan_when_writing_preview_then_lists_archive_actions( + test_case: JanitorPlanOutputTestCase, +) -> None: + stream: StringIO = StringIO() + + write_plan(plan=build_direct_archive_plan(), stream=stream, use_color=test_case.use_color) + + output: str = stream.getvalue() + for fragment in test_case.expected_output_fragments: + assert fragment in output + for fragment in test_case.unexpected_output_fragments: + assert fragment not in output diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py new file mode 100644 index 0000000000..a97d044d23 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py @@ -0,0 +1,98 @@ +"""Tests for janitor settings resolution from discovered project config.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.commands._helpers.janitor_runtime.invocation import ( + resolve_janitor_invocation, + resolve_janitor_settings, +) +from sqlbuild.cli.commands.models import ( + JanitorCommandRequest, + JanitorInvocation, + JanitorSettings, +) +from tests.unit.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorSettingsResolutionTestCase, +) + +DIRECT_PROJECT_TOML: str = ( + 'name = "orders_direct"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +VIRTUAL_PROJECT_TOML: str = ( + 'name = "orders_virtual"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[settings]\n" + "virtual_environments = true\n\n" + "[connection]\n" + 'database = "warehouse.duckdb"\n\n' + "[targets.dev]\n" + 'schema = "dev"\n\n' + "[targets.dev.state]\n" + 'backend = "duckdb"\n' + 'schema = "sqlbuild_state"\n\n' + "[targets.dev.state.connection]\n" + 'database = "state.duckdb"\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorSettingsResolutionTestCase( + description="direct mode without explicit retention resolves 14", + project_toml=DIRECT_PROJECT_TOML + "\n[janitor]\nenabled = true\n", + cli_retention_days=None, + expected_retention_days=14, + expected_archive_retention_days=14, + ), + JanitorSettingsResolutionTestCase( + description="virtual mode without explicit retention resolves 30", + project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\n", + cli_retention_days=None, + expected_retention_days=30, + expected_archive_retention_days=14, + ), + JanitorSettingsResolutionTestCase( + description="explicit retention applies to direct mode", + project_toml=DIRECT_PROJECT_TOML + + "\n[janitor]\nenabled = true\nretention_days = 9\narchive_retention_days = 2\n", + cli_retention_days=None, + expected_retention_days=9, + expected_archive_retention_days=2, + ), + JanitorSettingsResolutionTestCase( + description="explicit retention applies to virtual mode", + project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\nretention_days = 9\n", + cli_retention_days=None, + expected_retention_days=9, + expected_archive_retention_days=14, + ), + JanitorSettingsResolutionTestCase( + description="cli retention override wins in virtual mode", + project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\nretention_days = 9\n", + cli_retention_days=4, + expected_retention_days=4, + expected_archive_retention_days=14, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_config_when_resolving_janitor_settings_then_uses_mode_specific_default( + test_case: JanitorSettingsResolutionTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_toml, encoding="utf-8") + request: JanitorCommandRequest = JanitorCommandRequest( + project_dir=tmp_path, no_color=True, retention_days=test_case.cli_retention_days + ) + + invocation: JanitorInvocation = resolve_janitor_invocation(request=request) + settings: JanitorSettings = resolve_janitor_settings(request=request, invocation=invocation) + + assert settings.retention_days == test_case.expected_retention_days + assert settings.archive_retention_days == test_case.expected_archive_retention_days diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index db48cfebc5..ef91b1d8cd 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -262,7 +262,7 @@ class LoadProjectConfigTestCase: expected_vars: dict[str, str] expected_targets: dict[str, dict[str, object]] expected_janitor_enabled: bool - expected_retention_days: int + expected_retention_days: int | None expected_janitor_max_checkpoints: int expected_janitor_delete_tracked_only: bool expected_janitor_exclude_patterns: tuple[str, ...] @@ -271,6 +271,7 @@ class LoadProjectConfigTestCase: expected_function_database: str | None = None expected_function_schema: str | None = None expected_janitor_direct_state_history_versions: int = 20 + expected_janitor_archive_retention_days: int = 14 expected_current_state_full_refresh: str = "deny" expected_historical_full_refresh: str = "require_confirmation" expected_snapshot_schema_change: str = "append_new_columns" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 11adb31dd0..8205c0b06c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -667,7 +667,7 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise } }, expected_janitor_enabled=False, - expected_retention_days=30, + expected_retention_days=None, expected_janitor_max_checkpoints=20, expected_janitor_delete_tracked_only=True, expected_janitor_exclude_patterns=(), @@ -735,7 +735,8 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise [janitor] enabled = true -retention_days = 14 +retention_days = 21 +archive_retention_days = 3 max_checkpoints = 3 delete_tracked_only = false exclude_patterns = ["partition_*"] @@ -814,7 +815,8 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise } }, expected_janitor_enabled=True, - expected_retention_days=14, + expected_retention_days=21, + expected_janitor_archive_retention_days=3, expected_janitor_max_checkpoints=3, expected_janitor_delete_tracked_only=False, expected_janitor_exclude_patterns=("partition_*",), @@ -891,6 +893,9 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e } == test_case.expected_targets assert config.janitor.enabled is test_case.expected_janitor_enabled assert config.janitor.retention_days == test_case.expected_retention_days + assert ( + config.janitor.archive_retention_days == test_case.expected_janitor_archive_retention_days + ) assert config.janitor.max_checkpoints == test_case.expected_janitor_max_checkpoints assert config.janitor.delete_tracked_only is test_case.expected_janitor_delete_tracked_only assert config.janitor.exclude_patterns == test_case.expected_janitor_exclude_patterns @@ -1669,6 +1674,17 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed( """.strip(), expected_error_fragment="janitor.direct_state_history_versions must be >= 0", ), + LoadProjectConfigErrorTestCase( + description="raises when janitor archive retention days is negative", + project_file_contents=""" +name = "demo" +adapter = "duckdb" + +[janitor] +archive_retention_days = -1 +""".strip(), + expected_error_fragment="janitor.archive_retention_days must be >= 0", + ), LoadProjectConfigErrorTestCase( description="raises when project settings contain unknown key", project_file_contents=""" diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/__init__.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/__init__.py new file mode 100644 index 0000000000..7acab39e3f --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/__init__.py @@ -0,0 +1 @@ +"""Janitor helper unit tests.""" diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py new file mode 100644 index 0000000000..6a11a18d0d --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class ArchiveNameParseTestCase: + description: str + name: str + expected_archived_at: datetime + expected_logical_name: str + + +@dataclass(frozen=True) +class ArchiveNameRejectTestCase: + description: str + name: str + expected_lookalike: bool + + +@dataclass(frozen=True) +class ArchiveNameBuildTestCase: + description: str + original_name: str + archived_at: datetime + identifier_limit: int + expected_prefix: str + expected_length: int + expected_logical_name_is_original: bool + + +@dataclass(frozen=True) +class RelationAddressingTestCase: + description: str + name: str + listing: tuple[str, ...] + expected_reason: str | None diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py new file mode 100644 index 0000000000..464e7bd297 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.executor.janitor._helpers.archive_names import ( + build_archive_name, + is_archive_lookalike_name, + parse_archive_name, +) +from sqlbuild.executor.janitor.models import JanitorParsedArchiveName +from tests.unit.src.sqlbuild.executor.janitor._helpers._test_types import ( + ArchiveNameBuildTestCase, + ArchiveNameParseTestCase, + ArchiveNameRejectTestCase, +) + +ARCHIVED_AT: datetime = datetime(2026, 9, 24, 10, 15, 0, tzinfo=UTC) +LONG_NAME: str = "fulfillment_orders_" + "y" * 80 + + +@pytest.mark.parametrize( + "test_case", + [ + ArchiveNameParseTestCase( + description="canonical archive name parses", + name="_SQB_ARCHIVE__20260924T101500Z__orders", + expected_archived_at=ARCHIVED_AT, + expected_logical_name="orders", + ), + ArchiveNameParseTestCase( + description="lowercase folded archive name parses", + name="_sqb_archive__20260924t101500z__orders", + expected_archived_at=ARCHIVED_AT, + expected_logical_name="orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_strict_archive_name_when_parsing_then_returns_timestamp_and_logical_name( + test_case: ArchiveNameParseTestCase, +) -> None: + parsed: JanitorParsedArchiveName | None = parse_archive_name(test_case.name) + + assert parsed == JanitorParsedArchiveName( + archived_at=test_case.expected_archived_at, + logical_name=test_case.expected_logical_name, + ) + assert is_archive_lookalike_name(test_case.name) is True + + +@pytest.mark.parametrize( + "test_case", + [ + ArchiveNameRejectTestCase( + description="impossible calendar timestamp is malformed", + name="_SQB_ARCHIVE__20261324T101500Z__orders", + expected_lookalike=True, + ), + ArchiveNameRejectTestCase( + description="missing timezone marker is malformed", + name="_SQB_ARCHIVE__20260924T101500__orders", + expected_lookalike=True, + ), + ArchiveNameRejectTestCase( + description="single separator after prefix is malformed", + name="_SQB_ARCHIVE_20260924T101500Z__orders", + expected_lookalike=True, + ), + ArchiveNameRejectTestCase( + description="empty original name is malformed", + name="_SQB_ARCHIVE__20260924T101500Z__", + expected_lookalike=True, + ), + ArchiveNameRejectTestCase( + description="ordinary relation is neither archive nor look-alike", + name="orders_archive", + expected_lookalike=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_non_strict_name_when_parsing_archive_name_then_returns_none( + test_case: ArchiveNameRejectTestCase, +) -> None: + parsed: JanitorParsedArchiveName | None = parse_archive_name(test_case.name) + + assert parsed is None + assert is_archive_lookalike_name(test_case.name) is test_case.expected_lookalike + + +@pytest.mark.parametrize( + "test_case", + [ + ArchiveNameBuildTestCase( + description="short name keeps the full original name", + original_name="orders", + archived_at=ARCHIVED_AT, + identifier_limit=255, + expected_prefix="_sqb_archive__20260924t101500z__orders", + expected_length=len("_sqb_archive__20260924t101500z__orders"), + expected_logical_name_is_original=True, + ), + ArchiveNameBuildTestCase( + description="long name is fitted while the timestamp stays intact", + original_name=LONG_NAME, + archived_at=ARCHIVED_AT.replace(microsecond=654321), + identifier_limit=63, + expected_prefix="_sqb_archive__20260924t101500z__fulfillment_orders_", + expected_length=63, + expected_logical_name_is_original=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_original_name_when_building_archive_name_then_round_trips_through_parser( + test_case: ArchiveNameBuildTestCase, +) -> None: + archive_name: str = build_archive_name( + original_name=test_case.original_name, + archived_at=test_case.archived_at, + identifier_limit=test_case.identifier_limit, + ) + + parsed: JanitorParsedArchiveName | None = parse_archive_name(archive_name) + assert archive_name.startswith(test_case.expected_prefix) + assert len(archive_name) == test_case.expected_length + assert parsed is not None + assert parsed.archived_at == ARCHIVED_AT + assert (parsed.logical_name == test_case.original_name) is ( + test_case.expected_logical_name_is_original + ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py new file mode 100644 index 0000000000..f5bc3a9feb --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_relation_addressing.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.executor.janitor._helpers.relation_addressing import ( + case_colliding_names, + unaddressable_relation_reason, +) +from sqlbuild.executor.janitor.constants import ( + CASE_COLLISION_REASON, + UNQUOTED_ADDRESSING_REASON, +) +from tests.unit.src.sqlbuild.executor.janitor._helpers._test_types import ( + RelationAddressingTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RelationAddressingTestCase( + description="plain lowercase name is addressable", + name="old_orders", + listing=("old_orders", "customers"), + expected_reason=None, + ), + RelationAddressingTestCase( + description="lowercase name with digits dollar and leading underscore is addressable", + name="_orders$2026", + listing=("_orders$2026",), + expected_reason=None, + ), + RelationAddressingTestCase( + description="uppercase name is skipped", + name="OLD_ORDERS", + listing=("OLD_ORDERS",), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + RelationAddressingTestCase( + description="mixed-case name is skipped", + name="OldOrders", + listing=("OldOrders",), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + RelationAddressingTestCase( + description="name with punctuation is skipped", + name="old-orders;drop", + listing=("old-orders;drop",), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + RelationAddressingTestCase( + description="name with a leading digit is skipped", + name="2026_orders", + listing=("2026_orders",), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + RelationAddressingTestCase( + description="lowercase name colliding case-insensitively is skipped", + name="old_orders", + listing=("old_orders", "OLD_ORDERS"), + expected_reason=CASE_COLLISION_REASON, + ), + RelationAddressingTestCase( + description="uppercase side of a case collision is skipped", + name="OLD_ORDERS", + listing=("old_orders", "OLD_ORDERS"), + expected_reason=UNQUOTED_ADDRESSING_REASON, + ), + ], + ids=lambda case: case.description, +) +def test_given_catalog_name_when_checking_addressing_then_returns_expected_reason( + test_case: RelationAddressingTestCase, +) -> None: + reason: str | None = unaddressable_relation_reason( + name=test_case.name, colliding_names=case_colliding_names(test_case.listing) + ) + + assert reason == test_case.expected_reason diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 3209ab37ae..68a1d40b84 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -72,3 +72,113 @@ def relation_info( created_at=created_at, last_altered_at=last_altered_at, ) + + +@dataclass(frozen=True) +class JanitorArchivePlanTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + direct_mode: bool = True + retention_days: int = 7 + archive_retention_days: int = 14 + source_schema: str | None = None + exclude_patterns: tuple[str, ...] = field(default_factory=tuple) + identifier_limit: int = 255 + expected_archive_source_names: tuple[str, ...] = field(default_factory=tuple) + expected_existing_archive_deletion_names: tuple[str, ...] = field(default_factory=tuple) + expected_same_run_deletion_source_names: tuple[str, ...] = field(default_factory=tuple) + expected_retained_archive_names: tuple[str, ...] = field(default_factory=tuple) + expected_skipped_relations: tuple[tuple[str, str], ...] = field(default_factory=tuple) + expected_suppressed_archive_deletion_names: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorArchiveExecutionTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + archive_retention_days: int + expected_renamed_origins: tuple[str, ...] + expected_view_rename_origins: tuple[str, ...] + expected_dropped_targets: tuple[str, ...] + expected_dropped_view_targets: tuple[str, ...] + expected_event_insert_count: int + expected_event_table_create_count: int + + +@dataclass(frozen=True) +class JanitorArchiveEventFailureTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + expected_error_fragment: str + expected_renamed_origins: tuple[str, ...] + expected_dropped_targets: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorAddressingPlanTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + direct_mode: bool + expected_archive_source_names: tuple[str, ...] = field(default_factory=tuple) + expected_archive_deletion_names: tuple[str, ...] = field(default_factory=tuple) + expected_skipped_relations: tuple[tuple[str, str], ...] = field(default_factory=tuple) + expected_candidate_names: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorRetentionResolutionTestCase: + description: str + override: int | None + configured: int | None + virtual_environments: bool + expected_retention_days: int + + +@dataclass(frozen=True) +class JanitorFoldedIdentifierPlanTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + direct_mode: bool = True + delete_tracked_only: bool = False + tracked_relations: tuple[tuple[str | None, str | None, str], ...] = field(default_factory=tuple) + source_database: str | None = None + source_schema: str | None = None + protected_relation_keys: frozenset[JanitorRelationKey] = frozenset() + expected_candidate_display_names: tuple[str, ...] = field(default_factory=tuple) + expected_archive_display_names: tuple[str, ...] = field(default_factory=tuple) + expected_archive_deletion_display_names: tuple[str, ...] = field(default_factory=tuple) + expected_skipped_relations: tuple[tuple[str, str], ...] = field(default_factory=tuple) + expected_blocked_sources: tuple[str, ...] = field(default_factory=tuple) + expected_suppressed_names: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class JanitorSeparateAgeMetadataPlanTestCase: + description: str + relation_ages: dict[str, datetime] + expected_candidate_names: tuple[str, ...] + expected_skipped_relations: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class JanitorAgeReadScopeTestCase: + description: str + relation_infos: tuple[RelationInfo, ...] + direct_mode: bool + delete_tracked_only: bool + tracked_relations: tuple[tuple[str | None, str | None, str], ...] + exclude_patterns: tuple[str, ...] + protected_relation_keys: frozenset[JanitorRelationKey] + expected_age_requests: tuple[tuple[str, ...], ...] + retention_days: int = 7 + + +@dataclass(frozen=True) +class JanitorCaseSafetyTestCase: + description: str + relation_names: tuple[str, ...] + direct_mode: bool + tracked_names: tuple[str, ...] + exclude_patterns: tuple[str, ...] + expected_skipped_relations: tuple[tuple[str, str], ...] + expected_dropped_targets: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 125b93c2bb..9b142958f4 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -4,6 +4,8 @@ import base64 from collections.abc import Callable +from dataclasses import replace +from datetime import datetime from pathlib import Path from typing import Any, ClassVar, cast @@ -30,6 +32,7 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS from sqlbuild.spec.contracts.models import SchemaSeedEntry, SeedCsvSettings, SourceEntry @@ -43,10 +46,14 @@ def __init__( relation_infos: tuple[RelationInfo, ...], supports_age_metadata: bool = True, tracked_relations: tuple[tuple[str | None, str | None, str], ...] = (), + identifier_limit: int = 255, ) -> None: self.relation_infos: tuple[RelationInfo, ...] = relation_infos self.age_metadata_supported: bool = supports_age_metadata + self.identifier_limit: int = identifier_limit self.dropped_targets: list[str] = [] + self.dropped_view_targets: list[str] = [] + self.renamed_targets: list[tuple[str, str]] = [] self.executed_sql: list[str] = [] self.tracked_relations: tuple[tuple[str | None, str | None, str], ...] = tracked_relations self._tracked_rows: tuple[tuple[Any, ...], ...] = tuple( @@ -101,6 +108,9 @@ def __init__( def supports_relation_age_metadata(self) -> bool: return self.age_metadata_supported + def maximum_identifier_length(self) -> int: + return self.identifier_limit + def default_schema(self) -> str | None: return "analytics" @@ -126,7 +136,9 @@ def _execute(self, connection: Any, sql: str) -> Any: True: (), False: self._tracked_rows, } - return _FakeResult(rows=rows_by_query_kind["LIMIT 0" in sql]) + return _FakeResult( + rows=rows_by_query_kind["LIMIT 0" in sql or JANITOR_EVENTS_TABLE_NAME in sql] + ) def list_relations( self, @@ -213,6 +225,17 @@ def drop( del connection, if_exists, statement_recorder self.dropped_targets.append(destination) + def drop_view( + self, + connection: Any, + *, + destination: str, + if_exists: bool = True, + statement_recorder: StatementRecorder, + ) -> None: + del connection, if_exists, statement_recorder + self.dropped_view_targets.append(destination) + def create_table_as( self, connection: Any, @@ -336,7 +359,8 @@ def rename( destination: str, statement_recorder: StatementRecorder, ) -> None: - raise NotImplementedError + del connection, statement_recorder + self.renamed_targets.append((origin, destination)) def swap( self, @@ -438,6 +462,92 @@ def drop( raise RuntimeError(self.message) +class FoldingJanitorAdapter(FakeJanitorAdapter): + """Warehouse double that filters case-insensitively and lists lowercase identifiers.""" + + def list_relations( + self, + connection: Any, + *, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[RelationInfo, ...]: + requested_schemas: frozenset[str] = frozenset(schema.lower() for schema in schemas or ()) + requested_names: frozenset[str] = frozenset(name.lower() for name in names or ()) + matching: tuple[RelationInfo, ...] = tuple( + filter( + lambda relation: ( + (relation.database or "").lower() == (database or "").lower() + and ( + not requested_schemas + or (relation.schema or "").lower() in requested_schemas + ) + and (not requested_names or relation.name.lower() in requested_names) + ), + self._available_relations, + ) + ) + return tuple( + replace( + relation, + database=database, + schema=(relation.schema or "").lower(), + name=relation.name.lower(), + ) + for relation in matching + ) + + +class SeparateAgeMetadataJanitorAdapter(FakeJanitorAdapter): + """Warehouse double that lists without timestamps and supplies ages separately.""" + + def __init__( + self, + *, + relation_infos: tuple[RelationInfo, ...], + relation_ages: dict[str, datetime], + tracked_relations: tuple[tuple[str | None, str | None, str], ...] = (), + ) -> None: + super().__init__(relation_infos=relation_infos, tracked_relations=tracked_relations) + self.relation_ages: dict[str, datetime] = relation_ages + self.age_metadata_requests: list[tuple[str, ...]] = [] + + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + self.age_metadata_requests.append(tuple(relation.name for relation in relations)) + return tuple( + replace(relation, last_altered_at=self.relation_ages.get(relation.name)) + for relation in relations + ) + + +class FailingJanitorEventAdapter(FakeJanitorAdapter): + def _execute(self, connection: Any, sql: str) -> Any: + is_event_insert: bool = sql.startswith("INSERT INTO") and JANITOR_EVENTS_TABLE_NAME in sql + _EVENT_WRITE_GUARDS[is_event_insert](sql) + return super()._execute(connection=connection, sql=sql) + + +def _allow_statement(sql: str) -> None: + del sql + + +def _fail_event_write(sql: str) -> None: + del sql + raise RuntimeError("simulated janitor event write failure") + + +_EVENT_WRITE_GUARDS: dict[bool, Callable[[str], None]] = { + False: _allow_statement, + True: _fail_event_write, +} + + class _FakeResult: def __init__(self, *, rows: tuple[tuple[Any, ...], ...]) -> None: self.rows: tuple[tuple[Any, ...], ...] = rows @@ -446,7 +556,13 @@ def fetchall(self) -> list[tuple[Any, ...]]: return list(self.rows) -def build_project(*, source_schema: str | None = None) -> CompiledProject: +def build_project( + *, + source_schema: str | None = None, + source_database: str | None = None, + destination_database: str | None = None, + destination_schema: str = "analytics", +) -> CompiledProject: source: CompiledSource = CompiledSource( key=CompiledObjectKey( resource_type=CompiledResourceType.SOURCE, @@ -454,7 +570,12 @@ def build_project(*, source_schema: str | None = None) -> CompiledProject: ), deps=(), name="raw_orders", - source_entry=SourceEntry(name="raw_orders", schema=source_schema or "", table="orders"), + source_entry=SourceEntry( + name="raw_orders", + database=source_database, + schema=source_schema or "", + table="orders", + ), source_file=cast(Any, object()), ) sources: tuple[CompiledSource, ...] = {True: (), False: (source,)}[source_schema is None] @@ -472,10 +593,10 @@ def build_project(*, source_schema: str | None = None) -> CompiledProject: query_sql="select 1", config=CompileModelConfig(), destination=CompiledRelationLocation( - database=None, - schema="analytics", + database=destination_database, + schema=destination_schema, name="orders", - qualified_name="analytics.orders", + qualified_name=f"{destination_schema}.orders", ), ), ), @@ -488,10 +609,10 @@ def build_project(*, source_schema: str | None = None) -> CompiledProject: schema_entry=SchemaSeedEntry(name="countries"), schema_file=cast(Any, object()), destination=CompiledRelationLocation( - database=None, - schema="analytics", + database=destination_database, + schema=destination_schema, name="countries", - qualified_name="analytics.countries", + qualified_name=f"{destination_schema}.countries", ), ), ), diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py index 160c2bebb4..394a338b22 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py @@ -2,23 +2,37 @@ from __future__ import annotations +from datetime import UTC, datetime, timedelta + import pytest from sqlbuild.executor.janitor.main.execute import execute_janitor_plan +from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import ( JanitorDeleteCandidate, JanitorDetachedVirtualEnvironmentCandidate, + JanitorDirectModeSettings, + JanitorExecutionResult, JanitorPlan, JanitorRelationKey, ) +from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorArchiveEventFailureTestCase, + JanitorArchiveExecutionTestCase, JanitorExecutionOrderTestCase, + relation_info, ) from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( FailingDropAdapter, + FailingJanitorEventAdapter, + FakeJanitorAdapter, + build_project, relation_info_for_test, ) +OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) + @pytest.mark.parametrize( "test_case", @@ -73,3 +87,138 @@ def test_given_physical_drop_failure_when_executing_janitor_then_state_cleanup_i assert test_case.expected_error_fragment in str(exc_info.value) assert tuple(deleted_state_items) == test_case.expected_deleted_state_items assert adapter.dropped_targets == ["dev__sqb_physical.orders__v_old"] + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveExecutionTestCase( + description="renames stale table and view, drops expired archive, and audits each", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("old_customers_view", relation_type="VIEW", created_at=OLD_TIME), + relation_info("_sqb_archive__20200101t000000z__old_products"), + ), + archive_retention_days=14, + expected_renamed_origins=("analytics.old_orders",), + expected_view_rename_origins=("analytics.old_customers_view",), + expected_dropped_targets=("analytics._sqb_archive__20200101t000000z__old_products",), + expected_dropped_view_targets=(), + expected_event_insert_count=3, + expected_event_table_create_count=1, + ), + JanitorArchiveExecutionTestCase( + description="zero archive retention archives then drops in the same run", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("old_customers_view", relation_type="VIEW", created_at=OLD_TIME), + ), + archive_retention_days=0, + expected_renamed_origins=("analytics.old_orders",), + expected_view_rename_origins=("analytics.old_customers_view",), + expected_dropped_targets=("analytics._sqb_archive__",), + expected_dropped_view_targets=("analytics._sqb_archive__",), + expected_event_insert_count=4, + expected_event_table_create_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_direct_archive_plan_when_executing_then_renames_drops_and_writes_audit_events( + test_case: JanitorArchiveExecutionTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter(relation_infos=test_case.relation_infos) + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings( + enabled=True, archive_retention_days=test_case.archive_retention_days + ), + ) + + result: JanitorExecutionResult = execute_janitor_plan( + plan=plan, adapter=adapter, connection=object() + ) + + assert tuple(origin for origin, _ in adapter.renamed_targets) == ( + test_case.expected_renamed_origins + ) + view_rename_statements: tuple[str, ...] = tuple( + filter(lambda sql: sql.startswith("ALTER VIEW "), adapter.executed_sql) + ) + view_renames: tuple[str, ...] = tuple( + sql.removeprefix("ALTER VIEW ").split(" RENAME TO ")[0] for sql in view_rename_statements + ) + assert view_renames == test_case.expected_view_rename_origins + assert len(adapter.dropped_targets) == len(test_case.expected_dropped_targets) + for dropped, expected_prefix in zip( + adapter.dropped_targets, test_case.expected_dropped_targets, strict=True + ): + assert dropped.startswith(expected_prefix) + assert len(adapter.dropped_view_targets) == len(test_case.expected_dropped_view_targets) + for dropped, expected_prefix in zip( + adapter.dropped_view_targets, test_case.expected_dropped_view_targets, strict=True + ): + assert dropped.startswith(expected_prefix) + event_statements: tuple[str, ...] = tuple( + filter(lambda sql: JANITOR_EVENTS_TABLE_NAME in sql, adapter.executed_sql) + ) + event_inserts: tuple[str, ...] = tuple( + filter(lambda sql: sql.startswith("INSERT INTO"), event_statements) + ) + event_creates: tuple[str, ...] = tuple( + filter(lambda sql: sql.startswith("CREATE TABLE"), event_statements) + ) + assert len(event_inserts) == test_case.expected_event_insert_count + assert len(event_creates) == test_case.expected_event_table_create_count + assert tuple(candidate.key.name for candidate in result.archived) == tuple( + origin.rsplit(".", maxsplit=1)[-1] + for origin in (*test_case.expected_renamed_origins, *test_case.expected_view_rename_origins) + ) + assert len(result.deleted_archives) == len(test_case.expected_dropped_targets) + len( + test_case.expected_dropped_view_targets + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchiveEventFailureTestCase( + description="audit write failure after rename stops before further actions", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("_sqb_archive__20200101t000000z__old_products"), + ), + expected_error_fragment="simulated janitor event write failure", + expected_renamed_origins=("analytics.old_orders",), + expected_dropped_targets=(), + ) + ], + ids=lambda case: case.description, +) +def test_given_audit_write_failure_when_archiving_then_rename_persists_and_error_propagates( + test_case: JanitorArchiveEventFailureTestCase, +) -> None: + adapter: FailingJanitorEventAdapter = FailingJanitorEventAdapter( + relation_infos=test_case.relation_infos + ) + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings(enabled=True), + ) + + with pytest.raises(RuntimeError) as exc_info: + execute_janitor_plan(plan=plan, adapter=adapter, connection=object()) + + assert test_case.expected_error_fragment in str(exc_info.value) + assert tuple(origin for origin, _ in adapter.renamed_targets) == ( + test_case.expected_renamed_origins + ) + assert tuple(adapter.dropped_targets) == test_case.expected_dropped_targets diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index acb1fc712f..cc77f5b520 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -280,7 +280,7 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp expected_dropped_targets=("analytics.old_orders",), ), JanitorExecuteTestCase( - description="direct mode reports eligible candidates without dropping", + description="direct mode archives eligible candidates instead of dropping them", relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), expected_dropped_targets=(), direct_mode=True, diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py new file mode 100644 index 0000000000..0a5c041f8d --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py @@ -0,0 +1,301 @@ +"""Tests for direct-mode janitor archive planning.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from itertools import chain + +import pytest + +from sqlbuild.executor.janitor.constants import ( + CASE_COLLISION_REASON, + MALFORMED_ARCHIVE_REASON, + UNQUOTED_ADDRESSING_REASON, +) +from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild.executor.janitor.models import ( + JanitorArchivedRelation, + JanitorDirectModeSettings, + JanitorPlan, + JanitorRelationKey, + JanitorSkippedRelation, +) +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorAddressingPlanTestCase, + JanitorArchivePlanTestCase, + relation_info, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FakeJanitorAdapter, + build_project, +) + +OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) +RECENT_ARCHIVE_NAME: str = ( + f"_SQB_ARCHIVE__{(datetime.now(UTC) - timedelta(days=1)):%Y%m%dT%H%M%SZ}__old_customers" +).lower() +LONG_ORDERS_NAME: str = "orders_" + "x" * 60 + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchivePlanTestCase( + description="stale relation is archived and kept until archive retention", + relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), + expected_archive_source_names=("old_orders",), + ), + JanitorArchivePlanTestCase( + description="zero archive retention deletes the new archive in the same run", + relation_infos=(relation_info("old_orders"),), + retention_days=0, + archive_retention_days=0, + expected_archive_source_names=("old_orders",), + expected_same_run_deletion_source_names=("old_orders",), + ), + JanitorArchivePlanTestCase( + description="expired archive is deleted and recent archive is retained", + relation_infos=( + relation_info("_sqb_archive__20200101t000000z__old_products"), + relation_info(RECENT_ARCHIVE_NAME), + ), + expected_existing_archive_deletion_names=( + "_sqb_archive__20200101t000000z__old_products", + ), + expected_retained_archive_names=(RECENT_ARCHIVE_NAME,), + ), + JanitorArchivePlanTestCase( + description="case-folded archive name is recognized by strict grammar", + relation_infos=(relation_info("_sqb_archive__20200101t000000z__old_products"),), + expected_existing_archive_deletion_names=( + "_sqb_archive__20200101t000000z__old_products", + ), + ), + JanitorArchivePlanTestCase( + description="malformed archive look-alikes are reported and never archived or deleted", + relation_infos=( + relation_info("_sqb_archive__20201399t000000z__orders", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE_20200101T000000Z__orders", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE__2020010T000000Z__orders", created_at=OLD_TIME), + relation_info("_SQB_ARCHIVE__20200101T000000Z__", created_at=OLD_TIME), + relation_info("_sqb_archive_notes", created_at=OLD_TIME), + ), + retention_days=0, + archive_retention_days=0, + expected_skipped_relations=( + ("_sqb_archive__20201399t000000z__orders", MALFORMED_ARCHIVE_REASON), + ("_SQB_ARCHIVE_20200101T000000Z__orders", MALFORMED_ARCHIVE_REASON), + ("_SQB_ARCHIVE__2020010T000000Z__orders", MALFORMED_ARCHIVE_REASON), + ("_SQB_ARCHIVE__20200101T000000Z__", MALFORMED_ARCHIVE_REASON), + ("_sqb_archive_notes", MALFORMED_ARCHIVE_REASON), + ), + ), + JanitorArchivePlanTestCase( + description="archive matching an exclude pattern is skipped", + relation_infos=(relation_info("_sqb_archive__20200101t000000z__keep_orders"),), + exclude_patterns=("*keep_*",), + expected_skipped_relations=( + ( + "_sqb_archive__20200101t000000z__keep_orders", + "relation matches exclude pattern '*keep_*'", + ), + ), + ), + JanitorArchivePlanTestCase( + description="blocked schema suppresses expired archive deletion", + relation_infos=(relation_info("_sqb_archive__20200101t000000z__old_products"),), + source_schema="analytics", + expected_suppressed_archive_deletion_names=( + "_sqb_archive__20200101t000000z__old_products", + ), + ), + JanitorArchivePlanTestCase( + description="virtual mode neither archives nor expires archive-named relations", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("_sqb_archive__20200101t000000z__old_products", created_at=OLD_TIME), + ), + direct_mode=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_direct_warehouse_when_planning_janitor_then_plans_archives_by_strict_name( + test_case: JanitorArchivePlanTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter( + relation_infos=test_case.relation_infos, + identifier_limit=test_case.identifier_limit, + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(source_schema=test_case.source_schema), + adapter=adapter, + connection=object(), + retention_days=test_case.retention_days, + delete_tracked_only=False, + exclude_patterns=test_case.exclude_patterns, + direct_settings=JanitorDirectModeSettings( + enabled=test_case.direct_mode, + archive_retention_days=test_case.archive_retention_days, + ), + ) + + assert ( + tuple(candidate.key.name for candidate in plan.archive_candidates) + == test_case.expected_archive_source_names + ) + existing_deletions: tuple[JanitorArchivedRelation, ...] = tuple( + filter(lambda archive: archive.original_key is None, plan.archive_deletion_candidates) + ) + same_run_original_keys: tuple[JanitorRelationKey, ...] = tuple( + filter(None, (archive.original_key for archive in plan.archive_deletion_candidates)) + ) + assert ( + tuple(archive.key.name for archive in existing_deletions) + == test_case.expected_existing_archive_deletion_names + ) + assert ( + tuple(key.name for key in same_run_original_keys) + == test_case.expected_same_run_deletion_source_names + ) + assert ( + tuple(archive.key.name for archive in plan.retained_archives) + == test_case.expected_retained_archive_names + ) + archive_skipped: tuple[JanitorSkippedRelation, ...] = tuple( + filter( + lambda skipped: skipped.key.name.lower().startswith("_sqb_archive"), + plan.skipped_relations, + ) + ) + assert ( + tuple((skipped.key.name, skipped.reason) for skipped in archive_skipped) + == test_case.expected_skipped_relations + ) + suppressed: tuple[JanitorArchivedRelation, ...] = tuple( + chain.from_iterable( + blocked.suppressed_archive_deletions for blocked in plan.blocked_schemas + ) + ) + assert ( + tuple(archive.key.name for archive in suppressed) + == test_case.expected_suppressed_archive_deletion_names + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorArchivePlanTestCase( + description="long relation name is fitted with the archive timestamp intact", + relation_infos=(relation_info(LONG_ORDERS_NAME, created_at=OLD_TIME),), + identifier_limit=48, + expected_archive_source_names=(LONG_ORDERS_NAME,), + ), + ], + ids=lambda case: case.description, +) +def test_given_long_relation_name_when_planning_archive_then_fits_name_with_timestamp_intact( + test_case: JanitorArchivePlanTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter( + relation_infos=test_case.relation_infos, + identifier_limit=test_case.identifier_limit, + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=test_case.retention_days, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings(enabled=True), + ) + + assert ( + tuple(candidate.key.name for candidate in plan.archive_candidates) + == test_case.expected_archive_source_names + ) + archive_name: str = plan.archive_candidates[0].archive_key.name + expected_prefix: str = ( + f"_SQB_ARCHIVE__{plan.archive_candidates[0].archived_at:%Y%m%dT%H%M%SZ}__orders_" + ).lower() + assert len(archive_name) == test_case.identifier_limit + assert archive_name.startswith(expected_prefix) + assert archive_name != f"{expected_prefix[:-7]}{LONG_ORDERS_NAME}" + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorAddressingPlanTestCase( + description="unaddressable stale relations and archives are skipped, plain ones act", + relation_infos=( + relation_info("old_orders", created_at=OLD_TIME), + relation_info("LegacyCustomers", created_at=OLD_TIME), + relation_info("old-products", created_at=OLD_TIME), + relation_info("stale_items", created_at=OLD_TIME), + relation_info("STALE_ITEMS", created_at=OLD_TIME), + relation_info("_sqb_archive__20200101t000000z__old_products"), + relation_info("_SQB_ARCHIVE__20200101T000000Z__old_customers"), + relation_info("_sqb_archive__20200101t000000z__old_invoices"), + relation_info("_SQB_ARCHIVE__20200101T000000Z__OLD_INVOICES"), + ), + direct_mode=True, + expected_candidate_names=("old_orders",), + expected_archive_source_names=("old_orders",), + expected_archive_deletion_names=("_sqb_archive__20200101t000000z__old_products",), + expected_skipped_relations=( + ("LegacyCustomers", UNQUOTED_ADDRESSING_REASON), + ("old-products", UNQUOTED_ADDRESSING_REASON), + ("stale_items", CASE_COLLISION_REASON), + ("STALE_ITEMS", UNQUOTED_ADDRESSING_REASON), + ("_SQB_ARCHIVE__20200101T000000Z__old_customers", UNQUOTED_ADDRESSING_REASON), + ("_sqb_archive__20200101t000000z__old_invoices", CASE_COLLISION_REASON), + ("_SQB_ARCHIVE__20200101T000000Z__OLD_INVOICES", UNQUOTED_ADDRESSING_REASON), + ), + ), + JanitorAddressingPlanTestCase( + description="virtual mode keeps deleting mixed-case relations as before", + relation_infos=(relation_info("LegacyCustomers", created_at=OLD_TIME),), + direct_mode=False, + expected_candidate_names=("LegacyCustomers",), + ), + ], + ids=lambda case: case.description, +) +def test_given_unaddressable_names_when_planning_direct_janitor_then_they_are_skipped( + test_case: JanitorAddressingPlanTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter(relation_infos=test_case.relation_infos) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + ) + + assert ( + tuple(candidate.key.name for candidate in plan.archive_candidates) + == test_case.expected_archive_source_names + ) + assert ( + tuple(archive.key.name for archive in plan.archive_deletion_candidates) + == test_case.expected_archive_deletion_names + ) + assert ( + tuple((skipped.key.name, skipped.reason) for skipped in plan.skipped_relations) + == test_case.expected_skipped_relations + ) + assert ( + tuple(candidate.key.name for candidate in plan.candidates) + == test_case.expected_candidate_names + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py new file mode 100644 index 0000000000..f12ffb05f1 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py @@ -0,0 +1,111 @@ +"""Janitor safety for relation names that differ only by case.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.executor.janitor.constants import CASE_COLLISION_REASON +from sqlbuild.executor.janitor.main.execute import execute_janitor_plan +from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorCaseSafetyTestCase, + relation_info, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FakeJanitorAdapter, + build_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorCaseSafetyTestCase( + description="virtual mode never drops an excluded table through a quoted twin", + relation_names=("orders__v0", "Orders__v0"), + direct_mode=False, + tracked_names=("orders__v0",), + exclude_patterns=("orders__v0",), + expected_skipped_relations=( + ("analytics.orders__v0", "relation matches exclude pattern 'orders__v0'"), + ("analytics.Orders__v0", "relation matches exclude pattern 'orders__v0'"), + ), + expected_dropped_targets=(), + ), + JanitorCaseSafetyTestCase( + description="virtual mode skips relations whose folded names collide", + relation_names=("orders__v0", "Orders__v0"), + direct_mode=False, + tracked_names=("orders__v0",), + exclude_patterns=(), + expected_skipped_relations=( + ("analytics.orders__v0", CASE_COLLISION_REASON), + ("analytics.Orders__v0", CASE_COLLISION_REASON), + ), + expected_dropped_targets=(), + ), + JanitorCaseSafetyTestCase( + description="virtual mode still drops a non-colliding tracked relation", + relation_names=("orders__v0",), + direct_mode=False, + tracked_names=("orders__v0",), + exclude_patterns=(), + expected_skipped_relations=(), + expected_dropped_targets=("analytics.orders__v0",), + ), + JanitorCaseSafetyTestCase( + description="virtual exclude patterns match regardless of case", + relation_names=("orders__v0",), + direct_mode=False, + tracked_names=("orders__v0",), + exclude_patterns=("ORDERS__*",), + expected_skipped_relations=( + ("analytics.orders__v0", "relation matches exclude pattern 'ORDERS__*'"), + ), + expected_dropped_targets=(), + ), + JanitorCaseSafetyTestCase( + description="direct exclude patterns match regardless of case", + relation_names=("old_orders",), + direct_mode=True, + tracked_names=("old_orders",), + exclude_patterns=("ANALYTICS.OLD_*",), + expected_skipped_relations=( + ("analytics.old_orders", "relation matches exclude pattern 'ANALYTICS.OLD_*'"), + ), + expected_dropped_targets=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_case_variant_relations_when_running_janitor_then_excluded_tables_are_not_dropped( + test_case: JanitorCaseSafetyTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter( + relation_infos=tuple(relation_info(name) for name in test_case.relation_names), + tracked_relations=tuple((None, "analytics", name) for name in test_case.tracked_names), + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=0, + exclude_patterns=test_case.exclude_patterns, + direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + ) + execute_janitor_plan(plan=plan, adapter=adapter, connection=object()) + + assert ( + tuple( + (skipped.key.display_name(), skipped.reason) + for skipped in filter( + lambda skipped: skipped.key.name != "_sqlbuild_fingerprints", + plan.skipped_relations, + ) + ) + == test_case.expected_skipped_relations + ) + assert tuple(adapter.dropped_targets) == test_case.expected_dropped_targets + assert adapter.renamed_targets == [] diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py new file mode 100644 index 0000000000..b9c009d4ee --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py @@ -0,0 +1,201 @@ +"""Janitor planning against warehouses that report folded identifier case.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime, timedelta +from itertools import chain + +import pytest + +from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild.executor.janitor.models import ( + JanitorDeleteCandidate, + JanitorDirectModeSettings, + JanitorPlan, + JanitorRelationKey, + JanitorRelationScope, +) +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorFoldedIdentifierPlanTestCase, + relation_info, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + FoldingJanitorAdapter, + build_project, +) + +OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) +ARCHIVE_TIMESTAMP_RE: re.Pattern[str] = re.compile(r"__[0-9]{8}t[0-9]{6}z__") + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorFoldedIdentifierPlanTestCase( + description="lowercase listing finds stale relation and protects desired relations", + relation_infos=( + relation_info( + "orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + relation_info( + "countries", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + relation_info( + "old_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + ), + expected_candidate_display_names=("ANALYTICS.DEV_ORDERS.old_orders",), + expected_archive_display_names=("ANALYTICS.DEV_ORDERS._sqb_archive____old_orders",), + ), + JanitorFoldedIdentifierPlanTestCase( + description="tracked-only filter matches fingerprints across identifier case", + relation_infos=( + relation_info( + "old_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + relation_info( + "stray_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + ), + delete_tracked_only=True, + tracked_relations=(("ANALYTICS", "DEV_ORDERS", "OLD_ORDERS"),), + expected_candidate_display_names=("ANALYTICS.DEV_ORDERS.old_orders",), + expected_archive_display_names=("ANALYTICS.DEV_ORDERS._sqb_archive____old_orders",), + expected_skipped_relations=( + ("ANALYTICS.DEV_ORDERS.stray_orders", "relation is not tracked by SQLBuild"), + ( + "ANALYTICS.DEV_ORDERS._sqlbuild_fingerprints", + "relation matches exclude pattern '_sqlbuild_fingerprints'", + ), + ), + ), + JanitorFoldedIdentifierPlanTestCase( + description="sources in a different schema of the same database do not block", + relation_infos=( + relation_info( + "old_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + ), + source_database="ANALYTICS", + source_schema="RAW", + expected_candidate_display_names=("ANALYTICS.DEV_ORDERS.old_orders",), + expected_archive_display_names=("ANALYTICS.DEV_ORDERS._sqb_archive____old_orders",), + ), + JanitorFoldedIdentifierPlanTestCase( + description="source in the managed schema blocks even when spelled differently", + relation_infos=( + relation_info( + "old_orders", database="ANALYTICS", schema="DEV_ORDERS", created_at=OLD_TIME + ), + ), + source_database="analytics", + source_schema="dev_orders", + expected_blocked_sources=("raw_orders",), + expected_suppressed_names=("ANALYTICS.DEV_ORDERS.old_orders",), + ), + JanitorFoldedIdentifierPlanTestCase( + description="expired archive is found in a lowercase listing", + relation_infos=( + relation_info( + "_sqb_archive__20200101t000000z__old_products", + database="ANALYTICS", + schema="DEV_ORDERS", + ), + ), + expected_archive_deletion_display_names=( + "ANALYTICS.DEV_ORDERS._sqb_archive__20200101t000000z__old_products", + ), + ), + JanitorFoldedIdentifierPlanTestCase( + description="virtual protection matches physical relations across identifier case", + relation_infos=( + relation_info( + "orders__v1", + database="ANALYTICS", + schema="DEV_ORDERS__SQB_PHYSICAL", + created_at=OLD_TIME, + ), + relation_info( + "orders__v0", + database="ANALYTICS", + schema="DEV_ORDERS__SQB_PHYSICAL", + created_at=OLD_TIME, + ), + ), + direct_mode=False, + protected_relation_keys=frozenset( + ( + JanitorRelationKey( + database="ANALYTICS", + schema="DEV_ORDERS__SQB_PHYSICAL", + name="ORDERS__V1", + ), + ) + ), + expected_candidate_display_names=("ANALYTICS.DEV_ORDERS__SQB_PHYSICAL.orders__v0",), + expected_skipped_relations=( + ( + "ANALYTICS.DEV_ORDERS__SQB_PHYSICAL.orders__v1", + "relation is referenced by a retained virtual checkpoint", + ), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_folded_catalog_identifiers_when_planning_janitor_then_keys_compare_consistently( + test_case: JanitorFoldedIdentifierPlanTestCase, +) -> None: + adapter: FoldingJanitorAdapter = FoldingJanitorAdapter( + relation_infos=test_case.relation_infos, + tracked_relations=test_case.tracked_relations, + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project( + source_database=test_case.source_database, + source_schema=test_case.source_schema, + destination_database="ANALYTICS", + destination_schema="DEV_ORDERS", + ), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=test_case.delete_tracked_only, + relation_scope=JanitorRelationScope( + protected_relation_keys=test_case.protected_relation_keys + ), + direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + ) + + assert ( + tuple(candidate.key.display_name() for candidate in plan.candidates) + == test_case.expected_candidate_display_names + ) + assert ( + tuple( + ARCHIVE_TIMESTAMP_RE.sub("____", candidate.archive_key.display_name()) + for candidate in plan.archive_candidates + ) + == test_case.expected_archive_display_names + ) + assert ( + tuple(archive.key.display_name() for archive in plan.archive_deletion_candidates) + == test_case.expected_archive_deletion_display_names + ) + assert ( + tuple((skipped.key.display_name(), skipped.reason) for skipped in plan.skipped_relations) + == test_case.expected_skipped_relations + ) + blocked_sources: tuple[str, ...] = tuple( + chain.from_iterable(blocked.source_names for blocked in plan.blocked_schemas) + ) + suppressed: tuple[JanitorDeleteCandidate, ...] = tuple( + chain.from_iterable(blocked.suppressed_candidates for blocked in plan.blocked_schemas) + ) + assert blocked_sources == test_case.expected_blocked_sources + assert ( + tuple(candidate.key.display_name() for candidate in suppressed) + == test_case.expected_suppressed_names + ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py new file mode 100644 index 0000000000..1c42809a90 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py @@ -0,0 +1,179 @@ +"""Janitor planning against adapters that read relation ages separately from listing.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from sqlbuild.executor.janitor.main.plan import build_janitor_plan +from sqlbuild.executor.janitor.models import ( + JanitorDirectModeSettings, + JanitorPlan, + JanitorRelationKey, + JanitorRelationScope, +) +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorAgeReadScopeTestCase, + JanitorSeparateAgeMetadataPlanTestCase, + relation_info, +) +from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( + SeparateAgeMetadataJanitorAdapter, + build_project, +) + +OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) +RECENT_TIME: datetime = datetime.now(UTC) - timedelta(days=1) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorSeparateAgeMetadataPlanTestCase( + description="separately read ages decide which stale relations are old enough", + relation_ages={"old_orders": OLD_TIME, "recent_orders": RECENT_TIME}, + expected_candidate_names=("old_orders",), + expected_skipped_relations=( + ("analytics.recent_orders", "relation is newer than 7 days"), + ), + ), + JanitorSeparateAgeMetadataPlanTestCase( + description="relations without separately read ages are skipped as unknown", + relation_ages={}, + expected_candidate_names=(), + expected_skipped_relations=( + ("analytics.old_orders", "relation age is unavailable"), + ("analytics.recent_orders", "relation age is unavailable"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_reading_ages_separately_when_planning_janitor_then_uses_those_ages( + test_case: JanitorSeparateAgeMetadataPlanTestCase, +) -> None: + adapter: SeparateAgeMetadataJanitorAdapter = SeparateAgeMetadataJanitorAdapter( + relation_infos=(relation_info("old_orders"), relation_info("recent_orders")), + relation_ages=test_case.relation_ages, + ) + + plan: JanitorPlan = build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=7, + delete_tracked_only=False, + direct_settings=JanitorDirectModeSettings(enabled=True), + ) + + assert ( + tuple(candidate.key.name for candidate in plan.candidates) + == test_case.expected_candidate_names + ) + assert ( + tuple((skipped.key.display_name(), skipped.reason) for skipped in plan.skipped_relations) + == test_case.expected_skipped_relations + ) + assert len(adapter.age_metadata_requests) == 1 + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorAgeReadScopeTestCase( + description="direct mode reads ages only for unfiltered stale relations", + relation_infos=( + relation_info("orders"), + relation_info("old_orders"), + relation_info("stray_orders"), + relation_info("tmp_orders"), + relation_info("_sqlbuild_fingerprints"), + relation_info("Mixed_Orders"), + ), + direct_mode=True, + delete_tracked_only=True, + tracked_relations=( + (None, "analytics", "old_orders"), + (None, "analytics", "tmp_orders"), + (None, "analytics", "Mixed_Orders"), + ), + exclude_patterns=("tmp_*",), + protected_relation_keys=frozenset(), + expected_age_requests=(("old_orders",),), + ), + JanitorAgeReadScopeTestCase( + description="virtual mode reads ages only for unprotected stale relations", + relation_infos=( + relation_info("orders"), + relation_info("orders__v0"), + relation_info("orders__v1"), + relation_info("tmp_orders"), + ), + direct_mode=False, + delete_tracked_only=False, + tracked_relations=(), + exclude_patterns=("tmp_*",), + protected_relation_keys=frozenset( + (JanitorRelationKey(database=None, schema="analytics", name="orders__v1"),) + ), + expected_age_requests=(("orders__v0",),), + ), + JanitorAgeReadScopeTestCase( + description="no age read happens when nothing survives the filters", + relation_infos=(relation_info("orders"), relation_info("stray_orders")), + direct_mode=True, + delete_tracked_only=True, + tracked_relations=(), + exclude_patterns=(), + protected_relation_keys=frozenset(), + expected_age_requests=(), + ), + JanitorAgeReadScopeTestCase( + description="zero retention does not read ages", + relation_infos=(relation_info("orders"), relation_info("old_orders")), + direct_mode=True, + delete_tracked_only=False, + tracked_relations=(), + exclude_patterns=(), + protected_relation_keys=frozenset(), + expected_age_requests=(), + retention_days=0, + ), + JanitorAgeReadScopeTestCase( + description="zero retention in virtual mode does not read ages", + relation_infos=(relation_info("orders"), relation_info("orders__v0")), + direct_mode=False, + delete_tracked_only=False, + tracked_relations=(), + exclude_patterns=(), + protected_relation_keys=frozenset(), + expected_age_requests=(), + retention_days=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_filtered_relations_when_planning_janitor_then_ages_are_read_for_candidates_only( + test_case: JanitorAgeReadScopeTestCase, +) -> None: + adapter: SeparateAgeMetadataJanitorAdapter = SeparateAgeMetadataJanitorAdapter( + relation_infos=test_case.relation_infos, + relation_ages={}, + tracked_relations=test_case.tracked_relations, + ) + + build_janitor_plan( + project=build_project(), + adapter=adapter, + connection=object(), + retention_days=test_case.retention_days, + delete_tracked_only=test_case.delete_tracked_only, + exclude_patterns=test_case.exclude_patterns, + relation_scope=JanitorRelationScope( + protected_relation_keys=test_case.protected_relation_keys + ), + direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), + ) + + assert tuple(adapter.age_metadata_requests) == test_case.expected_age_requests diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py new file mode 100644 index 0000000000..69253bd50d --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py @@ -0,0 +1,70 @@ +"""Tests for mode-specific janitor retention resolution.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.executor.janitor.main.resolve_retention_days import resolve_janitor_retention_days +from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( + JanitorRetentionResolutionTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JanitorRetentionResolutionTestCase( + description="unset retention in direct mode defaults to 14", + override=None, + configured=None, + virtual_environments=False, + expected_retention_days=14, + ), + JanitorRetentionResolutionTestCase( + description="unset retention in virtual mode defaults to 30", + override=None, + configured=None, + virtual_environments=True, + expected_retention_days=30, + ), + JanitorRetentionResolutionTestCase( + description="explicit retention applies in direct mode", + override=None, + configured=7, + virtual_environments=False, + expected_retention_days=7, + ), + JanitorRetentionResolutionTestCase( + description="explicit retention applies in virtual mode", + override=None, + configured=7, + virtual_environments=True, + expected_retention_days=7, + ), + JanitorRetentionResolutionTestCase( + description="cli override wins over explicit config", + override=3, + configured=7, + virtual_environments=True, + expected_retention_days=3, + ), + JanitorRetentionResolutionTestCase( + description="cli zero override wins over the mode default", + override=0, + configured=None, + virtual_environments=True, + expected_retention_days=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_retention_inputs_when_resolving_janitor_retention_then_returns_expected_days( + test_case: JanitorRetentionResolutionTestCase, +) -> None: + resolved: int = resolve_janitor_retention_days( + override=test_case.override, + configured=test_case.configured, + virtual_environments=test_case.virtual_environments, + ) + + assert resolved == test_case.expected_retention_days