diff --git a/Makefile b/Makefile index 1043a3ee74..124fe03891 100644 --- a/Makefile +++ b/Makefile @@ -130,6 +130,7 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/init \ tests/e2e/src/sqlbuild/cli/commands/main/rules \ tests/e2e/src/sqlbuild/cli/commands/main/lineage \ + tests/e2e/src/sqlbuild/cli/commands/main/model_migrations \ tests/e2e/src/sqlbuild/cli/commands/main/motherduck \ tests/e2e/src/sqlbuild/cli/commands/main/playground \ tests/e2e/src/sqlbuild/cli/commands/main/postgres \ diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index db9a9d663b..afca25076e 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -36,6 +36,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RowDiffCoverage, @@ -57,6 +58,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, @@ -702,6 +704,29 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + ) + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + + def supports_transactional_ddl(self) -> bool: + return False + def render_query_with_cursor_bounds( self, *, @@ -2195,6 +2220,21 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: transient=self.state_tables_transient, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the model migration event table when it is missing.""" + + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 889dd0ff1c..35690b7107 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -41,6 +41,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RowDiffColumnResult, @@ -61,6 +62,7 @@ CursorKind, FrameworkType, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, TablePromotionMode, ) @@ -851,6 +853,18 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1524,6 +1538,29 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + ) + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + + def supports_transactional_ddl(self) -> bool: + return True + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index c436c727c0..f7524f205f 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -15,6 +15,7 @@ ColumnInfo, CursorValue, ExpressionInferenceProfile, + MigrationStagePlan, RowDiffTolerance, RowDiffTolerances, SnapshotChangeTarget, @@ -341,6 +342,30 @@ def render_durable_clone( """Render SQL statements that clone/copy into a durable independent destination.""" ... + @abstractmethod + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + """Render statements that create a fresh, independent stage holding the origin's data.""" + ... + + @abstractmethod + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + """Capture statements that re-point identity-bound dependent views at this table name.""" + ... + + @abstractmethod + def supports_transactional_ddl(self) -> bool: + """Return whether renames and state inserts can commit or roll back together.""" + ... + @abstractmethod def render_query_with_cursor_bounds( self, @@ -885,6 +910,11 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: """Render DDL that creates the janitor audit event table when it is missing.""" ... + @abstractmethod + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the model migration event table when it is missing.""" + ... + @abstractmethod def render_prune_fingerprint_history_sql( self, diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index d376cbebc0..81426da31b 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -15,6 +15,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LifeCycleEventKind, + MigrationTransfer, RetentionChangePhase, RetentionScope, SnapshotLatestVersionStyle, @@ -184,6 +185,16 @@ class RenderedRetentionChange: statements: tuple[str, ...] +@dataclass(frozen=True) +class MigrationStagePlan: + """Fresh migration stage statements, plus a copy used only for recognized clone refusals.""" + + transfer: MigrationTransfer + statements: tuple[str, ...] + fallback_statements: tuple[str, ...] = () + is_clone_refusal: Callable[[BaseException], bool] | None = None + + @dataclass(frozen=True) class FunctionInfo: """Metadata for one discovered warehouse function or routine.""" diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 56b0f4de32..68e116ffea 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -36,6 +36,19 @@ class PromotionStrategy(StrEnum): CREATE_NEW = "create_new" +class MigrationTransfer(StrEnum): + """How a model migration stage receives the origin's data.""" + + CLONE = "clone" + COPY = "copy" + + @property + def label(self) -> str: + """Return the human-readable plan label for this transfer.""" + + return "zero-copy clone" if self == MigrationTransfer.CLONE else "physical copy" + + class TablePromotionMode(StrEnum): IMMEDIATE = "immediate" STAGED = "staged" diff --git a/src/sqlbuild/adapters/bigquery/_helpers/clone_refusal.py b/src/sqlbuild/adapters/bigquery/_helpers/clone_refusal.py new file mode 100644 index 0000000000..ee761f9b04 --- /dev/null +++ b/src/sqlbuild/adapters/bigquery/_helpers/clone_refusal.py @@ -0,0 +1,40 @@ +"""Recognize BigQuery refusals to clone a table where a physical copy can still succeed.""" + +from __future__ import annotations + +from sqlbuild.adapters.bigquery.constants import ( + CLONE_REFUSAL_ERROR_CODE, + CLONE_REFUSAL_PHRASES, + CLONE_REFUSAL_REASONS, + MAX_CLONE_REFUSAL_ERROR_CHAIN, +) + + +def is_bigquery_clone_refusal(error: BaseException) -> bool: + """Return whether a driver error in the cause chain is a clone-capability refusal.""" + + current: BaseException | None = error + depth: int = 0 + while current is not None and depth < MAX_CLONE_REFUSAL_ERROR_CHAIN: + if _is_refusal(current): + return True + current = current.__cause__ or current.__context__ + depth += 1 + return False + + +def _is_refusal(error: BaseException) -> bool: + details: object | None = getattr(error, "errors", None) + if getattr(error, "code", None) != CLONE_REFUSAL_ERROR_CODE or not isinstance(details, list): + return False + reasons: set[str] = { + str(detail.get("reason", "")) for detail in details if isinstance(detail, dict) + } + messages: str = " ".join( + str(detail.get("message", "")) for detail in details if isinstance(detail, dict) + ).lower() + return ( + bool(reasons) + and reasons <= CLONE_REFUSAL_REASONS + and any(phrase in messages for phrase in CLONE_REFUSAL_PHRASES) + ) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 7715a4fbde..73b6943b25 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -46,6 +46,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RenderedRetentionChange, @@ -73,6 +74,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, RetentionChangePhase, RetentionScope, @@ -93,6 +95,7 @@ from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.bigquery._helpers.clone_refusal import is_bigquery_clone_refusal from sqlbuild.adapters.bigquery._helpers.statement_telemetry import affected_rows from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection from sqlbuild.adapters.bigquery.classes.bigquery_cursor import _BigQueryCursor @@ -321,6 +324,18 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1712,6 +1727,33 @@ def render_durable_clone( f"CLONE {self._quote_identifier_path(origin)}", ) + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + quoted_stage: str = self._quote_identifier_path(stage) + quoted_origin: str = self._quote_identifier_path(origin) + return MigrationStagePlan( + transfer=MigrationTransfer.CLONE, + statements=(f"CREATE TABLE {quoted_stage} CLONE {quoted_origin}",), + fallback_statements=(f"CREATE TABLE {quoted_stage} COPY {quoted_origin}",), + is_clone_refusal=is_bigquery_clone_refusal, + ) + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + + def supports_transactional_ddl(self) -> bool: + return False + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index 1454478697..8419dafa9f 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -17,3 +17,17 @@ SELECT_STATEMENT_TYPE: str = "SELECT" TABLE_NAME_WILDCARD: str = "*" TIMESTAMP_TYPE_TOKEN: str = "TIMESTAMP" + +CLONE_REFUSAL_ERROR_CODE: int = 400 +CLONE_REFUSAL_REASONS: frozenset[str] = frozenset({"invalid", "invalidQuery"}) +CLONE_REFUSAL_PHRASES: tuple[str, ...] = ( + "cannot clone", + "can't clone", + "clone is not supported", + "clones are not supported", + "cloning is not supported", + "not supported for clone", + "clones and snapshots", + "clone or snapshot", +) +MAX_CLONE_REFUSAL_ERROR_CHAIN: int = 8 diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 1b67dc4f65..245a4f7a29 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -44,6 +44,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RenderedRetentionChange, @@ -71,6 +72,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, RelationType, RetentionChangePhase, @@ -415,6 +417,18 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1751,6 +1765,29 @@ def render_durable_clone( del origin_is_transient return (f"CREATE TABLE {destination} DEEP CLONE {origin}",) + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} DEEP CLONE {origin}",), + ) + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + + def supports_transactional_ddl(self) -> bool: + return False + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/postgres/_helpers/__init__.py b/src/sqlbuild/adapters/postgres/_helpers/__init__.py new file mode 100644 index 0000000000..47b31f3a24 --- /dev/null +++ b/src/sqlbuild/adapters/postgres/_helpers/__init__.py @@ -0,0 +1 @@ +"""PostgreSQL adapter helpers.""" diff --git a/src/sqlbuild/adapters/postgres/_helpers/view_rebind.py b/src/sqlbuild/adapters/postgres/_helpers/view_rebind.py new file mode 100644 index 0000000000..0ae6a65481 --- /dev/null +++ b/src/sqlbuild/adapters/postgres/_helpers/view_rebind.py @@ -0,0 +1,32 @@ +"""Re-create a dependent PostgreSQL view with its original options.""" + +from __future__ import annotations + +import re + +_CHECK_OPTION_KEY: str = "check_option" +_PLAIN_OPTION_VALUE: re.Pattern[str] = re.compile(r"[A-Za-z0-9_.]+") + + +def render_postgres_view_rebind(*, view: str, definition: str, reloptions: tuple[str, ...]) -> str: + """Render CREATE OR REPLACE VIEW that keeps security and check options from reloptions.""" + + options: list[str] = [] + check_option: str | None = None + option: str + for option in reloptions: + key, _, value = option.partition("=") + if key == _CHECK_OPTION_KEY: + check_option = value.upper() + continue + options.append(f"{key}={_option_value(value)}") + with_clause: str = f" WITH ({', '.join(options)})" if options else "" + check_clause: str = f" WITH {check_option} CHECK OPTION" if check_option else "" + body: str = definition.strip().rstrip(";").rstrip() + return f"CREATE OR REPLACE VIEW {view}{with_clause} AS {body}{check_clause}" + + +def _option_value(value: str) -> str: + if _PLAIN_OPTION_VALUE.fullmatch(value): + return value + return "'" + value.replace("'", "''") + "'" diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 0e2edbcdf2..e4760b67bc 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -46,6 +46,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RowDiffColumnResult, @@ -70,6 +71,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, @@ -83,6 +85,7 @@ ) from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.postgres._helpers.view_rebind import render_postgres_view_rebind from sqlbuild.adapters.postgres.classes.postgres_connection import _PostgresConnection from sqlbuild.adapters.postgres.constants import TABLE_FUNCTION_RETURN_TYPE from sqlbuild.compiler.compile.types import FunctionLanguage @@ -577,6 +580,57 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + ) + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del database + query: str = ( + "SELECT DISTINCT dependent_namespace.nspname, dependent.relname, " + "pg_get_viewdef(dependent.oid), " + "array_to_string(dependent.reloptions, chr(31)) " + "FROM pg_depend AS dependency " + "JOIN pg_rewrite AS rewrite ON rewrite.oid = dependency.objid " + "JOIN pg_class AS dependent ON dependent.oid = rewrite.ev_class " + "JOIN pg_namespace AS dependent_namespace " + "ON dependent_namespace.oid = dependent.relnamespace " + "JOIN pg_class AS referenced ON referenced.oid = dependency.refobjid " + "JOIN pg_namespace AS referenced_namespace " + "ON referenced_namespace.oid = referenced.relnamespace " + "WHERE dependency.classid = 'pg_rewrite'::regclass " + "AND dependency.refclassid = 'pg_class'::regclass " + "AND dependent.relkind = 'v' " + "AND dependent.oid <> referenced.oid " + f"AND referenced_namespace.nspname = {_quote_sql_string(schema)} " + f"AND referenced.relname = {_quote_sql_string(name)} " + "ORDER BY 1, 2" + ) + cursor: Any = connection.execute(query) + return tuple( + render_postgres_view_rebind( + view=f"{self.render_identifier(str(row[0]))}.{self.render_identifier(str(row[1]))}", + definition=str(row[2]), + reloptions=tuple(str(row[3]).split(chr(31))) if row[3] else (), + ) + for row in cursor.fetchall() + ) + + def supports_transactional_ddl(self) -> bool: + return True + def render_query_with_cursor_bounds( self, *, @@ -1530,6 +1584,18 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_read_latest_source_freshness_sql( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 21cce1ee90..b62a05ef61 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -41,6 +41,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RenderedRetentionChange, @@ -68,6 +69,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, RetentionChangePhase, RetentionScope, @@ -497,6 +499,19 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: transient=self.state_tables_transient, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -2230,6 +2245,38 @@ def render_durable_clone( table_kind: str = "TRANSIENT TABLE" if origin_is_transient else "TABLE" return (f"CREATE OR REPLACE {table_kind} {destination} CLONE {origin}",) + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + transient: bool = origin_is_transient if stage_is_transient is None else stage_is_transient + if origin_is_transient and not transient: + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=( + f"CREATE TABLE {stage} LIKE {origin} COPY GRANTS", + f"INSERT INTO {stage} SELECT * FROM {origin}", + ), + ) + table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" + return MigrationStagePlan( + transfer=MigrationTransfer.CLONE, + statements=(f"CREATE {table_kind} {stage} CLONE {origin} COPY GRANTS",), + ) + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + + def supports_transactional_ddl(self) -> bool: + return False + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 62e09380e0..36d8af3132 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -48,6 +48,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RowDiffColumnResult, @@ -72,6 +73,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, @@ -631,6 +633,25 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: table_name=JANITOR_EVENTS_TABLE_NAME, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + create_sql: str = build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ).replace("CREATE TABLE IF NOT EXISTS", "CREATE TABLE", 1) + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=MIGRATION_TABLE_NAME, + ) + def render_read_latest_source_freshness_sql( self, *, @@ -1772,6 +1793,29 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"SELECT * INTO {stage} FROM {origin}",), + ) + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + + def supports_transactional_ddl(self) -> bool: + return True + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py index 4a6b8b98b0..74857928e6 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any +from typing import Any, TextIO from sqlbuild.cli.commands._helpers.build_execution.run_context import ( write_build_run_context, @@ -166,6 +166,9 @@ def execute_build_plan( providers=providers, ), callbacks=BuildCallbacks( + on_migration_progress=lambda message: _write_progress_line( + stream=invocation.progress_stream, message=message + ), on_node_start=lambda name, resource_kind: preparation.callbacks.on_node_start( name=name, resource_kind=resource_kind ), @@ -215,3 +218,8 @@ def execute_build_plan( result=result, python_results=preparation.python_lifecycle.python_results, ) + + +def _write_progress_line(*, stream: TextIO, message: str) -> None: + stream.write(f"{message}\n") + stream.flush() diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py new file mode 100644 index 0000000000..3fcabe3ac4 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -0,0 +1,100 @@ +"""Plan safety policies enforced before a direct build executes anything.""" + +from __future__ import annotations + +import sys + +from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( + enforce_model_execution_limit, + executable_model_count, +) +from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( + enforce_snapshot_full_refresh_policy, +) +from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( + enforce_retention_decrease_policy, +) +from sqlbuild.cli.commands._helpers.build_planning.table_type import ( + enforce_table_type_downgrade_policy, +) +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.cli.commands.models import BuildCommandRequest, BuildInvocation +from sqlbuild.compiler.migrations.types import MigrationDecision +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput + + +def enforce_build_plan_policies( + *, request: BuildCommandRequest, invocation: BuildInvocation, plan: PlanOutput +) -> None: + """Apply migration, execution-limit, and storage safety gates in their required order.""" + + _enforce_model_migration_policy(plan=plan) + enforce_model_execution_limit( + model_count=executable_model_count(plan=plan), + target_name=invocation.effective_target_name, + limits=invocation.execution_limits, + ) + enforce_snapshot_full_refresh_policy( + plan=plan, + snapshots_config=invocation.discovered_inputs.project_config.snapshots, + allow_snapshot_full_refresh=request.allow_snapshot_full_refresh, + input_stream=sys.stdin, + output_stream=sys.stdout, + ) + enforce_table_type_downgrade_policy( + plan=plan, + allow_table_type_downgrade=request.allow_table_type_downgrade, + input_stream=sys.stdin, + output_stream=sys.stdout, + ) + enforce_retention_decrease_policy( + plan=plan, + allow_retention_decrease=request.allow_retention_decrease, + input_stream=sys.stdin, + output_stream=sys.stdout, + ) + + +def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: + """Refuse to build while any planned migration conflicts or is incompatible.""" + + blocked: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in plan.migration_entries if entry.blocks_build + ) + if not blocked: + return + missing: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in blocked if entry.decision == MigrationDecision.ORIGIN_MISSING + ) + if missing: + raise CliUserError( + "model migration origin does not exist for " + + ", ".join( + f"'{entry.model_name}' (migrate_from " + f"{entry.origin.qualified_name or entry.origin.name})" + for entry in missing + ) + + " and no recorded migration into it was found; if the migration already happened " + "elsewhere or is no longer needed, remove migrate_from from the model header", + code="M102", + help="Run sqb plan to see every model migration decision.", + ) + conflicts: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in blocked if entry.decision == MigrationDecision.CONFLICT + ) + names: str = ", ".join(f"'{entry.model_name}'" for entry in blocked) + if conflicts: + raise CliUserError( + f"model migration conflict for {names}: the destination already exists with its " + "own build history and no recorded migration", + code="M103", + help=( + "Set migrate_force true on the model to replace the destination; the replaced " + "table remains recoverable through warehouse time travel." + ), + ) + raise CliUserError( + f"model migration source is incompatible with {names}", + code="M104", + help="Run sqb plan to see the blocking schema differences.", + ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 1c5e10c7b6..aa1d7a50ea 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -110,6 +110,7 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) defer_to=args.defer_to, defer_sources_to=args.defer_sources_to, selected_target=args.target, + as_target=args.as_target, cursor_overrides=CursorOverrides( start_ts=args.start_cursor_ts, end_ts=args.end_cursor_ts, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index 37d29e7e7c..b0789d403a 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -179,6 +179,15 @@ def _add_plan_and_build_parsers( plan_parser.add_argument("--defer-to", default=None) plan_parser.add_argument("--defer-sources-to", default=None) plan_parser.add_argument("--target", default=None) + plan_parser.add_argument( + "--as", + dest="as_target", + default=None, + help=( + "Preview the plan as another configured target through the active target's " + "connection; inspection only" + ), + ) plan_parser.add_argument("--json", action="store_true", default=False) plan_parser.add_argument("--full-refresh", action="store_true", default=False) plan_parser.add_argument("--virtual-env", default=None) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index b4187a1732..2a1677f1d9 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -13,6 +13,7 @@ enforce_no_defer_to_in_virtual_mode, enforce_virtual_only_flags_in_virtual_mode, ) +from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( AdapterConnectionContext, PlanCommandRequest, @@ -46,6 +47,7 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: selected_target=request.selected_target, cli_changes_only=request.changes_only, ) + _validate_preview_target(discovered_inputs=discovered_inputs, as_target=request.as_target) enforce_no_defer_to_in_virtual_mode( discovered_inputs=discovered_inputs, command_name="plan", @@ -89,3 +91,23 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: should_load_sources=should_load_sources, virtual_mode=bool(discovered_inputs.project_config.settings.virtual_environments), ) + + +def _validate_preview_target( + *, discovered_inputs: DiscoveredProjectInputs, as_target: str | None +) -> None: + if as_target is None: + return + if discovered_inputs.project_config.settings.virtual_environments: + raise CliUserError( + "plan --as is supported only in direct mode", + help="Virtual environments preview other environments with --virtual-env.", + ) + configured: set[str] = set(discovered_inputs.project_config.targets) | set( + discovered_inputs.local_config.targets + ) + if as_target not in configured: + raise CliUserError( + f"unknown target '{as_target}' for plan --as", + help=f"Configured targets: {', '.join(sorted(configured)) or 'none'}.", + ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/planning.py b/src/sqlbuild/cli/commands/_helpers/plan/planning.py index b95c53a488..e760773b14 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/planning.py @@ -79,11 +79,19 @@ def compile_plan_pipeline( output_stream=StringIO(), ) return virtual_result + if request.as_target is not None: + invocation.progress_stream.write( + f"Previewing plan as target '{request.as_target}' through the active target's " + "connection (inspection only).\n" + ) + invocation.progress_stream.flush() result: CompilePipelineResult = run_compile_pipeline( discovered_inputs=invocation.discovered_inputs, adapter=invocation.adapter, options=CompilePipelineOptions( - selected_target=request.selected_target, + selected_target=( + request.as_target if request.as_target is not None else request.selected_target + ), no_sql_validation=request.no_sql_validation, no_cache=request.no_cache, defer_to=request.defer_to, diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index 38ca9e3714..c33697d784 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -51,6 +51,7 @@ "fail_fast": False, "full_refresh": False, "virtual_env": None, + "as_target": None, "skip_locked": False, "include_stale_upstreams": False, "changes_only": False, @@ -225,6 +226,7 @@ class CliNamespace: fail_fast: bool full_refresh: bool virtual_env: str | None + as_target: str | None skip_locked: bool include_stale_upstreams: bool changes_only: bool diff --git a/src/sqlbuild/cli/commands/main/execution/_build.py b/src/sqlbuild/cli/commands/main/execution/_build.py index baa3137623..1f2458c546 100644 --- a/src/sqlbuild/cli/commands/main/execution/_build.py +++ b/src/sqlbuild/cli/commands/main/execution/_build.py @@ -25,21 +25,11 @@ from sqlbuild.cli.commands._helpers.build_planning.defer_clone import ( run_defer_clone_boundary_prephase, ) -from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( - enforce_model_execution_limit, - executable_model_count, -) -from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( - enforce_snapshot_full_refresh_policy, -) from sqlbuild.cli.commands._helpers.build_planning.invocation import resolve_build_invocation -from sqlbuild.cli.commands._helpers.build_planning.planning import compile_build_plan -from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( - enforce_retention_decrease_policy, -) -from sqlbuild.cli.commands._helpers.build_planning.table_type import ( - enforce_table_type_downgrade_policy, +from sqlbuild.cli.commands._helpers.build_planning.plan_policies import ( + enforce_build_plan_policies, ) +from sqlbuild.cli.commands._helpers.build_planning.planning import compile_build_plan from sqlbuild.cli.commands._helpers.cost.collection import ( finalize_build_cost, ) @@ -163,29 +153,10 @@ def _run_build( invocation=invocation, pipeline_result=pipeline_result, ) - enforce_model_execution_limit( - model_count=executable_model_count(plan=pipeline_result.plan_output), - target_name=invocation.effective_target_name, - limits=invocation.execution_limits, - ) - enforce_snapshot_full_refresh_policy( - plan=pipeline_result.plan_output, - snapshots_config=invocation.discovered_inputs.project_config.snapshots, - allow_snapshot_full_refresh=request.allow_snapshot_full_refresh, - input_stream=sys.stdin, - output_stream=sys.stdout, - ) - enforce_table_type_downgrade_policy( - plan=pipeline_result.plan_output, - allow_table_type_downgrade=request.allow_table_type_downgrade, - input_stream=sys.stdin, - output_stream=sys.stdout, - ) - enforce_retention_decrease_policy( + enforce_build_plan_policies( + request=request, + invocation=invocation, plan=pipeline_result.plan_output, - allow_retention_decrease=request.allow_retention_decrease, - input_stream=sys.stdin, - output_stream=sys.stdout, ) write_build_compile_target( request=request, diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 5d8dabd540..d7e0504971 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -977,6 +977,7 @@ class PlanCommandRequest: defer_to: str | None = None defer_sources_to: str | None = None selected_target: str | None = None + as_target: str | None = None cursor_overrides: CursorOverrides | None = None json_output: bool = False full_refresh: bool = False diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 23381f91b7..2da44895a5 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -98,6 +98,7 @@ def format_plan_json( "warnings": warnings, "retention": retention, "table_type_conversions": table_type_conversions, + "migrations": _serialize_model_migrations(plan), } if plan.metadata: result["metadata"] = plan.metadata @@ -469,3 +470,31 @@ def _serialize_provider_usage(usage: PlanProviderUsage) -> dict[str, object]: "module": usage.annotation_module, } return payload + + +def _serialize_model_migrations(plan: PlanOutput) -> list[dict[str, object]]: + return [ + { + "kind": "model_migration", + "model": entry.model_name, + "discovery": entry.discovery.value, + "decision": entry.decision.value, + "compatibility": entry.compatibility.value, + "compatibility_findings": list(entry.compatibility_findings), + "origin_model": entry.origin_model, + "origin": entry.origin.qualified_name or entry.origin.name, + "destination": entry.destination.qualified_name or entry.destination.name, + "target": entry.target_name, + "origin_version_hash": entry.origin_version_hash, + "transfer": entry.transfer.value if entry.transfer is not None else None, + "transfer_fallback": ( + entry.transfer_fallback.value if entry.transfer_fallback is not None else None + ), + "storage_transition": entry.storage_transition, + "promotion": entry.promotion.value if entry.promotion is not None else None, + "completed_at": ( + entry.completed_at.isoformat() if entry.completed_at is not None else None + ), + } + for entry in plan.migration_entries + ] diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index 4f80eaef92..60f240c2ea 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -22,6 +22,9 @@ from sqlbuild.compiler.planner.main.execution.model_materialization_label import ( model_materialization_label, ) +from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import ( + format_pre_build_work, +) from sqlbuild.compiler.planner.models import ( CascadeResult, CursorBounds, @@ -71,6 +74,7 @@ PlanReason.SCHEMA_CHANGED, PlanReason.RUN_DESPITE_UNCHANGED, PlanReason.FIRST_RUN, + PlanReason.RENAMED, ) _REASON_GROUP_LABELS: dict[PlanReason, str] = { @@ -80,6 +84,7 @@ PlanReason.SCHEMA_CHANGED: "Schema changed", PlanReason.RUN_DESPITE_UNCHANGED: "Runs despite unchanged", PlanReason.FIRST_RUN: "First run", + PlanReason.RENAMED: "Renamed", } _ANSI_ESCAPE_PATTERN: re.Pattern[str] = re.compile(r"\033\[[0-9;]*m") @@ -90,50 +95,6 @@ } -def _format_retention( - *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions -) -> list[str]: - if plan.table_type_entries: - lines.append("Table type conversions") - for table_type_entry in visible_entries( - entries=plan.table_type_entries, options=display_options - ): - actual_type: str = table_type_entry.actual_type or "unknown" - lines.append( - f" {table_type_entry.model_name} desired={table_type_entry.desired_type} " - f"actual={actual_type} source={table_type_entry.source}" - ) - if table_type_entry.irreversible_warning is not None: - lines.append(f" WARNING: {table_type_entry.irreversible_warning}") - lines = append_overflow_line( - lines=lines, - total_count=len(plan.table_type_entries), - visible_count=len( - visible_entries(entries=plan.table_type_entries, options=display_options) - ), - indent=" ", - options=display_options, - ) - if plan.retention_entries: - lines.append("Retention") - for entry in plan.retention_entries: - scope: str = ".".join( - part - for part in (entry.request.database, entry.request.schema, entry.request.name) - if part - ) - actual: str = "missing" if entry.actual_days is None else f"{entry.actual_days}d" - effective: str = "missing" if entry.effective_days is None else f"{entry.effective_days}d" - lines.append( - f" {scope} desired={entry.request.desired_days}d actual={actual} " - f"effective={effective} source={entry.source} direction={entry.direction.value} " - f"phase={entry.phase.value}" - ) - if entry.irreversible_warning is not None: - lines.append(f" WARNING: {entry.irreversible_warning}") - return lines - - def format_plan( *, plan: PlanOutput, @@ -174,7 +135,9 @@ def format_plan( include_direct_freshness_diagnostics=(include_direct_freshness_diagnostics), display_options=resolved_display_options, ) - lines = _format_retention(lines=lines, plan=plan, display_options=resolved_display_options) + lines = format_pre_build_work( + lines=lines, plan=plan, display_options=resolved_display_options + ) result: str = "\n".join(lines) return result if use_color else _strip_ansi(result) @@ -225,7 +188,7 @@ def format_plan( display_options=resolved_display_options, skipped_header_style=style.muted, ) - lines = _format_retention(lines=lines, plan=plan, display_options=resolved_display_options) + lines = format_pre_build_work(lines=lines, plan=plan, display_options=resolved_display_options) lines = _format_python_plan_entries( lines=lines, @@ -1184,6 +1147,8 @@ def _plan_reason_text(reason: PlanReason) -> str: return "schema changed" if reason == PlanReason.FIRST_RUN: return "first run" + if reason == PlanReason.RENAMED: + return "renamed" if reason == PlanReason.FULL_REFRESH: return "full refresh" if reason == PlanReason.RUN_DESPITE_UNCHANGED: @@ -1437,7 +1402,7 @@ def _format_warnings( include_direct_freshness_diagnostics: bool, display_options: DisplayOptions, ) -> list[str]: - """Append the warnings section.""" + """Append the errors and warnings sections.""" selection_diagnostics: bool | None = direct_selection_diagnostics_enabled(plan) include_stale_input_warnings: bool = ( @@ -1445,7 +1410,7 @@ def _format_warnings( if selection_diagnostics is not None else include_direct_freshness_diagnostics ) - warning_entries: list[PlanWarning] = [ + diagnostics: list[PlanWarning] = [ warning for warning in plan.warnings if warning.severity != WarningSeverity.INFO @@ -1454,18 +1419,46 @@ def _format_warnings( or not warning.message.startswith(_STALE_INPUT_WARNING_TITLE) ) ] - if not warning_entries: + style: CliStyle = CliStyle(use_color=True) + lines = _format_diagnostic_section( + lines=lines, + entries=[entry for entry in diagnostics if entry.severity == WarningSeverity.ERROR], + title="Errors", + heading_style=style.error_strong, + line_style=style.error, + display_options=display_options, + ) + return _format_diagnostic_section( + lines=lines, + entries=[entry for entry in diagnostics if entry.severity != WarningSeverity.ERROR], + title="Warnings", + heading_style=style.warning_strong, + line_style=style.warning, + display_options=display_options, + ) + + +def _format_diagnostic_section( + *, + lines: list[str], + entries: list[PlanWarning], + title: str, + heading_style: Callable[[str], str], + line_style: Callable[[str], str], + display_options: DisplayOptions, +) -> list[str]: + """Append one severity-grouped plan diagnostic section.""" + + if not entries: return lines style: CliStyle = CliStyle(use_color=True) lines.append("") - lines.append(style.warning_strong(f"Warnings ({len(warning_entries)})")) - shown_warnings: Sequence[PlanWarning] = visible_entries( - entries=warning_entries, options=display_options - ) + lines.append(heading_style(f"{title} ({len(entries)})")) + shown: Sequence[PlanWarning] = visible_entries(entries=entries, options=display_options) warning: PlanWarning warning_index: int - for warning_index, warning in enumerate(shown_warnings): - connector: str = tree_connector(style=style, last=warning_index == len(shown_warnings) - 1) + for warning_index, warning in enumerate(shown): + connector: str = tree_connector(style=style, last=warning_index == len(shown) - 1) message_lines: list[str] = warning.message.split("\n") if warning.model_name is not None: lines.append(f"{connector} {style.object_name(warning.model_name)}") @@ -1475,16 +1468,16 @@ def _format_warnings( style=style, last=message_index == len(message_lines) - 1, ) - lines.append(f" {child_connector} {style.warning(message_line)}") + lines.append(f" {child_connector} {line_style(message_line)}") else: - lines.append(f"{connector} {style.warning(message_lines[0])}") + lines.append(f"{connector} {line_style(message_lines[0])}") continuation: str for continuation in message_lines[1:]: - lines.append(f" {style.warning(continuation)}") + lines.append(f" {line_style(continuation)}") return append_overflow_line( lines=lines, - total_count=len(warning_entries), - visible_count=len(shown_warnings), + total_count=len(entries), + visible_count=len(shown), indent=" ", options=display_options, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 30b56e3067..189916b29d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -38,6 +38,7 @@ validate_custom_materialization_config, validate_incremental_config, validate_microbatch_project_capability, + validate_model_migration_config, validate_non_incremental_config, validate_placeholder_config, validate_snapshot_config, @@ -757,6 +758,7 @@ def _validate_model_input( custom_materialization_names=context.custom_materialization_names, ) validate_storage_policies(config=config, model_name=model_name) + validate_model_migration_config(config=config, model_name=model_name) validate_placeholder_config( config=config, model_name=model_name, diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index d8a99917d1..6ef14cc18f 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -10,6 +10,8 @@ from sqlbuild.compiler.compile.constants import ( MICROBATCH_LIMIT_ACTION_KEY, MICROBATCH_LIMIT_MAX_BATCHES_KEY, + MIGRATE_FORCE_CONFIG_KEY, + MIGRATE_FROM_CONFIG_KEY, SQL_WILDCARD_TOKEN, WATERMARK_CURSOR_INPUT_BLOCK_KEYS, ) @@ -949,6 +951,31 @@ def validate_non_incremental_config( ) +def validate_model_migration_config(*, config: CompileModelConfig, model_name: str) -> None: + """Validate migrate_from and migrate_force on history-bearing materializations.""" + + migrate_from: object | None = config.values.get(MIGRATE_FROM_CONFIG_KEY) + migrate_force: object | None = config.values.get(MIGRATE_FORCE_CONFIG_KEY) + if migrate_from is None and migrate_force is None: + return + materialized: str | None = get_config_str(values=config.values, key="materialized") + if materialized not in {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT}: + raise CompileInputError( + f"model '{model_name}': migrate_from is only valid for incremental and snapshot " + f"models; '{materialized}' models are rebuilt under their new name" + ) + if migrate_from is None: + raise CompileInputError(f"model '{model_name}': migrate_force requires migrate_from") + if not isinstance(migrate_from, str) or not migrate_from.strip(): + raise CompileInputError( + f"model '{model_name}': migrate_from must be a model name or a qualified relation" + ) + if migrate_from.strip().lower() == model_name.lower(): + raise CompileInputError(f"model '{model_name}': migrate_from cannot name the model itself") + if migrate_force is not None and not isinstance(migrate_force, bool): + raise CompileInputError(f"model '{model_name}': migrate_force must be true or false") + + def validate_snapshot_config( *, config: CompileModelConfig, diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 494ef088f9..8eb5364e16 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -17,6 +17,8 @@ MODEL_AUDIT_OVERRIDE_KEYS: frozenset[str] = frozenset({"by_type", "by_column"}) MODEL_FULL_REFRESH_CONFIG_KEY: str = "full_refresh" CURSOR_INPUTS_CONFIG_KEY: str = "cursor_inputs" +MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" +MIGRATE_FORCE_CONFIG_KEY: str = "migrate_force" CURSOR_FILTER_INPUTS_CONFIG_KEY: str = "cursor_filter_inputs" CURSOR_WATERMARK_INPUTS_CONFIG_KEY: str = "cursor_watermark_inputs" MAX_MICROBATCHES_CONFIG_KEY: str = "max_microbatches" diff --git a/src/sqlbuild/compiler/migrations/__init__.py b/src/sqlbuild/compiler/migrations/__init__.py new file mode 100644 index 0000000000..eaf52e580e --- /dev/null +++ b/src/sqlbuild/compiler/migrations/__init__.py @@ -0,0 +1 @@ +"""Direct-mode model migration state package.""" diff --git a/src/sqlbuild/compiler/migrations/_helpers/__init__.py b/src/sqlbuild/compiler/migrations/_helpers/__init__.py new file mode 100644 index 0000000000..114aa7085f --- /dev/null +++ b/src/sqlbuild/compiler/migrations/_helpers/__init__.py @@ -0,0 +1 @@ +"""Model migration state helper package.""" diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py new file mode 100644 index 0000000000..8e4a05257f --- /dev/null +++ b/src/sqlbuild/compiler/migrations/_helpers/sql.py @@ -0,0 +1,151 @@ +"""Portable SQL for append-only model migration events.""" + +from __future__ import annotations + +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) +from sqlbuild.compiler.migrations.constants import ( + MIGRATION_COLUMN_TYPES, + MIGRATION_COLUMNS, + MIGRATION_TABLE_NAME, +) +from sqlbuild.compiler.migrations.exceptions import MigrationStateError +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal +from sqlbuild.sql_values.types import StateSqlValueType + + +def qualified_migration_table( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: + table: str | None = render_qualified_name( + database=database, schema=schema, name=MIGRATION_TABLE_NAME + ) + if table is None: + raise MigrationStateError("model migration state requires a target schema") + return table + + +def build_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool, +) -> str: + return render_state_table_create_sql( + qualified_name=qualified_migration_table( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=MIGRATION_COLUMNS, + column_types=MIGRATION_COLUMN_TYPES, + required_columns=frozenset(), + render_framework_type=render_framework_type, + transient=transient, + ) + + +def build_existing_event_sql( + *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] +) -> str: + table: str = _event_table(event=event, render_qualified_name=render_qualified_name) + event_id_literal: str = render_state_sql_literal( + value=event.event_id, declared_type=StateSqlValueType.STRING + ) + return f"SELECT event_id FROM {table} WHERE event_id = {event_id_literal}" + + +def build_insert_sql( + *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] +) -> str: + table: str = _event_table(event=event, render_qualified_name=render_qualified_name) + values: tuple[object | None, ...] = _event_values(event) + literals: str = ", ".join( + render_state_sql_literal(value=value, declared_type=MIGRATION_COLUMN_TYPES[column]) + for column, value in zip(MIGRATION_COLUMNS, values, strict=True) + ) + return f"INSERT INTO {table} ({', '.join(MIGRATION_COLUMNS)}) VALUES ({literals})" + + +def _event_table(*, event: MigrationEvent, render_qualified_name: Callable[..., str | None]) -> str: + if event.destination.schema is None: + raise MigrationStateError("model migration events require a destination schema") + return qualified_migration_table( + database=event.destination.database, + schema=event.destination.schema, + render_qualified_name=render_qualified_name, + ) + + +def build_read_sql( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: + table: str = qualified_migration_table( + database=database, schema=schema, render_qualified_name=render_qualified_name + ) + return f"SELECT {', '.join(MIGRATION_COLUMNS)} FROM {table} ORDER BY created_at, event_id" + + +def decode_event_row(row: tuple[Any, ...]) -> MigrationEvent: + values: dict[str, Any] = dict(zip(MIGRATION_COLUMNS, row, strict=False)) + raw_created_at: Any = values["created_at"] + created_at: datetime = ( + raw_created_at + if isinstance(raw_created_at, datetime) + else datetime.fromisoformat(str(raw_created_at)) + ) + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=UTC) + return MigrationEvent( + event_id=str(values["event_id"]), + target_name=_optional_text(values["target_name"]), + origin_model=_optional_text(values["origin_model"]), + origin=MigrationRelation( + database=_optional_text(values["origin_database"]), + schema=_optional_text(values["origin_schema"]), + name=str(values["origin_name"]), + ), + destination_model=str(values["destination_model"]), + destination=MigrationRelation( + database=_optional_text(values["destination_database"]), + schema=_optional_text(values["destination_schema"]), + name=str(values["destination_name"]), + ), + origin_version_hash=str(values["origin_version_hash"] or ""), + discovery=MigrationDiscovery(str(values["discovery"])), + decision=MigrationDecision(str(values["decision"])), + run_id=str(values["run_id"]), + created_at=created_at, + ) + + +def _event_values(event: MigrationEvent) -> tuple[object | None, ...]: + return ( + event.event_id, + event.target_name, + event.origin_model, + event.origin.database, + event.origin.schema, + event.origin.name, + event.destination_model, + event.destination.database, + event.destination.schema, + event.destination.name, + event.origin_version_hash, + event.discovery.value, + event.decision.value, + event.run_id, + event.created_at, + ) + + +def _optional_text(value: object | None) -> str | None: + return None if value is None else str(value) diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py new file mode 100644 index 0000000000..c08a9dcc3e --- /dev/null +++ b/src/sqlbuild/compiler/migrations/constants.py @@ -0,0 +1,32 @@ +"""Stable constants for append-only model migration state.""" + +from __future__ import annotations + +from sqlbuild.sql_values.types import StateSqlValueType + +MIGRATION_TABLE_NAME: str = "_sqlbuild_migrations" +MIGRATION_WRITE_ATTEMPTS: int = 3 +MIGRATION_WRITE_RETRY_BASE_SECONDS: float = 0.05 + +MIGRATION_COLUMNS: tuple[str, ...] = ( + "event_id", + "target_name", + "origin_model", + "origin_database", + "origin_schema", + "origin_name", + "destination_model", + "destination_database", + "destination_schema", + "destination_name", + "origin_version_hash", + "discovery", + "decision", + "run_id", + "created_at", +) + +MIGRATION_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in MIGRATION_COLUMNS}, + "created_at": StateSqlValueType.TEXT_TIMESTAMP, +} diff --git a/src/sqlbuild/compiler/migrations/exceptions.py b/src/sqlbuild/compiler/migrations/exceptions.py new file mode 100644 index 0000000000..faca42ec60 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/exceptions.py @@ -0,0 +1,15 @@ +"""Expected model migration state exception types.""" + +from __future__ import annotations + + +class MigrationStateError(ValueError): + """Raised when model migration state inputs or rows are invalid.""" + + code: str = "M000" + + def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: + super().__init__(message) + self.message = message + self.code = code if code is not None else self.code + self.help = help diff --git a/src/sqlbuild/compiler/migrations/main/__init__.py b/src/sqlbuild/compiler/migrations/main/__init__.py new file mode 100644 index 0000000000..c0e9893620 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/__init__.py @@ -0,0 +1 @@ +"""Model migration state entry modules.""" diff --git a/src/sqlbuild/compiler/migrations/main/_newest_event.py b/src/sqlbuild/compiler/migrations/main/_newest_event.py new file mode 100644 index 0000000000..888f0d2a9e --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/_newest_event.py @@ -0,0 +1,25 @@ +"""Project the newest migration event that mentions one relation.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation + + +def newest_migration_event_mentioning( + *, events: Iterable[MigrationEvent], relation: MigrationRelation +) -> MigrationEvent | None: + """Return the latest event naming the relation as origin or destination.""" + + newest: MigrationEvent | None = None + event: MigrationEvent + for event in events: + if not event.mentions(relation): + continue + if newest is None or (event.created_at, event.event_id) > ( + newest.created_at, + newest.event_id, + ): + newest = event + return newest diff --git a/src/sqlbuild/compiler/migrations/main/_read_events.py b/src/sqlbuild/compiler/migrations/main/_read_events.py new file mode 100644 index 0000000000..af847b5aff --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/_read_events.py @@ -0,0 +1,33 @@ +"""Read append-only model migration events from one schema.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.compiler.migrations._helpers.sql import build_read_sql, decode_event_row +from sqlbuild.compiler.migrations.exceptions import MigrationStateError +from sqlbuild.compiler.migrations.models import MigrationEvent + + +def read_migration_events( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], +) -> tuple[MigrationEvent, ...]: + """Read every migration event recorded in one schema's state table.""" + + sql: str = build_read_sql( + database=database, schema=schema, render_qualified_name=render_qualified_name + ) + try: + rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() + except Exception as error: + raise MigrationStateError( + f"Unable to read model migration events from schema '{schema}': {error}" + ) from error + return tuple(decode_event_row(tuple(row)) for row in rows) diff --git a/src/sqlbuild/compiler/migrations/main/create_table_sql.py b/src/sqlbuild/compiler/migrations/main/create_table_sql.py new file mode 100644 index 0000000000..c2464b188c --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/create_table_sql.py @@ -0,0 +1,27 @@ +"""Model migration state table SQL entrypoint.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.compiler.migrations._helpers.sql import build_create_table_sql + + +def build_migration_state_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + """Build DDL that creates the model migration event table when it is missing.""" + + return build_create_table_sql( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + render_framework_type=render_framework_type, + transient=transient, + ) diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py new file mode 100644 index 0000000000..7c9ecf2f68 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py @@ -0,0 +1,27 @@ +"""Build stable identifiers for immutable model migration events.""" + +from __future__ import annotations + +import hashlib +import json + +from sqlbuild.compiler.migrations.models import MigrationRelation + + +def deterministic_migration_event_id( + *, + run_id: str, + target_name: str | None, + origin: MigrationRelation, + destination: MigrationRelation, +) -> str: + """Return the content-addressed ID for one migration performed by one run.""" + + identity: tuple[object, ...] = ( + run_id, + target_name, + origin.identity, + destination.identity, + ) + encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/migrations/main/relation_for_location.py b/src/sqlbuild/compiler/migrations/main/relation_for_location.py new file mode 100644 index 0000000000..b298a89606 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/relation_for_location.py @@ -0,0 +1,12 @@ +"""Name compiled relation locations in migration events.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.migrations.models import MigrationRelation + + +def migration_relation_for_location(location: CompiledRelationLocation) -> MigrationRelation: + """Return the migration relation that names one compiled relation location.""" + + return MigrationRelation(database=location.database, schema=location.schema, name=location.name) diff --git a/src/sqlbuild/compiler/migrations/main/write_event.py b/src/sqlbuild/compiler/migrations/main/write_event.py new file mode 100644 index 0000000000..fd0c04edb3 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/write_event.py @@ -0,0 +1,52 @@ +"""Append one model migration event idempotently.""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.compiler.migrations._helpers.sql import build_existing_event_sql, build_insert_sql +from sqlbuild.compiler.migrations.constants import ( + MIGRATION_WRITE_ATTEMPTS, + MIGRATION_WRITE_RETRY_BASE_SECONDS, +) +from sqlbuild.compiler.migrations.models import MigrationEvent + + +def write_migration_event( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + event: MigrationEvent, + render_qualified_name: Callable[..., str | None], + create_table_sql: str | None, + attempts: int = MIGRATION_WRITE_ATTEMPTS, +) -> None: + """Create the state table from adapter DDL when given, then insert the event if absent.""" + + existing_sql: str = build_existing_event_sql( + event=event, render_qualified_name=render_qualified_name + ) + insert_sql: str = build_insert_sql(event=event, render_qualified_name=render_qualified_name) + attempt: int + for attempt in range(attempts): + try: + if create_table_sql is not None: + _ = execute(connection=connection, sql=create_table_sql) + if not execute(connection=connection, sql=existing_sql).fetchall(): + _ = execute(connection=connection, sql=insert_sql) + return + except Exception as error: + if attempt + 1 == attempts: + raise + logging.getLogger("sqlbuild.migrations").warning( + "model migration event write attempt %s/%s failed for '%s'; retrying: %s", + attempt + 1, + attempts, + event.destination_model, + error, + ) + time.sleep(MIGRATION_WRITE_RETRY_BASE_SECONDS * (attempt + 1)) diff --git a/src/sqlbuild/compiler/migrations/models.py b/src/sqlbuild/compiler/migrations/models.py new file mode 100644 index 0000000000..cc83d5a4e3 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/models.py @@ -0,0 +1,58 @@ +"""Model migration state domain models.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery + + +@dataclass(frozen=True) +class MigrationRelation: + """One physical relation named by a migration event.""" + + database: str | None + schema: str | None + name: str + + @property + def identity(self) -> tuple[str | None, str, str]: + """Return the case-insensitive relation identity.""" + + return ( + None if self.database is None else self.database.lower(), + "" if self.schema is None else self.schema.lower(), + self.name.lower(), + ) + + def matches(self, other: MigrationRelation) -> bool: + """Compare relations, treating an unknown database as a wildcard.""" + + left: tuple[str | None, str, str] = self.identity + right: tuple[str | None, str, str] = other.identity + if left[1:] != right[1:]: + return False + return left[0] is None or right[0] is None or left[0] == right[0] + + +@dataclass(frozen=True) +class MigrationEvent: + """One immutable fact that a origin relation was cloned over a destination.""" + + event_id: str + target_name: str | None + origin_model: str | None + origin: MigrationRelation + destination_model: str + destination: MigrationRelation + origin_version_hash: str + discovery: MigrationDiscovery + decision: MigrationDecision + run_id: str + created_at: datetime + + def mentions(self, relation: MigrationRelation) -> bool: + """Return whether this event names the relation as origin or destination.""" + + return self.origin.matches(relation) or self.destination.matches(relation) diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py new file mode 100644 index 0000000000..6fc0791987 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/types.py @@ -0,0 +1,84 @@ +"""Stable model migration value types.""" + +from __future__ import annotations + +from enum import StrEnum + + +class MigrationDecision(StrEnum): + """Per-run decision for one declared or discovered model migration.""" + + DONE = "done" + MIGRATE = "migrate" + REDO = "redo" + SUPERSEDED_REPLACE = "superseded_replace" + FORCED_REPLACE = "forced_replace" + CONFLICT = "conflict" + ORIGIN_MISSING = "origin_missing" + RENAMED = "renamed" + + @property + def moves_data(self) -> bool: + """Return whether this decision clones the origin over the destination.""" + + return self in _MOVING_DECISIONS + + @property + def blocks_build(self) -> bool: + """Return whether this decision stops a build regardless of compatibility.""" + + return self in _BLOCKING_DECISIONS + + @property + def checks_compatibility(self) -> bool: + """Return whether this decision evaluated the origin against the destination.""" + + return self.moves_data or self == MigrationDecision.CONFLICT + + @property + def label(self) -> str: + """Return the human-readable plan label for this decision.""" + + return self.value.replace("_", " ") + + +_MOVING_DECISIONS: frozenset[MigrationDecision] = frozenset( + { + MigrationDecision.MIGRATE, + MigrationDecision.REDO, + MigrationDecision.SUPERSEDED_REPLACE, + MigrationDecision.FORCED_REPLACE, + } +) +_BLOCKING_DECISIONS: frozenset[MigrationDecision] = frozenset( + {MigrationDecision.CONFLICT, MigrationDecision.ORIGIN_MISSING} +) + + +class MigrationDiscovery(StrEnum): + """How a migration was requested.""" + + MANUAL = "manual" + AUTOMATIC = "automatic" + + +class MigrationPromotion(StrEnum): + """How a completed migration stage replaces or creates the destination.""" + + SWAP = "swap" + RENAME = "rename" + TRANSACTIONAL_RENAME = "transactional_rename" + + @property + def label(self) -> str: + """Return the human-readable plan label for this promotion.""" + + return self.value.replace("_", " ") + + +class MigrationCompatibility(StrEnum): + """Outcome of checking the origin relation against the destination model.""" + + COMPATIBLE = "compatible" + INCOMPATIBLE = "incompatible" + NOT_CHECKED = "not_checked" diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 57cda287d5..8554b8ca5c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -183,6 +183,14 @@ def detect_model_changes( fingerprint_metadata_json=metadata_json, fingerprint_version_hash=expected_version_hash, ) + if not relation_exists and model_name in snapshot.renamed_models: + return ChangeDetectionResult( + model_name=model_name, + change_kind=ChangeKind.RENAMED, + backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + fingerprint_metadata_json=metadata_json, + fingerprint_version_hash=expected_version_hash, + ) query_changed: bool = False config_changed: bool = ( diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py index 216292d61f..f6952a740a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py @@ -5,6 +5,7 @@ import json from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY +from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY def version_identity_metadata_payload(metadata_json: str | None) -> object: @@ -20,4 +21,5 @@ def version_identity_metadata_payload(metadata_json: str | None) -> object: return None identity_payload: dict[str, object] = dict(payload) identity_payload.pop(AUDIT_GATE_METADATA_KEY, None) + identity_payload.pop(MIGRATION_FINGERPRINT_METADATA_KEY, None) return identity_payload diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py b/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py new file mode 100644 index 0000000000..ccf1deae87 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py @@ -0,0 +1,175 @@ +"""Compatibility of a migration origin relation with its destination model.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import ColumnInfo, NormalizedType, RelationInfo +from sqlbuild.adapter.contract.types import TypeFamily +from sqlbuild.adapter.type_system.main.normalize_type import normalize_type +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.migrations.types import MigrationCompatibility +from sqlbuild.compiler.planner._helpers.changes.detect import detect_model_changes +from sqlbuild.compiler.planner.constants import VIEW_RELATION_TYPE_MARKER +from sqlbuild.compiler.planner.models import ( + ChangeDetectionResult, + MigrationCompatibilityResult, + SchemaFinding, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import ( + MaterializationType, + OnSchemaChange, + SchemaChangeKind, + SnapshotSchemaChangePolicy, +) +from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.models import SnapshotsConfig + +_DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS +_NUMERIC_FAMILIES: frozenset[TypeFamily] = frozenset( + {TypeFamily.INTEGER, TypeFamily.DECIMAL, TypeFamily.FLOAT} +) +_TEMPORAL_FAMILIES: frozenset[TypeFamily] = frozenset( + {TypeFamily.TIMESTAMP, TypeFamily.DATE, TypeFamily.DATETIME} +) + + +def check_migration_compatibility( + *, + model: CompiledModel, + origin_relation: RelationInfo, + origin_columns: tuple[ColumnInfo, ...], + sql_analysis_enabled: bool, + column_dialect: str | None, + snapshots_config: SnapshotsConfig, +) -> MigrationCompatibilityResult: + """Check the origin exactly as a normal build would check an existing destination.""" + + if VIEW_RELATION_TYPE_MARKER in origin_relation.relation_type.upper(): + return MigrationCompatibilityResult( + status=MigrationCompatibility.INCOMPATIBLE, + findings=("origin relation is a view; only tables carry migratable history",), + ) + change: ChangeDetectionResult = detect_model_changes( + model=model, + snapshot=WarehouseSnapshot( + existing_relations={model.name: origin_relation}, + existing_columns={model.name: origin_columns}, + column_dialect=column_dialect, + ), + sql_analysis_enabled=sql_analysis_enabled, + query_change_tracking=False, + full_refresh=False, + ) + if not _has_expected_columns(model=model): + return MigrationCompatibilityResult(status=MigrationCompatibility.NOT_CHECKED) + materialized: str | None = get_config_str(values=model.config.values, key="materialized") + blocking: tuple[str, ...] = ( + _snapshot_blocking_findings( + model=model, + findings=change.schema_findings, + dialect=column_dialect, + snapshots_config=snapshots_config, + ) + if materialized == MaterializationType.SNAPSHOT + else _incremental_blocking_findings( + model=model, findings=change.schema_findings, dialect=column_dialect + ) + ) + if blocking: + return MigrationCompatibilityResult( + status=MigrationCompatibility.INCOMPATIBLE, findings=blocking + ) + return MigrationCompatibilityResult(status=MigrationCompatibility.COMPATIBLE) + + +def _has_expected_columns(*, model: CompiledModel) -> bool: + if model.schema_entry is not None and any( + column.type is not None for column in model.schema_entry.columns + ): + return True + return bool(model.inferred_columns) + + +def _incremental_blocking_findings( + *, model: CompiledModel, findings: tuple[SchemaFinding, ...], dialect: str | None +) -> tuple[str, ...]: + raw_policy: str | None = get_config_str(values=model.config.values, key="on_schema_change") + policy: OnSchemaChange = OnSchemaChange(raw_policy) if raw_policy else _DEFAULT_ON_SCHEMA_CHANGE + blocking: list[str] = [] + finding: SchemaFinding + for finding in findings: + if policy == OnSchemaChange.FAIL: + blocking.append(f"{_describe(finding)} (on_schema_change fail)") + continue + if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED and not _types_convertible( + expected=finding.expected_type, actual=finding.actual_type, dialect=dialect + ): + blocking.append(f"{_describe(finding)} (incompatible types)") + return tuple(blocking) + + +def _snapshot_blocking_findings( + *, + model: CompiledModel, + findings: tuple[SchemaFinding, ...], + dialect: str | None, + snapshots_config: SnapshotsConfig, +) -> tuple[str, ...]: + raw_policy: str | None = get_config_str( + values=model.config.values, key="snapshot_schema_change" + ) + policy: SnapshotSchemaChangePolicy = SnapshotSchemaChangePolicy( + raw_policy or snapshots_config.schema_change + ) + blocking: list[str] = [] + finding: SchemaFinding + for finding in findings: + if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED and not _string_widening( + expected=finding.expected_type, actual=finding.actual_type, dialect=dialect + ): + blocking.append(f"{_describe(finding)} (snapshot type change)") + elif ( + finding.kind == SchemaChangeKind.COLUMN_ADDED + and policy == SnapshotSchemaChangePolicy.DENY + ): + blocking.append(f"{_describe(finding)} (snapshot_schema_change deny)") + return tuple(blocking) + + +def _types_convertible(*, expected: str | None, actual: str | None, dialect: str | None) -> bool: + if expected is None or actual is None: + return True + expected_family: TypeFamily = normalize_type(type_sql=expected, dialect=dialect).family + actual_family: TypeFamily = normalize_type(type_sql=actual, dialect=dialect).family + if TypeFamily.OTHER in {expected_family, actual_family}: + return True + if expected_family == actual_family: + return True + if {expected_family, actual_family} <= _NUMERIC_FAMILIES: + return True + return {expected_family, actual_family} <= _TEMPORAL_FAMILIES + + +def _string_widening(*, expected: str | None, actual: str | None, dialect: str | None) -> bool: + if expected is None or actual is None: + return True + target: NormalizedType = normalize_type(type_sql=actual, dialect=dialect) + incoming: NormalizedType = normalize_type(type_sql=expected, dialect=dialect) + if target == incoming: + return True + if target.family != TypeFamily.STRING or incoming.family != TypeFamily.STRING: + return False + if target.length is None: + return True + return incoming.length is not None and incoming.length <= target.length + + +def _describe(finding: SchemaFinding) -> str: + if finding.kind == SchemaChangeKind.COLUMN_ADDED: + return f"column {finding.column_name} is missing from the origin" + if finding.kind == SchemaChangeKind.COLUMN_REMOVED: + return f"origin column {finding.column_name} is not produced by the model" + return ( + f"column {finding.column_name} is {finding.actual_type} in the origin but the model " + f"produces {finding.expected_type}" + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py new file mode 100644 index 0000000000..8b69f97752 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -0,0 +1,589 @@ +"""Fingerprint-based discovery of renamed models within the project's schemas.""" + +from __future__ import annotations + +import json +from collections import defaultdict +from typing import Any + +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledObjectKey, + CompiledRelationLocation, +) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning +from sqlbuild.compiler.migrations.main.relation_for_location import ( + migration_relation_for_location, +) +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint +from sqlbuild.compiler.planner.classes.migration_state_inspection import ( + MigrationStateInspection, +) +from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY +from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( + build_function_local_hashes, +) +from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( + build_model_version_identity_metadata_json, +) +from sqlbuild.compiler.planner.models import ( + ModelMigrationDeclaration, + ModelMigrationDiscovery, + ModelMigrationRequest, + PlannerRuntime, + PlannerScope, + PlanWarning, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity +from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities +from sqlbuild.spec.contracts.main.get_config_str import get_config_str + +_HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( + {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} +) +_IDENTITY_MATERIALIZATIONS: frozenset[str] = frozenset( + {MaterializationType.TABLE, MaterializationType.VIEW} +) +_CONFIG_KEY: str = "config" +_MATERIALIZED_KEY: str = "materialized" + + +def discover_model_migrations( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + manual_requests: tuple[ModelMigrationRequest, ...], + declarations: tuple[ModelMigrationDeclaration, ...], + state: MigrationStateInspection, + project_schemas: set[str], +) -> ModelMigrationDiscovery: + """Add unique one-to-one fingerprint matches to the explicit migration requests.""" + + models: tuple[CompiledModel, ...] = _topological_models(runtime=runtime) + declared: frozenset[str] = frozenset(item.model_name for item in declarations) + triggered: bool = bool( + _new_model_names(models=models, scope=scope, snapshot=snapshot) - declared + ) + if not triggered and not manual_requests: + return ModelMigrationDiscovery() + metadata_jsons: dict[str, str] = _metadata_jsons(runtime=runtime, models=models) + dialect: str | None = runtime.adapter.sql_analysis_dialect() + state.inspect_schemas(schemas=project_schemas) + declared_renames: dict[str, str] = _declared_renames(declarations=declarations, state=state) + event_renames: dict[str, str] = _event_renames(runtime=runtime, models=models, state=state) + seed_variants: tuple[dict[str, str], ...] = ( + {**event_renames, **declared_renames}, + dict(declared_renames), + ) + if not triggered: + return ModelMigrationDiscovery( + requests=manual_requests, + destination_fingerprints=_destination_fingerprints( + requests=manual_requests, + variants=seed_variants, + metadata_jsons=metadata_jsons, + dialect=dialect, + ), + ) + unbuilt: frozenset[str] = frozenset( + model.name + for model in models + if model.name not in declared and state.named_fingerprint(model_name=model.name) is None + ) + resumed: dict[str, MigrationEvent] = _resumed_events( + runtime=runtime, models=models, unbuilt=unbuilt, state=state + ) + candidates: tuple[Fingerprint, ...] = _origin_candidates( + runtime=runtime, + state=state, + models=models, + excluded=_claimed_origins(declarations=declarations, resumed=resumed), + ) + variants: tuple[dict[str, str], ...] + matches: dict[str, Fingerprint] + ambiguous: dict[str, str] + variants, matches, ambiguous = _match( + models=models, + new_names=unbuilt - frozenset(resumed), + seed_variants=seed_variants, + candidates=candidates, + metadata_jsons=metadata_jsons, + dialect=dialect, + ) + requests: tuple[ModelMigrationRequest, ...] = ( + *manual_requests, + *_automatic_requests( + runtime=runtime, scope=scope, models=models, matches=matches, resumed=resumed + ), + ) + return ModelMigrationDiscovery( + requests=requests, + warnings=_ambiguity_warnings(models=models, scope=scope, ambiguous=ambiguous), + destination_fingerprints=_destination_fingerprints( + requests=requests, variants=variants, metadata_jsons=metadata_jsons, dialect=dialect + ), + ) + + +def _declared_renames( + *, declarations: tuple[ModelMigrationDeclaration, ...], state: MigrationStateInspection +) -> dict[str, str]: + """Map each explicitly migrated model to the model name its origin was built as.""" + + renames: dict[str, str] = {} + declaration: ModelMigrationDeclaration + for declaration in declarations: + origin_fingerprint: Fingerprint | None = state.model_fingerprint( + location=declaration.origin_location, model_name=declaration.origin_model + ) + origin_model: str | None = declaration.origin_model or ( + origin_fingerprint.node_name if origin_fingerprint is not None else None + ) + if origin_model is not None: + renames[declaration.model_name] = origin_model + return renames + + +def _event_renames( + *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...], state: MigrationStateInspection +) -> dict[str, str]: + """Map project models to the removed model their latest recorded move came from.""" + + project_names: frozenset[str] = frozenset(model.name for model in models) + target_name: str | None = runtime.project.effective_target_name + latest: dict[str, MigrationEvent] = {} + event: MigrationEvent + for event in sorted(state.events, key=lambda item: (item.created_at, item.event_id)): + if event.target_name in (None, target_name): + latest[event.destination_model] = event + return { + name: event.origin_model + for name, event in latest.items() + if name in project_names + and event.origin_model is not None + and event.origin_model not in project_names + } + + +def _resumed_events( + *, + runtime: PlannerRuntime, + models: tuple[CompiledModel, ...], + unbuilt: frozenset[str], + state: MigrationStateInspection, +) -> dict[str, MigrationEvent]: + """Return recorded moves or renames into models that have not been built since.""" + + target_name: str | None = runtime.project.effective_target_name + resumed: dict[str, MigrationEvent] = {} + model: CompiledModel + for model in models: + if model.name not in unbuilt or not (_stores_history(model) or _hands_over_identity(model)): + continue + destination: MigrationRelation = migration_relation_for_location(model.destination) + newest: MigrationEvent | None = newest_migration_event_mentioning( + events=state.events, relation=destination + ) + if ( + newest is not None + and newest.destination.matches(destination) + and newest.target_name in (None, target_name) + and (newest.decision == MigrationDecision.RENAMED) is _hands_over_identity(model) + ): + resumed[model.name] = newest + return resumed + + +def _claimed_origins( + *, + declarations: tuple[ModelMigrationDeclaration, ...], + resumed: dict[str, MigrationEvent], +) -> frozenset[tuple[str, str]]: + """Return relation keys already claimed by explicit or recorded migrations.""" + + declared: set[tuple[str, str]] = { + _relation_key(schema=item.origin_location.schema, name=item.origin_location.name) + for item in declarations + } + recorded: set[tuple[str, str]] = { + _relation_key(schema=event.origin.schema, name=event.origin.name) + for event in resumed.values() + } + return frozenset(declared | recorded) + + +def _relation_key(*, schema: str | None, name: str) -> tuple[str, str]: + return ((schema or "").lower(), name.lower()) + + +def _automatic_requests( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + models: tuple[CompiledModel, ...], + matches: dict[str, Fingerprint], + resumed: dict[str, MigrationEvent], +) -> tuple[ModelMigrationRequest, ...]: + """Return automatic requests for selected destinations only.""" + + requests: list[ModelMigrationRequest] = [] + model: CompiledModel + for model in models: + if model.key not in scope.selected_keys: + continue + event: MigrationEvent | None = resumed.get(model.name) + match: Fingerprint | None = matches.get(model.name) + if event is not None: + requests.append( + ModelMigrationRequest( + model=model, + discovery=MigrationDiscovery.AUTOMATIC, + origin_location=_event_origin_location(runtime=runtime, event=event), + origin_model=event.origin_model, + identity_only=_hands_over_identity(model), + ) + ) + elif match is not None and ( + _moves_history(model=model, origin=match) + or _renames_identity(model=model, origin=match) + ): + requests.append( + ModelMigrationRequest( + model=model, + discovery=MigrationDiscovery.AUTOMATIC, + origin_location=_fingerprint_location(runtime=runtime, fingerprint=match), + origin_model=match.node_name, + identity_only=_hands_over_identity(model), + ) + ) + return tuple(requests) + + +def _event_origin_location( + *, runtime: PlannerRuntime, event: MigrationEvent +) -> CompiledRelationLocation: + return CompiledRelationLocation( + database=event.origin.database, + schema=event.origin.schema, + name=event.origin.name, + qualified_name=runtime.adapter.render_qualified_name( + database=event.origin.database, schema=event.origin.schema, name=event.origin.name + ), + ) + + +def _ambiguity_warnings( + *, models: tuple[CompiledModel, ...], scope: PlannerScope, ambiguous: dict[str, str] +) -> tuple[PlanWarning, ...]: + selected: frozenset[str] = frozenset( + model.name for model in models if model.key in scope.selected_keys + ) + return tuple( + PlanWarning( + model_name=model_name, + severity=WarningSeverity.WARNING, + message=( + f"'{model_name}' matches {detail}; no automatic migration was inferred. " + "Add migrate_from to choose one." + ), + code="M107", + ) + for model_name, detail in sorted(ambiguous.items()) + if model_name in selected + ) + + +def _topological_models(*, runtime: PlannerRuntime) -> tuple[CompiledModel, ...]: + by_name: dict[str, CompiledModel] = {model.name: model for model in runtime.project.models} + ordered: list[CompiledModel] = [] + visited: set[str] = set() + pending: list[tuple[str, bool]] = [(name, False) for name in sorted(by_name, reverse=True)] + while pending: + name: str + expanded: bool + name, expanded = pending.pop() + if expanded: + ordered.append(by_name[name]) + continue + if name in visited: + continue + visited.add(name) + pending.append((name, True)) + pending.extend( + (dep.name, False) + for dep in sorted(by_name[name].deps, key=lambda key: key.name, reverse=True) + if dep.resource_type == CompiledResourceType.MODEL + and dep.name in by_name + and dep.name not in visited + ) + return tuple(ordered) + + +def _new_model_names( + *, models: tuple[CompiledModel, ...], scope: PlannerScope, snapshot: WarehouseSnapshot +) -> frozenset[str]: + relevant: set[CompiledObjectKey] = set(scope.selected_keys) + frontier: list[CompiledObjectKey] = list(scope.selected_keys) + while frontier: + key: CompiledObjectKey = frontier.pop() + upstream: CompiledObjectKey + for upstream in scope.upstream_deps.get(key, ()): + if upstream not in relevant: + relevant.add(upstream) + frontier.append(upstream) + return frozenset( + model.name + for model in models + if model.key in relevant and model.name not in snapshot.fingerprints.models + ) + + +def _metadata_jsons( + *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] +) -> dict[str, str]: + function_hashes: dict[str, str] = build_function_local_hashes( + functions=runtime.project.functions + ) + hook_hashes: dict[str, str] = { + name: identity.version_hash + for name, identity in build_hook_identities(runtime.project.hook_functions).items() + } + return { + model.name: build_model_version_identity_metadata_json( + model=model, function_local_hashes=function_hashes, hook_version_hashes=hook_hashes + ) + for model in models + } + + +def _origin_candidates( + *, + runtime: PlannerRuntime, + state: MigrationStateInspection, + models: tuple[CompiledModel, ...], + excluded: frozenset[tuple[str, str]], +) -> tuple[Fingerprint, ...]: + project_model_names: frozenset[str] = frozenset(model.name for model in models) + orphans: tuple[Fingerprint, ...] = tuple( + fingerprint + for fingerprint in state.orphan_fingerprints(project_model_names=project_model_names) + if stored_migration_fingerprint(fingerprint) is not None + and fingerprint.target_name is not None + ) + locations: tuple[CompiledRelationLocation, ...] = tuple( + _fingerprint_location(runtime=runtime, fingerprint=fingerprint) for fingerprint in orphans + ) + state.inspect_relations(locations=locations) + return tuple( + fingerprint + for fingerprint, location in zip(orphans, locations, strict=True) + if state.relation(location) is not None + and _relation_key(schema=location.schema, name=location.name) not in excluded + and not _superseded(events=state.events, location=location) + ) + + +def _superseded(*, events: tuple[MigrationEvent, ...], location: CompiledRelationLocation) -> bool: + relation: MigrationRelation = migration_relation_for_location(location) + newest: MigrationEvent | None = newest_migration_event_mentioning( + events=events, relation=relation + ) + return newest is not None and newest.origin.matches(relation) + + +def _match( + *, + models: tuple[CompiledModel, ...], + new_names: frozenset[str], + seed_variants: tuple[dict[str, str], ...], + candidates: tuple[Fingerprint, ...], + metadata_jsons: dict[str, str], + dialect: str | None, +) -> tuple[tuple[dict[str, str], ...], dict[str, Fingerprint], dict[str, str]]: + """Match unbuilt models to candidates under every rename-map variant.""" + + by_fingerprint: dict[str, list[Fingerprint]] = defaultdict(list) + candidate: Fingerprint + for candidate in candidates: + by_fingerprint[stored_migration_fingerprint(candidate) or ""].append(candidate) + blocked: set[str] = set() + ambiguous: dict[str, str] = {} + for _ in range(len(candidates) + 1): + variants: tuple[dict[str, str], ...] = tuple(dict(seed) for seed in seed_variants) + matches: dict[str, Fingerprint] = {} + claims: dict[str, list[str]] = defaultdict(list) + model: CompiledModel + for model in models: + if model.name not in new_names: + continue + options: list[Fingerprint] = _match_options( + fingerprints=_variant_fingerprints( + model=model, + metadata_json=metadata_jsons[model.name], + variants=variants, + dialect=dialect, + ), + by_fingerprint=by_fingerprint, + blocked=blocked, + ) + if len(options) > 1: + ambiguous[model.name] = ( + "several earlier models " + f"({', '.join(sorted(option.node_name for option in options))})" + ) + continue + if len(options) == 1: + variant: dict[str, str] + for variant in variants: + variant[model.name] = options[0].node_name + matches[model.name] = options[0] + claims[options[0].node_name].append(model.name) + contested: dict[str, list[str]] = { + origin: names for origin, names in claims.items() if len(names) > 1 + } + if not contested: + return variants, matches, ambiguous + blocked.update(contested) + ambiguous.update(_contested_details(contested)) + return tuple(dict(seed) for seed in seed_variants), {}, ambiguous + + +def _variant_fingerprints( + *, + model: CompiledModel, + metadata_json: str, + variants: tuple[dict[str, str], ...], + dialect: str | None, +) -> tuple[str, ...]: + """Return the distinct migration fingerprints of one model under each rename map.""" + + fingerprints: list[str | None] = [ + build_migration_fingerprint( + query_sql=model.query_sql, + metadata_json=metadata_json, + ref_identities=variant, + dialect=dialect, + ) + for variant in variants + ] + return tuple(dict.fromkeys(item for item in fingerprints if item is not None)) + + +def _match_options( + *, + fingerprints: tuple[str, ...], + by_fingerprint: dict[str, list[Fingerprint]], + blocked: set[str], +) -> list[Fingerprint]: + options: dict[str, Fingerprint] = {} + fingerprint: str + for fingerprint in fingerprints: + options.update( + { + option.node_name: option + for option in by_fingerprint.get(fingerprint, []) + if option.node_name not in blocked + } + ) + return list(options.values()) + + +def _contested_details(contested: dict[str, list[str]]) -> dict[str, str]: + """Describe each model that shares its only matching earlier model with others.""" + + details: dict[str, str] = {} + origin: str + names: list[str] + for origin, names in contested.items(): + name: str + for name in names: + others: list[str] = [f"'{other}'" for other in sorted(names) if other != name] + details[name] = f"earlier model '{origin}', which also matches {', '.join(others)}" + return details + + +def _destination_fingerprints( + *, + requests: tuple[ModelMigrationRequest, ...], + variants: tuple[dict[str, str], ...], + metadata_jsons: dict[str, str], + dialect: str | None, +) -> dict[str, tuple[str, ...]]: + fingerprints: dict[str, tuple[str, ...]] = {} + request: ModelMigrationRequest + for request in requests: + metadata_json: str | None = metadata_jsons.get(request.model.name) + if metadata_json is None: + continue + fingerprints[request.model.name] = _variant_fingerprints( + model=request.model, metadata_json=metadata_json, variants=variants, dialect=dialect + ) + return fingerprints + + +def _moves_history(*, model: CompiledModel, origin: Fingerprint) -> bool: + return _stores_history(model) and _stored_materialization(origin) in _HISTORY_MATERIALIZATIONS + + +def _renames_identity(*, model: CompiledModel, origin: Fingerprint) -> bool: + return ( + _hands_over_identity(model) + and _stored_materialization(origin) in _IDENTITY_MATERIALIZATIONS + ) + + +def _hands_over_identity(model: CompiledModel) -> bool: + return ( + get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) + in _IDENTITY_MATERIALIZATIONS + ) + + +def _stores_history(model: CompiledModel) -> bool: + return ( + get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) + in _HISTORY_MATERIALIZATIONS + ) + + +def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: + """Return the migration fingerprint recorded in a model fingerprint row, if any.""" + + payload: Any = _metadata_payload(fingerprint) + value: Any = payload.get(MIGRATION_FINGERPRINT_METADATA_KEY) if payload else None + return value if isinstance(value, str) else None + + +def _stored_materialization(fingerprint: Fingerprint) -> str | None: + payload: Any = _metadata_payload(fingerprint) + config: Any = payload.get(_CONFIG_KEY) if payload else None + value: Any = config.get(_MATERIALIZED_KEY) if isinstance(config, dict) else None + return value if isinstance(value, str) else None + + +def _metadata_payload(fingerprint: Fingerprint) -> dict[str, Any] | None: + try: + payload: Any = json.loads(fingerprint.metadata_json) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def _fingerprint_location( + *, runtime: PlannerRuntime, fingerprint: Fingerprint +) -> CompiledRelationLocation: + name: str = fingerprint.target_name or fingerprint.node_name + return CompiledRelationLocation( + database=fingerprint.target_database, + schema=fingerprint.target_schema, + name=name, + qualified_name=runtime.adapter.render_qualified_name( + database=fingerprint.target_database, schema=fingerprint.target_schema, name=name + ), + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py new file mode 100644 index 0000000000..423a009235 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py @@ -0,0 +1,343 @@ +"""Local migration fingerprints that identify the same table across renames.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from dataclasses import replace +from typing import Any + +from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.planner._helpers.changes.metadata import version_identity_metadata_payload +from sqlbuild.compiler.planner.constants import ( + MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS, + MIGRATION_LOCAL_NAME_PREFIX, + MIGRATION_MODEL_NAME_METADATA_KEY, + MIGRATION_REF_PLACEHOLDER_PREFIX, +) +from sqlbuild.compiler.planner.models import ModelPlanEntry +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql + +_REF_MARKER: re.Pattern[str] = re.compile(r"""__ref\(\s*(?:"([^"]+)"|'([^']+)')\s*\)""") +_IDENTIFIER_NAME_KEY: str = "name" +_IDENTIFIER_QUOTED_KEY: str = "quoted" +_ALIAS_KEY: str = "alias" +_WITH_KEY: str = "with" +_CTES_KEY: str = "ctes" +_SPAN_KEY: str = "span" +_COMMENTS_SUFFIX: str = "comments" +_CONFIG_KEY: str = "config" +_GENERIC_DIALECT: str = "generic" +_TABLE_KEY: str = "table" +_RECURSIVE_KEY: str = "recursive" +_CTE_BODY_KEY: str = "this" +_ALIASED_RELATION_KEYS: frozenset[str] = frozenset({_TABLE_KEY, "subquery"}) +_QUALIFIED_REFERENCE_KEYS: frozenset[str] = frozenset({"column", "star"}) +_RELATION_QUALIFIER_KEYS: tuple[str, ...] = ("schema", "catalog") + + +def build_migration_fingerprint( + *, + query_sql: str, + metadata_json: str, + ref_identities: Mapping[str, str], + dialect: str | None, +) -> str | None: + """Hash normalized SQL and non-storage metadata with refs resolved through renames.""" + + normalized_sql: str | None = _normalized_query_sql( + query_sql=query_sql, ref_identities=ref_identities, dialect=dialect + ) + if normalized_sql is None: + return None + payload: str = json.dumps( + { + "metadata": _identity_metadata( + metadata_json=metadata_json, ref_identities=ref_identities + ), + "sql": normalized_sql, + }, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _identity_metadata(*, metadata_json: str, ref_identities: Mapping[str, str]) -> Any: + payload: Any = version_identity_metadata_payload(metadata_json) + if not isinstance(payload, dict): + return payload + identity: dict[str, Any] = {str(key): value for key, value in payload.items()} + _ = identity.pop(MIGRATION_MODEL_NAME_METADATA_KEY, None) + config: Any = identity.get(_CONFIG_KEY) + if isinstance(config, dict): + identity[_CONFIG_KEY] = { + key: _resolved_cursor_inputs(value=value, ref_identities=ref_identities) + if key == CURSOR_INPUTS_CONFIG_KEY + else value + for key, value in config.items() + if key not in MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS + } + return identity + + +def _resolved_cursor_inputs(*, value: Any, ref_identities: Mapping[str, str]) -> Any: + if not isinstance(value, dict): + return value + return {ref_identities.get(str(name), str(name)): column for name, column in value.items()} + + +def _normalized_query_sql( + *, query_sql: str, ref_identities: Mapping[str, str], dialect: str | None +) -> str | None: + placeholder_sql: str = _REF_MARKER.sub( + lambda match: _ref_placeholder( + identity=ref_identities.get( + match.group(1) or match.group(2), match.group(1) or match.group(2) + ) + ), + query_sql, + ) + polyglot: Any = import_polyglot_sql() + effective_dialect: str = dialect or _GENERIC_DIALECT + try: + parsed: dict[str, Any] = polyglot.parse_one( + placeholder_sql, dialect=effective_dialect + ).to_dict() + except polyglot.PolyglotError: + return None + stripped: Any = _strip_formatting(parsed) + if not _has_unambiguous_local_scopes(stripped): + return None + definitions: list[tuple[str, bool]] = _local_names(stripped) + ordered_names: list[str] = list(dict.fromkeys(name for name, _ in definitions)) + local_names: dict[str, str] = { + name: f"{MIGRATION_LOCAL_NAME_PREFIX}{index}" + for index, name in enumerate(ordered_names, start=1) + } + cte_names: dict[str, str] = {name: local_names[name] for name, is_cte in definitions if is_cte} + canonical: Any = _canonical_local_names(node=stripped, names=local_names, cte_names=cte_names) + generated: list[str] = polyglot.generate(canonical, dialect=effective_dialect) + return "\n".join(generated) + + +def _ref_placeholder(*, identity: str) -> str: + digest: str = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] + return f"{MIGRATION_REF_PLACEHOLDER_PREFIX}{digest}" + + +def _strip_formatting(node: Any) -> Any: + if isinstance(node, dict): + return { + key: [] if key.endswith(_COMMENTS_SUFFIX) else _strip_formatting(value) + for key, value in node.items() + if key != _SPAN_KEY + } + if isinstance(node, list): + return [_strip_formatting(value) for value in node] + return node + + +def _has_unambiguous_local_scopes(root: Any) -> bool: + """Accept one top-level WITH whose names never shadow physical relations.""" + + statements: list[Any] = list(root.values()) if isinstance(root, dict) else [] + top_level: Any = statements[0].get(_WITH_KEY) if len(statements) == 1 else None + clauses: list[Any] = _with_clauses(root) + if clauses and (not isinstance(top_level, dict) or len(clauses) > 1): + return False + if isinstance(top_level, dict) and top_level.get(_RECURSIVE_KEY): + return False + ctes: list[Any] = [ + cte for cte in (top_level or {}).get(_CTES_KEY) or () if isinstance(cte, dict) + ] + cte_names: list[str] = _identifier_names([cte.get(_ALIAS_KEY) for cte in ctes]) + if len(cte_names) != len(ctes) or len(set(cte_names)) != len(cte_names): + return False + index: int + cte: Any + for index, cte in enumerate(ctes): + if set(_relation_names(cte.get(_CTE_BODY_KEY))) & set(cte_names[index:]): + return False + physical: set[str] = set(_relation_names(root)) - set(cte_names) + aliases: set[str] = {name for name, is_cte in _local_names(root) if not is_cte} + return not physical & aliases + + +def _with_clauses(node: Any) -> list[Any]: + found: list[Any] = [] + if isinstance(node, list): + item: Any + for item in node: + found.extend(_with_clauses(item)) + return found + if not isinstance(node, dict): + return found + key: str + value: Any + for key, value in node.items(): + if key == _WITH_KEY and isinstance(value, dict): + found.append(value) + found.extend(_with_clauses(value)) + return found + + +def _relation_names(node: Any) -> list[str]: + """Return lower-cased unqualified relation names referenced under a node.""" + + found: list[str] = [] + if isinstance(node, list): + item: Any + for item in node: + found.extend(_relation_names(item)) + return found + if not isinstance(node, dict): + return found + key: str + value: Any + for key, value in node.items(): + if key == _TABLE_KEY and _is_relation(value) and _unqualified(value): + found.extend(_identifier_names([value.get(_IDENTIFIER_NAME_KEY)])) + found.extend(_relation_names(value)) + return found + + +def _unqualified(relation: dict[str, Any]) -> bool: + return all(relation.get(key) is None for key in _RELATION_QUALIFIER_KEYS) + + +def _local_names(node: Any) -> list[tuple[str, bool]]: + """Return (name, is_cte) for CTE and relation alias definitions in first-seen order.""" + + found: list[tuple[str, bool]] = [] + if isinstance(node, list): + item: Any + for item in node: + found.extend(_local_names(item)) + return found + if not isinstance(node, dict): + return found + with_clause: Any = node.get(_WITH_KEY) + if isinstance(with_clause, dict): + cte_aliases: list[Any] = [ + cte.get(_ALIAS_KEY) for cte in with_clause.get(_CTES_KEY) or () if isinstance(cte, dict) + ] + found.extend((name, True) for name in _identifier_names(cte_aliases)) + key: str + value: Any + for key, value in node.items(): + if key in _ALIASED_RELATION_KEYS and _is_relation(value): + found.extend((name, False) for name in _identifier_names([value.get(_ALIAS_KEY)])) + found.extend(_local_names(value)) + return found + + +def _identifier_names(identifiers: list[Any]) -> list[str]: + return [ + str(identifier[_IDENTIFIER_NAME_KEY]).lower() + for identifier in identifiers + if _is_identifier(identifier) + ] + + +def _canonical_local_names( + *, node: Any, names: Mapping[str, str], cte_names: Mapping[str, str] +) -> Any: + """Rename CTE and alias names only where they define or qualify a relation.""" + + if isinstance(node, list): + return [ + _canonical_local_names(node=value, names=names, cte_names=cte_names) for value in node + ] + if not isinstance(node, dict): + return node + rewritten: dict[str, Any] = { + key: _canonical_local_names(node=value, names=names, cte_names=cte_names) + for key, value in node.items() + } + key: str + value: Any + for key, value in node.items(): + if key in _QUALIFIED_REFERENCE_KEYS and isinstance(value, dict): + rewritten[key] = { + **rewritten[key], + _TABLE_KEY: _renamed(identifier=value.get(_TABLE_KEY), names=names), + } + elif key in _ALIASED_RELATION_KEYS and _is_relation(value): + rewritten[key] = { + **rewritten[key], + _ALIAS_KEY: _renamed(identifier=value.get(_ALIAS_KEY), names=names), + **_cte_reference(relation=value, cte_names=cte_names), + } + elif key == _WITH_KEY and isinstance(value, dict): + rewritten[key] = { + **rewritten[key], + _CTES_KEY: [ + {**cte, _ALIAS_KEY: _renamed(identifier=cte.get(_ALIAS_KEY), names=names)} + for cte in rewritten[key].get(_CTES_KEY) or () + ], + } + return rewritten + + +def _cte_reference(*, relation: dict[str, Any], cte_names: Mapping[str, str]) -> dict[str, Any]: + """Rename an unqualified relation name that refers to a CTE.""" + + name: Any = relation.get(_IDENTIFIER_NAME_KEY) + if not _unqualified(relation) or not _is_identifier(name): + return {} + return {_IDENTIFIER_NAME_KEY: _renamed(identifier=name, names=cte_names)} + + +def _renamed(*, identifier: Any, names: Mapping[str, str]) -> Any: + if not _is_identifier(identifier): + return identifier + canonical: str | None = names.get(str(identifier[_IDENTIFIER_NAME_KEY]).lower()) + if canonical is None: + return identifier + return {**identifier, _IDENTIFIER_NAME_KEY: canonical, _IDENTIFIER_QUOTED_KEY: False} + + +def _is_relation(node: Any) -> bool: + return isinstance(node, dict) and not _is_identifier(node) + + +def _is_identifier(node: Any) -> bool: + return ( + isinstance(node, dict) + and isinstance(node.get(_IDENTIFIER_NAME_KEY), str) + and _IDENTIFIER_QUOTED_KEY in node + ) + + +def with_migration_fingerprints( + *, + entries: tuple[ModelPlanEntry, ...], + models_by_name: Mapping[str, CompiledModel], + dialect: str | None, +) -> tuple[ModelPlanEntry, ...]: + """Attach the current-name migration fingerprint each build stores with its fingerprint.""" + + attached: list[ModelPlanEntry] = [] + entry: ModelPlanEntry + for entry in entries: + model: CompiledModel | None = models_by_name.get(entry.name) + if model is None or entry.fingerprint_metadata_json is None: + attached.append(entry) + continue + attached.append( + replace( + entry, + migration_fingerprint=build_migration_fingerprint( + query_sql=model.query_sql, + metadata_json=entry.fingerprint_metadata_json, + ref_identities={}, + dialect=dialect, + ), + ) + ) + return tuple(attached) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py new file mode 100644 index 0000000000..8020c59751 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -0,0 +1,750 @@ +"""Per-run decisions for direct-mode model migrations.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from datetime import UTC, datetime + +from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo +from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer +from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY +from sqlbuild.compiler.compile.models import CompiledModel, CompiledRelationLocation +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning +from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location +from sqlbuild.compiler.migrations.models import MigrationEvent +from sqlbuild.compiler.migrations.types import ( + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, + MigrationPromotion, +) +from sqlbuild.compiler.planner._helpers.migrations.compatibility import ( + MigrationCompatibilityResult, + check_migration_compatibility, +) +from sqlbuild.compiler.planner._helpers.migrations.discovery import ( + discover_model_migrations, + stored_migration_fingerprint, +) +from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( + effectively_full_refreshed_model_names, +) +from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( + gather_redirected_cursor_snapshots, +) +from sqlbuild.compiler.planner.classes.migration_state_inspection import ( + MigrationStateInspection, +) +from sqlbuild.compiler.planner.constants import ( + MIGRATION_MODEL_NAME_METADATA_KEY, + QUALIFIED_RELATION_MAX_PARTS, +) +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.planner.models import ( + CursorSnapshotScope, + DeferralInputs, + ModelCursorSnapshot, + ModelMigrationDeclaration, + ModelMigrationDiscovery, + ModelMigrationPlanEntry, + ModelMigrationPlanning, + ModelMigrationRequest, + PlannerOverrides, + PlannerRuntime, + PlannerScope, + PlanWarning, + WarehouseFingerprints, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity +from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.models import SnapshotsConfig +from sqlbuild.spec.contracts.types import TableType + + +def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: + """Collect selected models that declare migrate_from.""" + + requests: list[ModelMigrationRequest] = [] + key: object + for key in scope.execution_order: + if key not in scope.selected_keys or getattr(key, "resource_type", None) != ( + CompiledResourceType.MODEL + ): + continue + model: CompiledModel | None = scope.models_by_name.get(getattr(key, "name", "")) + request: ModelMigrationRequest | None = None if model is None else _declared_request(model) + if request is not None: + requests.append(request) + return tuple(requests) + + +def _declared_request(model: CompiledModel) -> ModelMigrationRequest | None: + raw_origin: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) + if not isinstance(raw_origin, str): + return None + return ModelMigrationRequest( + model=model, + discovery=MigrationDiscovery.MANUAL, + raw_origin=raw_origin.strip(), + force=model.config.values.get(MIGRATE_FORCE_CONFIG_KEY) is True, + ) + + +def _project_declarations( + *, runtime: PlannerRuntime, state: MigrationStateInspection +) -> tuple[ModelMigrationDeclaration, ...]: + """Resolve every project model's migrate_from, selected or not.""" + + declarations: list[ModelMigrationDeclaration] = [] + model: CompiledModel + for model in runtime.project.models: + request: ModelMigrationRequest | None = _declared_request(model) + if request is None: + continue + origin: CompiledRelationLocation + origin_model: str | None + origin, origin_model = _resolve_origin(request=request, runtime=runtime, state=state) + declarations.append( + ModelMigrationDeclaration( + model_name=model.name, origin_location=origin, origin_model=origin_model + ) + ) + return tuple(declarations) + + +def plan_model_migrations( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + overrides: PlannerOverrides, + deferral: DeferralInputs, +) -> ModelMigrationPlanning: + """Decide every declared or discovered migration and project the post-migration snapshot.""" + + manual: tuple[ModelMigrationRequest, ...] = manual_migration_requests(scope=scope) + schemas: set[str] = project_schemas(runtime=runtime) + state: MigrationStateInspection = MigrationStateInspection( + adapter=runtime.adapter, + connection=runtime.connection, + database=planning_database(runtime=runtime), + ) + declares_migrations: bool = any( + _declared_request(model) is not None for model in runtime.project.models + ) + if declares_migrations: + state.inspect_schemas(schemas=schemas) + discovery: ModelMigrationDiscovery = discover_model_migrations( + runtime=runtime, + scope=scope, + snapshot=snapshot, + manual_requests=manual, + declarations=_project_declarations(runtime=runtime, state=state), + state=state, + project_schemas=schemas, + ) + if not discovery.requests: + return ModelMigrationPlanning(snapshot=snapshot, warnings=discovery.warnings) + resolved: list[tuple[ModelMigrationRequest, CompiledRelationLocation, str | None]] = [ + (request, *_resolve_origin(request=request, runtime=runtime, state=state)) + for request in discovery.requests + ] + origin: CompiledRelationLocation + for _, origin, _ in resolved: + _require_same_database(state=state, location=origin) + state.inspect_schemas( + schemas={location.schema for _, location, _ in resolved if location.schema is not None} + ) + state.inspect_relations(locations=tuple(location for _, location, _ in resolved)) + snapshots_config: SnapshotsConfig = ( + runtime.project_config.snapshots + if runtime.project_config is not None + else SnapshotsConfig() + ) + entries: list[ModelMigrationPlanEntry] = [] + handovers: dict[str, Fingerprint | None] = {} + request: ModelMigrationRequest + origin: CompiledRelationLocation + origin_model: str | None + for request, origin, origin_model in resolved: + entry: ModelMigrationPlanEntry + handover: Fingerprint | None + applies: bool + entry, handover, applies = _decide( + runtime=runtime, + request=request, + origin=origin, + origin_model=origin_model, + snapshot=snapshot, + state=state, + snapshots_config=snapshots_config, + ) + entries.append(entry) + if applies: + handovers[request.model.name] = _equivalent_definition( + model=request.model, + handover=handover, + destination_fingerprints=discovery.destination_fingerprints.get( + request.model.name, () + ), + ) + renamed: frozenset[str] = _renamed_models( + runtime=runtime, entries=tuple(entries), handovers=handovers + ) + return ModelMigrationPlanning( + snapshot=replace( + _overlay_snapshot( + runtime=runtime, + scope=scope, + snapshot=snapshot, + entries=tuple(entries), + handovers=_effective_handovers( + entries=tuple(entries), handovers=handovers, renamed=renamed + ), + state=state, + overrides=overrides, + deferral=deferral, + ), + renamed_models=renamed, + ), + entries=tuple(entries), + warnings=( + *discovery.warnings, + *(warning for entry in entries if (warning := _entry_warning(entry)) is not None), + ), + ) + + +def _renamed_models( + *, + runtime: PlannerRuntime, + entries: tuple[ModelMigrationPlanEntry, ...], + handovers: dict[str, Fingerprint | None], +) -> frozenset[str]: + """Return renamed tables and views whose handed-over definition matches their own.""" + + query_sql: dict[str, str] = {model.name: model.query_sql for model in runtime.project.models} + return frozenset( + entry.model_name + for entry in entries + if entry.decision == MigrationDecision.RENAMED + and (handover := handovers.get(entry.model_name)) is not None + and handover.definition == query_sql.get(entry.model_name) + ) + + +def _effective_handovers( + *, + entries: tuple[ModelMigrationPlanEntry, ...], + handovers: dict[str, Fingerprint | None], + renamed: frozenset[str], +) -> dict[str, Fingerprint | None]: + """Drop the handover of any rename whose definition no longer matches.""" + + unmatched: frozenset[str] = frozenset( + entry.model_name + for entry in entries + if entry.decision == MigrationDecision.RENAMED and entry.model_name not in renamed + ) + return {name: None if name in unmatched else handover for name, handover in handovers.items()} + + +def _equivalent_definition( + *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprints: tuple[str, ...] +) -> Fingerprint | None: + if handover is None or stored_migration_fingerprint(handover) not in destination_fingerprints: + return handover + return replace( + handover, + definition=model.query_sql, + definition_hash=compute_query_hash(model.query_sql), + ) + + +def planning_database(*, runtime: PlannerRuntime) -> str | None: + """Return the single database that direct planning inspects for migration state.""" + + return next( + ( + model.destination.database + for model in runtime.project.models + if model.destination.database is not None + ), + runtime.project.effective_target_database, + ) + + +def project_schemas(*, runtime: PlannerRuntime) -> set[str]: + """Return every destination schema owned by project models.""" + + return { + model.destination.schema + for model in runtime.project.models + if model.destination.schema is not None + } + + +def _require_same_database( + *, state: MigrationStateInspection, location: CompiledRelationLocation +) -> None: + if ( + location.database is None + or state.database is None + or location.database.lower() == state.database.lower() + ): + return + raise PlannerInputError( + f"migration origin {location.qualified_name or location.name} is in database " + f"'{location.database}', but direct planning inspects '{state.database}'; " + "cross-database migrations are not supported" + ) + + +def _relation_key(location: CompiledRelationLocation) -> tuple[str, str]: + return ((location.schema or "").lower(), location.name.lower()) + + +def _resolve_origin( + *, request: ModelMigrationRequest, runtime: PlannerRuntime, state: MigrationStateInspection +) -> tuple[CompiledRelationLocation, str | None]: + if request.origin_location is not None: + return request.origin_location, request.origin_model + raw: str = request.raw_origin or "" + destination: CompiledRelationLocation = request.model.destination + parts: list[str] = [part.strip().strip('"`') for part in raw.split(".")] + if any(not part for part in parts) or len(parts) > QUALIFIED_RELATION_MAX_PARTS: + raise PlannerInputError( + f"model '{request.model.name}': migrate_from '{raw}' is not a model name or a " + "qualified relation" + ) + if len(parts) == 1: + return _resolve_named_origin( + name=parts[0], destination=destination, runtime=runtime, state=state + ) + database: str | None = ( + parts[0] if len(parts) == QUALIFIED_RELATION_MAX_PARTS else destination.database + ) + schema: str = parts[-2] + return ( + _location(runtime=runtime, database=database, schema=schema, name=parts[-1]), + None, + ) + + +def _resolve_named_origin( + *, + name: str, + destination: CompiledRelationLocation, + runtime: PlannerRuntime, + state: MigrationStateInspection, +) -> tuple[CompiledRelationLocation, str | None]: + project_model: CompiledModel | None = next( + (model for model in runtime.project.models if model.name == name), None + ) + if project_model is not None: + return project_model.destination, name + fingerprint: Fingerprint | None = state.named_fingerprint(model_name=name) + if fingerprint is not None and fingerprint.target_name is not None: + return ( + _location( + runtime=runtime, + database=fingerprint.target_database, + schema=fingerprint.target_schema, + name=fingerprint.target_name, + ), + name, + ) + return ( + _location( + runtime=runtime, database=destination.database, schema=destination.schema, name=name + ), + name, + ) + + +def _location( + *, runtime: PlannerRuntime, database: str | None, schema: str | None, name: str +) -> CompiledRelationLocation: + return CompiledRelationLocation( + database=database, + schema=schema, + name=name, + qualified_name=runtime.adapter.render_qualified_name( + database=database, schema=schema, name=name + ), + ) + + +def _decide( + *, + runtime: PlannerRuntime, + request: ModelMigrationRequest, + origin: CompiledRelationLocation, + origin_model: str | None, + snapshot: WarehouseSnapshot, + state: MigrationStateInspection, + snapshots_config: SnapshotsConfig, +) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: + model: CompiledModel = request.model + destination: CompiledRelationLocation = model.destination + if _relation_key(origin) == _relation_key(destination): + raise PlannerInputError( + f"model '{model.name}': migrate_from names the model's own relation" + ) + newest: MigrationEvent | None = newest_migration_event_mentioning( + events=state.events, relation=migration_relation_for_location(destination) + ) + origin_relation: RelationInfo | None = state.relation(origin) + origin_fingerprint: Fingerprint | None = state.model_fingerprint( + location=origin, model_name=origin_model + ) + base: ModelMigrationPlanEntry = ModelMigrationPlanEntry( + model_name=model.name, + discovery=request.discovery, + decision=MigrationDecision.MIGRATE, + compatibility=MigrationCompatibility.NOT_CHECKED, + origin_model=origin_model, + origin=origin, + destination=destination, + target_name=runtime.project.effective_target_name, + origin_version_hash=( + origin_fingerprint.version_hash if origin_fingerprint is not None else "" + ), + origin_is_transient=bool(origin_relation is not None and origin_relation.is_transient), + ) + if request.identity_only: + return _decide_rename( + model=model, + base=base, + newest=newest, + origin=origin, + origin_fingerprint=origin_fingerprint, + ) + if ( + newest is not None + and newest.destination.matches(migration_relation_for_location(destination)) + and newest.origin.matches(migration_relation_for_location(origin)) + ): + return ( + replace( + base, + decision=MigrationDecision.DONE, + completed_at=newest.created_at, + origin_version_hash=newest.origin_version_hash, + target_name=newest.target_name, + ), + _done_handover( + model=model, + event=newest, + snapshot=snapshot, + origin_fingerprint=origin_fingerprint, + ), + _done_handover_applies(model=model, event=newest, snapshot=snapshot), + ) + if origin_relation is None: + return replace(base, decision=MigrationDecision.ORIGIN_MISSING), None, False + compatibility: MigrationCompatibilityResult = check_migration_compatibility( + model=model, + origin_relation=origin_relation, + origin_columns=state.relation_columns(origin), + sql_analysis_enabled=runtime.project.settings.sql_analysis, + column_dialect=snapshot.column_dialect, + snapshots_config=snapshots_config, + ) + decision: MigrationDecision + if model.name not in snapshot.existing_relations: + decision = MigrationDecision.MIGRATE + elif newest is not None and newest.origin.matches(migration_relation_for_location(destination)): + decision = MigrationDecision.SUPERSEDED_REPLACE + elif newest is None and model.name not in snapshot.fingerprints.models: + decision = MigrationDecision.REDO + elif request.force: + decision = MigrationDecision.FORCED_REPLACE + else: + decision = MigrationDecision.CONFLICT + entry: ModelMigrationPlanEntry = replace( + base, + decision=decision, + compatibility=compatibility.status, + compatibility_findings=compatibility.findings, + ) + if not decision.moves_data or entry.blocks_build: + return entry, None, False + return ( + _with_execution(runtime=runtime, model=model, entry=entry), + _handover_fingerprint( + model=model, + fingerprint=origin_fingerprint, + version_hash=entry.origin_version_hash, + ), + True, + ) + + +def _decide_rename( + *, + model: CompiledModel, + base: ModelMigrationPlanEntry, + newest: MigrationEvent | None, + origin: CompiledRelationLocation, + origin_fingerprint: Fingerprint | None, +) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: + """Hand a renamed table or view's identity to its unbuilt successor; no data moves.""" + + recorded: bool = ( + newest is not None + and newest.decision == MigrationDecision.RENAMED + and newest.destination.matches(migration_relation_for_location(model.destination)) + and newest.origin.matches(migration_relation_for_location(origin)) + ) + return ( + replace( + base, + decision=MigrationDecision.RENAMED, + completed_at=newest.created_at if recorded and newest is not None else None, + ), + _handover_fingerprint(model=model, fingerprint=origin_fingerprint, version_hash=""), + True, + ) + + +def _with_execution( + *, runtime: PlannerRuntime, model: CompiledModel, entry: ModelMigrationPlanEntry +) -> ModelMigrationPlanEntry: + """Describe how the executor will stage and promote this migration.""" + + stage_is_transient: bool | None = ( + model.config.table_type.value == TableType.TRANSIENT + if runtime.adapter.adapter_name == BuiltinAdapter.SNOWFLAKE + else None + ) + stage: MigrationStagePlan = runtime.adapter.render_migration_stage( + origin=entry.origin.qualified_name or entry.origin.name, + stage=entry.destination.qualified_name or entry.destination.name, + origin_is_transient=entry.origin_is_transient, + stage_is_transient=stage_is_transient, + ) + return replace( + entry, + stage_is_transient=stage_is_transient, + transfer=stage.transfer, + transfer_fallback=MigrationTransfer.COPY if stage.fallback_statements else None, + promotion=_promotion(runtime=runtime, decision=entry.decision), + ) + + +def _promotion(*, runtime: PlannerRuntime, decision: MigrationDecision) -> MigrationPromotion: + if runtime.adapter.supports_transactional_ddl(): + return MigrationPromotion.TRANSACTIONAL_RENAME + if decision != MigrationDecision.MIGRATE and ( + runtime.adapter.adapter_name == BuiltinAdapter.SNOWFLAKE + ): + return MigrationPromotion.SWAP + return MigrationPromotion.RENAME + + +def _done_handover_applies( + *, model: CompiledModel, event: MigrationEvent, snapshot: WarehouseSnapshot +) -> bool: + own: Fingerprint | None = snapshot.fingerprints.models.get(model.name) + return own is None or _utc(own.ts) < _utc(event.created_at) + + +def _utc(value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def _done_handover( + *, + model: CompiledModel, + event: MigrationEvent, + snapshot: WarehouseSnapshot, + origin_fingerprint: Fingerprint | None, +) -> Fingerprint | None: + if not _done_handover_applies(model=model, event=event, snapshot=snapshot): + return snapshot.fingerprints.models.get(model.name) + return _handover_fingerprint( + model=model, fingerprint=origin_fingerprint, version_hash=event.origin_version_hash + ) + + +def _handover_fingerprint( + *, model: CompiledModel, fingerprint: Fingerprint | None, version_hash: str +) -> Fingerprint | None: + if fingerprint is None: + return None + metadata_json: str = fingerprint.metadata_json + try: + payload: object = json.loads(metadata_json) + except json.JSONDecodeError: + payload = None + if isinstance(payload, dict) and MIGRATION_MODEL_NAME_METADATA_KEY in payload: + payload[MIGRATION_MODEL_NAME_METADATA_KEY] = model.name + metadata_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return replace( + fingerprint, + node_name=model.name, + target_database=model.destination.database, + target_schema=model.destination.schema, + target_name=model.destination.name, + version_hash=version_hash or fingerprint.version_hash, + metadata_json=metadata_json, + ) + + +def _overlay_snapshot( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + entries: tuple[ModelMigrationPlanEntry, ...], + handovers: dict[str, Fingerprint | None], + state: MigrationStateInspection, + overrides: PlannerOverrides, + deferral: DeferralInputs, +) -> WarehouseSnapshot: + if not handovers: + return snapshot + relations: dict[str, RelationInfo] = dict(snapshot.existing_relations) + columns: dict[str, tuple[ColumnInfo, ...]] = dict(snapshot.existing_columns) + model_fingerprints: dict[str, Fingerprint] = dict(snapshot.fingerprints.models) + redirected: dict[str, str] = {} + entry: ModelMigrationPlanEntry + for entry in entries: + if entry.model_name not in handovers: + continue + handover: Fingerprint | None = handovers[entry.model_name] + if handover is None: + _ = model_fingerprints.pop(entry.model_name, None) + else: + model_fingerprints[entry.model_name] = handover + if not entry.decision.moves_data: + continue + origin_relation: RelationInfo | None = state.relation(entry.origin) + if origin_relation is None: + continue + relations[entry.model_name] = replace( + origin_relation, + database=entry.destination.database, + schema=entry.destination.schema, + name=entry.destination.name, + is_transient=( + origin_relation.is_transient + if entry.stage_is_transient is None + else entry.stage_is_transient + ), + ) + columns[entry.model_name] = state.relation_columns(entry.origin) + if entry.origin.qualified_name is not None and _has_cursor( + runtime=runtime, model_name=entry.model_name + ): + redirected[entry.model_name] = entry.origin.qualified_name + cursor_snapshots: dict[str, ModelCursorSnapshot] = { + **snapshot.cursor_snapshots, + **gather_redirected_cursor_snapshots( + project=runtime.project, + adapter=runtime.adapter, + connection=runtime.connection, + existing_relations=relations, + target_relations=redirected, + cursor_scope=CursorSnapshotScope( + model_keys=scope.selected_keys, + runtime_producer_keys=scope.selected_keys, + invocation_time=runtime.invocation_time, + start_cursor_config=( + runtime.project_config.cursors.start + if runtime.project_config is not None + else None + ), + cursor_overrides=overrides.cursor_overrides, + ), + full_refresh_model_names=effectively_full_refreshed_model_names( + project=runtime.project, cli_full_refresh=overrides.full_refresh + ), + deferred_locations=deferral.deferred_locations, + on_progress=runtime.on_progress, + ), + } + return replace( + snapshot, + existing_relations=relations, + existing_columns=columns, + fingerprints=WarehouseFingerprints( + models=model_fingerprints, + functions=snapshot.fingerprints.functions, + seeds=snapshot.fingerprints.seeds, + python_nodes=snapshot.fingerprints.python_nodes, + ), + cursor_snapshots=cursor_snapshots, + ) + + +def _has_cursor(*, runtime: PlannerRuntime, model_name: str) -> bool: + model: CompiledModel | None = next( + (model for model in runtime.project.models if model.name == model_name), None + ) + return ( + model is not None + and get_config_str(values=model.config.values, key="materialized") + == MaterializationType.INCREMENTAL + and get_config_str(values=model.config.values, key="cursor") is not None + ) + + +def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: + origin: str = entry.origin.qualified_name or entry.origin.name + destination: str = entry.destination.qualified_name or entry.destination.name + if entry.decision == MigrationDecision.DONE: + if entry.discovery != MigrationDiscovery.MANUAL: + return None + completed: str = entry.completed_at.isoformat() if entry.completed_at else "unknown time" + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.WARNING, + message=( + f"migration from {origin} completed at {completed} on target " + f"'{entry.target_name}'; migrate_from can be removed from '{entry.model_name}'" + ), + code="M101", + ) + if entry.decision == MigrationDecision.ORIGIN_MISSING: + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.ERROR, + message=( + f"model '{entry.model_name}': migrate_from origin {origin} does not exist and " + "no recorded migration into it was found; if the migration already happened " + "elsewhere or is no longer needed, remove migrate_from from the model header" + ), + code="M102", + ) + if entry.decision == MigrationDecision.CONFLICT: + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.ERROR, + message=( + f"migration conflict: {destination} already exists with its own build history " + f"and has no recorded migration from {origin}; set migrate_force true to " + "replace it (the replaced table is kept under a _sqb_archive__ name until " + "janitor expires it)" + ), + code="M103", + ) + if entry.blocks_build: + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.ERROR, + message=( + f"migration from {origin} is incompatible with '{entry.model_name}': " + + "; ".join(entry.compatibility_findings) + ), + code="M104", + ) + return None diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index d9c5ab26f3..7acdf31041 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -67,6 +67,9 @@ def resolve_model_plan_action( if change_result.change_kind == ChangeKind.FIRST_RUN: return PlanAction.CREATE_TABLE, PlanReason.FIRST_RUN + if change_result.change_kind == ChangeKind.RENAMED: + return PlanAction.CREATE_TABLE, PlanReason.RENAMED + if change_result.backfill.action == BackfillAction.FULL: reason: PlanReason = _backfill_reason(change_result) return PlanAction.CREATE_TABLE, reason @@ -296,6 +299,8 @@ def _view_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> return PlanReason.FULL_REFRESH if change_result.change_kind == ChangeKind.FIRST_RUN: return PlanReason.FIRST_RUN + if change_result.change_kind == ChangeKind.RENAMED: + return PlanReason.RENAMED if change_result.change_kind == ChangeKind.QUERY_CHANGED: return PlanReason.QUERY_CHANGED if change_result.change_kind == ChangeKind.CONFIG_CHANGED: diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py index f179e64aca..fe0d02ed5e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py @@ -5,6 +5,7 @@ from dataclasses import replace from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.planner._helpers.migrations.fingerprint import with_migration_fingerprints from sqlbuild.compiler.planner._helpers.output.plan_output import build_plan_output from sqlbuild.compiler.planner._helpers.planning.retention import plan_retention, plan_table_types from sqlbuild.compiler.planner._helpers.pruning.selection_staleness import ( @@ -79,10 +80,17 @@ def with_storage_policies( warehouse: PlannerWarehouseState, scopes: PlannerScopeResolution, ) -> PlanOutput: - """Attach selected Snowflake table-type and retention work to the plan.""" + """Attach migrations and selected Snowflake table-type and retention work to the plan.""" return replace( plan_output, + model_entries=with_migration_fingerprints( + entries=plan_output.model_entries, + models_by_name={model.name: model for model in runtime.project.models}, + dialect=runtime.adapter.sql_analysis_dialect(), + ), + migration_entries=warehouse.migration_entries, + warnings=(*warehouse.migration_warnings, *plan_output.warnings), table_type_entries=plan_table_types( runtime=runtime, warehouse=warehouse, scope=scopes.selected_scope ), diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index 86ac842e34..0e809de25b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -4,6 +4,7 @@ import time +from sqlbuild.compiler.planner._helpers.migrations.planning import plan_model_migrations from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, @@ -12,6 +13,7 @@ from sqlbuild.compiler.planner.models import ( CursorSnapshotScope, DeferralInputs, + ModelMigrationPlanning, PlannerOverrides, PlannerRelationsContext, PlannerRuntime, @@ -55,6 +57,13 @@ def gather_planner_warehouse_state( cursor_overrides=overrides.cursor_overrides, ), ) + migrations: ModelMigrationPlanning = plan_model_migrations( + runtime=runtime, + scope=scopes.selected_scope, + snapshot=snapshot, + overrides=overrides, + deferral=deferral, + ) inspection_relations: PlannerRelationsContext = build_planner_relations_context( project=runtime.project, adapter=runtime.adapter, @@ -69,4 +78,9 @@ def gather_planner_warehouse_state( f"Inspected warehouse state. ({time.monotonic() - warehouse_start:.2f}s)" ) runtime.on_progress("Generating plan...") - return PlannerWarehouseState(snapshot=snapshot, inspection_relations=inspection_relations) + return PlannerWarehouseState( + snapshot=migrations.snapshot, + inspection_relations=inspection_relations, + migration_entries=migrations.entries, + migration_warnings=migrations.warnings, + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index ebc3905442..c377c299eb 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -5,7 +5,7 @@ import logging import time from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime from typing import Any @@ -141,6 +141,7 @@ class _CursorGatherInputs: invocation_time: datetime | None start_cursor_config: StartCursorsConfig | None cursor_overrides: CursorOverrides | None + target_relation_overrides: dict[str, str] | None = None def build_warehouse_snapshot( @@ -280,6 +281,52 @@ def gather_warehouse_snapshot( ) +def gather_redirected_cursor_snapshots( + *, + project: CompiledProject, + adapter: BaseAdapter, + connection: Any, + existing_relations: dict[str, RelationInfo], + target_relations: dict[str, str], + cursor_scope: CursorSnapshotScope, + full_refresh_model_names: frozenset[str], + deferred_locations: dict[str, CompiledRelationLocation] | None = None, + on_progress: Callable[[str], None] | None = None, +) -> dict[str, ModelCursorSnapshot]: + """Gather cursor snapshots for models whose target history lives in another relation.""" + + if not target_relations: + return {} + model_keys: frozenset[CompiledObjectKey] = frozenset( + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) + for name in target_relations + ) + snapshots: dict[str, ModelCursorSnapshot] = _gather_cursor_snapshots( + project=project, + adapter=adapter, + connection=connection, + execute=adapter.execute, + existing_relations=existing_relations, + on_progress=on_progress, + inputs=_CursorGatherInputs( + selected_keys=model_keys, + full_refresh_model_names=full_refresh_model_names, + deferred_locations=deferred_locations, + runtime_producer_keys=cursor_scope.runtime_producer_keys, + invocation_time=cursor_scope.invocation_time, + start_cursor_config=cursor_scope.start_cursor_config, + cursor_overrides=cursor_scope.cursor_overrides, + target_relation_overrides=target_relations, + ), + ) + model_map: dict[str, CompiledModel] = {model.name: model for model in project.models} + return { + name: replace(snapshot, target_relation=model_map[name].destination.qualified_name) + for name, snapshot in snapshots.items() + if name in model_map + } + + def _relevant_state_keys( *, project: CompiledProject, @@ -696,7 +743,11 @@ def _collect_cursor_models( target_tag: str | None = None target_relation: str | None = None target_relation_info: RelationInfo | None = existing_relations.get(model.name) - if ( + redirected_target: str | None = (inputs.target_relation_overrides or {}).get(model.name) + if redirected_target is not None: + target_tag = f"{model.name}__target__max" + target_relation = redirected_target + elif ( model.destination.qualified_name is not None and target_relation_info is not None and target_relation_info.name == model.name diff --git a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py new file mode 100644 index 0000000000..dea9293152 --- /dev/null +++ b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py @@ -0,0 +1,205 @@ +"""Read-only warehouse evidence gathered once for all model migrations in one plan.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL +from sqlbuild.compiler.fingerprints.main.read import read_latest_fingerprints +from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet +from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME +from sqlbuild.compiler.migrations.main._read_events import read_migration_events +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation + + +class MigrationStateInspection: + """Relations, fingerprints, and migration events for one planning database.""" + + def __init__(self, *, adapter: BaseAdapter, connection: Any, database: str | None) -> None: + self._adapter: BaseAdapter = adapter + self._connection: Any = connection + self._database: str | None = database + self._read_schemas: set[str] = set() + self._fingerprints: dict[str, tuple[Fingerprint, ...]] = {} + self._events: list[MigrationEvent] = [] + self._relations: dict[tuple[str, str], RelationInfo] = {} + self._columns: dict[tuple[str, str], tuple[ColumnInfo, ...]] = {} + + @property + def database(self) -> str | None: + """Return the database whose namespaces this inspection reads.""" + + return self._database + + @property + def events(self) -> tuple[MigrationEvent, ...]: + """Return every migration event read so far.""" + + return tuple(self._events) + + def inspect_schemas(self, *, schemas: set[str]) -> None: + """Read state tables once for every schema not inspected yet.""" + + pending: tuple[str, ...] = tuple( + sorted(schema for schema in schemas if schema.lower() not in self._read_schemas) + ) + if not pending: + return + self._read_schemas.update(schema.lower() for schema in pending) + listed: tuple[RelationInfo, ...] = self._adapter.list_relations( + connection=self._connection, + database=self._database, + schemas=pending, + names=(MIGRATION_TABLE_NAME, FINGERPRINT_TABLE_NAME), + ) + state_tables: set[tuple[str, str]] = { + ((relation.schema or "").lower(), relation.name.lower()) for relation in listed + } + schema: str + for schema in pending: + if (schema.lower(), MIGRATION_TABLE_NAME) in state_tables: + self._events.extend( + read_migration_events( + connection=self._connection, + execute=self._adapter.execute, + database=self._database, + schema=schema, + render_qualified_name=self._adapter.render_qualified_name, + ) + ) + if (schema.lower(), FINGERPRINT_TABLE_NAME) in state_tables: + self._fingerprints[schema.lower()] = self._read_model_fingerprints(schema=schema) + + def _read_model_fingerprints(self, *, schema: str) -> tuple[Fingerprint, ...]: + fingerprint_set: FingerprintSet = read_latest_fingerprints( + connection=self._connection, + execute=self._adapter.execute, + table_exists=True, + database=self._database, + schema=schema, + render_qualified_name=self._adapter.render_qualified_name, + render_read_latest_sql=self._adapter.render_read_latest_fingerprints_sql, + ) + return tuple( + fingerprint + for fingerprint in (fingerprint_set.fingerprints_by_identity or {}).values() + if fingerprint.node_type == NODE_TYPE_MODEL + ) + + def inspect_relations(self, *, locations: tuple[CompiledRelationLocation, ...]) -> None: + """List candidate relations and their columns with one metadata read.""" + + pending: tuple[CompiledRelationLocation, ...] = tuple( + location + for location in locations + if location.schema is not None and _relation_key(location) not in self._relations + ) + if not pending: + return + listed: tuple[RelationInfo, ...] = self._adapter.list_relations( + connection=self._connection, + database=self._database, + schemas=tuple(sorted({location.schema or "" for location in pending})), + names=tuple(sorted({location.name for location in pending})), + ) + wanted: set[tuple[str, str]] = {_relation_key(location) for location in pending} + matched: tuple[RelationInfo, ...] = tuple( + relation for relation in listed if _listed_key(relation) in wanted + ) + columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + self._adapter.get_columns_for_relations(connection=self._connection, relations=matched) + if matched + else {} + ) + relation: RelationInfo + for relation in matched: + self._relations[_listed_key(relation)] = relation + self._columns[_listed_key(relation)] = columns.get(relation.identity, ()) + + def relation(self, location: CompiledRelationLocation) -> RelationInfo | None: + """Return the listed relation for a location.""" + + return self._relations.get(_relation_key(location)) + + def relation_columns(self, location: CompiledRelationLocation) -> tuple[ColumnInfo, ...]: + """Return listed columns for a location.""" + + return self._columns.get(_relation_key(location), ()) + + def model_fingerprint( + self, *, location: CompiledRelationLocation, model_name: str | None + ) -> Fingerprint | None: + """Return the latest model fingerprint that built the location or named model.""" + + candidates: tuple[Fingerprint, ...] = self._fingerprints.get( + (location.schema or "").lower(), () + ) + target: MigrationRelation = MigrationRelation( + database=location.database, schema=location.schema, name=location.name + ) + by_relation: tuple[Fingerprint, ...] = tuple( + fingerprint + for fingerprint in candidates + if _fingerprint_built(fingerprint=fingerprint, target=target) + ) + if by_relation: + return max(by_relation, key=_fingerprint_order) + if model_name is None: + return None + named: tuple[Fingerprint, ...] = tuple( + fingerprint for fingerprint in candidates if fingerprint.node_name == model_name + ) + return max(named, key=_fingerprint_order) if named else None + + def named_fingerprint(self, *, model_name: str) -> Fingerprint | None: + """Return the newest model fingerprint with this node name in any inspected schema.""" + + matches: list[Fingerprint] = [] + fingerprints: tuple[Fingerprint, ...] + for fingerprints in self._fingerprints.values(): + matches.extend( + fingerprint for fingerprint in fingerprints if fingerprint.node_name == model_name + ) + return max(matches, key=_fingerprint_order) if matches else None + + def orphan_fingerprints( + self, *, project_model_names: frozenset[str] + ) -> tuple[Fingerprint, ...]: + """Return latest model fingerprints whose names are no longer in the project.""" + + orphans: list[Fingerprint] = [] + fingerprints: tuple[Fingerprint, ...] + for fingerprints in self._fingerprints.values(): + orphans.extend( + fingerprint + for fingerprint in fingerprints + if fingerprint.node_name not in project_model_names + ) + return tuple(orphans) + + +def _relation_key(location: CompiledRelationLocation) -> tuple[str, str]: + return ((location.schema or "").lower(), location.name.lower()) + + +def _listed_key(relation: RelationInfo) -> tuple[str, str]: + return ((relation.schema or "").lower(), relation.name.lower()) + + +def _fingerprint_built(*, fingerprint: Fingerprint, target: MigrationRelation) -> bool: + if fingerprint.target_name is None: + return False + return target.matches( + MigrationRelation( + database=fingerprint.target_database, + schema=fingerprint.target_schema, + name=fingerprint.target_name, + ) + ) + + +def _fingerprint_order(fingerprint: Fingerprint) -> tuple[object, str]: + return (fingerprint.ts.isoformat(), fingerprint.run_id) diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index dc7fa0bb6a..b4cdf5513d 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -58,3 +58,25 @@ SCENARIO_PLAN_INTERNAL: str = "S599" WHOLE_DAY_CURSOR_GRAINS: frozenset[str] = frozenset({"day", "month", "year"}) +VIEW_RELATION_TYPE_MARKER: str = "VIEW" +QUALIFIED_RELATION_MAX_PARTS: int = 3 +MIGRATION_MODEL_NAME_METADATA_KEY: str = "model_name" +MIGRATION_FINGERPRINT_METADATA_KEY: str = "migration_fingerprint" +MIGRATION_REF_PLACEHOLDER_PREFIX: str = "__sqb_migration_ref__" +MIGRATION_LOCAL_NAME_PREFIX: str = "__sqb_local_" +MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS: frozenset[str] = frozenset( + { + "batch_size", + "full_refresh", + "incremental_mode", + "incremental_strategy", + "lookback", + "materialized", + "max_microbatches", + "microbatch_limit", + "microbatch_strategy", + "on_schema_change", + "snapshot_full_refresh", + "snapshot_schema_change", + } +) diff --git a/src/sqlbuild/compiler/planner/main/pre_build/__init__.py b/src/sqlbuild/compiler/planner/main/pre_build/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py new file mode 100644 index 0000000000..bb4ec60431 --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py @@ -0,0 +1,118 @@ +"""Plan text for storage and migration work that runs before model builds.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput +from sqlbuild.presentation.main.append_overflow_line import append_overflow_line +from sqlbuild.presentation.main.visible_entries import visible_entries +from sqlbuild.presentation.models import DisplayOptions + + +def format_pre_build_work( + *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions +) -> list[str]: + """Format retention, table-type, and migration work that runs before model builds.""" + + if plan.table_type_entries: + lines.append("Table type conversions") + for table_type_entry in visible_entries( + entries=plan.table_type_entries, options=display_options + ): + actual_type: str = table_type_entry.actual_type or "unknown" + lines.append( + f" {table_type_entry.model_name} desired={table_type_entry.desired_type} " + f"actual={actual_type} source={table_type_entry.source}" + ) + if table_type_entry.irreversible_warning is not None: + lines.append(f" WARNING: {table_type_entry.irreversible_warning}") + lines = append_overflow_line( + lines=lines, + total_count=len(plan.table_type_entries), + visible_count=len( + visible_entries(entries=plan.table_type_entries, options=display_options) + ), + indent=" ", + options=display_options, + ) + if plan.retention_entries: + lines.append("Retention") + for entry in plan.retention_entries: + scope: str = ".".join( + part + for part in (entry.request.database, entry.request.schema, entry.request.name) + if part + ) + actual: str = "missing" if entry.actual_days is None else f"{entry.actual_days}d" + effective: str = "missing" if entry.effective_days is None else f"{entry.effective_days}d" + lines.append( + f" {scope} desired={entry.request.desired_days}d actual={actual} " + f"effective={effective} source={entry.source} direction={entry.direction.value} " + f"phase={entry.phase.value}" + ) + if entry.irreversible_warning is not None: + lines.append(f" WARNING: {entry.irreversible_warning}") + return _format_model_migrations(lines=lines, plan=plan, display_options=display_options) + + +def _format_model_migrations( + *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions +) -> list[str]: + title: str + renamed: bool + for title, renamed in (("Migrations", False), ("Renamed", True)): + entries: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry + for entry in plan.migration_entries + if (entry.decision == MigrationDecision.RENAMED) is renamed + ) + if not entries: + continue + lines.append(f"{title} ({len(entries)})") + visible: Sequence[ModelMigrationPlanEntry] = visible_entries( + entries=entries, options=display_options + ) + entry: ModelMigrationPlanEntry + for entry in visible: + lines.extend(_migration_lines(entry)) + lines = append_overflow_line( + lines=lines, + total_count=len(entries), + visible_count=len(visible), + indent=" ", + options=display_options, + ) + return lines + + +def _migration_lines(entry: ModelMigrationPlanEntry) -> list[str]: + origin: str = entry.origin.qualified_name or entry.origin.name + destination: str = entry.destination.qualified_name or entry.destination.name + if entry.decision == MigrationDecision.RENAMED: + return [f" {entry.model_name} {origin} -> {destination} (identity handed over)"] + rows: list[str] = [f" {entry.model_name} {entry.decision.label} {origin} -> {destination}"] + if entry.decision.checks_compatibility: + rows.append(f" compatibility {entry.compatibility.value}") + if entry.transfer is not None: + details: tuple[str | None, ...] = ( + entry.transfer.label + + ( + "" + if entry.transfer_fallback is None + else f" ({entry.transfer_fallback.label} if refused)" + ), + entry.storage_transition, + None if entry.promotion is None else f"promote by {entry.promotion.label}", + ) + rows.append(f" transfer {', '.join(detail for detail in details if detail)}") + if entry.discovery != MigrationDiscovery.MANUAL: + rows.append(f" discovery {entry.discovery.value}") + if entry.completed_at is not None: + rows.append( + f" completed {entry.completed_at.isoformat()} on target " + f"'{entry.target_name or 'default'}'" + ) + rows.extend(f" ! {finding}" for finding in entry.compatibility_findings) + return rows diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 58ff77e55e..7121851f5a 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -11,6 +11,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RetentionRequest +from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.auditing.models import MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -30,6 +31,12 @@ from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, FunctionLanguage from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.migrations.types import ( + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, + MigrationPromotion, +) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.types import ( BackfillAction, @@ -639,6 +646,7 @@ class WarehouseSnapshot: cursor_snapshots: dict[str, ModelCursorSnapshot] = field(default_factory=dict) source_freshness_state_schemas: frozenset[str] = field(default_factory=frozenset) column_dialect: str | None = None + renamed_models: frozenset[str] = field(default_factory=frozenset) @dataclass(frozen=True) @@ -958,6 +966,46 @@ class TableTypePlanEntry: irreversible_warning: str | None = None +@dataclass(frozen=True) +class ModelMigrationPlanEntry: + """One declared or discovered model migration and its per-run decision.""" + + model_name: str + discovery: MigrationDiscovery + decision: MigrationDecision + compatibility: MigrationCompatibility + origin_model: str | None + origin: CompiledRelationLocation + destination: CompiledRelationLocation + target_name: str | None + origin_version_hash: str = "" + origin_is_transient: bool = False + stage_is_transient: bool | None = None + transfer: MigrationTransfer | None = None + transfer_fallback: MigrationTransfer | None = None + promotion: MigrationPromotion | None = None + completed_at: datetime | None = None + compatibility_findings: tuple[str, ...] = () + message: str | None = None + + @property + def storage_transition(self) -> str | None: + """Return the Snowflake table-type transition label, when both types are known.""" + + if self.stage_is_transient is None: + return None + labels: tuple[str, str] = ("permanent", "transient") + return f"{labels[self.origin_is_transient]} -> {labels[self.stage_is_transient]}" + + @property + def blocks_build(self) -> bool: + """Return whether this migration must stop a build before any execution.""" + + return self.decision.blocks_build or ( + self.decision.moves_data and self.compatibility == MigrationCompatibility.INCOMPATIBLE + ) + + @dataclass(frozen=True) class PlanOutputExtras: """Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.""" @@ -991,6 +1039,7 @@ class ModelPlanEntry: fingerprint_query_sql: str resolved_sql: str logical_ddl: str + migration_fingerprint: str | None = None incremental_strategy: str | None = None incremental_mode: str | None = None microbatch_strategy: str | None = None @@ -1365,6 +1414,7 @@ class PlanOutput: warnings: tuple[PlanWarning, ...] = field(default_factory=tuple) retention_entries: tuple[RetentionPlanEntry, ...] = field(default_factory=tuple) table_type_entries: tuple[TableTypePlanEntry, ...] = field(default_factory=tuple) + migration_entries: tuple[ModelMigrationPlanEntry, ...] = field(default_factory=tuple) upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = field( default_factory=dict ) @@ -1451,6 +1501,56 @@ class PlannerWarehouseState: snapshot: WarehouseSnapshot inspection_relations: PlannerRelationsContext + migration_entries: tuple[ModelMigrationPlanEntry, ...] = () + migration_warnings: tuple[PlanWarning, ...] = () + + +@dataclass(frozen=True) +class ModelMigrationRequest: + """One destination model whose data should come from an earlier relation.""" + + model: CompiledModel + discovery: MigrationDiscovery + raw_origin: str | None = None + origin_location: CompiledRelationLocation | None = None + origin_model: str | None = None + force: bool = False + identity_only: bool = False + + +@dataclass(frozen=True) +class ModelMigrationDeclaration: + """One project model's explicit migrate_from, resolved to a relation.""" + + model_name: str + origin_location: CompiledRelationLocation + origin_model: str | None + + +@dataclass(frozen=True) +class ModelMigrationDiscovery: + """Manual and automatically discovered migration requests for one plan.""" + + requests: tuple[ModelMigrationRequest, ...] = () + warnings: tuple[PlanWarning, ...] = () + destination_fingerprints: dict[str, tuple[str, ...]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class MigrationCompatibilityResult: + """Compatibility status plus human-readable blocking findings.""" + + status: MigrationCompatibility + findings: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ModelMigrationPlanning: + """Migration decisions plus the warehouse snapshot as it will look after moves.""" + + snapshot: WarehouseSnapshot + entries: tuple[ModelMigrationPlanEntry, ...] = () + warnings: tuple[PlanWarning, ...] = () @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index f7ba982083..080c37f6ab 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -75,6 +75,7 @@ class GraphResourceKind(StrEnum): class ChangeKind(StrEnum): FIRST_RUN = "first_run" + RENAMED = "renamed" QUERY_CHANGED = "query_changed" CONFIG_CHANGED = "config_changed" SCHEMA_CHANGED = "schema_changed" @@ -145,6 +146,7 @@ class PlanAction(StrEnum): class PlanReason(StrEnum): FIRST_RUN = "first_run" + RENAMED = "renamed" FULL_REFRESH = "full_refresh" QUERY_CHANGED = "query_changed" FUNCTION_CHANGED = "function_changed" diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 6ad0eb9052..1d49045db1 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -33,6 +33,7 @@ ) from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.migrations.main._apply import apply_model_migrations from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult @@ -72,6 +73,13 @@ def execute_build_plan( schema_prepared=schema_prepared, ) + _ = apply_model_migrations( + plan=plan, + adapter=adapter, + connection=scheduler_connection, + run_id=runtime.run_id, + on_progress=resolved_callbacks.on_migration_progress, + ) _ = apply_retention_phase( plan=plan, adapter=adapter, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 9f3f62dc27..1b412b8696 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -108,6 +108,7 @@ class BuildCallbacks: python_identity_recorder: PythonIdentityRecorder | None = None on_scheduler_state: Callable[[SchedulerState], None] | None = None on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None + on_migration_progress: Callable[[str], None] | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_names.py b/src/sqlbuild/executor/janitor/_helpers/archive_names.py index b9382bfa27..af806b6cdb 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_names.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_names.py @@ -31,13 +31,17 @@ def archive_timestamp(value: datetime) -> datetime: return aware.replace(microsecond=0) -def build_archive_name(*, original_name: str, archived_at: datetime, identifier_limit: int) -> str: - """Build the fitted, case-folded archive name, keeping the prefix and timestamp intact.""" +def build_archive_name( + *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str | None = None +) -> str: + """Build the fitted, case-folded archive name, keeping prefix, timestamp, and kind intact.""" + kind_segment: str = "" if kind is None else f"{kind}{ARCHIVE_NAME_SEPARATOR}" fixed_prefix: str = ( f"{ARCHIVE_NAME_PREFIX}" f"{archive_timestamp(archived_at).strftime(ARCHIVE_TIMESTAMP_FORMAT)}" f"{ARCHIVE_NAME_SEPARATOR}" + f"{kind_segment}" ) logical_part: str = fit_artifact_logical_name( logical_name=original_name, diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 5cd2663418..7fd9271740 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -3,6 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME +from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.audit_results.constants import AUDIT_RESULTS_TABLE_NAME from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME @@ -16,6 +17,7 @@ NODE_RESULTS_TABLE_NAME, AUDIT_RESULTS_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, + MIGRATION_TABLE_NAME, ) DIRECT_DEFAULT_RETENTION_DAYS: int = 14 diff --git a/src/sqlbuild/executor/janitor/main/_build_archive_name.py b/src/sqlbuild/executor/janitor/main/_build_archive_name.py new file mode 100644 index 0000000000..e7f55b8153 --- /dev/null +++ b/src/sqlbuild/executor/janitor/main/_build_archive_name.py @@ -0,0 +1,20 @@ +"""Public janitor archive naming for relations archived outside the janitor.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlbuild.executor.janitor._helpers.archive_names import build_archive_name + + +def build_janitor_archive_name( + *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str +) -> str: + """Build a strict archive name that janitor expires by its embedded timestamp.""" + + return build_archive_name( + original_name=original_name, + archived_at=archived_at, + identifier_limit=identifier_limit, + kind=kind, + ) diff --git a/src/sqlbuild/executor/migrations/__init__.py b/src/sqlbuild/executor/migrations/__init__.py new file mode 100644 index 0000000000..3a4ddf9cb2 --- /dev/null +++ b/src/sqlbuild/executor/migrations/__init__.py @@ -0,0 +1 @@ +"""Model migration executor package.""" diff --git a/src/sqlbuild/executor/migrations/_helpers/__init__.py b/src/sqlbuild/executor/migrations/_helpers/__init__.py new file mode 100644 index 0000000000..2c57a3bbe3 --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/__init__.py @@ -0,0 +1 @@ +"""Model migration executor helpers.""" diff --git a/src/sqlbuild/executor/migrations/_helpers/naming.py b/src/sqlbuild/executor/migrations/_helpers/naming.py new file mode 100644 index 0000000000..7c4a2ae6c0 --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/naming.py @@ -0,0 +1,86 @@ +"""Fresh stage and displaced-destination names for one migration attempt.""" + +from __future__ import annotations + +import itertools +from datetime import datetime, timedelta +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( + resolve_qualified_name_parts, +) +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor.main._build_archive_name import build_janitor_archive_name +from sqlbuild.executor.migrations.constants import ( + MIGRATION_NAME_ATTEMPTS, + MIGRATION_PREVIOUS_ARCHIVE_KIND, + MIGRATION_STAGE_ARCHIVE_KIND, +) +from sqlbuild.executor.migrations.models import MigrationArtifactNames + + +def resolve_artifact_names( + *, adapter: BaseAdapter, connection: Any, entry: ModelMigrationPlanEntry, now: datetime +) -> MigrationArtifactNames: + """Pick the latest second at or before now, never later, whose archive names are unused.""" + + schema: str | None = entry.destination.schema + if schema is None: + raise ExecutorInputError( + f"model '{entry.model_name}': model migrations require a destination schema" + ) + database: str | None = entry.destination.database + identifier_limit: int = adapter.maximum_identifier_length() + candidates: tuple[tuple[str, str], ...] = tuple( + ( + build_janitor_archive_name( + original_name=entry.destination.name, + archived_at=now - timedelta(seconds=offset), + identifier_limit=identifier_limit, + kind=MIGRATION_STAGE_ARCHIVE_KIND, + ), + build_janitor_archive_name( + original_name=entry.destination.name, + archived_at=now - timedelta(seconds=offset), + identifier_limit=identifier_limit, + kind=MIGRATION_PREVIOUS_ARCHIVE_KIND, + ), + ) + for offset in range(MIGRATION_NAME_ATTEMPTS) + ) + listed: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=database, + schemas=(schema,), + names=(entry.destination.name, *itertools.chain.from_iterable(candidates)), + ) + existing: frozenset[str] = frozenset( + relation.name.lower() + for relation in listed + if (relation.schema or "").lower() == schema.lower() + ) + stage_name: str + displaced_name: str + for stage_name, displaced_name in candidates: + if stage_name not in existing and displaced_name not in existing: + return MigrationArtifactNames( + stage_name=stage_name, + stage_qualified=resolve_qualified_name_parts( + adapter=adapter, database=database, schema=schema, name=stage_name + ), + displaced_name=displaced_name, + displaced_qualified=resolve_qualified_name_parts( + adapter=adapter, database=database, schema=schema, name=displaced_name + ), + destination_qualified=resolve_qualified_name_parts( + adapter=adapter, database=database, schema=schema, name=entry.destination.name + ), + destination_exists=entry.destination.name.lower() in existing, + ) + raise ExecutorInputError( + f"model '{entry.model_name}': no unused migration stage name was found in the last " + f"{MIGRATION_NAME_ATTEMPTS} seconds" + ) diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py new file mode 100644 index 0000000000..ad6e6a8eef --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -0,0 +1,149 @@ +"""Promote a verified migration stage and record the completed migration.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS +from sqlbuild.compiler.migrations.main.deterministic_event_id import ( + deterministic_migration_event_id, +) +from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location +from sqlbuild.compiler.migrations.main.write_event import write_migration_event +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.executor.migrations.models import MigrationArtifactNames +from sqlbuild.executor.run.main.promote_staged_relation import promote_staged_relation + + +def promote_and_record( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelMigrationPlanEntry, + names: MigrationArtifactNames, + event: MigrationEvent, +) -> None: + """Promote the stage, keeping a displaced destination, then append the migration event.""" + + if not adapter.supports_transactional_ddl(): + _promote(adapter=adapter, connection=connection, names=names) + record_event(adapter=adapter, connection=connection, event=event) + return + _ = adapter.execute( + connection=connection, + sql=adapter.render_create_migration_state_table_sql( + database=event.destination.database, schema=event.destination.schema or "" + ), + ) + with adapter.transaction(connection): + rebinds: tuple[str, ...] = ( + adapter.capture_dependent_view_rebinds( + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema or "", + name=entry.destination.name, + ) + if names.destination_exists + else () + ) + _promote(adapter=adapter, connection=connection, names=names) + statement: str + for statement in rebinds: + _ = adapter.execute(connection=connection, sql=statement) + record_event( + adapter=adapter, connection=connection, event=event, create_table=False, attempts=1 + ) + + +def record_event( + *, + adapter: BaseAdapter, + connection: Any, + event: MigrationEvent, + create_table: bool = True, + attempts: int = MIGRATION_WRITE_ATTEMPTS, +) -> None: + """Append one migration event through the adapter's state-table rendering.""" + + write_migration_event( + connection=connection, + execute=adapter.execute, + event=event, + render_qualified_name=adapter.render_qualified_name, + create_table_sql=( + adapter.render_create_migration_state_table_sql( + database=event.destination.database, schema=event.destination.schema or "" + ) + if create_table + else None + ), + attempts=attempts, + ) + + +def _promote(*, adapter: BaseAdapter, connection: Any, names: MigrationArtifactNames) -> None: + promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified=names.destination_qualified, + staged_qualified=names.stage_qualified, + displaced_qualified=names.displaced_qualified, + target_exists=names.destination_exists, + statement_recorder=StatementRecorder(), + ) + + +def migration_event(*, entry: ModelMigrationPlanEntry, run_id: str) -> MigrationEvent: + """Build the append-only event recording one completed migration.""" + + origin: MigrationRelation = migration_relation_for_location(entry.origin) + destination: MigrationRelation = migration_relation_for_location(entry.destination) + return MigrationEvent( + event_id=deterministic_migration_event_id( + run_id=run_id, + target_name=entry.target_name, + origin=origin, + destination=destination, + ), + target_name=entry.target_name, + origin_model=entry.origin_model, + origin=origin, + destination_model=entry.model_name, + destination=destination, + origin_version_hash=entry.origin_version_hash, + discovery=entry.discovery, + decision=entry.decision, + run_id=run_id, + created_at=datetime.now(tz=UTC), + ) + + +def record_renames( + *, + adapter: BaseAdapter, + connection: Any, + entries: tuple[ModelMigrationPlanEntry, ...], + run_id: str, +) -> None: + """Record each newly handed-over table or view rename so retries keep the identity.""" + + entry: ModelMigrationPlanEntry + for entry in entries: + if entry.decision != MigrationDecision.RENAMED or entry.completed_at is not None: + continue + adapter.ensure_schema( + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + statement_recorder=StatementRecorder(), + ) + record_event( + adapter=adapter, + connection=connection, + event=migration_event(entry=entry, run_id=run_id), + ) diff --git a/src/sqlbuild/executor/migrations/_helpers/staging.py b/src/sqlbuild/executor/migrations/_helpers/staging.py new file mode 100644 index 0000000000..b954e24ad0 --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/staging.py @@ -0,0 +1,175 @@ +"""Create and verify an isolated migration stage without touching the live destination.""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.migrations.models import MigrationArtifactNames + + +def create_stage( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelMigrationPlanEntry, + names: MigrationArtifactNames, +) -> MigrationTransfer: + """Clone or copy the origin into the fresh stage, falling back to a copy if clone is refused.""" + + plan: MigrationStagePlan = adapter.render_migration_stage( + origin=entry.origin.qualified_name or entry.origin.name, + stage=names.stage_qualified, + origin_is_transient=entry.origin_is_transient, + stage_is_transient=entry.stage_is_transient, + ) + try: + _execute_all(adapter=adapter, connection=connection, statements=plan.statements) + except Exception as error: + if ( + not plan.fallback_statements + or plan.is_clone_refusal is None + or not plan.is_clone_refusal(error) + ): + raise + if _stage_exists(adapter=adapter, connection=connection, entry=entry, names=names): + raise ExecutorInputError( + f"migration clone into {names.stage_qualified} was refused but the stage now " + "exists; refusing to adopt a stage of unknown completeness" + ) from error + logging.getLogger("sqlbuild.migrations").warning( + "migration clone into '%s' was refused; copying instead: %s", + names.stage_name, + error, + ) + _execute_all(adapter=adapter, connection=connection, statements=plan.fallback_statements) + return MigrationTransfer.COPY + return plan.transfer + + +def verify_stage( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelMigrationPlanEntry, + names: MigrationArtifactNames, + transfer: MigrationTransfer, +) -> None: + """Refuse promotion unless the stage is complete, typed as planned, and matches the origin.""" + + stage: RelationInfo = _listed( + adapter=adapter, + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + name=names.stage_name, + missing=f"migration stage {names.stage_qualified} was not created", + ) + origin: RelationInfo = _listed( + adapter=adapter, + connection=connection, + database=entry.origin.database or entry.destination.database, + schema=entry.origin.schema, + name=entry.origin.name, + missing=f"migration origin {entry.origin.qualified_name} no longer exists", + ) + if ( + entry.stage_is_transient is not None + and stage.is_transient is not None + and stage.is_transient != entry.stage_is_transient + ): + raise ExecutorInputError( + f"migration stage {names.stage_qualified} was not created with the destination's " + "table type; refusing promotion" + ) + columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + adapter.get_columns_for_relations(connection=connection, relations=(stage, origin)) + ) + if _column_names(columns.get(stage.identity, ())) != _column_names( + columns.get(origin.identity, ()) + ): + raise ExecutorInputError( + f"migration stage {names.stage_qualified} columns differ from the origin; " + "refusing promotion" + ) + if transfer != MigrationTransfer.COPY: + return + stage_rows: int | None = _row_count( + adapter=adapter, connection=connection, relation=names.stage_qualified + ) + origin_rows: int | None = _row_count( + adapter=adapter, + connection=connection, + relation=entry.origin.qualified_name or entry.origin.name, + ) + if stage_rows != origin_rows: + raise ExecutorInputError( + f"migration stage {names.stage_qualified} holds {stage_rows} rows but the origin " + f"holds {origin_rows}; refusing to promote an incomplete copy" + ) + + +def _stage_exists( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelMigrationPlanEntry, + names: MigrationArtifactNames, +) -> bool: + listed: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=entry.destination.database, + schemas=(entry.destination.schema,) if entry.destination.schema is not None else None, + names=(names.stage_name,), + ) + return any(relation.name.lower() == names.stage_name.lower() for relation in listed) + + +def _execute_all(*, adapter: BaseAdapter, connection: Any, statements: tuple[str, ...]) -> None: + statement: str + for statement in statements: + _ = adapter.execute(connection=connection, sql=statement) + + +def _listed( + *, + adapter: BaseAdapter, + connection: Any, + database: str | None, + schema: str | None, + name: str, + missing: str, +) -> RelationInfo: + listed: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=database, + schemas=(schema,) if schema is not None else None, + names=(name,), + ) + relation: RelationInfo | None = next( + ( + relation + for relation in listed + if relation.name.lower() == name.lower() + and (relation.schema or "").lower() == (schema or "").lower() + ), + None, + ) + if relation is None: + raise ExecutorInputError(missing) + return relation + + +def _column_names(columns: tuple[ColumnInfo, ...]) -> tuple[str, ...]: + return tuple(column.name.lower() for column in columns) + + +def _row_count(*, adapter: BaseAdapter, connection: Any, relation: str) -> int | None: + cursor: Any = adapter.execute(connection=connection, sql=f"SELECT COUNT(*) FROM {relation}") + row: Any = cursor.fetchone() + return None if row is None or row[0] is None else int(row[0]) diff --git a/src/sqlbuild/executor/migrations/constants.py b/src/sqlbuild/executor/migrations/constants.py new file mode 100644 index 0000000000..2798088f67 --- /dev/null +++ b/src/sqlbuild/executor/migrations/constants.py @@ -0,0 +1,7 @@ +"""Model migration executor constants.""" + +from __future__ import annotations + +MIGRATION_STAGE_ARCHIVE_KIND: str = "migration_stage" +MIGRATION_PREVIOUS_ARCHIVE_KIND: str = "migration_previous" +MIGRATION_NAME_ATTEMPTS: int = 30 diff --git a/src/sqlbuild/executor/migrations/main/__init__.py b/src/sqlbuild/executor/migrations/main/__init__.py new file mode 100644 index 0000000000..5c0117eccc --- /dev/null +++ b/src/sqlbuild/executor/migrations/main/__init__.py @@ -0,0 +1 @@ +"""Model migration executor entrypoints.""" diff --git a/src/sqlbuild/executor/migrations/main/_apply.py b/src/sqlbuild/executor/migrations/main/_apply.py new file mode 100644 index 0000000000..d341b483a5 --- /dev/null +++ b/src/sqlbuild/executor/migrations/main/_apply.py @@ -0,0 +1,120 @@ +"""Execute planned direct-mode model migrations before any build node runs.""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.migrations._helpers.naming import resolve_artifact_names +from sqlbuild.executor.migrations._helpers.promotion import ( + migration_event, + promote_and_record, + record_renames, +) +from sqlbuild.executor.migrations._helpers.staging import create_stage, verify_stage +from sqlbuild.executor.migrations.models import MigrationArtifactNames + + +def apply_model_migrations( + *, + plan: PlanOutput, + adapter: BaseAdapter, + connection: Any, + run_id: str, + on_progress: Callable[[str], None] | None = None, +) -> None: + """Stage, verify, and promote each pending migration, then record its event.""" + + blocked: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in plan.migration_entries if entry.blocks_build + ) + if blocked: + raise ExecutorInputError( + "model migrations are blocked for " + + ", ".join(f"'{entry.model_name}' ({entry.decision.value})" for entry in blocked), + code="M103", + ) + pending: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in plan.migration_entries if entry.decision.moves_data + ) + entry: ModelMigrationPlanEntry + for entry in pending: + _ = _apply_one( + entry=entry, + adapter=adapter, + connection=connection, + run_id=run_id, + on_progress=on_progress, + ) + _ = record_renames( + adapter=adapter, connection=connection, entries=plan.migration_entries, run_id=run_id + ) + + +def _apply_one( + *, + entry: ModelMigrationPlanEntry, + adapter: BaseAdapter, + connection: Any, + run_id: str, + on_progress: Callable[[str], None] | None, +) -> None: + origin_label: str = entry.origin.qualified_name or entry.origin.name + destination_label: str = entry.destination.qualified_name or entry.destination.name + if on_progress is not None: + on_progress(f"Migrating {origin_label} -> {destination_label} ({entry.decision.value})...") + started: float = time.monotonic() + try: + transfer: MigrationTransfer = _stage_promote_and_record( + entry=entry, adapter=adapter, connection=connection, run_id=run_id + ) + except Exception as error: + raise ExecutorInputError( + f"model migration {origin_label} -> {destination_label} failed: {error}", + code="M106", + help=( + "Re-run the build; migration decisions are re-evaluated from recorded events. " + "Any abandoned _sqb_archive__ stage is left for janitor to expire." + ), + ) from error + if on_progress is not None: + on_progress( + f"Migrated {origin_label} -> {destination_label} by {transfer.label}" + f"{' (clone refused)' if transfer != entry.transfer else ''}. " + f"({time.monotonic() - started:.2f}s)" + ) + + +def _stage_promote_and_record( + *, entry: ModelMigrationPlanEntry, adapter: BaseAdapter, connection: Any, run_id: str +) -> MigrationTransfer: + adapter.ensure_schema( + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + statement_recorder=StatementRecorder(), + ) + names: MigrationArtifactNames = resolve_artifact_names( + adapter=adapter, connection=connection, entry=entry, now=datetime.now(tz=UTC) + ) + transfer: MigrationTransfer = create_stage( + adapter=adapter, connection=connection, entry=entry, names=names + ) + _ = verify_stage( + adapter=adapter, connection=connection, entry=entry, names=names, transfer=transfer + ) + _ = promote_and_record( + adapter=adapter, + connection=connection, + entry=entry, + names=names, + event=migration_event(entry=entry, run_id=run_id), + ) + return transfer diff --git a/src/sqlbuild/executor/migrations/models.py b/src/sqlbuild/executor/migrations/models.py new file mode 100644 index 0000000000..7a6994abee --- /dev/null +++ b/src/sqlbuild/executor/migrations/models.py @@ -0,0 +1,17 @@ +"""Model migration executor models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MigrationArtifactNames: + """Fresh janitor-archive names for one migration attempt's stage and displaced destination.""" + + stage_name: str + stage_qualified: str + displaced_name: str + displaced_qualified: str + destination_qualified: str + destination_exists: bool diff --git a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py index 8edc02b125..97abb0c853 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py @@ -95,49 +95,73 @@ def promote_full_refresh_rebuild( target_kind="relation", ), ) as lifecycle: - if not target_exists: - adapter.rename( - connection=connection, - origin=relations.rebuild_qualified, - destination=relations.target_qualified, - statement_recorder=statement_recorder, - ) - else: + if target_exists: adapter.drop( connection=connection, destination=relations.previous_qualified, if_exists=True, statement_recorder=statement_recorder, ) - if adapter.adapter_name == BuiltinAdapter.SNOWFLAKE: - adapter.swap( - connection=connection, - left=relations.target_qualified, - right=relations.rebuild_qualified, - statement_recorder=statement_recorder, - ) - adapter.rename( - connection=connection, - origin=relations.rebuild_qualified, - destination=relations.previous_qualified, - statement_recorder=statement_recorder, - ) - else: - adapter.rename( - connection=connection, - origin=relations.target_qualified, - destination=relations.previous_qualified, - statement_recorder=statement_recorder, - ) - adapter.rename( - connection=connection, - origin=relations.rebuild_qualified, - destination=relations.target_qualified, - statement_recorder=statement_recorder, - ) + promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified=relations.target_qualified, + staged_qualified=relations.rebuild_qualified, + displaced_qualified=relations.previous_qualified, + target_exists=target_exists, + statement_recorder=statement_recorder, + ) lifecycle.completed(metadata={"changed_count": 1}) +def promote_staged_relation( + *, + adapter: BaseAdapter, + connection: Any, + target_qualified: str, + staged_qualified: str, + displaced_qualified: str, + target_exists: bool, + statement_recorder: StatementRecorder, +) -> None: + """Promote a staged relation without dropping anything; a live target moves aside.""" + + if not target_exists: + adapter.rename( + connection=connection, + origin=staged_qualified, + destination=target_qualified, + statement_recorder=statement_recorder, + ) + return + if adapter.adapter_name == BuiltinAdapter.SNOWFLAKE: + adapter.swap( + connection=connection, + left=target_qualified, + right=staged_qualified, + statement_recorder=statement_recorder, + ) + adapter.rename( + connection=connection, + origin=staged_qualified, + destination=displaced_qualified, + statement_recorder=statement_recorder, + ) + return + adapter.rename( + connection=connection, + origin=target_qualified, + destination=displaced_qualified, + statement_recorder=statement_recorder, + ) + adapter.rename( + connection=connection, + origin=staged_qualified, + destination=target_qualified, + statement_recorder=statement_recorder, + ) + + def _artifact_name(*, logical_name: str, fixed_prefix: str, identifier_limit: int) -> str: available: int = identifier_limit - len(fixed_prefix) if len(logical_name) <= available: diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py index 99183ae4fd..9cc0e2759b 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py @@ -11,6 +11,7 @@ from sqlbuild.compiler.auditing.models import AuditGateIdentity, AuditIdentity from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY +from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.models import ( @@ -84,6 +85,20 @@ def model_fingerprint_metadata_with_audit_gate( return json.dumps(metadata, sort_keys=True, separators=(",", ":"), default=str) +def model_fingerprint_metadata_with_migration_fingerprint( + *, metadata_json: str, migration_fingerprint: str | None +) -> str: + """Return model fingerprint metadata JSON carrying the rename-matching fingerprint.""" + + if migration_fingerprint is None: + return metadata_json + metadata: object = json.loads(metadata_json or "{}") + if not isinstance(metadata, dict): + return metadata_json + metadata[MIGRATION_FINGERPRINT_METADATA_KEY] = migration_fingerprint + return json.dumps(metadata, sort_keys=True, separators=(",", ":"), default=str) + + def same_target_audit_gate_reuse_decision( *, metadata_json: str | None, model_audits: tuple[AuditPlanEntry, ...] ) -> AuditGateReuseDecision: diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index 4712de9e75..b823a163d4 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -15,6 +15,7 @@ from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( model_fingerprint_metadata_with_audit_gate, + model_fingerprint_metadata_with_migration_fingerprint, ) @@ -46,6 +47,9 @@ def try_write_fingerprint( audit_results=audit_results, run_id=run_id, ) + metadata_json = model_fingerprint_metadata_with_migration_fingerprint( + metadata_json=metadata_json, migration_fingerprint=entry.migration_fingerprint + ) fingerprint: Fingerprint = Fingerprint( node_type=NODE_TYPE_MODEL, node_name=entry.name, diff --git a/src/sqlbuild/executor/run/main/promote_staged_relation.py b/src/sqlbuild/executor/run/main/promote_staged_relation.py new file mode 100644 index 0000000000..02465892ae --- /dev/null +++ b/src/sqlbuild/executor/run/main/promote_staged_relation.py @@ -0,0 +1,34 @@ +"""Public staged-relation promotion shared by build-aside rebuilds and model migrations.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.executor.run._helpers.materializations.full_refresh import ( + promote_staged_relation as _promote_staged_relation, +) + + +def promote_staged_relation( + *, + adapter: BaseAdapter, + connection: Any, + target_qualified: str, + staged_qualified: str, + displaced_qualified: str, + target_exists: bool, + statement_recorder: StatementRecorder, +) -> None: + """Promote a staged relation into its target, moving a live target to the displaced name.""" + + _ = _promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified=target_qualified, + staged_qualified=staged_qualified, + displaced_qualified=displaced_qualified, + target_exists=target_exists, + statement_recorder=statement_recorder, + ) diff --git a/src/sqlbuild/virtual/executor/_helpers/build.py b/src/sqlbuild/virtual/executor/_helpers/build.py index cd893bc423..3a61334f74 100644 --- a/src/sqlbuild/virtual/executor/_helpers/build.py +++ b/src/sqlbuild/virtual/executor/_helpers/build.py @@ -19,6 +19,7 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup +from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -533,6 +534,20 @@ def _execute_leased_virtual_build( return result, ingress_python_results, read_side_results +def _reject_model_migrations(*, project: CompiledProject) -> None: + declared: tuple[str, ...] = tuple( + model.name + for model in project.models + if model.config.values.get(MIGRATE_FROM_CONFIG_KEY) is not None + ) + if declared: + raise PlannerInputError( + "migrate_from is supported only in direct mode; virtual environments do not run " + f"model migrations (declared by: {', '.join(declared)})", + code="M105", + ) + + def _resolve_virtual_build( *, project_dir: Path, @@ -886,6 +901,7 @@ def _plan_virtual_build( reads: _VirtualBuildStateReads, deferred_relations: dict[str, RelationInfo], ) -> _VirtualBuildPlan: + _reject_model_migrations(project=graph.project) adapter: BaseAdapter = runtime.adapter options: VirtualBuildOptions = runtime.options hooks: VirtualBuildHooks = runtime.hooks diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py new file mode 100644 index 0000000000..9a8644d8fb --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -0,0 +1,14 @@ +"""Test case types for model migration CLI e2e tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MigrationLifecycleE2ETestCase: + description: str + expected_step_decisions: tuple[str, ...] + expected_final_ids: tuple[int, ...] + expected_previous_archive_ids: tuple[tuple[int, ...], ...] + expected_events: tuple[tuple[str, str, str], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py new file mode 100644 index 0000000000..070c2dfd3b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -0,0 +1,127 @@ +"""Project builders for model migration CLI e2e tests.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +DATABASE_FILE: str = "orders.duckdb" +_PROJECT_TOML: str = ( + f'name = "orders_project"\nadapter = "duckdb"\n\n[connection]\ndatabase = "{DATABASE_FILE}"\n' +) +_SOURCES_YML: str = "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + + +def orders_sql(*, migrate_from: str) -> str: + """Return an incremental orders model with an optional migrate_from header.""" + + migration: str = {"": ""}.get(migrate_from, f' migrate_from "{migrate_from}",\n') + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + f"{migration}" + ");\n\n" + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")\n' + ) + + +def write_orders_project(*, tmp_path: Path, name: str, migrate_from: str) -> Path: + """Write a project containing exactly one incremental orders model.""" + + project_dir: Path = tmp_path / "orders_project" + stale: Path + for stale in (project_dir / "models").glob("*.sql"): + stale.unlink() + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": _SOURCES_YML, + f"models/{name}.sql": orders_sql(migrate_from=migrate_from), + }, + ) + + +def load_raw_orders(*, project_dir: Path, last_day: int) -> None: + """Replace raw orders with one order per January day up to the given day.""" + + execute_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "CREATE OR REPLACE TABLE main.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) AS order_date, " + f"100 + i AS amount_cents FROM range(1, {last_day + 1}) AS t(i)" + ), + ) + + +def plan_decisions(*, project_dir: Path) -> tuple[str, ...]: + """Run sqb plan --json in a subprocess and return the migration decisions.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--json"), project_dir=project_dir + ) + payload: dict[str, Any] = json.loads(result.stdout) + return tuple(str(migration["decision"]) for migration in payload["migrations"]) + + +def build(*, project_dir: Path) -> subprocess.CompletedProcess[str]: + """Run sqb build in a subprocess.""" + + return run_sqb(command=("--no-color", "build"), project_dir=project_dir) + + +def order_ids(*, project_dir: Path, relation: str) -> tuple[int, ...]: + """Return the sorted order IDs stored in one relation.""" + + return tuple( + int(row[0]) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=f"SELECT order_id FROM {relation} ORDER BY 1", + ) + ) + + +def previous_archive_ids(*, project_dir: Path) -> tuple[tuple[int, ...], ...]: + """Return the order IDs held by each displaced-destination archive.""" + + names: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' " + "AND contains(table_name, '__migration_previous__') ORDER BY 1" + ), + ) + return tuple(order_ids(project_dir=project_dir, relation=f"main.{row[0]}") for row in names) + + +def migration_events(*, project_dir: Path) -> tuple[tuple[str, str, str], ...]: + """Return (origin, destination, decision) for every recorded migration event in order.""" + + return tuple( + (str(row[0]), str(row[1]), str(row[2])) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT origin_name, destination_name, decision FROM main._sqlbuild_migrations " + "ORDER BY created_at, event_id" + ), + ) + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_model_migration_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_model_migration_lifecycle.py new file mode 100644 index 0000000000..a8a3f9d51a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_model_migration_lifecycle.py @@ -0,0 +1,76 @@ +"""CLI e2e coverage for a model renamed away and back through staged migrations.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + MigrationLifecycleE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + build, + load_raw_orders, + migration_events, + order_ids, + plan_decisions, + previous_archive_ids, + write_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationLifecycleE2ETestCase( + description="rename away then back keeps history and archives the stale original", + expected_step_decisions=("migrate", "superseded_replace", "done"), + expected_final_ids=(1, 2, 3, 4, 5), + expected_previous_archive_ids=((1, 2, 3),), + expected_events=( + ("stg_orders", "stg_customer_orders", "migrate"), + ("stg_customer_orders", "stg_orders", "superseded_replace"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_model_renamed_away_and_back_when_building_then_history_follows_the_model( + tmp_path: Path, test_case: MigrationLifecycleE2ETestCase +) -> None: + """The real CLI stages, promotes, archives, and records each move.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path, name="stg_orders", migrate_from="") + load_raw_orders(project_dir=project_dir, last_day=3) + initial: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + _ = write_orders_project( + tmp_path=tmp_path, name="stg_customer_orders", migrate_from="stg_orders" + ) + load_raw_orders(project_dir=project_dir, last_day=4) + away_decisions: tuple[str, ...] = plan_decisions(project_dir=project_dir) + away: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + _ = write_orders_project( + tmp_path=tmp_path, name="stg_orders", migrate_from="stg_customer_orders" + ) + load_raw_orders(project_dir=project_dir, last_day=5) + back_decisions: tuple[str, ...] = plan_decisions(project_dir=project_dir) + back: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + done_decisions: tuple[str, ...] = plan_decisions(project_dir=project_dir) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert away.returncode == 0, away.stdout + away.stderr + assert back.returncode == 0, back.stdout + back.stderr + assert away_decisions + back_decisions + done_decisions == test_case.expected_step_decisions + assert order_ids(project_dir=project_dir, relation="main.stg_orders") == ( + test_case.expected_final_ids + ) + assert previous_archive_ids(project_dir=project_dir) == ( + test_case.expected_previous_archive_ids + ) + assert migration_events(project_dir=project_dir) == test_case.expected_events + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index 91164c51e3..de8b43bd2f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -260,3 +260,22 @@ class PostgresDbtSeedChangeE2ETestCase: expected_changed_total: int expected_changed_fragments: tuple[str, ...] expected_rerun_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PostgresModelMigrationE2ETestCase: + description: str + expected_destination_ids: tuple[tuple[object, ...], ...] + expected_view_ids: tuple[tuple[object, ...], ...] + expected_archive_ids: tuple[tuple[object, ...], ...] + expected_view_options: tuple[tuple[object, ...], ...] + expected_events: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class PostgresMigrationRollbackE2ETestCase: + description: str + expected_failure_fragment: str + expected_destination_ids_after_failure: tuple[tuple[object, ...], ...] + expected_previous_archives_after_failure: int + expected_final_destination_ids: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py index 85659c4ec4..dae8b3c63a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py @@ -13,6 +13,7 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, prepare_source_loader_strategies, prepare_waffle_shop, stringify_warehouse_rows, @@ -631,3 +632,175 @@ def append_postgres_dbt_seed_change_order(*, project_dir: Path, order_id: int, a seed_path.read_text(encoding="utf-8") + f"{order_id},{amount}\n", encoding="utf-8", ) + + +ORIGIN_MODEL: str = "stg_orders" +DESTINATION_MODEL: str = "stg_customer_orders" +_JANITOR_CONFIG: str = "\n[janitor]\nenabled = true\narchive_retention_days = 0\n" +_MIGRATION_TEXT_COLUMNS: tuple[str, ...] = ( + "event_id", + "target_name", + "origin_model", + "origin_database", + "origin_schema", + "origin_name", + "destination_model", + "destination_database", + "destination_schema", + "destination_name", + "origin_version_hash", + "discovery", + "decision", + "run_id", +) + + +def orders_sql(*, migrate_from: str = "", where: str = "") -> str: + """Return an incremental orders model with an optional forced migration header.""" + + migration: str = {"": ""}.get( + migrate_from, f' migrate_from "{migrate_from}",\n migrate_force true,\n' + ) + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + f"{migration}" + ");\n\n" + f'SELECT order_id, order_date, amount_cents FROM __source("raw_orders"){where}\n' + ) + + +def report_view_sql() -> str: + """Return a project view that reads the destination by name.""" + + return f'MODEL (materialized view);\n\nSELECT order_id FROM __ref("{DESTINATION_MODEL}")\n' + + +def write_migration_project( + *, + tmp_path: Path, + schema_name: str, + raw_schema_name: str, + config: dict[str, object], + models: dict[str, str], +) -> Path: + """Write a Postgres project whose sources live outside the managed schema.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": build_postgres_project_toml( + project_name="postgres_model_migrations", schema_name=schema_name, config=config + ) + + _JANITOR_CONFIG, + "sources/raw.yml": ( + f"sources:\n - name: raw_orders\n schema: {raw_schema_name}\n" + " table: raw_orders\n" + ), + } + project_dir: Path = tmp_path / "postgres_model_migrations" + models_dir: Path = project_dir / "models" + stale: Path + for stale in models_dir.glob("*.sql"): + stale.unlink() + files.update({f"models/{name}.sql": sql for name, sql in models.items()}) + return prepare_inline_project( + tmp_path=tmp_path, project_name="postgres_model_migrations", repo_files=files + ) + + +def load_raw_orders(*, raw_schema_name: str, config: dict[str, object]) -> None: + """Create five raw orders, one per January day, in the source schema.""" + + execute_postgres_sql(sql=f"CREATE SCHEMA IF NOT EXISTS {raw_schema_name}", config=config) + execute_postgres_sql( + sql=( + f"CREATE TABLE {raw_schema_name}.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + (i - 1) * INTERVAL '1 day' AS order_date, " + "100 + i AS amount_cents FROM generate_series(1, 5) AS t(i)" + ), + config=config, + ) + + +def create_unwritable_migration_table(*, schema_name: str, config: dict[str, object]) -> None: + """Occupy the migration state name with a read-only view so the event insert fails.""" + + columns: str = ", ".join( + f"CAST(NULL AS TEXT) AS {column}" for column in _MIGRATION_TEXT_COLUMNS + ) + execute_postgres_sql( + sql=( + f"CREATE VIEW {schema_name}._sqlbuild_migrations AS SELECT {columns}, " + "CAST(NULL AS TIMESTAMP) AS created_at WHERE false" + ), + config=config, + ) + + +def archive_names( + *, schema_name: str, kind: str, config: dict[str, object] +) -> tuple[tuple[object, ...], ...]: + """Return every migration archive of one kind in the managed schema.""" + + return fetch_postgres_rows( + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema_name}' " + "AND table_name LIKE '\\_sqb\\_archive\\_\\_%' " + f"AND position('__{kind}__' IN table_name) > 0 ORDER BY 1" + ), + config=config, + ) + + +def ordered_ids(*, relation: str, config: dict[str, object]) -> tuple[tuple[object, ...], ...]: + """Return the sorted order IDs readable through one relation.""" + + return fetch_postgres_rows(sql=f"SELECT order_id FROM {relation} ORDER BY 1", config=config) + + +def create_external_dependent_views( + *, raw_schema_name: str, schema_name: str, config: dict[str, object] +) -> None: + """Create plain, security-option, and check-option views on the destination.""" + + destination: str = f"{schema_name}.{DESTINATION_MODEL}" + statements: tuple[str, ...] = ( + f"CREATE VIEW {raw_schema_name}.orders_dashboard AS SELECT order_id FROM {destination}", + f"GRANT SELECT ON {raw_schema_name}.orders_dashboard TO PUBLIC", + ( + f"CREATE VIEW {raw_schema_name}.orders_invoker " + "WITH (security_barrier=true, security_invoker=true) AS " + f"SELECT order_id FROM {destination}" + ), + ( + f"CREATE VIEW {raw_schema_name}.orders_checked AS " + f"SELECT order_id, order_date, amount_cents FROM {destination} " + "WHERE order_id > 0 WITH CASCADED CHECK OPTION" + ), + ) + statement: str + for statement in statements: + execute_postgres_sql(sql=statement, config=config) + + +def view_catalog( + *, raw_schema_name: str, config: dict[str, object] +) -> tuple[tuple[object, ...], ...]: + """Return (view, options, owner, privileges) for every view in the source schema.""" + + return fetch_postgres_rows( + sql=( + "SELECT view.relname, coalesce(array_to_string(view.reloptions, ','), ''), " + "pg_get_userbyid(view.relowner), coalesce(view.relacl::text, '') " + "FROM pg_class AS view JOIN pg_namespace AS namespace " + "ON namespace.oid = view.relnamespace " + f"WHERE namespace.nspname = '{raw_schema_name}' AND view.relkind = 'v' ORDER BY 1" + ), + config=config, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py new file mode 100644 index 0000000000..5b615b11b3 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py @@ -0,0 +1,243 @@ +"""PostgreSQL e2e coverage for staged model migrations and identity-bound dependent views.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.postgres._test_types import ( + PostgresMigrationRollbackE2ETestCase, + PostgresModelMigrationE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.postgres.helpers import ( + DESTINATION_MODEL, + ORIGIN_MODEL, + archive_names, + build_unique_schema_name, + cleanup_postgres_schema, + create_external_dependent_views, + create_unwritable_migration_table, + ensure_postgres_schema_ready, + execute_postgres_sql, + fetch_postgres_rows, + load_raw_orders, + ordered_ids, + orders_sql, + report_view_sql, + view_catalog, + write_migration_project, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +_TWO_ORDERS: str = " WHERE order_id <= 2" +_ALL_IDS: tuple[tuple[object, ...], ...] = ((1,), (2,), (3,), (4,), (5,)) + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresModelMigrationE2ETestCase( + description="forced replace rebinds identity-bound views and archives the old table", + expected_destination_ids=_ALL_IDS, + expected_view_ids=_ALL_IDS, + expected_archive_ids=((1,), (2,)), + expected_view_options=( + ("orders_checked", "check_option=cascaded"), + ("orders_dashboard", ""), + ("orders_invoker", "security_barrier=true,security_invoker=true"), + ), + expected_events=((ORIGIN_MODEL, DESTINATION_MODEL, "forced_replace"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_views_on_destination_when_forcing_migration_then_views_follow_the_new_table( + tmp_path: Path, + test_case: PostgresModelMigrationE2ETestCase, + postgres_e2e_config: dict[str, object], +) -> None: + """Views bound by OID read the promoted table, so janitor can drop the archive.""" + + schema_name: str = build_unique_schema_name(prefix="sqlbuild_migration") + raw_schema_name: str = f"{schema_name}_raw" + ensure_postgres_schema_ready(schema_name=schema_name, config=postgres_e2e_config) + load_raw_orders(raw_schema_name=raw_schema_name, config=postgres_e2e_config) + project_dir: Path = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={ + ORIGIN_MODEL: orders_sql(), + DESTINATION_MODEL: orders_sql(where=_TWO_ORDERS), + "orders_report": report_view_sql(), + }, + ) + try: + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + create_external_dependent_views( + raw_schema_name=raw_schema_name, schema_name=schema_name, config=postgres_e2e_config + ) + views_before: tuple[tuple[object, ...], ...] = view_catalog( + raw_schema_name=raw_schema_name, config=postgres_e2e_config + ) + project_dir = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={ + DESTINATION_MODEL: orders_sql(migrate_from=ORIGIN_MODEL), + "orders_report": report_view_sql(), + }, + ) + migrated: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + views_after: tuple[tuple[object, ...], ...] = view_catalog( + raw_schema_name=raw_schema_name, config=postgres_e2e_config + ) + archives: tuple[tuple[object, ...], ...] = archive_names( + schema_name=schema_name, kind="migration_previous", config=postgres_e2e_config + ) + archive_ids: tuple[tuple[object, ...], ...] = ordered_ids( + relation=f"{schema_name}.{archives[0][0]}", config=postgres_e2e_config + ) + janitor: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "janitor", "--auto-approve", "--retention-days", "30"), + project_dir=project_dir, + ) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert migrated.returncode == 0, migrated.stdout + migrated.stderr + assert views_after == views_before + assert tuple((row[0], row[1]) for row in views_after) == test_case.expected_view_options + assert ( + ordered_ids(relation=f"{raw_schema_name}.orders_checked", config=postgres_e2e_config) + == test_case.expected_view_ids + ) + assert len(archives) == 1 + assert archive_ids == test_case.expected_archive_ids + assert janitor.returncode == 0, janitor.stdout + janitor.stderr + assert ( + archive_names( + schema_name=schema_name, kind="migration_previous", config=postgres_e2e_config + ) + == () + ) + assert ( + ordered_ids(relation=f"{schema_name}.{DESTINATION_MODEL}", config=postgres_e2e_config) + == test_case.expected_destination_ids + ) + assert ( + ordered_ids(relation=f"{raw_schema_name}.orders_dashboard", config=postgres_e2e_config) + == test_case.expected_view_ids + ) + assert ( + ordered_ids(relation=f"{schema_name}.orders_report", config=postgres_e2e_config) + == test_case.expected_view_ids + ) + assert ( + ordered_ids(relation=f"{schema_name}.{ORIGIN_MODEL}", config=postgres_e2e_config) + == _ALL_IDS + ) + assert ( + fetch_postgres_rows( + sql=( + "SELECT origin_name, destination_name, decision " + f"FROM {schema_name}._sqlbuild_migrations" + ), + config=postgres_e2e_config, + ) + == test_case.expected_events + ) + finally: + cleanup_postgres_schema(schema_name=raw_schema_name, config=postgres_e2e_config) + cleanup_postgres_schema(schema_name=schema_name, config=postgres_e2e_config) + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresMigrationRollbackE2ETestCase( + description="failed event insert rolls the promotion back and keeps the destination", + expected_failure_fragment="M106", + expected_destination_ids_after_failure=((1,), (2,)), + expected_previous_archives_after_failure=0, + expected_final_destination_ids=_ALL_IDS, + ) + ], + ids=lambda case: case.description, +) +def test_given_unwritable_event_table_when_migrating_then_promotion_rolls_back( + tmp_path: Path, + test_case: PostgresMigrationRollbackE2ETestCase, + postgres_e2e_config: dict[str, object], +) -> None: + """Promotion and the migration event commit or roll back together on PostgreSQL.""" + + schema_name: str = build_unique_schema_name(prefix="sqlbuild_migration_rollback") + raw_schema_name: str = f"{schema_name}_raw" + ensure_postgres_schema_ready(schema_name=schema_name, config=postgres_e2e_config) + load_raw_orders(raw_schema_name=raw_schema_name, config=postgres_e2e_config) + project_dir: Path = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={ + ORIGIN_MODEL: orders_sql(), + DESTINATION_MODEL: orders_sql(where=_TWO_ORDERS), + }, + ) + try: + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + create_unwritable_migration_table(schema_name=schema_name, config=postgres_e2e_config) + project_dir = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={DESTINATION_MODEL: orders_sql(migrate_from=ORIGIN_MODEL)}, + ) + failed: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + ids_after_failure: tuple[tuple[object, ...], ...] = ordered_ids( + relation=f"{schema_name}.{DESTINATION_MODEL}", config=postgres_e2e_config + ) + previous_after_failure: int = len( + archive_names( + schema_name=schema_name, kind="migration_previous", config=postgres_e2e_config + ) + ) + execute_postgres_sql( + sql=f"DROP VIEW {schema_name}._sqlbuild_migrations", config=postgres_e2e_config + ) + retried: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert failed.returncode != 0 + assert test_case.expected_failure_fragment in failed.stdout + failed.stderr + assert ids_after_failure == test_case.expected_destination_ids_after_failure + assert previous_after_failure == test_case.expected_previous_archives_after_failure + assert retried.returncode == 0, retried.stdout + retried.stderr + assert ( + ordered_ids(relation=f"{schema_name}.{DESTINATION_MODEL}", config=postgres_e2e_config) + == test_case.expected_final_destination_ids + ) + finally: + cleanup_postgres_schema(schema_name=raw_schema_name, config=postgres_e2e_config) + cleanup_postgres_schema(schema_name=schema_name, config=postgres_e2e_config) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/__init__.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/__init__.py new file mode 100644 index 0000000000..5d734971d6 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/__init__.py @@ -0,0 +1 @@ +"""Integration tests for direct-mode model migrations.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py new file mode 100644 index 0000000000..034efd68b6 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -0,0 +1,134 @@ +"""Test case types for model migration integration tests.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +import pytest + + +@dataclass(frozen=True) +class MigrationOutcomeTestCase: + description: str + expected_decisions: tuple[str, ...] + expected_events: tuple[tuple[str, str, str], ...] = () + expected_destination_ids: tuple[int, ...] = () + expected_output_fragment: str = "" + expected_reason: str = "" + + +@dataclass(frozen=True) +class BackAndForthMigrationTestCase: + description: str + expected_day_decisions: tuple[tuple[str, ...], ...] + expected_day_five_ids: tuple[int, ...] + expected_final_ids: tuple[int, ...] + expected_events: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class MigrationCompileErrorTestCase: + description: str + model_sql: str + expected_fragment: str + + +@dataclass(frozen=True) +class MigrationCompatibilityTestCase: + description: str + origin_setup_sql: str + destination_extra_config: str + destination_select_sql: str + expected_compatibility: str + expected_build_exit_code: int + expected_fragment: str + + +@dataclass(frozen=True) +class MigrationInterruptionTestCase: + description: str + install_failure: Callable[[pytest.MonkeyPatch], None] + rerun_with_force: bool + expected_first_exit_code: int + expected_decision_after_failure: str + expected_final_decisions: tuple[str, ...] + + +@dataclass(frozen=True) +class AutomaticMigrationTestCase: + description: str + expected_migrations: tuple[tuple[str | None, str, str, str], ...] = () + expected_events: tuple[tuple[str, str, str], ...] = () + expected_reason: str = "" + expected_warning: str = "" + + +@dataclass(frozen=True) +class PlanAsTargetTestCase: + description: str + preview_target: str + expected_exit_code: int + expected_decisions: tuple[str, ...] = () + expected_fragment: str = "" + + +@dataclass(frozen=True) +class VirtualModeMigrationTestCase: + description: str + expected_exit_code: int + expected_fragment: str + + +@dataclass(frozen=True) +class StagedReplaceTestCase: + description: str + install_failure: Callable[[pytest.MonkeyPatch], None] + expected_first_exit_code: int + expected_relations_after_failure: tuple[str, ...] + expected_previous_ids_after_failure: tuple[tuple[int, ...], ...] + expected_decision_after_failure: str + expected_final_previous_ids: tuple[tuple[int, ...], ...] + expected_final_abandoned_stages: int + expected_final_events: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class StagedPlanOutputTestCase: + description: str + install_stage: Callable[[pytest.MonkeyPatch], None] + expected_transfer_fallback: str | None + expected_build_fragment: str + expected_transfer: str + expected_promotion: str + expected_storage_transition: str | None + expected_text_fragment: str + + +@dataclass(frozen=True) +class MigrationArchiveExpiryTestCase: + description: str + janitor_retention_days: str + expected_archives_before: int + expected_relations_after: tuple[str, ...] + expected_migration_state_kept: bool + + +@dataclass(frozen=True) +class IdentityHandoverTestCase: + description: str + middle_materialization: str + install_failure: Callable[..., None] + failing_model: str + expected_first_exit_code: int + expected_first_view_reason: str + expected_retry_totals_reason: str + expected_totals_rows: int + expected_marked_total: int + expected_events: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class RenamedPlanTextTestCase: + description: str + expected_fragment: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py new file mode 100644 index 0000000000..1a3364ef7b --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -0,0 +1,744 @@ +"""Project builders and CLI runners for model migration integration tests.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from textwrap import dedent +from typing import Any + +import duckdb +import pytest +from _pytest.capture import CaptureResult + +from sqlbuild.adapter.contract.models import MigrationStagePlan +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.executor.build._helpers import scheduler as scheduler_module + +DATABASE_FILE: str = "orders.duckdb" +PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + + [connection] + database = "{DATABASE_FILE}" + """ +).lstrip() +TARGETS_PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + default_target = "dev" + + [connection] + database = "{DATABASE_FILE}" + + [targets.dev] + schema = "dev" + + [targets.prod] + schema = "prod" + """ +).lstrip() +VIRTUAL_PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + default_target = "dev" + + [settings] + virtual_environments = true + + [connection] + database = "{DATABASE_FILE}" + + [targets.dev] + schema = "dev" + + [targets.dev.state] + backend = "duckdb" + schema = "sqlbuild_state" + + [targets.dev.state.connection] + database = "state.duckdb" + """ +).lstrip() +JANITOR_PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + + default_target = "dev" + + [connection] + database = "{DATABASE_FILE}" + + [targets.dev] + schema = "dev" + + [janitor] + enabled = true + archive_retention_days = 0 + delete_tracked_only = false + """ +).lstrip() +RAW_SOURCES_YML: str = dedent( + """ + sources: + - name: raw_orders + schema: main + table: raw_orders + """ +).lstrip() + + +_FORCE_LINES: dict[bool, str] = {True: " migrate_force true,\n", False: ""} +_DAILY_TOTALS_HEADER: str = ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_date,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + "{extra_config}" + ");\n\n" +) +_SNAPSHOT_SQL: str = ( + "MODEL (\n" + " materialized snapshot,\n" + " unique_key [order_id],\n" + " snapshot_strategy timestamp,\n" + " updated_at order_date,\n" + "{migration}" + ");\n\n" + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")\n' +) +ORIGIN_MODEL: str = "stg_orders" +DESTINATION_MODEL: str = "stg_customer_orders" + + +@dataclass(frozen=True) +class CliRun: + """Captured result of one in-process sqb invocation.""" + + exit_code: int + output: str + + +def incremental_orders_sql( + *, + migrate_from: str | None = None, + migrate_force: bool = False, + extra_config: str = "", + select_sql: str = 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")', +) -> str: + """Return an incremental orders model with optional migration header keys.""" + + origin_lines: dict[str | None, str] = {None: ""} + migration_lines: str = origin_lines.get( + migrate_from, f' migrate_from "{migrate_from}",\n' + ) + _FORCE_LINES.get(migrate_force, "") + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + f"{extra_config}" + f"{migration_lines}" + ");\n\n" + f"{select_sql}\n" + ) + + +def write_project( + *, project_dir: Path, models: dict[str, str], project_toml: str = PROJECT_TOML +) -> None: + """Write the project config, sources, and exactly the given model files.""" + + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "sqlbuild_project.toml").write_text(project_toml, encoding="utf-8") + sources_dir: Path = project_dir / "sources" + sources_dir.mkdir(exist_ok=True) + (sources_dir / "raw.yml").write_text(RAW_SOURCES_YML, encoding="utf-8") + models_dir: Path = project_dir / "models" + models_dir.mkdir(exist_ok=True) + existing: Path + for existing in models_dir.glob("*.sql"): + existing.unlink() + name: str + sql: str + for name, sql in models.items(): + (models_dir / f"{name}.sql").write_text(sql, encoding="utf-8") + + +def load_raw_orders(*, project_dir: Path, first_day: int, last_day: int) -> None: + """Replace raw orders with one order per day in the inclusive January day range.""" + + execute( + project_dir=project_dir, + sql=( + "CREATE OR REPLACE TABLE main.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) AS order_date, " + "100 + i AS amount_cents " + f"FROM range({first_day}, {last_day + 1}) AS t(i)" + ), + ) + + +def execute(*, project_dir: Path, sql: str) -> None: + """Run one mutating statement against the project database.""" + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / DATABASE_FILE)) + try: + _ = connection.execute(sql) + finally: + connection.close() + + +def query(*, project_dir: Path, sql: str) -> list[tuple[Any, ...]]: + """Read rows from the project database.""" + + connection: duckdb.DuckDBPyConnection = duckdb.connect( + str(project_dir / DATABASE_FILE), read_only=True + ) + try: + return connection.execute(sql).fetchall() + finally: + connection.close() + + +def order_ids(*, project_dir: Path, relation: str) -> tuple[int, ...]: + """Return the sorted order IDs stored in one relation.""" + + return tuple( + int(row[0]) + for row in query(project_dir=project_dir, sql=f"SELECT order_id FROM {relation} ORDER BY 1") + ) + + +def migration_events( + *, project_dir: Path, schema: str = "main" +) -> tuple[tuple[str, str, str], ...]: + """Return (origin, destination, decision) for every recorded event in creation order.""" + + return tuple( + (str(row[0]), str(row[1]), str(row[2])) + for row in query( + project_dir=project_dir, + sql=( + "SELECT origin_name, destination_name, decision " + f"FROM {schema}._sqlbuild_migrations ORDER BY created_at, event_id" + ), + ) + ) + + +def relation_names(*, project_dir: Path, schema: str) -> list[str]: + """Return every relation name in one schema.""" + + return [ + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' ORDER BY 1" + ), + ) + ] + + +def run_sqb( + *, project_dir: Path, args: tuple[str, ...], capsys: pytest.CaptureFixture[str] +) -> CliRun: + """Run sqb in-process and capture combined output.""" + + _ = capsys.readouterr() + exit_code: int = main(["--project-dir", str(project_dir), "--no-color", *args]) + captured: CaptureResult[str] = capsys.readouterr() + return CliRun(exit_code=exit_code, output=captured.out + captured.err) + + +def build(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: + """Run sqb build, reporting an escaped exception as a failed run.""" + + try: + return run_sqb(project_dir=project_dir, args=("build",), capsys=capsys) + except Exception as error: + return CliRun(exit_code=1, output=str(error)) + + +def build_ok(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: + """Run sqb build and require success.""" + + result: CliRun = build(project_dir=project_dir, capsys=capsys) + assert result.exit_code == 0, result.output + return result + + +def plan_json( + *, project_dir: Path, capsys: pytest.CaptureFixture[str], args: tuple[str, ...] = () +) -> dict[str, Any]: + """Run sqb plan --json and parse stdout.""" + + _ = capsys.readouterr() + exit_code: int = main( + ["--project-dir", str(project_dir), "--no-color", "plan", "--json", *args] + ) + captured: CaptureResult[str] = capsys.readouterr() + assert exit_code == 0, captured.out + captured.err + return json.loads(captured.out) + + +def model_entry(*, plan: dict[str, Any], name: str) -> dict[str, Any]: + """Return one serialized model entry from plan JSON.""" + + models_by_name: dict[str, dict[str, Any]] = {model["name"]: model for model in plan["models"]} + return models_by_name[name] + + +def migration_decisions(plan: dict[str, Any]) -> tuple[str, ...]: + """Return every planned migration decision in plan order.""" + + return tuple(migration["decision"] for migration in plan["migrations"]) + + +def planned_migrations(plan: dict[str, Any]) -> tuple[tuple[str | None, str, str, str], ...]: + """Return sorted (origin, destination, discovery, decision) for every planned migration.""" + + return tuple( + sorted( + ( + migration["origin_model"], + migration["model"], + migration["discovery"], + migration["decision"], + ) + for migration in plan["migrations"] + ) + ) + + +def warning_messages(plan: dict[str, Any]) -> tuple[str, ...]: + """Return every plan warning message.""" + + return tuple(str(warning["message"]) for warning in plan["warnings"]) + + +def row_count(*, project_dir: Path, relation: str) -> int: + """Return the number of rows stored in one relation.""" + + return int(query(project_dir=project_dir, sql=f"SELECT count(*) FROM {relation}")[0][0]) + + +def build_origin(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Build the origin orders model over five days of raw orders.""" + + write_project(project_dir=project_dir, models={ORIGIN_MODEL: incremental_orders_sql()}) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) + _ = build_ok(project_dir=project_dir, capsys=capsys) + + +def rename_model( + *, + project_dir: Path, + name: str, + migrate_from: str | None, + migrate_force: bool = False, + extra_config: str = "", +) -> None: + """Replace the project with one incremental orders model under a new name.""" + + write_project( + project_dir=project_dir, + models={ + name: incremental_orders_sql( + migrate_from=migrate_from, migrate_force=migrate_force, extra_config=extra_config + ) + }, + ) + + +def snapshot_orders_sql(*, migration_lines: str) -> str: + """Return a timestamp snapshot of raw orders with optional migration header lines.""" + + return _SNAPSHOT_SQL.format(migration=migration_lines) + + +def _raise_interruption(*_: object, **__: object) -> None: + raise RuntimeError("simulated interruption") + + +def fail_clone(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the migration stage statement fail before any stage exists.""" + + monkeypatch.setattr( + DuckDbAdapter, + "render_migration_stage", + lambda self, **_: MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=("SELECT * FROM main.simulated_missing_relation",), + ), + ) + + +def refuse_clone_then_copy(monkeypatch: pytest.MonkeyPatch) -> None: + """Plan a clone with a copy fallback, as BigQuery does, and refuse the clone.""" + + monkeypatch.setattr( + DuckDbAdapter, + "render_migration_stage", + lambda self, *, origin, stage, **_: MigrationStagePlan( + transfer=MigrationTransfer.CLONE, + statements=("SELECT error('simulated clone refusal')",), + fallback_statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + is_clone_refusal=lambda error: "simulated clone refusal" in str(error), + ), + ) + + +def fail_partial_stage(monkeypatch: pytest.MonkeyPatch) -> None: + """Leave an empty, incomplete stage behind and then fail the copy.""" + + monkeypatch.setattr( + DuckDbAdapter, + "render_migration_stage", + lambda self, *, origin, stage, **_: MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=( + f"CREATE TABLE {stage} AS SELECT * FROM {origin} WHERE false", + "SELECT * FROM main.simulated_missing_relation", + ), + ), + ) + + +def fail_verification(monkeypatch: pytest.MonkeyPatch) -> None: + """Complete the stage, then refuse promotion.""" + + monkeypatch.setattr( + "sqlbuild.executor.migrations.main._apply.verify_stage", _raise_interruption + ) + + +def fail_clone_into(*, monkeypatch: pytest.MonkeyPatch, origin: str) -> None: + """Make only the stage copied from one origin fail; every other stage runs normally.""" + + original: Callable[..., MigrationStagePlan] = DuckDbAdapter.render_migration_stage + failing: dict[str, MigrationStagePlan] = { + origin: MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=("SELECT * FROM main.simulated_missing_relation",), + ) + } + monkeypatch.setattr( + DuckDbAdapter, + "render_migration_stage", + lambda self, *, origin, **kwargs: ( + failing.get(origin) or original(self, origin=origin, **kwargs) + ), + ) + + +def disable_transactional_ddl(monkeypatch: pytest.MonkeyPatch) -> None: + """Promote without a transaction, as Snowflake, BigQuery, and Databricks do.""" + + monkeypatch.setattr(DuckDbAdapter, "supports_transactional_ddl", lambda self: False) + + +def fail_promotion_rename(monkeypatch: pytest.MonkeyPatch) -> None: + """Fail when the stage is renamed into place, after any live destination moved aside.""" + + original: Callable[..., None] = DuckDbAdapter.rename + failing: dict[bool, Callable[..., None]] = {True: _raise_interruption} + monkeypatch.setattr( + DuckDbAdapter, + "rename", + lambda self, *, origin, **kwargs: failing.get("migration_stage" in origin, original)( + self, origin=origin, **kwargs + ), + ) + + +def fail_non_transactional_promotion_rename(monkeypatch: pytest.MonkeyPatch) -> None: + """Leave the destination moved aside and missing, as a non-transactional crash would.""" + + disable_transactional_ddl(monkeypatch) + fail_promotion_rename(monkeypatch) + + +def fail_non_transactional_record(monkeypatch: pytest.MonkeyPatch) -> None: + """Crash after a committed promotion but before the event, as non-transactional adapters can.""" + + disable_transactional_ddl(monkeypatch) + fail_record(monkeypatch) + + +def fail_model_build(*, monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: + """Make one incremental model's build raise after migrations have run.""" + + failing: dict[str, Callable[..., Any]] = {model_name: _raise_interruption} + incremental: Callable[..., Any] = scheduler_module.execute_incremental_entry + table: Callable[..., Any] = scheduler_module.execute_table_entry + monkeypatch.setattr( + scheduler_module, + "execute_incremental_entry", + lambda **kwargs: failing.get(kwargs["context"].entry.name, incremental)(**kwargs), + ) + monkeypatch.setattr( + scheduler_module, + "execute_table_entry", + lambda **kwargs: failing.get(kwargs["context"].entry.name, table)(**kwargs), + ) + + +def fail_record(monkeypatch: pytest.MonkeyPatch) -> None: + """Crash after the clone but before the migration event is recorded.""" + + monkeypatch.setattr( + "sqlbuild.executor.migrations._helpers.promotion.write_migration_event", + _raise_interruption, + ) + + +def fail_after_migrations(monkeypatch: pytest.MonkeyPatch) -> None: + """Crash after migrations are recorded but before any model builds.""" + + monkeypatch.setattr( + "sqlbuild.executor.build.main._execute.apply_retention_phase", _raise_interruption + ) + + +def enriched_view_sql(*, upstream: str, alias: str, materialized: str = "view") -> str: + """Return a view or table that selects every order column through a table alias.""" + + return ( + f"MODEL (materialized {materialized});\n\n" + f"SELECT {alias}.order_id, {alias}.order_date, {alias}.amount_cents " + f'FROM __ref("{upstream}") AS {alias}\n' + ) + + +def daily_totals_sql(*, upstream: str, cte: str, comment: str = "", extra_config: str = "") -> str: + """Return an incremental daily revenue model that imports its upstream through a CTE.""" + + return ( + f"{_DAILY_TOTALS_HEADER.format(extra_config=extra_config)}" + f"{comment}" + f'WITH {cte} AS (SELECT * FROM __ref("{upstream}"))\n' + f"SELECT\n {cte}.order_date,\n SUM({cte}.amount_cents) AS total_cents\n" + f"FROM {cte}\nGROUP BY {cte}.order_date\n" + ) + + +def original_order_models() -> dict[str, str]: + """Return the connected staging, view, and daily totals models before renaming.""" + + return { + "stg_orders": incremental_orders_sql(), + "orders_enriched": enriched_view_sql(upstream="stg_orders", alias="o"), + "daily_order_totals": daily_totals_sql(upstream="orders_enriched", cte="orders_enriched"), + } + + +def renamed_order_models() -> dict[str, str]: + """Return the same connected models renamed, reformatted, and with new storage settings.""" + + return { + "stg_customer_orders": incremental_orders_sql(extra_config=" lookback 1d,\n"), + "customer_orders_enriched": enriched_view_sql( + upstream="stg_customer_orders", alias="customer_order" + ), + "customer_daily_order_totals": daily_totals_sql( + upstream="customer_orders_enriched", + cte="customer_orders_enriched", + comment="-- Daily revenue per order date.\n", + ), + } + + +def build_original_order_models(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Build the connected models over five days of raw orders.""" + + write_project( + project_dir=project_dir, + models=original_order_models(), + ) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) + _ = build_ok(project_dir=project_dir, capsys=capsys) + + +def warehouse_state(*, project_dir: Path) -> tuple[tuple[Any, ...], ...]: + """Return (schema, relation, row count) for every relation in the dev and prod schemas.""" + + tables: list[tuple[Any, ...]] = query( + project_dir=project_dir, + sql=( + "SELECT table_schema, table_name FROM information_schema.tables " + "WHERE table_schema IN ('dev', 'prod') ORDER BY 1, 2" + ), + ) + return tuple( + (schema, name, row_count(project_dir=project_dir, relation=f"{schema}.{name}")) + for schema, name in tables + ) + + +def prepare_prod_rename(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Build the origin on prod, then rename it with migrate_from in the project.""" + + write_project( + project_dir=project_dir, + models={ORIGIN_MODEL: incremental_orders_sql()}, + project_toml=TARGETS_PROJECT_TOML, + ) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=4) + built: CliRun = run_sqb( + project_dir=project_dir, args=("build", "--target", "prod"), capsys=capsys + ) + assert built.exit_code == 0, built.output + write_project( + project_dir=project_dir, + models={DESTINATION_MODEL: incremental_orders_sql(migrate_from=ORIGIN_MODEL)}, + project_toml=TARGETS_PROJECT_TOML, + ) + + +def archived_relations(*, project_dir: Path, kind: str, schema: str = "main") -> tuple[str, ...]: + """Return every janitor-archive relation of one migration kind in one schema.""" + + return tuple( + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' " + "AND starts_with(table_name, '_sqb_archive__') " + f"AND contains(table_name, '__{kind}__') ORDER BY 1" + ), + ) + ) + + +def archived_order_ids(*, project_dir: Path, kind: str) -> tuple[tuple[int, ...], ...]: + """Return the order IDs held by each archive of one migration kind.""" + + return tuple( + order_ids(project_dir=project_dir, relation=f"main.{name}") + for name in archived_relations(project_dir=project_dir, kind=kind) + ) + + +def prepare_forced_replace( + *, project_dir: Path, capsys: pytest.CaptureFixture[str], project_toml: str = PROJECT_TOML +) -> None: + """Build an origin and an independent destination, then force the origin over it.""" + + write_project( + project_dir=project_dir, + models={ + ORIGIN_MODEL: incremental_orders_sql(), + DESTINATION_MODEL: incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE order_id <= 2" + ) + ), + }, + project_toml=project_toml, + ) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) + _ = build_ok(project_dir=project_dir, capsys=capsys) + write_project( + project_dir=project_dir, + models={ + DESTINATION_MODEL: incremental_orders_sql(migrate_from=ORIGIN_MODEL, migrate_force=True) + }, + project_toml=project_toml, + ) + + +def no_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """Install no failure.""" + + del monkeypatch + + +def live_relations(*, project_dir: Path, schema: str = "main") -> tuple[str, ...]: + """Return relations in one schema that are neither state tables nor archives.""" + + return tuple( + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' AND NOT starts_with(table_name, '_') ORDER BY 1" + ), + ) + ) + + +_REPLAY_FULL: str = " replay_on_change full,\n" + + +def original_replay_order_models(*, middle: str) -> dict[str, str]: + """Return the staging, enriched, and daily totals chain with full replay on the totals.""" + + return { + "stg_orders": incremental_orders_sql(), + "orders_enriched": enriched_view_sql(upstream="stg_orders", alias="o", materialized=middle), + "daily_order_totals": daily_totals_sql( + upstream="orders_enriched", cte="orders_enriched", extra_config=_REPLAY_FULL + ), + } + + +def renamed_replay_order_models(*, middle: str) -> dict[str, str]: + """Return the same full-replay chain with every model renamed.""" + + return { + "stg_customer_orders": incremental_orders_sql(), + "customer_orders_enriched": enriched_view_sql( + upstream="stg_customer_orders", alias="customer_order", materialized=middle + ), + "customer_daily_order_totals": daily_totals_sql( + upstream="customer_orders_enriched", + cte="customer_orders_enriched", + extra_config=_REPLAY_FULL, + ), + } + + +def fail_nothing_in_build(*, monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: + """Install no build failure.""" + + del monkeypatch, model_name + + +def state_tables(*, project_dir: Path, schema: str) -> tuple[str, ...]: + """Return the SQLBuild state tables present in one schema.""" + + return tuple( + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' AND starts_with(table_name, '_sqlbuild_') " + "ORDER BY 1" + ), + ) + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py new file mode 100644 index 0000000000..912a5e1ab5 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py @@ -0,0 +1,457 @@ +"""Integration coverage for fingerprint-based automatic migrations through the real CLI.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + AutomaticMigrationTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + CliRun, + build, + build_ok, + build_original_order_models, + daily_totals_sql, + execute, + fail_clone_into, + fail_model_build, + incremental_orders_sql, + load_raw_orders, + migration_events, + model_entry, + order_ids, + original_order_models, + plan_json, + planned_migrations, + relation_names, + renamed_order_models, + row_count, + run_sqb, + warning_messages, + write_project, +) + +_FIRST_RUN: str = "first_run" + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="incremental, view, and incremental renamed together", + expected_migrations=( + ("daily_order_totals", "customer_daily_order_totals", "automatic", "migrate"), + ("orders_enriched", "customer_orders_enriched", "automatic", "renamed"), + ("stg_orders", "stg_customer_orders", "automatic", "migrate"), + ), + expected_events=( + ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("orders_enriched", "customer_orders_enriched", "renamed"), + ("stg_orders", "stg_customer_orders", "migrate"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_coordinated_renames_when_building_then_history_moves_automatically( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=3, last_day=8) + write_project(project_dir=tmp_path, models=renamed_order_models()) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + rebuild_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] != _FIRST_RUN + assert model_entry(plan=plan, name="customer_daily_order_totals")["reason"] != _FIRST_RUN + assert "Migrated main.stg_orders -> main.stg_customer_orders" in result.output + assert order_ids(project_dir=tmp_path, relation="main.stg_customer_orders") == tuple( + range(1, 9) + ) + assert row_count(project_dir=tmp_path, relation="main.customer_daily_order_totals") == 8 + assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events + assert planned_migrations(rebuild_plan) == () + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="downstream rename matches after an upstream logic change", + expected_migrations=( + ("daily_order_totals", "customer_daily_order_totals", "automatic", "migrate"), + ), + expected_events=(("daily_order_totals", "customer_daily_order_totals", "migrate"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_upstream_logic_change_when_renaming_then_downstream_still_matches( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + models: dict[str, str] = original_order_models() + models["stg_orders"] = incremental_orders_sql( + select_sql=( + "SELECT order_id, order_date, amount_cents * 2 AS amount_cents " + 'FROM __source("raw_orders")' + ) + ) + models["customer_daily_order_totals"] = daily_totals_sql( + upstream="orders_enriched", cte="enriched" + ) + del models["daily_order_totals"] + write_project(project_dir=tmp_path, models=models) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert model_entry(plan=plan, name="customer_daily_order_totals")["reason"] != _FIRST_RUN + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="renamed model with a new filter starts fresh", + expected_reason="first_run", + ) + ], + ids=lambda case: case.description, +) +def test_given_changed_logic_when_renaming_then_destination_is_first_run( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={ + "stg_customer_orders": incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE amount_cents > 0" + ) + ) + }, + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] == ( + test_case.expected_reason + ) + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="two identical removed models make the match ambiguous", + expected_reason="first_run", + expected_warning=( + "'stg_customer_orders' matches several earlier models (orders_copy, stg_orders); " + "no automatic migration was inferred. Add migrate_from to choose one." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_identical_orphans_when_renaming_then_warns_and_does_not_migrate( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={"stg_orders": incremental_orders_sql(), "orders_copy": incremental_orders_sql()}, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project(project_dir=tmp_path, models={"stg_customer_orders": incremental_orders_sql()}) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert test_case.expected_warning in warning_messages(plan) + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] == ( + test_case.expected_reason + ) + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="dropped removed relation cannot migrate", + expected_reason="first_run", + ) + ], + ids=lambda case: case.description, +) +def test_given_dropped_old_relation_when_renaming_then_destination_is_first_run( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(project_dir=tmp_path, models={"stg_orders": incremental_orders_sql()}) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + execute(project_dir=tmp_path, sql="DROP TABLE main.stg_orders") + write_project(project_dir=tmp_path, models={"stg_customer_orders": incremental_orders_sql()}) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] == ( + test_case.expected_reason + ) + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="migrate_from wins over a matching fingerprint", + expected_migrations=(("legacy_orders", "stg_customer_orders", "manual", "migrate"),), + expected_events=(("legacy_orders", "stg_customer_orders", "migrate"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_explicit_migrate_from_when_fingerprint_matches_another_then_manual_wins( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={ + "stg_orders": incremental_orders_sql(), + "legacy_orders": incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE order_id <= 2" + ) + ), + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=4) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={"stg_customer_orders": incremental_orders_sql(migrate_from="legacy_orders")}, + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="qualified migrate_from removes its origin and destination from matching", + expected_migrations=((None, "stg_customer_orders", "manual", "migrate"),), + expected_events=(("legacy_orders", "stg_customer_orders", "migrate"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_qualified_migrate_from_when_another_orphan_matches_then_only_declared_moves( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={ + "stg_orders": incremental_orders_sql(), + "legacy_orders": incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE order_id <= 2" + ) + ), + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=4) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={"stg_customer_orders": incremental_orders_sql(migrate_from="main.legacy_orders")}, + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="recorded upstream move resolves the downstream rename on retry", + expected_migrations=( + ("daily_order_totals", "customer_daily_order_totals", "automatic", "migrate"), + ("orders_enriched", "customer_orders_enriched", "automatic", "renamed"), + ("stg_orders", "stg_customer_orders", "automatic", "done"), + ), + expected_events=( + ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("orders_enriched", "customer_orders_enriched", "renamed"), + ("stg_orders", "stg_customer_orders", "migrate"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_interrupted_coordinated_renames_when_rerunning_then_downstream_resumes( + test_case: AutomaticMigrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=3, last_day=8) + write_project(project_dir=tmp_path, models=renamed_order_models()) + + with monkeypatch.context() as patch: + fail_clone_into(monkeypatch=patch, origin="main.daily_order_totals") + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) + events_after_crash: tuple[tuple[str, str, str], ...] = migration_events(project_dir=tmp_path) + retry_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code == 1, interrupted.output + assert events_after_crash == (("stg_orders", "stg_customer_orders", "migrate"),) + assert planned_migrations(retry_plan) == test_case.expected_migrations + assert model_entry(plan=retry_plan, name="stg_customer_orders")["reason"] != _FIRST_RUN + assert order_ids(project_dir=tmp_path, relation="main.stg_customer_orders") == tuple( + range(1, 9) + ) + assert row_count(project_dir=tmp_path, relation="main.customer_daily_order_totals") == 8 + assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="selecting one of two new models still sees the whole-project ambiguity", + expected_reason="first_run", + expected_warning=( + "'stg_customer_orders' matches earlier model 'stg_orders', which also matches " + "'stg_web_orders'; no automatic migration was inferred. " + "Add migrate_from to choose one." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_one_orphan_matching_two_new_models_when_selecting_one_then_warns_ambiguous( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(project_dir=tmp_path, models={"stg_orders": incremental_orders_sql()}) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={ + "stg_customer_orders": incremental_orders_sql(), + "stg_web_orders": incremental_orders_sql(), + }, + ) + + plan: dict[str, Any] = plan_json( + project_dir=tmp_path, capsys=capsys, args=("--select", "stg_customer_orders") + ) + built: CliRun = run_sqb( + project_dir=tmp_path, args=("build", "--select", "stg_customer_orders"), capsys=capsys + ) + + assert planned_migrations(plan) == test_case.expected_migrations + assert warning_messages(plan) == (test_case.expected_warning,) + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] == ( + test_case.expected_reason + ) + assert built.exit_code == 0, built.output + assert "_sqlbuild_migrations" not in relation_names(project_dir=tmp_path, schema="main") + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="downstream failed after its clone while the upstream was rebuilt", + expected_migrations=( + ("daily_order_totals", "customer_daily_order_totals", "automatic", "done"), + ), + expected_events=( + ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("stg_orders", "stg_customer_orders", "migrate"), + ), + expected_reason="normal_incremental", + ) + ], + ids=lambda case: case.description, +) +def test_given_fingerprinted_upstream_when_retrying_failed_downstream_then_history_is_kept( + test_case: AutomaticMigrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + replay_full: str = " replay_on_change full,\n" + write_project( + project_dir=tmp_path, + models={ + "stg_orders": incremental_orders_sql(), + "daily_order_totals": daily_totals_sql( + upstream="stg_orders", cte="stg_orders", extra_config=replay_full + ), + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=3, last_day=8) + write_project( + project_dir=tmp_path, + models={ + "stg_customer_orders": incremental_orders_sql(), + "customer_daily_order_totals": daily_totals_sql( + upstream="stg_customer_orders", + cte="stg_customer_orders", + extra_config=replay_full, + ), + }, + ) + + with monkeypatch.context() as patch: + fail_model_build(monkeypatch=patch, model_name="customer_daily_order_totals") + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) + upstream_fingerprinted: dict[str, Any] = plan_json( + project_dir=tmp_path, capsys=capsys, args=("--select", "stg_customer_orders") + ) + retry_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code == 1, interrupted.output + assert planned_migrations(upstream_fingerprinted) == () + assert planned_migrations(retry_plan) == test_case.expected_migrations + assert model_entry(plan=retry_plan, name="stg_customer_orders")["reason"] != _FIRST_RUN + assert model_entry(plan=retry_plan, name="customer_daily_order_totals")["reason"] == ( + test_case.expected_reason + ) + assert row_count(project_dir=tmp_path, relation="main.customer_daily_order_totals") == 8 + assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py new file mode 100644 index 0000000000..2d0a1d4eef --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py @@ -0,0 +1,177 @@ +"""Integration coverage for identity handover of renamed views and tables.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + IdentityHandoverTestCase, + RenamedPlanTextTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + CliRun, + build, + build_ok, + execute, + fail_model_build, + fail_nothing_in_build, + load_raw_orders, + migration_events, + model_entry, + original_replay_order_models, + plan_json, + query, + renamed_replay_order_models, + row_count, + run_sqb, + write_project, +) + +_MARK_HISTORY: str = ( + "UPDATE main.daily_order_totals SET total_cents = -1 WHERE order_date = TIMESTAMP '2026-01-01'" +) +_READ_MARK: str = ( + "SELECT total_cents FROM main.customer_daily_order_totals " + "WHERE order_date = TIMESTAMP '2026-01-01'" +) +_EVENTS: tuple[tuple[str, str, str], ...] = ( + ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("orders_enriched", "customer_orders_enriched", "renamed"), + ("stg_orders", "stg_customer_orders", "migrate"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + IdentityHandoverTestCase( + middle_materialization="view", + description="renamed view between incrementals does not replay downstream history", + install_failure=fail_nothing_in_build, + failing_model="", + expected_first_exit_code=0, + expected_first_view_reason="renamed", + expected_retry_totals_reason="normal_incremental", + expected_totals_rows=8, + expected_marked_total=-1, + expected_events=_EVENTS, + ), + IdentityHandoverTestCase( + middle_materialization="view", + description="retry after the renamed view was built keeps downstream history", + install_failure=fail_model_build, + failing_model="customer_daily_order_totals", + expected_first_exit_code=1, + expected_first_view_reason="renamed", + expected_retry_totals_reason="normal_incremental", + expected_totals_rows=8, + expected_marked_total=-1, + expected_events=_EVENTS, + ), + IdentityHandoverTestCase( + middle_materialization="view", + description="retry before the renamed view was built keeps downstream history", + install_failure=fail_model_build, + failing_model="stg_customer_orders", + expected_first_exit_code=1, + expected_first_view_reason="renamed", + expected_retry_totals_reason="normal_incremental", + expected_totals_rows=8, + expected_marked_total=-1, + expected_events=_EVENTS, + ), + IdentityHandoverTestCase( + middle_materialization="table", + description="renamed table between incrementals is rebuilt without replaying history", + install_failure=fail_model_build, + failing_model="customer_daily_order_totals", + expected_first_exit_code=1, + expected_first_view_reason="renamed", + expected_retry_totals_reason="normal_incremental", + expected_totals_rows=8, + expected_marked_total=-1, + expected_events=_EVENTS, + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_chain_with_full_replay_when_building_then_migrated_history_survives( + test_case: IdentityHandoverTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A renamed view hands its identity downstream so replay_on_change keeps marked history.""" + + write_project( + project_dir=tmp_path, + models=original_replay_order_models(middle=test_case.middle_materialization), + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + execute(project_dir=tmp_path, sql=_MARK_HISTORY) + load_raw_orders(project_dir=tmp_path, first_day=3, last_day=8) + write_project( + project_dir=tmp_path, + models=renamed_replay_order_models(middle=test_case.middle_materialization), + ) + + first_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + with monkeypatch.context() as patch: + test_case.install_failure(monkeypatch=patch, model_name=test_case.failing_model) + first: CliRun = build(project_dir=tmp_path, capsys=capsys) + retry_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert first.exit_code == test_case.expected_first_exit_code, first.output + assert model_entry(plan=first_plan, name="customer_orders_enriched")["reason"] == ( + test_case.expected_first_view_reason + ) + assert model_entry(plan=first_plan, name="customer_daily_order_totals")["reason"] != ( + "upstream_changed" + ) + assert model_entry(plan=retry_plan, name="customer_daily_order_totals")["reason"] == ( + test_case.expected_retry_totals_reason + ) + assert ( + row_count(project_dir=tmp_path, relation="main.customer_daily_order_totals") + == test_case.expected_totals_rows + ) + assert query(project_dir=tmp_path, sql=_READ_MARK) == [(test_case.expected_marked_total,)] + assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + RenamedPlanTextTestCase( + description="plan lists renamed views separately from data migrations", + expected_fragment=( + "Renamed (1)\n└── customer_orders_enriched main.orders_enriched -> " + "main.customer_orders_enriched (identity handed over)" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_view_when_planning_then_text_lists_it_under_renamed( + test_case: RenamedPlanTextTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Renamed tables and views move no data and appear in their own plan section.""" + + write_project(project_dir=tmp_path, models=original_replay_order_models(middle="view")) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project(project_dir=tmp_path, models=renamed_replay_order_models(middle="view")) + + text: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + + assert text.exit_code == 0, text.output + assert test_case.expected_fragment in text.output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py new file mode 100644 index 0000000000..14423a36bd --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -0,0 +1,716 @@ +"""Integration coverage for manual migrate_from migrations through the real CLI.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + BackAndForthMigrationTestCase, + MigrationCompatibilityTestCase, + MigrationCompileErrorTestCase, + MigrationInterruptionTestCase, + MigrationOutcomeTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + DESTINATION_MODEL, + ORIGIN_MODEL, + CliRun, + build, + build_ok, + build_origin, + execute, + fail_after_migrations, + fail_clone, + fail_non_transactional_promotion_rename, + fail_non_transactional_record, + fail_partial_stage, + fail_record, + fail_verification, + incremental_orders_sql, + load_raw_orders, + migration_decisions, + migration_events, + model_entry, + order_ids, + plan_json, + query, + relation_names, + rename_model, + run_sqb, + snapshot_orders_sql, + write_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="first migration clones eight days of history", + expected_decisions=("migrate",), + expected_events=(("stg_orders", "stg_customer_orders", "migrate"),), + expected_destination_ids=tuple(range(1, 9)), + expected_output_fragment="Migrated main.stg_orders -> main.stg_customer_orders", + expected_reason="normal_incremental", + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_incremental_when_building_then_history_is_cloned_and_recorded( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_origin(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=4, last_day=8) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) == test_case.expected_decisions + assert plan["migrations"][0]["compatibility"] == "compatible" + assert model_entry(plan=plan, name=DESTINATION_MODEL)["reason"] == test_case.expected_reason + assert model_entry(plan=plan, name=DESTINATION_MODEL)["action"] == "incremental_delete_insert" + assert test_case.expected_output_fragment in result.output + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + assert order_ids(project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}") == tuple(range(1, 6)) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="completed forced migration is skipped with a removal hint", + expected_decisions=("done",), + expected_events=(("stg_orders", "stg_customer_orders", "migrate"),), + expected_destination_ids=tuple(range(1, 8)), + expected_output_fragment="migrate_from can be removed from 'stg_customer_orders'", + ) + ], + ids=lambda case: case.description, +) +def test_given_completed_migration_when_rebuilding_then_warns_and_skips( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_origin(project_dir=tmp_path, capsys=capsys) + rename_model( + project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL, migrate_force=True + ) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=7) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) == test_case.expected_decisions + assert test_case.expected_output_fragment in result.output + assert "Migrating" not in result.output + assert migration_events(project_dir=tmp_path) == test_case.expected_events + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + + +@pytest.mark.parametrize( + "test_case", + [ + BackAndForthMigrationTestCase( + description="day one, five, and nine renames", + expected_day_decisions=( + ("migrate",), + ("superseded_replace",), + ("superseded_replace",), + ), + expected_day_five_ids=tuple(range(1, 7)), + expected_final_ids=tuple(range(1, 10)), + expected_events=( + ("stg_orders", "stg_customer_orders", "migrate"), + ("stg_customer_orders", "stg_orders", "superseded_replace"), + ("stg_orders", "stg_customer_orders", "superseded_replace"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_back_and_forth_renames_when_building_each_day_then_every_move_converges( + test_case: BackAndForthMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(project_dir=tmp_path, models={ORIGIN_MODEL: incremental_orders_sql()}) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=4) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + day_one: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=6) + rename_model(project_dir=tmp_path, name=ORIGIN_MODEL, migrate_from=DESTINATION_MODEL) + day_five: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + after_day_five: tuple[int, ...] = order_ids( + project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}" + ) + + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=9) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + day_nine: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert ( + migration_decisions(day_one), + migration_decisions(day_five), + migration_decisions(day_nine), + ) == test_case.expected_day_decisions + assert after_day_five == test_case.expected_day_five_ids + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_final_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="destination without build history is replaced by a redo", + expected_decisions=("redo", "done"), + expected_events=(("stg_orders", "stg_customer_orders", "redo"),), + expected_destination_ids=tuple(range(1, 6)), + expected_output_fragment="Migrated main.stg_orders -> main.stg_customer_orders", + ) + ], + ids=lambda case: case.description, +) +def test_given_unrecorded_destination_without_history_when_building_then_redoes_the_clone( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_origin(project_dir=tmp_path, capsys=capsys) + execute( + project_dir=tmp_path, + sql=( + f"CREATE TABLE main.{DESTINATION_MODEL} AS SELECT 999 AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date, 1 AS amount_cents" + ), + ) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + after: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) + migration_decisions(after) == test_case.expected_decisions + assert test_case.expected_output_fragment in result.output + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="destination with its own build history conflicts until forced", + expected_decisions=("conflict", "forced_replace", "done"), + expected_events=(("stg_orders", "stg_customer_orders", "forced_replace"),), + expected_destination_ids=tuple(range(1, 6)), + expected_output_fragment="model migration conflict", + ) + ], + ids=lambda case: case.description, +) +def test_given_destination_with_build_history_when_building_then_conflict_until_forced( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={ + ORIGIN_MODEL: incremental_orders_sql(), + DESTINATION_MODEL: incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE order_id <= 2" + ) + ), + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + + conflict_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + conflict: CliRun = build(project_dir=tmp_path, capsys=capsys) + preserved: tuple[int, ...] = order_ids( + project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}" + ) + rename_model( + project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL, migrate_force=True + ) + forced_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + after_force: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert ( + migration_decisions(conflict_plan) + + migration_decisions(forced_plan) + + migration_decisions(after_force) + ) == test_case.expected_decisions + assert conflict.exit_code == 1 + assert test_case.expected_output_fragment in conflict.output + assert conflict.output.count("error[M103]") == 1 + assert "Errors (1)\n└── stg_customer_orders\n └── migration conflict:" in conflict.output + assert preserved == (1, 2) + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="missing origin without a recorded migration fails the build", + expected_decisions=("origin_missing",), + expected_output_fragment=( + "'stg_customer_orders' (migrate_from main.retired_orders) and no recorded " + "migration into it was found; if the migration already happened elsewhere or " + "is no longer needed, remove migrate_from from the model header" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_without_event_when_building_then_fails_before_building( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={DESTINATION_MODEL: incremental_orders_sql(migrate_from="retired_orders")}, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + text: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + result: CliRun = build(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) == test_case.expected_decisions + assert tuple(warning["severity"] for warning in plan["warnings"]) == ("error",) + assert plan["warnings"][0]["message"] == ( + "model 'stg_customer_orders': migrate_from origin main.retired_orders does not exist " + "and no recorded migration into it was found; if the migration already happened " + "elsewhere or is no longer needed, remove migrate_from from the model header" + ) + assert "origin missing main.retired_orders -> main.stg_customer_orders" in text.output + assert result.exit_code == 1 + assert result.output.count("error[M102]") == 1 + assert result.output.count("= help:") == 1 + assert "Errors (1)" in text.output + assert "Warnings (" not in text.output + assert "Errors (1)" in result.output + assert test_case.expected_output_fragment in result.output + assert relation_names(project_dir=tmp_path, schema="main") == ["raw_orders"] + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="recorded migration keeps a leftover migrate_from harmless", + expected_decisions=("done",), + expected_events=(("stg_orders", "stg_customer_orders", "migrate"),), + expected_destination_ids=tuple(range(1, 8)), + expected_output_fragment="migrate_from can be removed from 'stg_customer_orders'", + ) + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_with_recorded_migration_when_building_then_reports_done( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_origin(project_dir=tmp_path, capsys=capsys) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + execute(project_dir=tmp_path, sql=f"DROP TABLE main.{ORIGIN_MODEL}") + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=7) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) == test_case.expected_decisions + assert test_case.expected_output_fragment in result.output + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="replay_on_change full does not replay after the handover", + expected_decisions=("migrate",), + expected_destination_ids=tuple(range(1, 7)), + expected_reason="normal_incremental", + ) + ], + ids=lambda case: case.description, +) +def test_given_replay_on_change_full_when_migrating_then_destination_is_not_replayed( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={ORIGIN_MODEL: incremental_orders_sql(extra_config=" replay_on_change full,\n")}, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=5, last_day=6) + rename_model( + project_dir=tmp_path, + name=DESTINATION_MODEL, + migrate_from=ORIGIN_MODEL, + extra_config=" replay_on_change full,\n", + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + second_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) == test_case.expected_decisions + assert model_entry(plan=plan, name=DESTINATION_MODEL)["reason"] == test_case.expected_reason + assert model_entry(plan=plan, name=DESTINATION_MODEL)["action"] != "create_table" + assert ( + model_entry(plan=second_plan, name=DESTINATION_MODEL)["reason"] == test_case.expected_reason + ) + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationCompileErrorTestCase( + description="table materialization", + model_sql='MODEL (materialized table, migrate_from "stg_orders");\nSELECT 1 AS id\n', + expected_fragment="migrate_from is only valid for incremental and snapshot models", + ), + MigrationCompileErrorTestCase( + description="view materialization", + model_sql='MODEL (materialized view, migrate_from "stg_orders");\nSELECT 1 AS id\n', + expected_fragment="migrate_from is only valid for incremental and snapshot models", + ), + MigrationCompileErrorTestCase( + description="force without origin", + model_sql=incremental_orders_sql(extra_config=" migrate_force true,\n"), + expected_fragment="migrate_force requires migrate_from", + ), + MigrationCompileErrorTestCase( + description="origin names the model itself", + model_sql=incremental_orders_sql(migrate_from=DESTINATION_MODEL), + expected_fragment="migrate_from cannot name the model itself", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_migration_header_when_compiling_then_fails( + test_case: MigrationCompileErrorTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(project_dir=tmp_path, models={DESTINATION_MODEL: test_case.model_sql}) + + result: CliRun = run_sqb(project_dir=tmp_path, args=("compile",), capsys=capsys) + + assert result.exit_code == 1 + assert test_case.expected_fragment in result.output + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="snapshot history rows move to the renamed snapshot", + expected_decisions=("migrate",), + expected_events=(("order_snapshot", "order_history", "migrate"),), + expected_destination_ids=(1, 2, 3), + ) + ], + ids=lambda case: case.description, +) +def test_given_snapshot_model_when_migrating_then_history_rows_are_preserved( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, models={"order_snapshot": snapshot_orders_sql(migration_lines="")} + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={ + "order_history": snapshot_orders_sql( + migration_lines=' migrate_from "order_snapshot",\n' + ) + }, + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) == test_case.expected_decisions + assert ( + order_ids(project_dir=tmp_path, relation="main.order_history") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationCompatibilityTestCase( + description="incompatible column type blocks the move", + origin_setup_sql=( + "CREATE TABLE main.legacy_orders AS SELECT 'A-1' AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date, 101 AS amount_cents" + ), + destination_extra_config="", + destination_select_sql=( + "SELECT CAST(order_id AS INTEGER) AS order_id, order_date, amount_cents " + 'FROM __source("raw_orders")' + ), + expected_compatibility="incompatible", + expected_build_exit_code=1, + expected_fragment="incompatible", + ), + MigrationCompatibilityTestCase( + description="on_schema_change fail rejects a missing column", + origin_setup_sql=( + "CREATE TABLE main.legacy_orders AS SELECT 1 AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date" + ), + destination_extra_config=" on_schema_change fail,\n", + destination_select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")' + ), + expected_compatibility="incompatible", + expected_build_exit_code=1, + expected_fragment="on_schema_change fail", + ), + MigrationCompatibilityTestCase( + description="append_new_columns accepts a new model column", + origin_setup_sql=( + "CREATE TABLE main.legacy_orders AS SELECT 1 AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date" + ), + destination_extra_config="", + destination_select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")' + ), + expected_compatibility="compatible", + expected_build_exit_code=0, + expected_fragment="Migrated main.legacy_orders -> main.stg_customer_orders", + ), + MigrationCompatibilityTestCase( + description="numeric widening is compatible under sync_all_columns", + origin_setup_sql=( + "CREATE TABLE main.legacy_orders AS SELECT CAST(1 AS INTEGER) AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date, CAST(101 AS INTEGER) AS amount_cents" + ), + destination_extra_config=" on_schema_change sync_all_columns,\n", + destination_select_sql=( + "SELECT order_id, order_date, CAST(amount_cents AS BIGINT) AS amount_cents " + 'FROM __source("raw_orders")' + ), + expected_compatibility="compatible", + expected_build_exit_code=0, + expected_fragment="Migrated main.legacy_orders -> main.stg_customer_orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_origin_schema_when_migrating_then_normal_incremental_rules_decide( + test_case: MigrationCompatibilityTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project( + project_dir=tmp_path, + models={ + DESTINATION_MODEL: incremental_orders_sql( + migrate_from="main.legacy_orders", + extra_config=test_case.destination_extra_config, + select_sql=test_case.destination_select_sql, + ) + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + execute(project_dir=tmp_path, sql=test_case.origin_setup_sql) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build(project_dir=tmp_path, capsys=capsys) + + assert plan["migrations"][0]["compatibility"] == test_case.expected_compatibility + assert result.exit_code == test_case.expected_build_exit_code, result.output + assert test_case.expected_fragment in result.output + assert query(project_dir=tmp_path, sql="SELECT count(*) FROM main.legacy_orders") == [(1,)] + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationInterruptionTestCase( + description="failure while cloning redoes the move", + install_failure=fail_clone, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="partial stage is abandoned and a fresh stage is used on re-run", + install_failure=fail_partial_stage, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="failed stage verification never promotes the stage", + install_failure=fail_verification, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="transactional crash before recording rolls the promotion back", + install_failure=fail_record, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="non-transactional crash before recording redoes the move", + install_failure=fail_non_transactional_record, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="redo", + expected_final_decisions=("redo",), + ), + MigrationInterruptionTestCase( + description="non-transactional crash while renaming the stage migrates again", + install_failure=fail_non_transactional_promotion_rename, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="crash after recording continues from the migrated history", + install_failure=fail_after_migrations, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="done", + expected_final_decisions=("migrate",), + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupted_first_migration_when_rerunning_then_converges( + test_case: MigrationInterruptionTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + build_origin(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=4, last_day=7) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + + with monkeypatch.context() as patch: + test_case.install_failure(patch) + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) + after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + rename_model( + project_dir=tmp_path, + name=DESTINATION_MODEL, + migrate_from=ORIGIN_MODEL, + migrate_force=test_case.rerun_with_force, + ) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + converged: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code == test_case.expected_first_exit_code, interrupted.output + assert migration_decisions(after_failure) == (test_case.expected_decision_after_failure,) + assert migration_decisions(converged) == ("done",) + assert model_entry(plan=converged, name=DESTINATION_MODEL)["reason"] == "normal_incremental" + assert order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") == tuple( + range(1, 8) + ) + assert order_ids(project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}") == tuple(range(1, 6)) + assert tuple(event[2] for event in migration_events(project_dir=tmp_path)) == ( + test_case.expected_final_decisions + ) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="crash before recording a superseded replace repeats the replace", + expected_decisions=("superseded_replace",), + expected_events=( + ("stg_orders", "stg_customer_orders", "migrate"), + ("stg_customer_orders", "stg_orders", "superseded_replace"), + ), + expected_destination_ids=tuple(range(1, 6)), + ) + ], + ids=lambda case: case.description, +) +def test_given_interrupted_superseded_replace_when_rerunning_then_replaces_again( + test_case: MigrationOutcomeTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_project(project_dir=tmp_path, models={ORIGIN_MODEL: incremental_orders_sql()}) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + rename_model(project_dir=tmp_path, name=ORIGIN_MODEL, migrate_from=DESTINATION_MODEL) + + with monkeypatch.context() as patch: + fail_non_transactional_record(patch) + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) + after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code == 1 + assert migration_decisions(after_failure) == test_case.expected_decisions + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py new file mode 100644 index 0000000000..9e0e0add75 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py @@ -0,0 +1,120 @@ +"""Integration coverage for inspection-only plan --as previews of migrations.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + PlanAsTargetTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + DESTINATION_MODEL, + CliRun, + migration_decisions, + plan_json, + prepare_prod_rename, + run_sqb, + warehouse_state, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PlanAsTargetTestCase( + description="pending prod migration previews as migrate", + preview_target="prod", + expected_exit_code=0, + expected_decisions=("migrate",), + expected_fragment="migrate prod.stg_orders -> prod.stg_customer_orders", + ) + ], + ids=lambda case: case.description, +) +def test_given_pending_prod_migration_when_planning_as_prod_then_previews_without_mutation( + test_case: PlanAsTargetTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + before: tuple[tuple[Any, ...], ...] = warehouse_state(project_dir=tmp_path) + + preview: dict[str, Any] = plan_json( + project_dir=tmp_path, capsys=capsys, args=("--as", test_case.preview_target) + ) + active: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + text: CliRun = run_sqb( + project_dir=tmp_path, args=("plan", "--as", test_case.preview_target), capsys=capsys + ) + after: tuple[tuple[Any, ...], ...] = warehouse_state(project_dir=tmp_path) + + assert migration_decisions(preview) == test_case.expected_decisions + assert preview["migrations"][0]["destination"] == "prod.stg_customer_orders" + assert migration_decisions(active) == ("origin_missing",) + assert text.exit_code == test_case.expected_exit_code, text.output + assert "Previewing plan as target 'prod'" in text.output + assert test_case.expected_fragment in text.output + assert after == before + assert ("prod", "_sqlbuild_migrations", 0) not in after + assert all(name != DESTINATION_MODEL for _, name, _ in after) + + +@pytest.mark.parametrize( + "test_case", + [ + PlanAsTargetTestCase( + description="executed prod migration previews as done", + preview_target="prod", + expected_exit_code=0, + expected_decisions=("done",), + ) + ], + ids=lambda case: case.description, +) +def test_given_executed_prod_migration_when_planning_as_prod_then_reports_done( + test_case: PlanAsTargetTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + migrated: CliRun = run_sqb( + project_dir=tmp_path, args=("build", "--target", "prod"), capsys=capsys + ) + before: tuple[tuple[Any, ...], ...] = warehouse_state(project_dir=tmp_path) + + preview: dict[str, Any] = plan_json( + project_dir=tmp_path, capsys=capsys, args=("--as", test_case.preview_target) + ) + + assert migrated.exit_code == test_case.expected_exit_code, migrated.output + assert migration_decisions(preview) == test_case.expected_decisions + assert preview["migrations"][0]["target"] == test_case.preview_target + assert warehouse_state(project_dir=tmp_path) == before + + +@pytest.mark.parametrize( + "test_case", + [ + PlanAsTargetTestCase( + description="unknown target fails clearly", + preview_target="staging", + expected_exit_code=1, + expected_fragment="unknown target 'staging' for plan --as", + ) + ], + ids=lambda case: case.description, +) +def test_given_unknown_target_when_planning_as_it_then_fails_clearly( + test_case: PlanAsTargetTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + + result: CliRun = run_sqb( + project_dir=tmp_path, args=("plan", "--as", test_case.preview_target), capsys=capsys + ) + + assert result.exit_code == test_case.expected_exit_code + assert test_case.expected_fragment in result.output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py new file mode 100644 index 0000000000..8c18d03f8e --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py @@ -0,0 +1,238 @@ +"""Integration coverage for staged migration execution, archived displacement, and recovery.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + MigrationArchiveExpiryTestCase, + StagedPlanOutputTestCase, + StagedReplaceTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + DESTINATION_MODEL, + JANITOR_PROJECT_TOML, + ORIGIN_MODEL, + CliRun, + archived_order_ids, + archived_relations, + build, + build_ok, + fail_non_transactional_promotion_rename, + fail_partial_stage, + fail_promotion_rename, + live_relations, + migration_decisions, + migration_events, + no_failure, + order_ids, + plan_json, + prepare_forced_replace, + refuse_clone_then_copy, + run_sqb, + state_tables, +) + +_STAGE: str = "migration_stage" +_PREVIOUS: str = "migration_previous" +_FORCED_EVENT: tuple[str, str, str] = (ORIGIN_MODEL, DESTINATION_MODEL, "forced_replace") + + +@pytest.mark.parametrize( + "test_case", + [ + StagedReplaceTestCase( + description="successful forced replace keeps the displaced destination archived", + install_failure=no_failure, + expected_first_exit_code=0, + expected_relations_after_failure=("raw_orders", "stg_customer_orders", "stg_orders"), + expected_previous_ids_after_failure=((1, 2),), + expected_decision_after_failure="done", + expected_final_previous_ids=((1, 2),), + expected_final_abandoned_stages=0, + expected_final_events=(_FORCED_EVENT,), + ), + StagedReplaceTestCase( + description="transactional crash while promoting leaves the live destination intact", + install_failure=fail_promotion_rename, + expected_first_exit_code=1, + expected_relations_after_failure=("raw_orders", "stg_customer_orders", "stg_orders"), + expected_previous_ids_after_failure=(), + expected_decision_after_failure="forced_replace", + expected_final_previous_ids=((1, 2),), + expected_final_abandoned_stages=1, + expected_final_events=(_FORCED_EVENT,), + ), + StagedReplaceTestCase( + description="non-transactional crash between renames keeps the old data archived", + install_failure=fail_non_transactional_promotion_rename, + expected_first_exit_code=1, + expected_relations_after_failure=("raw_orders", "stg_orders"), + expected_previous_ids_after_failure=((1, 2),), + expected_decision_after_failure="migrate", + expected_final_previous_ids=((1, 2),), + expected_final_abandoned_stages=1, + expected_final_events=((ORIGIN_MODEL, DESTINATION_MODEL, "migrate"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_forced_replace_when_interrupted_then_retry_converges_without_losing_data( + test_case: StagedReplaceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The destination is never dropped and a retry never promotes an old stage.""" + + prepare_forced_replace(project_dir=tmp_path, capsys=capsys) + + with monkeypatch.context() as patch: + test_case.install_failure(patch) + first: CliRun = build(project_dir=tmp_path, capsys=capsys) + relations_after_failure: tuple[str, ...] = live_relations(project_dir=tmp_path) + previous_after_failure: tuple[tuple[int, ...], ...] = archived_order_ids( + project_dir=tmp_path, kind=_PREVIOUS + ) + after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert first.exit_code == test_case.expected_first_exit_code, first.output + assert relations_after_failure == test_case.expected_relations_after_failure + assert previous_after_failure == test_case.expected_previous_ids_after_failure + assert migration_decisions(after_failure) == (test_case.expected_decision_after_failure,) + assert order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") == tuple( + range(1, 6) + ) + assert order_ids(project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}") == tuple(range(1, 6)) + assert len(archived_relations(project_dir=tmp_path, kind=_STAGE)) == ( + test_case.expected_final_abandoned_stages + ) + assert archived_order_ids(project_dir=tmp_path, kind=_PREVIOUS) == ( + test_case.expected_final_previous_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_final_events + + +@pytest.mark.parametrize( + "test_case", + [ + StagedPlanOutputTestCase( + description="duckdb plans a physical copy promoted in one transaction", + install_stage=no_failure, + expected_transfer_fallback=None, + expected_build_fragment="by physical copy. (", + expected_transfer="copy", + expected_promotion="transactional_rename", + expected_storage_transition=None, + expected_text_fragment="└── transfer physical copy, promote by transactional rename", + ), + StagedPlanOutputTestCase( + description="refused clone plans its copy fallback and builds by copy", + install_stage=refuse_clone_then_copy, + expected_transfer_fallback="copy", + expected_build_fragment="by physical copy (clone refused).", + expected_transfer="clone", + expected_promotion="transactional_rename", + expected_storage_transition=None, + expected_text_fragment=( + "└── transfer zero-copy clone (physical copy if refused), " + "promote by transactional rename" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_forced_replace_when_planning_then_reports_transfer_and_promotion( + test_case: StagedPlanOutputTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Plan text and JSON state the planned transfer; build output reports the one used.""" + + prepare_forced_replace(project_dir=tmp_path, capsys=capsys) + test_case.install_stage(monkeypatch) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + text: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + built: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + + migration: dict[str, Any] = plan["migrations"][0] + assert migration["transfer"] == test_case.expected_transfer + assert migration["transfer_fallback"] == test_case.expected_transfer_fallback + assert migration["promotion"] == test_case.expected_promotion + assert migration["storage_transition"] == test_case.expected_storage_transition + assert "statement" not in migration + assert test_case.expected_text_fragment in text.output + assert test_case.expected_build_fragment in built.output + assert order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") == tuple( + range(1, 6) + ) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationArchiveExpiryTestCase( + description="janitor expires abandoned stages and displaced destinations only", + janitor_retention_days="30", + expected_archives_before=2, + expected_relations_after=("stg_customer_orders", "stg_orders"), + expected_migration_state_kept=True, + ), + MigrationArchiveExpiryTestCase( + description="janitor removing stale relations still keeps the migration state table", + janitor_retention_days="0", + expected_archives_before=2, + expected_relations_after=("stg_customer_orders",), + expected_migration_state_kept=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_migration_archives_when_janitor_runs_then_expires_them_by_name( + test_case: MigrationArchiveExpiryTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Archive names alone let janitor remove migration leftovers without touching live data.""" + + prepare_forced_replace(project_dir=tmp_path, capsys=capsys, project_toml=JANITOR_PROJECT_TOML) + with monkeypatch.context() as patch: + fail_partial_stage(patch) + _ = build(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + archives_before: int = len( + archived_relations(project_dir=tmp_path, kind=_STAGE, schema="dev") + ) + len(archived_relations(project_dir=tmp_path, kind=_PREVIOUS, schema="dev")) + + janitor: CliRun = run_sqb( + project_dir=tmp_path, + args=("janitor", "--auto-approve", "--retention-days", test_case.janitor_retention_days), + capsys=capsys, + ) + + assert janitor.exit_code == 0, janitor.output + assert archives_before == test_case.expected_archives_before + assert archived_relations(project_dir=tmp_path, kind=_STAGE, schema="dev") == () + assert archived_relations(project_dir=tmp_path, kind=_PREVIOUS, schema="dev") == (), ( + janitor.output + ) + assert live_relations(project_dir=tmp_path, schema="dev") == ( + test_case.expected_relations_after + ) + assert order_ids(project_dir=tmp_path, relation=f"dev.{DESTINATION_MODEL}") == tuple( + range(1, 6) + ) + assert ("_sqlbuild_migrations" in state_tables(project_dir=tmp_path, schema="dev")) is ( + test_case.expected_migration_state_kept + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py new file mode 100644 index 0000000000..149f232b6c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py @@ -0,0 +1,54 @@ +"""Integration coverage for rejecting model migrations in virtual environments.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + VirtualModeMigrationTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + VIRTUAL_PROJECT_TOML, + CliRun, + incremental_orders_sql, + load_raw_orders, + run_sqb, + write_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + VirtualModeMigrationTestCase( + description="virtual build rejects migrate_from", + expected_exit_code=1, + expected_fragment="migrate_from is supported only in direct mode", + ) + ], + ids=lambda case: case.description, +) +def test_given_migrate_from_when_building_virtually_then_rejects_before_building( + test_case: VirtualModeMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={"stg_customer_orders": incremental_orders_sql(migrate_from="stg_orders")}, + project_toml=VIRTUAL_PROJECT_TOML, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + initialized: CliRun = run_sqb(project_dir=tmp_path, args=("state", "init"), capsys=capsys) + + result: CliRun = run_sqb( + project_dir=tmp_path, args=("build", "--virtual-env", "feature"), capsys=capsys + ) + + assert initialized.exit_code == 0, initialized.output + assert result.exit_code == test_case.expected_exit_code, result.output + assert test_case.expected_fragment in result.output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/migrations/__init__.py b/tests/integration/src/sqlbuild/executor/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/__init__.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py new file mode 100644 index 0000000000..52eaba518e --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py @@ -0,0 +1,34 @@ +"""Test case types for migration executor helper integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class ArtifactNamingTestCase: + description: str + existing_relations: tuple[str, ...] + now: datetime + expected_stage_name: str + expected_displaced_name: str + expected_destination_exists: bool + + +@dataclass(frozen=True) +class CloneFallbackTestCase: + description: str + clone_statements: tuple[str, ...] + refused_messages: tuple[str, ...] + expected_transfer: str + expected_stage_rows: int + + +@dataclass(frozen=True) +class CloneFailureTestCase: + description: str + clone_statements: tuple[str, ...] + refused_messages: tuple[str, ...] + expected_error_fragment: str + expected_stage_exists: bool diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py new file mode 100644 index 0000000000..88b2e1653e --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -0,0 +1,84 @@ +"""Builders for migration executor helper integration tests.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import MigrationStagePlan +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.migrations.types import ( + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, +) +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.executor.migrations.models import MigrationArtifactNames + + +def orders_migration_entry() -> ModelMigrationPlanEntry: + """Return a planned forced replace of main.orders_v2 from main.orders.""" + + return ModelMigrationPlanEntry( + model_name="orders_v2", + discovery=MigrationDiscovery.MANUAL, + decision=MigrationDecision.FORCED_REPLACE, + compatibility=MigrationCompatibility.COMPATIBLE, + origin_model="orders", + origin=CompiledRelationLocation( + database=None, schema="main", name="orders", qualified_name="main.orders" + ), + destination=CompiledRelationLocation( + database=None, + schema="main", + name="orders_v2", + qualified_name="main.orders_v2", + ), + target_name=None, + ) + + +class CloneRefusingDuckDbAdapter(DuckDbAdapter): + """DuckDB adapter whose stage clone is scripted to fail with a copy fallback.""" + + def __init__( + self, *, clone_statements: tuple[str, ...], refused_messages: tuple[str, ...] + ) -> None: + super().__init__() + self.clone_statements: tuple[str, ...] = clone_statements + self.refused_messages: tuple[str, ...] = refused_messages + + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.CLONE, + statements=tuple( + statement.format(stage=stage, origin=origin) for statement in self.clone_statements + ), + fallback_statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + is_clone_refusal=self.is_clone_refusal, + ) + + def is_clone_refusal(self, error: BaseException) -> bool: + """Treat errors naming one of the scripted refusal messages as clone refusals.""" + + return any(message in str(error) for message in self.refused_messages) + + +def artifact_names_for(*, stage_name: str) -> MigrationArtifactNames: + """Return artifact names for a fresh stage beside main.orders_v2.""" + + return MigrationArtifactNames( + stage_name=stage_name, + stage_qualified=f"main.{stage_name}", + displaced_name="_sqb_archive__20260102t030405z__migration_previous__orders_v2", + displaced_qualified="main._sqb_archive__20260102t030405z__migration_previous__orders_v2", + destination_qualified="main.orders_v2", + destination_exists=False, + ) diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_naming.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_naming.py new file mode 100644 index 0000000000..322c9ca1c8 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_naming.py @@ -0,0 +1,76 @@ +"""Integration coverage for fresh migration stage and displaced-destination names.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.migrations._helpers.naming import resolve_artifact_names +from sqlbuild.executor.migrations.models import MigrationArtifactNames +from tests.integration.src.sqlbuild.executor.migrations._helpers._test_types import ( + ArtifactNamingTestCase, +) +from tests.integration.src.sqlbuild.executor.migrations._helpers.helpers import ( + orders_migration_entry, +) + +_NOW: datetime = datetime(2026, 1, 2, 3, 4, 5, 678000, tzinfo=UTC) + + +@pytest.mark.parametrize( + "test_case", + [ + ArtifactNamingTestCase( + description="unused names take the current second and see a missing destination", + existing_relations=(), + now=_NOW, + expected_stage_name="_sqb_archive__20260102t030405z__migration_stage__orders_v2", + expected_displaced_name=( + "_sqb_archive__20260102t030405z__migration_previous__orders_v2" + ), + expected_destination_exists=False, + ), + ArtifactNamingTestCase( + description="an abandoned stage from the same second steps back one second", + existing_relations=( + "orders_v2", + "_sqb_archive__20260102t030405z__migration_stage__orders_v2", + ), + now=_NOW, + expected_stage_name="_sqb_archive__20260102t030404z__migration_stage__orders_v2", + expected_displaced_name=( + "_sqb_archive__20260102t030404z__migration_previous__orders_v2" + ), + expected_destination_exists=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_existing_relations_when_resolving_names_then_never_reuses_an_existing_stage( + test_case: ArtifactNamingTestCase, +) -> None: + """Every attempt stages under a name that did not exist when the attempt began.""" + + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + name: str + for name in test_case.existing_relations: + _ = adapter.execute( + connection=connection, sql=f"CREATE TABLE main.{name} AS SELECT 1 AS id" + ) + + names: MigrationArtifactNames = resolve_artifact_names( + adapter=adapter, connection=connection, entry=orders_migration_entry(), now=test_case.now + ) + + assert names.stage_name == test_case.expected_stage_name + assert names.displaced_name == test_case.expected_displaced_name + assert names.destination_exists is test_case.expected_destination_exists + assert names.stage_qualified == f"main.{test_case.expected_stage_name}" + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_staging.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_staging.py new file mode 100644 index 0000000000..d056da8208 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_staging.py @@ -0,0 +1,122 @@ +"""Integration coverage for the clone-refusal copy fallback when creating a migration stage.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.executor.migrations._helpers.staging import create_stage +from sqlbuild.executor.migrations.models import MigrationArtifactNames +from tests.integration.src.sqlbuild.executor.migrations._helpers._test_types import ( + CloneFailureTestCase, + CloneFallbackTestCase, +) +from tests.integration.src.sqlbuild.executor.migrations._helpers.helpers import ( + CloneRefusingDuckDbAdapter, + artifact_names_for, + orders_migration_entry, +) + +_STAGE: str = "_sqb_archive__20260102t030405z__migration_stage__orders_v2" + + +@pytest.mark.parametrize( + "test_case", + [ + CloneFallbackTestCase( + description="recognized clone refusal falls back to a physical copy", + clone_statements=("SELECT error('clone refused by warehouse')",), + refused_messages=("clone refused by warehouse",), + expected_transfer=MigrationTransfer.COPY.value, + expected_stage_rows=2, + ), + CloneFallbackTestCase( + description="successful clone reports the planned clone transfer", + clone_statements=("CREATE TABLE {stage} AS SELECT * FROM {origin}",), + refused_messages=("clone refused by warehouse",), + expected_transfer=MigrationTransfer.CLONE.value, + expected_stage_rows=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_clone_outcome_when_creating_stage_then_reports_actual_transfer( + test_case: CloneFallbackTestCase, +) -> None: + """The returned transfer is the one that actually populated the stage.""" + + adapter: CloneRefusingDuckDbAdapter = CloneRefusingDuckDbAdapter( + clone_statements=test_case.clone_statements, refused_messages=test_case.refused_messages + ) + connection: Any = adapter.connect({"database": ":memory:"}) + _ = adapter.execute( + connection=connection, sql="CREATE TABLE main.orders AS SELECT * FROM range(2) t(id)" + ) + names: MigrationArtifactNames = artifact_names_for(stage_name=_STAGE) + + transfer: MigrationTransfer = create_stage( + adapter=adapter, connection=connection, entry=orders_migration_entry(), names=names + ) + + assert transfer.value == test_case.expected_transfer + assert adapter.execute( + connection=connection, sql=f"SELECT COUNT(*) FROM {names.stage_qualified}" + ).fetchall() == [(test_case.expected_stage_rows,)] + + +@pytest.mark.parametrize( + "test_case", + [ + CloneFailureTestCase( + description="unrecognized failure is re-raised without attempting a copy", + clone_statements=("SELECT error('connection reset by peer')",), + refused_messages=("clone refused by warehouse",), + expected_error_fragment="connection reset by peer", + expected_stage_exists=False, + ), + CloneFailureTestCase( + description="refusal after the stage appeared fails instead of adopting it", + clone_statements=( + "CREATE TABLE {stage} AS SELECT * FROM {origin}", + "SELECT error('clone refused by warehouse')", + ), + refused_messages=("clone refused by warehouse",), + expected_error_fragment="refusing to adopt a stage of unknown completeness", + expected_stage_exists=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_unsafe_clone_failure_when_creating_stage_then_raises_without_copy( + test_case: CloneFailureTestCase, +) -> None: + """Only clean, recognized refusals fall back; anything else surfaces to the caller.""" + + adapter: CloneRefusingDuckDbAdapter = CloneRefusingDuckDbAdapter( + clone_statements=test_case.clone_statements, refused_messages=test_case.refused_messages + ) + connection: Any = adapter.connect({"database": ":memory:"}) + _ = adapter.execute( + connection=connection, sql="CREATE TABLE main.orders AS SELECT * FROM range(2) t(id)" + ) + names: MigrationArtifactNames = artifact_names_for(stage_name=_STAGE) + + with pytest.raises(Exception, match=test_case.expected_error_fragment): + _ = create_stage( + adapter=adapter, connection=connection, entry=orders_migration_entry(), names=names + ) + + stage_rows: list[tuple[object, ...]] = adapter.execute( + connection=connection, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = 'main' AND table_name = '{_STAGE}'" + ), + ).fetchall() + assert bool(stage_rows) is test_case.expected_stage_exists + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 39275d97d8..3cc1f0296f 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -2,6 +2,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter +from sqlbuild.adapter.contract.types import MigrationTransfer @dataclass(frozen=True) @@ -76,3 +77,21 @@ class AdapterRelationAgeMetadataCapabilityTestCase: description: str adapter: StrictAdapter expected_supported: bool + + +@dataclass(frozen=True) +class AdapterMigrationStageTestCase: + description: str + adapter: BaseAdapter + origin_is_transient: bool + stage_is_transient: bool | None + expected_transfer: MigrationTransfer + expected_statements: tuple[str, ...] + expected_fallback_statements: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AdapterTransactionalDdlTestCase: + description: str + adapter: BaseAdapter + expected_transactional: bool diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index 5ed09e990c..581f8a21da 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -210,3 +210,10 @@ class BigQueryRelationAgeMetadataTestCase: metadata_rows: tuple[tuple[object, ...], ...] expected_timestamps: tuple[tuple[str, str], ...] relation_names: tuple[str, ...] = ("old_orders", "old_orders_view") + + +@dataclass(frozen=True) +class BigQueryCloneRefusalTestCase: + description: str + driver_error: Exception + expected_refusal: bool diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_clone_refusal.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_clone_refusal.py new file mode 100644 index 0000000000..b6b784fd98 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_clone_refusal.py @@ -0,0 +1,100 @@ +"""BigQuery classification of stage clone failures that may fall back to a physical copy.""" + +from __future__ import annotations + +import pytest +from google.api_core.exceptions import BadRequest, Forbidden, InternalServerError, TooManyRequests + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import MigrationStagePlan +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection +from tests.unit.src.sqlbuild.adapters.bigquery._test_types import BigQueryCloneRefusalTestCase +from tests.unit.src.sqlbuild.adapters.bigquery.helpers import FakeBigQueryFailingClient + + +@pytest.mark.parametrize( + "test_case", + [ + BigQueryCloneRefusalTestCase( + description="clone chain depth limit is a refusal", + driver_error=BadRequest( + "clone chain too deep", + errors=[ + { + "reason": "invalid", + "message": "Cannot clone table: exceeded the limit of 3 clones or " + "snapshots in a chain; use a copy operation instead.", + } + ], + ), + expected_refusal=True, + ), + BigQueryCloneRefusalTestCase( + description="unsupported clone source is a refusal", + driver_error=BadRequest( + "unsupported clone source", + errors=[{"reason": "invalidQuery", "message": "Clone is not supported for tables"}], + ), + expected_refusal=True, + ), + BigQueryCloneRefusalTestCase( + description="backend error is transient and not a refusal", + driver_error=InternalServerError( + "backend", + errors=[{"reason": "backendError", "message": "Cannot clone right now"}], + ), + expected_refusal=False, + ), + BigQueryCloneRefusalTestCase( + description="rate limit is transient and not a refusal", + driver_error=TooManyRequests( + "rate", + errors=[{"reason": "rateLimitExceeded", "message": "Cannot clone: rate limit"}], + ), + expected_refusal=False, + ), + BigQueryCloneRefusalTestCase( + description="permission denial is not a refusal", + driver_error=Forbidden( + "denied", + errors=[{"reason": "accessDenied", "message": "Access Denied: cannot clone"}], + ), + expected_refusal=False, + ), + BigQueryCloneRefusalTestCase( + description="bad request without clone wording is not a refusal", + driver_error=BadRequest( + "missing", + errors=[{"reason": "invalid", "message": "Dataset main was not found"}], + ), + expected_refusal=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_clone_failure_when_classifying_then_only_capability_refusals_fall_back( + test_case: BigQueryCloneRefusalTestCase, +) -> None: + """Refusals are 400 invalid errors with clone-limitation wording, seen through the adapter.""" + + adapter: BigQueryAdapter = BigQueryAdapter() + plan: MigrationStagePlan = adapter.render_migration_stage( + origin="analytics.orders", stage="analytics.orders_stage" + ) + connection: _BigQueryConnection = _BigQueryConnection( + client=FakeBigQueryFailingClient(query_error=test_case.driver_error), location="US" + ) + + with pytest.raises(AdapterUserError) as raised: + _ = adapter.execute(connection=connection, sql=plan.statements[0]) + + assert plan.is_clone_refusal is not None + assert plan.is_clone_refusal(raised.value) is test_case.expected_refusal + assert plan.fallback_statements == ( + "CREATE TABLE `analytics.orders_stage` COPY `analytics.orders`", + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/postgres/_test_types.py b/tests/unit/src/sqlbuild/adapters/postgres/_test_types.py index 4872190836..13ae7d8cae 100644 --- a/tests/unit/src/sqlbuild/adapters/postgres/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/postgres/_test_types.py @@ -118,3 +118,10 @@ class PostgresRenderSourceFreshnessQueryTestCase: source_is_subquery: bool where_sql: str expected_sql: str + + +@dataclass(frozen=True) +class PostgresDependentViewRebindTestCase: + description: str + catalog_rows: tuple[tuple[object, ...], ...] + expected_statements: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/postgres/test_dependent_view_rebinds.py b/tests/unit/src/sqlbuild/adapters/postgres/test_dependent_view_rebinds.py new file mode 100644 index 0000000000..8338541221 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/postgres/test_dependent_view_rebinds.py @@ -0,0 +1,89 @@ +"""Unit coverage for re-pointing PostgreSQL views bound to a promoted table.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from tests.unit.src.sqlbuild.adapters.postgres._test_types import ( + PostgresDependentViewRebindTestCase, +) +from tests.unit.src.sqlbuild.adapters.postgres.helpers import ( + FakePostgresConnection, + FakePostgresCursor, +) + +_DEFINITION: str = " SELECT orders.order_id\n FROM dev.orders;" +_BODY: str = "SELECT orders.order_id\n FROM dev.orders" + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresDependentViewRebindTestCase( + description="view without options is re-created plainly", + catalog_rows=(("reporting", "orders_dashboard", _DEFINITION, None),), + expected_statements=( + f'CREATE OR REPLACE VIEW "reporting"."orders_dashboard" AS {_BODY}', + ), + ), + PostgresDependentViewRebindTestCase( + description="security options are re-emitted in the with clause", + catalog_rows=( + ( + "reporting", + "orders_secure", + _DEFINITION, + "security_barrier=true\x1fsecurity_invoker=true", + ), + ), + expected_statements=( + 'CREATE OR REPLACE VIEW "reporting"."orders_secure" ' + f"WITH (security_barrier=true, security_invoker=true) AS {_BODY}", + ), + ), + PostgresDependentViewRebindTestCase( + description="check option is re-emitted after the query", + catalog_rows=( + ( + "reporting", + "orders_checked", + _DEFINITION, + "check_option=cascaded\x1fsecurity_barrier=false", + ), + ), + expected_statements=( + 'CREATE OR REPLACE VIEW "reporting"."orders_checked" ' + f"WITH (security_barrier=false) AS {_BODY} WITH CASCADED CHECK OPTION", + ), + ), + PostgresDependentViewRebindTestCase( + description="local check option alone adds no with clause", + catalog_rows=(("reporting", "orders_local", _DEFINITION, "check_option=local"),), + expected_statements=( + 'CREATE OR REPLACE VIEW "reporting"."orders_local" ' + f"AS {_BODY} WITH LOCAL CHECK OPTION", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_dependent_views_when_capturing_rebinds_then_view_options_survive( + test_case: PostgresDependentViewRebindTestCase, +) -> None: + """CREATE OR REPLACE VIEW resets omitted options, so every captured option is re-emitted.""" + + connection: FakePostgresConnection = FakePostgresConnection( + FakePostgresCursor(rows=test_case.catalog_rows) + ) + + statements: tuple[str, ...] = PostgresAdapter().capture_dependent_view_rebinds( + connection=connection, database=None, schema="dev", name="orders" + ) + + assert statements == test_case.expected_statements + assert "reloptions" in connection.executed_sql[0] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/_test_types.py b/tests/unit/src/sqlbuild/adapters/sqlserver/_test_types.py index b4eaeaff78..d4b2389d6f 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/_test_types.py @@ -142,3 +142,25 @@ class SqlServerPruneSqlTestCase: schema: str retain_versions: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SqlServerMigrationEventWriteTestCase: + description: str + existing_rows: tuple[tuple[object, ...], ...] + expected_statement_prefixes: tuple[str, ...] + + +@dataclass(frozen=True) +class SqlServerMigrationTableTestCase: + description: str + schema: str + expected_prefix: str + expected_suffix: str + + +@dataclass(frozen=True) +class SqlServerMigrationEventReadTestCase: + description: str + stored_created_at: object + expected_read_suffix: str diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/helpers.py b/tests/unit/src/sqlbuild/adapters/sqlserver/helpers.py index 7fab1b2779..d6fa9df15f 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/helpers.py @@ -7,3 +7,23 @@ def __init__(self) -> None: def execute(self, sql: str) -> None: self.executed_sql.append(sql) + + +class RowsCursor: + def __init__(self, rows: tuple[tuple[object, ...], ...]) -> None: + self.rows: tuple[tuple[object, ...], ...] = rows + + def fetchall(self) -> list[tuple[object, ...]]: + return list(self.rows) + + +class ScriptedSqlServerConnection: + """Record statements and answer every query with the same scripted rows.""" + + def __init__(self, rows: tuple[tuple[object, ...], ...] = ()) -> None: + self.rows: tuple[tuple[object, ...], ...] = rows + self.executed_sql: list[str] = [] + + def execute(self, sql: str) -> RowsCursor: + self.executed_sql.append(sql) + return RowsCursor(self.rows) diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py new file mode 100644 index 0000000000..302b428eac --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py @@ -0,0 +1,168 @@ +"""SQL Server rendering coverage for the append-only model migration state table.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.migrations.main._read_events import read_migration_events +from sqlbuild.compiler.migrations.main.write_event import write_migration_event +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from tests.unit.src.sqlbuild.adapters.sqlserver._test_types import ( + SqlServerMigrationEventReadTestCase, + SqlServerMigrationEventWriteTestCase, + SqlServerMigrationTableTestCase, +) +from tests.unit.src.sqlbuild.adapters.sqlserver.helpers import ScriptedSqlServerConnection + +_CREATED_AT: datetime = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) +_EVENT: MigrationEvent = MigrationEvent( + event_id="event-1", + target_name="dev", + origin_model="stg_orders", + origin=MigrationRelation(database=None, schema="dbo", name="stg_orders"), + destination_model="stg_customer_orders", + destination=MigrationRelation(database=None, schema="dbo", name="stg_customer_orders"), + origin_version_hash="hash-1", + discovery=MigrationDiscovery.MANUAL, + decision=MigrationDecision.MIGRATE, + run_id="run-1", + created_at=_CREATED_AT, +) +_GUARDED_CREATE: str = ( + "IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'dbo' " + "AND table_name = '_sqlbuild_migrations') CREATE TABLE dbo._sqlbuild_migrations (" +) +_EXISTING: str = "SELECT event_id FROM dbo._sqlbuild_migrations WHERE event_id = 'event-1'" +_INSERT: str = "INSERT INTO dbo._sqlbuild_migrations (event_id, target_name" + + +@pytest.mark.parametrize( + "test_case", + [ + SqlServerMigrationTableTestCase( + description="state table creation is guarded by a catalog check", + schema="dbo", + expected_prefix=_GUARDED_CREATE, + expected_suffix="run_id NVARCHAR(MAX), created_at DATETIME2)", + ) + ], + ids=lambda case: case.description, +) +def test_given_sqlserver_when_rendering_migration_table_then_uses_guarded_create( + test_case: SqlServerMigrationTableTestCase, +) -> None: + """SQL Server has no CREATE TABLE IF NOT EXISTS, so creation is guarded by a catalog check.""" + + sql: str = SqlServerAdapter().render_create_migration_state_table_sql( + database=None, schema=test_case.schema + ) + + assert sql.startswith(test_case.expected_prefix) + assert "IF NOT EXISTS dbo" not in sql + assert sql.endswith(test_case.expected_suffix) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlServerMigrationEventWriteTestCase( + description="new event is checked then inserted with values", + existing_rows=(), + expected_statement_prefixes=(_GUARDED_CREATE, _EXISTING, _INSERT), + ), + SqlServerMigrationEventWriteTestCase( + description="already recorded event is not inserted again", + existing_rows=(("event-1",),), + expected_statement_prefixes=(_GUARDED_CREATE, _EXISTING), + ), + ], + ids=lambda case: case.description, +) +def test_given_sqlserver_when_writing_migration_event_then_statements_are_tsql( + test_case: SqlServerMigrationEventWriteTestCase, +) -> None: + """Event writes avoid SELECT-without-FROM guards and ANSI TIMESTAMP casts.""" + + adapter: SqlServerAdapter = SqlServerAdapter() + connection: ScriptedSqlServerConnection = ScriptedSqlServerConnection( + rows=test_case.existing_rows + ) + + write_migration_event( + connection=connection, + execute=adapter.execute, + event=_EVENT, + render_qualified_name=adapter.render_qualified_name, + create_table_sql=adapter.render_create_migration_state_table_sql( + database=None, schema="dbo" + ), + ) + + assert tuple( + sql[: len(prefix)] + for sql, prefix in zip( + connection.executed_sql, test_case.expected_statement_prefixes, strict=True + ) + ) == (test_case.expected_statement_prefixes) + assert all("AS TIMESTAMP" not in sql for sql in connection.executed_sql) + assert all("NOT EXISTS (SELECT 1 FROM dbo." not in sql for sql in connection.executed_sql) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlServerMigrationEventReadTestCase( + description="naive datetime2 values decode as utc", + stored_created_at=datetime(2026, 1, 2, 3, 4, 5), + expected_read_suffix="FROM dbo._sqlbuild_migrations ORDER BY created_at, event_id", + ) + ], + ids=lambda case: case.description, +) +def test_given_sqlserver_datetime2_rows_when_reading_events_then_decodes_utc_events( + test_case: SqlServerMigrationEventReadTestCase, +) -> None: + """Reads use plain ORDER BY and decode naive DATETIME2 values as UTC.""" + + adapter: SqlServerAdapter = SqlServerAdapter() + connection: ScriptedSqlServerConnection = ScriptedSqlServerConnection( + rows=( + ( + "event-1", + "dev", + "stg_orders", + None, + "dbo", + "stg_orders", + "stg_customer_orders", + None, + "dbo", + "stg_customer_orders", + "hash-1", + "manual", + "migrate", + "run-1", + test_case.stored_created_at, + ), + ) + ) + + events: tuple[MigrationEvent, ...] = read_migration_events( + connection=connection, + execute=adapter.execute, + database=None, + schema="dbo", + render_qualified_name=adapter.render_qualified_name, + ) + + assert events == (_EVENT,) + assert connection.executed_sql[0].endswith(test_case.expected_read_suffix) + assert "LIMIT" not in connection.executed_sql[0] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/test_migration_stage.py b/tests/unit/src/sqlbuild/adapters/test_migration_stage.py new file mode 100644 index 0000000000..b3ab9e6955 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/test_migration_stage.py @@ -0,0 +1,195 @@ +"""Unit coverage for per-adapter model migration staging statements.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.models import MigrationStagePlan +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from tests.unit.src.sqlbuild.adapters._test_types import ( + AdapterMigrationStageTestCase, + AdapterTransactionalDdlTestCase, +) + +_STAGE: str = "dev._sqb_archive__20260102t030405z__migration_stage__stg_customer_orders" + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterMigrationStageTestCase( + description="snowflake permanent to permanent is a zero-copy clone with grants", + adapter=SnowflakeAdapter(), + origin_is_transient=False, + stage_is_transient=False, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=(f"CREATE TABLE {_STAGE} CLONE dev.stg_orders COPY GRANTS",), + ), + AdapterMigrationStageTestCase( + description="snowflake permanent to transient is a zero-copy transient clone", + adapter=SnowflakeAdapter(), + origin_is_transient=False, + stage_is_transient=True, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=( + f"CREATE TRANSIENT TABLE {_STAGE} CLONE dev.stg_orders COPY GRANTS", + ), + ), + AdapterMigrationStageTestCase( + description="snowflake transient to transient is a zero-copy transient clone", + adapter=SnowflakeAdapter(), + origin_is_transient=True, + stage_is_transient=True, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=( + f"CREATE TRANSIENT TABLE {_STAGE} CLONE dev.stg_orders COPY GRANTS", + ), + ), + AdapterMigrationStageTestCase( + description="snowflake transient to permanent is a physical copy into a permanent table", + adapter=SnowflakeAdapter(), + origin_is_transient=True, + stage_is_transient=False, + expected_transfer=MigrationTransfer.COPY, + expected_statements=( + f"CREATE TABLE {_STAGE} LIKE dev.stg_orders COPY GRANTS", + f"INSERT INTO {_STAGE} SELECT * FROM dev.stg_orders", + ), + ), + AdapterMigrationStageTestCase( + description="snowflake without a destination type keeps the origin type", + adapter=SnowflakeAdapter(), + origin_is_transient=True, + stage_is_transient=None, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=( + f"CREATE TRANSIENT TABLE {_STAGE} CLONE dev.stg_orders COPY GRANTS", + ), + ), + AdapterMigrationStageTestCase( + description="bigquery clones and falls back to a table copy when clone is refused", + adapter=BigQueryAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=(f"CREATE TABLE `{_STAGE}` CLONE `dev.stg_orders`",), + expected_fallback_statements=(f"CREATE TABLE `{_STAGE}` COPY `dev.stg_orders`",), + ), + AdapterMigrationStageTestCase( + description="databricks copies through an independent deep clone", + adapter=DatabricksAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"CREATE TABLE {_STAGE} DEEP CLONE dev.stg_orders",), + ), + AdapterMigrationStageTestCase( + description="postgres copies into a new table without replacing anything", + adapter=PostgresAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"CREATE TABLE {_STAGE} AS SELECT * FROM dev.stg_orders",), + ), + AdapterMigrationStageTestCase( + description="sqlserver copies with select into", + adapter=SqlServerAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"SELECT * INTO {_STAGE} FROM dev.stg_orders",), + ), + AdapterMigrationStageTestCase( + description="duckdb copies into a new table without replacing anything", + adapter=DuckDbAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"CREATE TABLE {_STAGE} AS SELECT * FROM dev.stg_orders",), + ), + AdapterMigrationStageTestCase( + description="motherduck copies like duckdb", + adapter=MotherDuckAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"CREATE TABLE {_STAGE} AS SELECT * FROM dev.stg_orders",), + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_when_rendering_migration_stage_then_creates_a_fresh_stage( + test_case: AdapterMigrationStageTestCase, +) -> None: + """Stage statements never replace or drop an existing relation.""" + + plan: MigrationStagePlan = test_case.adapter.render_migration_stage( + origin="dev.stg_orders", + stage=_STAGE, + origin_is_transient=test_case.origin_is_transient, + stage_is_transient=test_case.stage_is_transient, + ) + + assert plan.transfer == test_case.expected_transfer + assert plan.statements == test_case.expected_statements + assert plan.fallback_statements == test_case.expected_fallback_statements + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterTransactionalDdlTestCase( + description="snowflake ddl commits implicitly", + adapter=SnowflakeAdapter(), + expected_transactional=False, + ), + AdapterTransactionalDdlTestCase( + description="bigquery has no transactional ddl", + adapter=BigQueryAdapter(), + expected_transactional=False, + ), + AdapterTransactionalDdlTestCase( + description="databricks has no transactional ddl", + adapter=DatabricksAdapter(), + expected_transactional=False, + ), + AdapterTransactionalDdlTestCase( + description="postgres renames roll back with the transaction", + adapter=PostgresAdapter(), + expected_transactional=True, + ), + AdapterTransactionalDdlTestCase( + description="sqlserver renames roll back with the transaction", + adapter=SqlServerAdapter(), + expected_transactional=True, + ), + AdapterTransactionalDdlTestCase( + description="duckdb renames roll back with the transaction", + adapter=DuckDbAdapter(), + expected_transactional=True, + ), + AdapterTransactionalDdlTestCase( + description="motherduck follows duckdb transactions", + adapter=MotherDuckAdapter(), + expected_transactional=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_when_checking_transactional_ddl_then_reports_capability( + test_case: AdapterTransactionalDdlTestCase, +) -> None: + """Transactional promotion is only claimed where renames roll back.""" + + assert test_case.adapter.supports_transactional_ddl() is test_case.expected_transactional + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index c15779d1a8..070f29c2e0 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -897,6 +897,30 @@ "type change detected", ), ), + FormatPlanTestCase( + description="error-severity diagnostics render under an errors heading", + plan_output=build_plan_output( + warnings=( + build_warning( + model_name="stg_customer_orders", + message="migrate_from origin main.stg_orders does not exist", + severity=WarningSeverity.ERROR, + code="M102", + ), + build_warning( + model_name="stg_customers", + message="type change detected", + severity=WarningSeverity.WARNING, + ), + ), + ), + expected_fragments=( + "Errors (1)\n└── stg_customer_orders\n" + " └── migrate_from origin main.stg_orders does not exist", + "Warnings (1)\n└── stg_customers\n └── type change detected", + ), + unexpected_fragments=("Warnings (2)",), + ), FormatPlanTestCase( description="multiline warning messages terminate their child tree", plan_output=build_plan_output( diff --git a/tests/unit/src/sqlbuild/compiler/migrations/__init__.py b/tests/unit/src/sqlbuild/compiler/migrations/__init__.py new file mode 100644 index 0000000000..bfa43ba22d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/__init__.py @@ -0,0 +1 @@ +"""Unit tests for model migration state.""" diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/__init__.py b/tests/unit/src/sqlbuild/compiler/migrations/main/__init__.py new file mode 100644 index 0000000000..6f9ea464f6 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/__init__.py @@ -0,0 +1 @@ +"""Unit tests for model migration state entrypoints.""" diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py new file mode 100644 index 0000000000..8621359dd7 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -0,0 +1,20 @@ +"""Test case types for model migration state entrypoints.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NewestEventTestCase: + description: str + events: tuple[tuple[str, str], ...] + relation: str + expected_newest_index: int | None + + +@dataclass(frozen=True) +class RepeatedEventWriteTestCase: + description: str + write_count: int + expected_stored_count: int diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py new file mode 100644 index 0000000000..c39ce64147 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -0,0 +1,42 @@ +"""Event builders for model migration state unit tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from sqlbuild.compiler.migrations.main.deterministic_event_id import ( + deterministic_migration_event_id, +) +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery + +_STARTED_AT: datetime = datetime(2026, 1, 1, tzinfo=UTC) + + +def main_relation(name: str) -> MigrationRelation: + """Return a relation in the main schema with no database qualifier.""" + + return MigrationRelation(database=None, schema="main", name=name) + + +def migration_event(*, origin: str, destination: str, day: int) -> MigrationEvent: + """Return a manual migrate event recorded on the given day.""" + + return MigrationEvent( + event_id=deterministic_migration_event_id( + run_id=f"run-{day}", + target_name="dev", + origin=main_relation(origin), + destination=main_relation(destination), + ), + target_name="dev", + origin_model=origin, + origin=main_relation(origin), + destination_model=destination, + destination=main_relation(destination), + origin_version_hash=f"hash-{origin}", + discovery=MigrationDiscovery.MANUAL, + decision=MigrationDecision.MIGRATE, + run_id=f"run-{day}", + created_at=_STARTED_AT + timedelta(days=day), + ) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py new file mode 100644 index 0000000000..87d21f00ee --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -0,0 +1,112 @@ +"""Unit coverage for append-only migration event projection and storage.""" + +from __future__ import annotations + +import duckdb +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning +from sqlbuild.compiler.migrations.main._read_events import read_migration_events +from sqlbuild.compiler.migrations.main.write_event import write_migration_event +from sqlbuild.compiler.migrations.models import MigrationEvent +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( + NewestEventTestCase, + RepeatedEventWriteTestCase, +) +from tests.unit.src.sqlbuild.compiler.migrations.main.helpers import ( + main_relation, + migration_event, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NewestEventTestCase( + description="day nine destination sees the day five event as origin", + events=(("stg_orders", "stg_customer_orders"), ("stg_customer_orders", "stg_orders")), + relation="stg_customer_orders", + expected_newest_index=1, + ), + NewestEventTestCase( + description="day five destination sees the day one event as origin", + events=(("stg_orders", "stg_customer_orders"),), + relation="stg_orders", + expected_newest_index=0, + ), + NewestEventTestCase( + description="unrelated relation has no event", + events=(("stg_orders", "stg_customer_orders"),), + relation="dim_customers", + expected_newest_index=None, + ), + NewestEventTestCase( + description="relation identity is case insensitive", + events=(("STG_ORDERS", "Stg_Customer_Orders"),), + relation="stg_customer_orders", + expected_newest_index=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_event_history_when_projecting_newest_mention_then_returns_latest_event( + test_case: NewestEventTestCase, +) -> None: + events: tuple[MigrationEvent, ...] = tuple( + migration_event(origin=origin, destination=destination, day=index * 4 + 1) + for index, (origin, destination) in enumerate(test_case.events) + ) + events_by_index: dict[int | None, MigrationEvent] = dict(enumerate(events)) + + newest: MigrationEvent | None = newest_migration_event_mentioning( + events=reversed(events), relation=main_relation(test_case.relation) + ) + + assert newest == events_by_index.get(test_case.expected_newest_index) + + +@pytest.mark.parametrize( + "test_case", + [ + RepeatedEventWriteTestCase( + description="writing the same event twice stores it once", + write_count=2, + expected_stored_count=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_repeated_event_write_when_reading_then_event_is_stored_once( + test_case: RepeatedEventWriteTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + event: MigrationEvent = migration_event( + origin="stg_orders", destination="stg_customer_orders", day=1 + ) + + for _ in range(test_case.write_count): + write_migration_event( + connection=connection, + execute=adapter.execute, + event=event, + render_qualified_name=adapter.render_qualified_name, + create_table_sql=adapter.render_create_migration_state_table_sql( + database=None, schema="main" + ), + ) + stored: tuple[MigrationEvent, ...] = read_migration_events( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + ) + + assert len(stored) == test_case.expected_stored_count + assert stored == (event,) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py new file mode 100644 index 0000000000..5d3d916a71 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py @@ -0,0 +1,25 @@ +"""Test case types for migration fingerprint helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import BASE_CONFIG + + +@dataclass(frozen=True) +class MigrationFingerprintTestCase: + description: str + origin_sql: str + destination_sql: str + expected_match: bool + origin_config: dict[str, object] = field(default_factory=lambda: dict(BASE_CONFIG)) + destination_config: dict[str, object] = field(default_factory=lambda: dict(BASE_CONFIG)) + destination_ref_identities: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class IneligibleFingerprintTestCase: + description: str + query_sql: str + expected_fingerprint: str | None diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py new file mode 100644 index 0000000000..d5a94a4f24 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -0,0 +1,28 @@ +"""Fingerprint builders for migration fingerprint unit tests.""" + +from __future__ import annotations + +import json + +from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint + +BASE_CONFIG: dict[str, object] = { + "materialized": "incremental", + "incremental_strategy": "delete_insert", + "unique_key": "order_id", + "cursor": "order_date", +} +ORDERS_SQL: str = 'SELECT o.order_id, o.amount_cents FROM __ref("stg_orders") AS o' + + +def model_fingerprint( + *, model_name: str, query_sql: str, config: dict[str, object], ref_identities: dict[str, str] +) -> str | None: + """Return the DuckDB migration fingerprint for one model definition.""" + + return build_migration_fingerprint( + query_sql=query_sql, + metadata_json=json.dumps({"model_name": model_name, "config": config}), + ref_identities=ref_identities, + dialect="duckdb", + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py new file mode 100644 index 0000000000..211a7bb3f6 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py @@ -0,0 +1,234 @@ +"""Unit coverage for local migration fingerprints.""" + +from __future__ import annotations + +import pytest + +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( + IneligibleFingerprintTestCase, + MigrationFingerprintTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import ( + BASE_CONFIG, + ORDERS_SQL, + model_fingerprint, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationFingerprintTestCase( + expected_match=False, + description="alias rename does not rewrite the qualified column name", + origin_sql='SELECT a.a AS value FROM __ref("stg_orders") a', + destination_sql='SELECT b.b AS value FROM __ref("stg_orders") b', + ), + MigrationFingerprintTestCase( + expected_match=True, + description="cte and alias renames keep columns named like the cte", + origin_sql=( + 'WITH orders AS (SELECT orders FROM __ref("stg_orders")) ' + "SELECT orders.orders, totals.total FROM orders " + "JOIN (SELECT 1 AS total) AS totals ON TRUE" + ), + destination_sql=( + 'WITH customer_orders AS (SELECT orders FROM __ref("stg_orders")) ' + "SELECT customer_orders.orders, sums.total FROM customer_orders " + "JOIN (SELECT 1 AS total) AS sums ON TRUE" + ), + ), + MigrationFingerprintTestCase( + expected_match=True, + description="later cte reading an earlier renamed cte still matches", + origin_sql=( + 'WITH src AS (SELECT * FROM __ref("stg_orders")), totals AS ' + "(SELECT src.order_id FROM src) SELECT totals.order_id FROM totals" + ), + destination_sql=( + 'WITH base AS (SELECT * FROM __ref("stg_orders")), sums AS ' + "(SELECT base.order_id FROM base) SELECT sums.order_id FROM sums" + ), + ), + MigrationFingerprintTestCase( + expected_match=False, + description="column named like the renamed cte is still a column", + origin_sql=( + 'WITH orders AS (SELECT orders FROM __ref("stg_orders")) ' + "SELECT orders.orders FROM orders" + ), + destination_sql=( + 'WITH customer_orders AS (SELECT customer_orders FROM __ref("stg_orders")) ' + "SELECT customer_orders.customer_orders FROM customer_orders" + ), + ), + MigrationFingerprintTestCase( + expected_match=True, + description="qualified star follows the alias rename", + origin_sql='SELECT o.* FROM __ref("stg_orders") AS o', + destination_sql='SELECT customer_order.* FROM __ref("stg_orders") AS customer_order', + ), + MigrationFingerprintTestCase( + expected_match=True, + description="table_alias_renamed", + origin_sql=ORDERS_SQL, + destination_sql=( + "SELECT customer_order.order_id, customer_order.amount_cents " + 'FROM __ref("stg_orders") AS customer_order' + ), + ), + MigrationFingerprintTestCase( + expected_match=True, + description="import_cte_renamed_with_upstream", + origin_sql=( + 'WITH stg_orders AS (SELECT * FROM __ref("stg_orders")) ' + "SELECT stg_orders.order_id FROM stg_orders" + ), + destination_sql=( + 'WITH stg_customer_orders AS (SELECT * FROM __ref("stg_customer_orders")) ' + "SELECT stg_customer_orders.order_id FROM stg_customer_orders" + ), + destination_ref_identities={"stg_customer_orders": "stg_orders"}, + ), + MigrationFingerprintTestCase( + expected_match=True, + description="comments_and_formatting", + origin_sql=ORDERS_SQL, + destination_sql=( + "-- orders with amounts\n" + "SELECT\n o.order_id,\n /* cents */ o.amount_cents\n" + 'FROM __ref("stg_orders") AS o\n' + ), + ), + MigrationFingerprintTestCase( + expected_match=True, + description="storage_config_changed", + origin_sql=ORDERS_SQL, + destination_sql=ORDERS_SQL, + destination_config={**BASE_CONFIG, "incremental_strategy": "merge", "lookback": "1d"}, + ), + MigrationFingerprintTestCase( + expected_match=True, + description="cursor_inputs_follow_renamed_upstream", + origin_sql=ORDERS_SQL, + destination_sql=ORDERS_SQL.replace("stg_orders", "stg_customer_orders"), + origin_config={**BASE_CONFIG, "cursor_inputs": {"stg_orders": "order_date"}}, + destination_config={ + **BASE_CONFIG, + "cursor_inputs": {"stg_customer_orders": "order_date"}, + }, + destination_ref_identities={"stg_customer_orders": "stg_orders"}, + ), + MigrationFingerprintTestCase( + expected_match=False, + description="filter_added", + origin_sql=ORDERS_SQL, + destination_sql=f"{ORDERS_SQL} WHERE o.amount_cents > 0", + ), + MigrationFingerprintTestCase( + expected_match=False, + description="unresolved_upstream_rename", + origin_sql=ORDERS_SQL, + destination_sql=ORDERS_SQL.replace("stg_orders", "stg_customer_orders"), + ), + MigrationFingerprintTestCase( + expected_match=False, + description="column_renamed", + origin_sql=ORDERS_SQL, + destination_sql=( + 'SELECT o.order_id, o.amount_cents AS total_cents FROM __ref("stg_orders") AS o' + ), + ), + MigrationFingerprintTestCase( + expected_match=False, + description="unique_key_changed", + origin_sql=ORDERS_SQL, + destination_sql=ORDERS_SQL, + destination_config={**BASE_CONFIG, "unique_key": "amount_cents"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_definition_when_fingerprinting_then_matches_only_equivalent_logic( + test_case: MigrationFingerprintTestCase, +) -> None: + origin: str | None = model_fingerprint( + model_name="stg_orders", + query_sql=test_case.origin_sql, + config=test_case.origin_config, + ref_identities={}, + ) + destination: str | None = model_fingerprint( + model_name="stg_customer_orders", + query_sql=test_case.destination_sql, + config=test_case.destination_config, + ref_identities=test_case.destination_ref_identities, + ) + + assert origin is not None + assert destination is not None + assert (origin == destination) is test_case.expected_match + + +@pytest.mark.parametrize( + "test_case", + [ + IneligibleFingerprintTestCase( + description="unparseable query has no fingerprint", + query_sql="SELECT FROM WHERE (", + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="nested with shadows an outer physical relation", + query_sql=( + "WITH wrapper AS (WITH a AS (SELECT 1 AS value) SELECT value FROM a) " + "SELECT value FROM a" + ), + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="with nested in a subquery", + query_sql="SELECT x FROM (WITH q AS (SELECT 1 AS x) SELECT x FROM q) AS s", + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="cte body reads the physical relation it shadows", + query_sql="WITH orders AS (SELECT * FROM orders) SELECT * FROM orders", + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="cte body reads a later cte name", + query_sql=( + "WITH totals AS (SELECT * FROM orders), orders AS (SELECT 1 AS x) " + "SELECT * FROM totals" + ), + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="recursive with", + query_sql="WITH RECURSIVE n AS (SELECT 1 AS x) SELECT x FROM n", + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="alias reuses a physical relation name", + query_sql="SELECT o.x FROM orders AS o JOIN o ON TRUE", + expected_fingerprint=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_unparseable_or_ambiguous_sql_when_fingerprinting_then_returns_none( + test_case: IneligibleFingerprintTestCase, +) -> None: + fingerprint: str | None = model_fingerprint( + model_name="stg_orders", + query_sql=test_case.query_sql, + config=BASE_CONFIG, + ref_identities={}, + ) + + assert fingerprint == test_case.expected_fingerprint + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py index 6a11a18d0d..db317f90d9 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py @@ -28,6 +28,7 @@ class ArchiveNameBuildTestCase: expected_prefix: str expected_length: int expected_logical_name_is_original: bool + kind: str | None = None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py index 464e7bd297..a552c637d7 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py @@ -111,6 +111,26 @@ def test_given_non_strict_name_when_parsing_archive_name_then_returns_none( expected_length=63, expected_logical_name_is_original=False, ), + ArchiveNameBuildTestCase( + description="kind segment follows the timestamp and precedes the original name", + original_name="orders", + archived_at=ARCHIVED_AT, + identifier_limit=255, + expected_prefix="_sqb_archive__20260924t101500z__migration_stage__orders", + expected_length=len("_sqb_archive__20260924t101500z__migration_stage__orders"), + expected_logical_name_is_original=False, + kind="migration_stage", + ), + ArchiveNameBuildTestCase( + description="long name is fitted after an intact kind at the postgres limit", + original_name=LONG_NAME, + archived_at=ARCHIVED_AT, + identifier_limit=63, + expected_prefix="_sqb_archive__20260924t101500z__migration_previous__fu_", + expected_length=63, + expected_logical_name_is_original=False, + kind="migration_previous", + ), ], ids=lambda case: case.description, ) @@ -121,6 +141,7 @@ def test_given_original_name_when_building_archive_name_then_round_trips_through original_name=test_case.original_name, archived_at=test_case.archived_at, identifier_limit=test_case.identifier_limit, + kind=test_case.kind, ) parsed: JanitorParsedArchiveName | None = parse_archive_name(archive_name) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index 6abdb62087..f7c9f78ce3 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -499,3 +499,19 @@ class RuntimeContractValidationTestCase: contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = () expected_error_fragment: str | None = None expected_error_code: str | None = None + + +@dataclass(frozen=True) +class StagedPromotionTestCase: + description: str + adapter_name: str + target_exists: bool + expected_statements: tuple[str, ...] + + +@dataclass(frozen=True) +class FullRefreshPromotionTestCase: + description: str + adapter_name: str + target_exists: bool + expected_statements: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index a0a7713b2e..e2ce01e86b 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -6,6 +6,8 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -263,3 +265,36 @@ def create_table_as( del connection, destination, config, statement_recorder self.calls.append("create_table_as") self.sql = sql + + +class RecordingConnection: + def __init__(self) -> None: + self.executed: list[str] = [] + + def execute(self, sql: str) -> None: + self.executed.append(sql) + + +class RecordingSnowflakeAdapter(SnowflakeAdapter): + def _execute(self, *, connection: Any, sql: str) -> Any: + return connection.execute(sql) + + +class RecordingPostgresAdapter(PostgresAdapter): + def _execute(self, *, connection: Any, sql: str) -> Any: + return connection.execute(sql) + + +class RecordingDuckDbAdapter(DuckDbAdapter): + def _execute(self, *, connection: Any, sql: str) -> Any: + return connection.execute(sql) + + +def build_recording_adapter( + adapter_name: str, +) -> RecordingSnowflakeAdapter | RecordingPostgresAdapter | RecordingDuckDbAdapter: + return { + "snowflake": RecordingSnowflakeAdapter, + "postgres": RecordingPostgresAdapter, + "duckdb": RecordingDuckDbAdapter, + }[adapter_name]() diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py new file mode 100644 index 0000000000..bd65b3d594 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py @@ -0,0 +1,128 @@ +"""Tests for staged-relation promotion shared by full refresh and model migrations.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.executor.run._helpers.materializations.full_refresh import ( + promote_full_refresh_rebuild, + resolve_full_refresh_relations, +) +from sqlbuild.executor.run.main.promote_staged_relation import promote_staged_relation +from sqlbuild.executor.run.models import FullRefreshRelations +from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( + FullRefreshPromotionTestCase, + StagedPromotionTestCase, +) +from tests.unit.src.sqlbuild.executor.run._helpers.helpers import ( + RecordingConnection, + build_recording_adapter, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StagedPromotionTestCase( + description="missing target is promoted by renaming the stage", + adapter_name="duckdb", + target_exists=False, + expected_statements=("ALTER TABLE main.orders_stage RENAME TO orders",), + ), + StagedPromotionTestCase( + description="rename adapters move the live target aside before renaming the stage", + adapter_name="postgres", + target_exists=True, + expected_statements=( + "ALTER TABLE main.orders RENAME TO orders_previous", + "ALTER TABLE main.orders_stage RENAME TO orders", + ), + ), + StagedPromotionTestCase( + description="snowflake swaps then keeps the displaced target under the displaced name", + adapter_name="snowflake", + target_exists=True, + expected_statements=( + "ALTER TABLE main.orders SWAP WITH main.orders_stage", + "ALTER TABLE main.orders_stage RENAME TO main.orders_previous", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_staged_relation_when_promoting_then_nothing_is_dropped( + test_case: StagedPromotionTestCase, +) -> None: + """Promotion never drops the live target or the stage.""" + + adapter: BaseAdapter = build_recording_adapter(test_case.adapter_name) + connection: RecordingConnection = RecordingConnection() + + promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified="main.orders", + staged_qualified="main.orders_stage", + displaced_qualified="main.orders_previous", + target_exists=test_case.target_exists, + statement_recorder=StatementRecorder(), + ) + + assert tuple(connection.executed) == test_case.expected_statements + + +@pytest.mark.parametrize( + "test_case", + [ + FullRefreshPromotionTestCase( + description="first full refresh renames the rebuild into place", + adapter_name="duckdb", + target_exists=False, + expected_statements=("ALTER TABLE main.__sqb_rebuild__orders RENAME TO orders",), + ), + FullRefreshPromotionTestCase( + description="rename adapters drop the previous copy then rename twice", + adapter_name="duckdb", + target_exists=True, + expected_statements=( + "DROP TABLE IF EXISTS main.__sqb_prev__orders", + "ALTER TABLE main.orders RENAME TO __sqb_prev__orders", + "ALTER TABLE main.__sqb_rebuild__orders RENAME TO orders", + ), + ), + FullRefreshPromotionTestCase( + description="snowflake drops the previous copy then swaps", + adapter_name="snowflake", + target_exists=True, + expected_statements=( + "DROP TABLE IF EXISTS main.__sqb_prev__orders", + "DROP VIEW IF EXISTS main.__sqb_prev__orders", + "ALTER TABLE main.orders SWAP WITH main.__sqb_rebuild__orders", + "ALTER TABLE main.__sqb_rebuild__orders RENAME TO main.__sqb_prev__orders", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_full_refresh_rebuild_when_promoting_then_statements_are_unchanged( + test_case: FullRefreshPromotionTestCase, +) -> None: + """Full refresh keeps its drop-previous-then-promote sequence.""" + + adapter: BaseAdapter = build_recording_adapter(test_case.adapter_name) + connection: RecordingConnection = RecordingConnection() + relations: FullRefreshRelations = resolve_full_refresh_relations( + adapter=adapter, database=None, schema="main", target_name="orders" + ) + + promote_full_refresh_rebuild( + adapter=adapter, + connection=connection, + relations=relations, + target_exists=test_case.target_exists, + statement_recorder=StatementRecorder(), + ) + + assert tuple(connection.executed) == test_case.expected_statements