From 5b1119db0c5e6ba324ab1e3223d2a14b2c025c49 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:39:48 +0100 Subject: [PATCH 01/20] feat(migrations): migrate incremental and snapshot history with migrate_from Add migrate_from and migrate_force MODEL header keys for incremental and snapshot models. Direct-mode planning decides each migration per run from append-only _sqlbuild_migrations events plus live relation state (done, migrate, superseded replace, forced replace, conflict, origin missing), checks the origin against the model with the normal incremental schema rules, and plans the destination from the migrated history so it is not treated as a first run. Builds clone the origin over the destination with one adapter statement before any model runs and then record the event. --- .../adapter/contract/classes/base_adapter.py | 8 + .../contract/classes/duckdb_backed_adapter.py | 6 + .../contract/classes/strict_adapter.py | 7 + .../bigquery/classes/bigquery_adapter.py | 9 + .../databricks/classes/databricks_adapter.py | 6 + .../postgres/classes/postgres_adapter.py | 8 + .../snowflake/classes/snowflake_adapter.py | 7 + .../sqlserver/classes/sqlserver_adapter.py | 8 + .../_helpers/build_execution/execution.py | 10 +- .../_helpers/build_planning/plan_policies.py | 84 +++ .../cli/commands/main/execution/_build.py | 41 +- src/sqlbuild/cli/output/_helpers/plan_json.py | 24 + src/sqlbuild/cli/output/_helpers/plan_text.py | 53 +- .../compile/_helpers/attachment/core.py | 2 + .../_helpers/config/model_validation.py | 27 + src/sqlbuild/compiler/compile/constants.py | 2 + src/sqlbuild/compiler/migrations/__init__.py | 1 + .../compiler/migrations/_helpers/__init__.py | 1 + .../compiler/migrations/_helpers/sql.py | 146 +++++ src/sqlbuild/compiler/migrations/constants.py | 32 + .../compiler/migrations/exceptions.py | 15 + .../compiler/migrations/main/__init__.py | 1 + .../compiler/migrations/main/_newest_event.py | 25 + .../compiler/migrations/main/_read_events.py | 33 + .../migrations/main/deterministic_event_id.py | 27 + .../compiler/migrations/main/write_event.py | 60 ++ src/sqlbuild/compiler/migrations/models.py | 58 ++ src/sqlbuild/compiler/migrations/types.py | 58 ++ .../planner/_helpers/migrations/__init__.py | 0 .../_helpers/migrations/compatibility.py | 175 ++++++ .../planner/_helpers/migrations/planning.py | 573 ++++++++++++++++++ .../_helpers/planning/output_assembly.py | 4 +- .../_helpers/planning/warehouse_state.py | 20 +- .../planner/_helpers/warehouse/snapshot.py | 55 +- .../classes/migration_state_inspection.py | 205 +++++++ src/sqlbuild/compiler/planner/constants.py | 3 + .../planner/main/pre_build/__init__.py | 0 .../main/pre_build/pre_build_work_display.py | 95 +++ src/sqlbuild/compiler/planner/models.py | 65 ++ .../build/main/_apply_model_migrations.py | 124 ++++ src/sqlbuild/executor/build/main/_execute.py | 8 + src/sqlbuild/executor/build/models.py | 1 + .../virtual/executor/_helpers/build.py | 16 + .../main/model_migrations/__init__.py | 1 + .../main/model_migrations/_test_types.py | 33 + .../commands/main/model_migrations/helpers.py | 212 +++++++ .../test_manual_migrations.py | 511 ++++++++++++++++ .../unit/src/sqlbuild/adapters/_test_types.py | 9 + .../adapters/test_replace_with_clone.py | 108 ++++ .../sqlbuild/compiler/migrations/__init__.py | 1 + .../compiler/migrations/main/__init__.py | 1 + .../compiler/migrations/main/_test_types.py | 13 + .../migrations/main/test_migration_events.py | 123 ++++ 53 files changed, 3029 insertions(+), 86 deletions(-) create mode 100644 src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py create mode 100644 src/sqlbuild/compiler/migrations/__init__.py create mode 100644 src/sqlbuild/compiler/migrations/_helpers/__init__.py create mode 100644 src/sqlbuild/compiler/migrations/_helpers/sql.py create mode 100644 src/sqlbuild/compiler/migrations/constants.py create mode 100644 src/sqlbuild/compiler/migrations/exceptions.py create mode 100644 src/sqlbuild/compiler/migrations/main/__init__.py create mode 100644 src/sqlbuild/compiler/migrations/main/_newest_event.py create mode 100644 src/sqlbuild/compiler/migrations/main/_read_events.py create mode 100644 src/sqlbuild/compiler/migrations/main/deterministic_event_id.py create mode 100644 src/sqlbuild/compiler/migrations/main/write_event.py create mode 100644 src/sqlbuild/compiler/migrations/models.py create mode 100644 src/sqlbuild/compiler/migrations/types.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/planning.py create mode 100644 src/sqlbuild/compiler/planner/classes/migration_state_inspection.py create mode 100644 src/sqlbuild/compiler/planner/main/pre_build/__init__.py create mode 100644 src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py create mode 100644 src/sqlbuild/executor/build/main/_apply_model_migrations.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/__init__.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py create mode 100644 tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/main/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 174f4b76a0..2707d3209a 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -669,6 +669,14 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + def render_replace_with_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> str: + del origin, destination, origin_is_transient + raise AdapterUserError( + message=f"adapter '{self.adapter_name}' does not support model migrations" + ) + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 31f77c7009..1038dcb6ee 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1690,6 +1690,12 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + def render_replace_with_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> str: + del origin_is_transient + return f"CREATE OR REPLACE TABLE {destination} AS SELECT * FROM {origin}" + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index e76502d06d..ed1d73b57d 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -336,6 +336,13 @@ def render_durable_clone( """Render SQL statements that clone/copy into a durable independent destination.""" ... + @abstractmethod + def render_replace_with_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> str: + """Render one statement that replaces the destination with a durable source copy.""" + ... + @abstractmethod def render_query_with_cursor_bounds( self, diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index d37debc4bd..ae65cd3c58 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1853,6 +1853,15 @@ def render_durable_clone( f"CLONE {self._quote_identifier_path(origin)}", ) + def render_replace_with_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> str: + del origin_is_transient + return ( + f"CREATE OR REPLACE TABLE {self._quote_identifier_path(destination)} " + f"CLONE {self._quote_identifier_path(origin)}" + ) + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 743883bbc2..a654e2c967 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -1855,6 +1855,12 @@ def render_durable_clone( del origin_is_transient return (f"CREATE TABLE {destination} DEEP CLONE {origin}",) + def render_replace_with_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> str: + del origin_is_transient + return f"CREATE OR REPLACE TABLE {destination} DEEP CLONE {origin}" + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 6b123118f8..b26081bdbf 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -552,6 +552,14 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + def render_replace_with_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> str: + del origin, destination, origin_is_transient + raise AdapterUserError( + message=f"adapter '{self.adapter_name}' does not support model migrations" + ) + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index cbf3550519..37701b76da 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -2364,6 +2364,13 @@ def render_durable_clone( table_kind: str = "TRANSIENT TABLE" if origin_is_transient else "TABLE" return (f"CREATE OR REPLACE {table_kind} {destination} CLONE {origin}",) + def render_replace_with_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> str: + return self.render_durable_clone( + origin=origin, destination=destination, origin_is_transient=origin_is_transient + )[0] + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 4e60c013e1..4fbfbf05de 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -2108,6 +2108,14 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + def render_replace_with_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> str: + del origin, destination, origin_is_transient + raise AdapterUserError( + message=f"adapter '{self.adapter_name}' does not support model migrations" + ) + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py index 4a6b8b98b0..74857928e6 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any +from typing import Any, TextIO from sqlbuild.cli.commands._helpers.build_execution.run_context import ( write_build_run_context, @@ -166,6 +166,9 @@ def execute_build_plan( providers=providers, ), callbacks=BuildCallbacks( + on_migration_progress=lambda message: _write_progress_line( + stream=invocation.progress_stream, message=message + ), on_node_start=lambda name, resource_kind: preparation.callbacks.on_node_start( name=name, resource_kind=resource_kind ), @@ -215,3 +218,8 @@ def execute_build_plan( result=result, python_results=preparation.python_lifecycle.python_results, ) + + +def _write_progress_line(*, stream: TextIO, message: str) -> None: + stream.write(f"{message}\n") + stream.flush() diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py new file mode 100644 index 0000000000..de30fa3e8d --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -0,0 +1,84 @@ +"""Plan safety policies enforced before a direct build executes anything.""" + +from __future__ import annotations + +import sys + +from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( + enforce_model_execution_limit, + executable_model_count, +) +from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( + enforce_snapshot_full_refresh_policy, +) +from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( + enforce_retention_decrease_policy, +) +from sqlbuild.cli.commands._helpers.build_planning.table_type import ( + enforce_table_type_downgrade_policy, +) +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.cli.commands.models import BuildCommandRequest, BuildInvocation +from sqlbuild.compiler.migrations.types import MigrationDecision +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput + + +def enforce_build_plan_policies( + *, request: BuildCommandRequest, invocation: BuildInvocation, plan: PlanOutput +) -> None: + """Apply migration, execution-limit, and storage safety gates in their required order.""" + + _enforce_model_migration_policy(plan=plan) + enforce_model_execution_limit( + model_count=executable_model_count(plan=plan), + target_name=invocation.effective_target_name, + limits=invocation.execution_limits, + ) + enforce_snapshot_full_refresh_policy( + plan=plan, + snapshots_config=invocation.discovered_inputs.project_config.snapshots, + allow_snapshot_full_refresh=request.allow_snapshot_full_refresh, + input_stream=sys.stdin, + output_stream=sys.stdout, + ) + enforce_table_type_downgrade_policy( + plan=plan, + allow_table_type_downgrade=request.allow_table_type_downgrade, + input_stream=sys.stdin, + output_stream=sys.stdout, + ) + enforce_retention_decrease_policy( + plan=plan, + allow_retention_decrease=request.allow_retention_decrease, + input_stream=sys.stdin, + output_stream=sys.stdout, + ) + + +def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: + """Refuse to build while any planned migration conflicts or is incompatible.""" + + blocked: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in plan.migration_entries if entry.blocks_build + ) + if not blocked: + return + conflicts: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in blocked if entry.decision == MigrationDecision.CONFLICT + ) + names: str = ", ".join(f"'{entry.model_name}'" for entry in blocked) + if conflicts: + raise CliUserError( + f"model migration conflict for {names}: the destination already exists without a " + "recorded migration", + code="M103", + help=( + "Set migrate_force true on the model to replace the destination; the replaced " + "table remains recoverable through warehouse time travel." + ), + ) + raise CliUserError( + f"model migration source is incompatible with {names}", + code="M104", + help="Run sqb plan to see the blocking schema differences.", + ) diff --git a/src/sqlbuild/cli/commands/main/execution/_build.py b/src/sqlbuild/cli/commands/main/execution/_build.py index baa3137623..1f2458c546 100644 --- a/src/sqlbuild/cli/commands/main/execution/_build.py +++ b/src/sqlbuild/cli/commands/main/execution/_build.py @@ -25,21 +25,11 @@ from sqlbuild.cli.commands._helpers.build_planning.defer_clone import ( run_defer_clone_boundary_prephase, ) -from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( - enforce_model_execution_limit, - executable_model_count, -) -from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( - enforce_snapshot_full_refresh_policy, -) from sqlbuild.cli.commands._helpers.build_planning.invocation import resolve_build_invocation -from sqlbuild.cli.commands._helpers.build_planning.planning import compile_build_plan -from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( - enforce_retention_decrease_policy, -) -from sqlbuild.cli.commands._helpers.build_planning.table_type import ( - enforce_table_type_downgrade_policy, +from sqlbuild.cli.commands._helpers.build_planning.plan_policies import ( + enforce_build_plan_policies, ) +from sqlbuild.cli.commands._helpers.build_planning.planning import compile_build_plan from sqlbuild.cli.commands._helpers.cost.collection import ( finalize_build_cost, ) @@ -163,29 +153,10 @@ def _run_build( invocation=invocation, pipeline_result=pipeline_result, ) - enforce_model_execution_limit( - model_count=executable_model_count(plan=pipeline_result.plan_output), - target_name=invocation.effective_target_name, - limits=invocation.execution_limits, - ) - enforce_snapshot_full_refresh_policy( - plan=pipeline_result.plan_output, - snapshots_config=invocation.discovered_inputs.project_config.snapshots, - allow_snapshot_full_refresh=request.allow_snapshot_full_refresh, - input_stream=sys.stdin, - output_stream=sys.stdout, - ) - enforce_table_type_downgrade_policy( - plan=pipeline_result.plan_output, - allow_table_type_downgrade=request.allow_table_type_downgrade, - input_stream=sys.stdin, - output_stream=sys.stdout, - ) - enforce_retention_decrease_policy( + enforce_build_plan_policies( + request=request, + invocation=invocation, plan=pipeline_result.plan_output, - allow_retention_decrease=request.allow_retention_decrease, - input_stream=sys.stdin, - output_stream=sys.stdout, ) write_build_compile_target( request=request, diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 23381f91b7..89ed08db32 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -98,6 +98,7 @@ def format_plan_json( "warnings": warnings, "retention": retention, "table_type_conversions": table_type_conversions, + "migrations": _serialize_model_migrations(plan), } if plan.metadata: result["metadata"] = plan.metadata @@ -469,3 +470,26 @@ def _serialize_provider_usage(usage: PlanProviderUsage) -> dict[str, object]: "module": usage.annotation_module, } return payload + + +def _serialize_model_migrations(plan: PlanOutput) -> list[dict[str, object]]: + return [ + { + "kind": "model_migration", + "model": entry.model_name, + "discovery": entry.discovery.value, + "decision": entry.decision.value, + "compatibility": entry.compatibility.value, + "compatibility_findings": list(entry.compatibility_findings), + "origin_model": entry.origin_model, + "origin": entry.origin.qualified_name or entry.origin.name, + "destination": entry.destination.qualified_name or entry.destination.name, + "target": entry.target_name, + "origin_version_hash": entry.origin_version_hash, + "statement": entry.statement, + "completed_at": ( + entry.completed_at.isoformat() if entry.completed_at is not None else None + ), + } + for entry in plan.migration_entries + ] diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index 4f80eaef92..4bb16a23fd 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -22,6 +22,9 @@ from sqlbuild.compiler.planner.main.execution.model_materialization_label import ( model_materialization_label, ) +from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import ( + format_pre_build_work, +) from sqlbuild.compiler.planner.models import ( CascadeResult, CursorBounds, @@ -90,50 +93,6 @@ } -def _format_retention( - *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions -) -> list[str]: - if plan.table_type_entries: - lines.append("Table type conversions") - for table_type_entry in visible_entries( - entries=plan.table_type_entries, options=display_options - ): - actual_type: str = table_type_entry.actual_type or "unknown" - lines.append( - f" {table_type_entry.model_name} desired={table_type_entry.desired_type} " - f"actual={actual_type} source={table_type_entry.source}" - ) - if table_type_entry.irreversible_warning is not None: - lines.append(f" WARNING: {table_type_entry.irreversible_warning}") - lines = append_overflow_line( - lines=lines, - total_count=len(plan.table_type_entries), - visible_count=len( - visible_entries(entries=plan.table_type_entries, options=display_options) - ), - indent=" ", - options=display_options, - ) - if plan.retention_entries: - lines.append("Retention") - for entry in plan.retention_entries: - scope: str = ".".join( - part - for part in (entry.request.database, entry.request.schema, entry.request.name) - if part - ) - actual: str = "missing" if entry.actual_days is None else f"{entry.actual_days}d" - effective: str = "missing" if entry.effective_days is None else f"{entry.effective_days}d" - lines.append( - f" {scope} desired={entry.request.desired_days}d actual={actual} " - f"effective={effective} source={entry.source} direction={entry.direction.value} " - f"phase={entry.phase.value}" - ) - if entry.irreversible_warning is not None: - lines.append(f" WARNING: {entry.irreversible_warning}") - return lines - - def format_plan( *, plan: PlanOutput, @@ -174,7 +133,9 @@ def format_plan( include_direct_freshness_diagnostics=(include_direct_freshness_diagnostics), display_options=resolved_display_options, ) - lines = _format_retention(lines=lines, plan=plan, display_options=resolved_display_options) + lines = format_pre_build_work( + lines=lines, plan=plan, display_options=resolved_display_options + ) result: str = "\n".join(lines) return result if use_color else _strip_ansi(result) @@ -225,7 +186,7 @@ def format_plan( display_options=resolved_display_options, skipped_header_style=style.muted, ) - lines = _format_retention(lines=lines, plan=plan, display_options=resolved_display_options) + lines = format_pre_build_work(lines=lines, plan=plan, display_options=resolved_display_options) lines = _format_python_plan_entries( lines=lines, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 30b56e3067..189916b29d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -38,6 +38,7 @@ validate_custom_materialization_config, validate_incremental_config, validate_microbatch_project_capability, + validate_model_migration_config, validate_non_incremental_config, validate_placeholder_config, validate_snapshot_config, @@ -757,6 +758,7 @@ def _validate_model_input( custom_materialization_names=context.custom_materialization_names, ) validate_storage_policies(config=config, model_name=model_name) + validate_model_migration_config(config=config, model_name=model_name) validate_placeholder_config( config=config, model_name=model_name, diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index d8a99917d1..6ef14cc18f 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -10,6 +10,8 @@ from sqlbuild.compiler.compile.constants import ( MICROBATCH_LIMIT_ACTION_KEY, MICROBATCH_LIMIT_MAX_BATCHES_KEY, + MIGRATE_FORCE_CONFIG_KEY, + MIGRATE_FROM_CONFIG_KEY, SQL_WILDCARD_TOKEN, WATERMARK_CURSOR_INPUT_BLOCK_KEYS, ) @@ -949,6 +951,31 @@ def validate_non_incremental_config( ) +def validate_model_migration_config(*, config: CompileModelConfig, model_name: str) -> None: + """Validate migrate_from and migrate_force on history-bearing materializations.""" + + migrate_from: object | None = config.values.get(MIGRATE_FROM_CONFIG_KEY) + migrate_force: object | None = config.values.get(MIGRATE_FORCE_CONFIG_KEY) + if migrate_from is None and migrate_force is None: + return + materialized: str | None = get_config_str(values=config.values, key="materialized") + if materialized not in {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT}: + raise CompileInputError( + f"model '{model_name}': migrate_from is only valid for incremental and snapshot " + f"models; '{materialized}' models are rebuilt under their new name" + ) + if migrate_from is None: + raise CompileInputError(f"model '{model_name}': migrate_force requires migrate_from") + if not isinstance(migrate_from, str) or not migrate_from.strip(): + raise CompileInputError( + f"model '{model_name}': migrate_from must be a model name or a qualified relation" + ) + if migrate_from.strip().lower() == model_name.lower(): + raise CompileInputError(f"model '{model_name}': migrate_from cannot name the model itself") + if migrate_force is not None and not isinstance(migrate_force, bool): + raise CompileInputError(f"model '{model_name}': migrate_force must be true or false") + + def validate_snapshot_config( *, config: CompileModelConfig, diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 494ef088f9..8eb5364e16 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -17,6 +17,8 @@ MODEL_AUDIT_OVERRIDE_KEYS: frozenset[str] = frozenset({"by_type", "by_column"}) MODEL_FULL_REFRESH_CONFIG_KEY: str = "full_refresh" CURSOR_INPUTS_CONFIG_KEY: str = "cursor_inputs" +MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" +MIGRATE_FORCE_CONFIG_KEY: str = "migrate_force" CURSOR_FILTER_INPUTS_CONFIG_KEY: str = "cursor_filter_inputs" CURSOR_WATERMARK_INPUTS_CONFIG_KEY: str = "cursor_watermark_inputs" MAX_MICROBATCHES_CONFIG_KEY: str = "max_microbatches" diff --git a/src/sqlbuild/compiler/migrations/__init__.py b/src/sqlbuild/compiler/migrations/__init__.py new file mode 100644 index 0000000000..eaf52e580e --- /dev/null +++ b/src/sqlbuild/compiler/migrations/__init__.py @@ -0,0 +1 @@ +"""Direct-mode model migration state package.""" diff --git a/src/sqlbuild/compiler/migrations/_helpers/__init__.py b/src/sqlbuild/compiler/migrations/_helpers/__init__.py new file mode 100644 index 0000000000..114aa7085f --- /dev/null +++ b/src/sqlbuild/compiler/migrations/_helpers/__init__.py @@ -0,0 +1 @@ +"""Model migration state helper package.""" diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py new file mode 100644 index 0000000000..8b1b9e9e39 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/_helpers/sql.py @@ -0,0 +1,146 @@ +"""Portable SQL for append-only model migration events.""" + +from __future__ import annotations + +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.compiler.migrations.constants import ( + MIGRATION_COLUMN_TYPES, + MIGRATION_COLUMNS, + MIGRATION_TABLE_NAME, +) +from sqlbuild.compiler.migrations.exceptions import MigrationStateError +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal +from sqlbuild.sql_values.types import StateSqlValueType + + +def qualified_migration_table( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: + table: str | None = render_qualified_name( + database=database, schema=schema, name=MIGRATION_TABLE_NAME + ) + if table is None: + raise MigrationStateError("model migration state requires a target schema") + return table + + +def build_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool, +) -> str: + table: str = qualified_migration_table( + database=database, schema=schema, render_qualified_name=render_qualified_name + ) + string_type: str = render_framework_type(FrameworkType.STRING) + timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) + definitions: str = ", ".join( + f"{column} " + + ( + timestamp_type + if MIGRATION_COLUMN_TYPES[column] == StateSqlValueType.TIMESTAMP + else string_type + ) + for column in MIGRATION_COLUMNS + ) + table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" + return f"CREATE {table_kind} IF NOT EXISTS {table} ({definitions})" + + +def build_insert_sql( + *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] +) -> str: + if event.destination.schema is None: + raise MigrationStateError("model migration events require a destination schema") + table: str = qualified_migration_table( + database=event.destination.database, + schema=event.destination.schema, + render_qualified_name=render_qualified_name, + ) + values: tuple[object | None, ...] = _event_values(event) + literals: str = ", ".join( + render_state_sql_literal(value=value, declared_type=MIGRATION_COLUMN_TYPES[column]) + for column, value in zip(MIGRATION_COLUMNS, values, strict=True) + ) + event_id_literal: str = render_state_sql_literal( + value=event.event_id, declared_type=StateSqlValueType.STRING + ) + return ( + f"INSERT INTO {table} ({', '.join(MIGRATION_COLUMNS)}) SELECT {literals} " + f"WHERE NOT EXISTS (SELECT 1 FROM {table} WHERE event_id = {event_id_literal})" + ) + + +def build_read_sql( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: + table: str = qualified_migration_table( + database=database, schema=schema, render_qualified_name=render_qualified_name + ) + return f"SELECT {', '.join(MIGRATION_COLUMNS)} FROM {table} ORDER BY created_at, event_id" + + +def decode_event_row(row: tuple[Any, ...]) -> MigrationEvent: + values: dict[str, Any] = dict(zip(MIGRATION_COLUMNS, row, strict=False)) + raw_created_at: Any = values["created_at"] + created_at: datetime = ( + raw_created_at + if isinstance(raw_created_at, datetime) + else datetime.fromisoformat(str(raw_created_at)) + ) + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=UTC) + return MigrationEvent( + event_id=str(values["event_id"]), + target_name=_optional_text(values["target_name"]), + origin_model=_optional_text(values["origin_model"]), + origin=MigrationRelation( + database=_optional_text(values["origin_database"]), + schema=_optional_text(values["origin_schema"]), + name=str(values["origin_name"]), + ), + destination_model=str(values["destination_model"]), + destination=MigrationRelation( + database=_optional_text(values["destination_database"]), + schema=_optional_text(values["destination_schema"]), + name=str(values["destination_name"]), + ), + origin_version_hash=str(values["origin_version_hash"] or ""), + discovery=MigrationDiscovery(str(values["discovery"])), + decision=MigrationDecision(str(values["decision"])), + run_id=str(values["run_id"]), + created_at=created_at, + ) + + +def _event_values(event: MigrationEvent) -> tuple[object | None, ...]: + return ( + event.event_id, + event.target_name, + event.origin_model, + event.origin.database, + event.origin.schema, + event.origin.name, + event.destination_model, + event.destination.database, + event.destination.schema, + event.destination.name, + event.origin_version_hash, + event.discovery.value, + event.decision.value, + event.run_id, + event.created_at, + ) + + +def _optional_text(value: object | None) -> str | None: + return None if value is None else str(value) diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py new file mode 100644 index 0000000000..5d6efbbfc9 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/constants.py @@ -0,0 +1,32 @@ +"""Stable constants for append-only model migration state.""" + +from __future__ import annotations + +from sqlbuild.sql_values.types import StateSqlValueType + +MIGRATION_TABLE_NAME: str = "_sqlbuild_migrations" +MIGRATION_WRITE_ATTEMPTS: int = 3 +MIGRATION_WRITE_RETRY_BASE_SECONDS: float = 0.05 + +MIGRATION_COLUMNS: tuple[str, ...] = ( + "event_id", + "target_name", + "origin_model", + "origin_database", + "origin_schema", + "origin_name", + "destination_model", + "destination_database", + "destination_schema", + "destination_name", + "origin_version_hash", + "discovery", + "decision", + "run_id", + "created_at", +) + +MIGRATION_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in MIGRATION_COLUMNS}, + "created_at": StateSqlValueType.TIMESTAMP, +} diff --git a/src/sqlbuild/compiler/migrations/exceptions.py b/src/sqlbuild/compiler/migrations/exceptions.py new file mode 100644 index 0000000000..faca42ec60 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/exceptions.py @@ -0,0 +1,15 @@ +"""Expected model migration state exception types.""" + +from __future__ import annotations + + +class MigrationStateError(ValueError): + """Raised when model migration state inputs or rows are invalid.""" + + code: str = "M000" + + def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: + super().__init__(message) + self.message = message + self.code = code if code is not None else self.code + self.help = help diff --git a/src/sqlbuild/compiler/migrations/main/__init__.py b/src/sqlbuild/compiler/migrations/main/__init__.py new file mode 100644 index 0000000000..c0e9893620 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/__init__.py @@ -0,0 +1 @@ +"""Model migration state entry modules.""" diff --git a/src/sqlbuild/compiler/migrations/main/_newest_event.py b/src/sqlbuild/compiler/migrations/main/_newest_event.py new file mode 100644 index 0000000000..888f0d2a9e --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/_newest_event.py @@ -0,0 +1,25 @@ +"""Project the newest migration event that mentions one relation.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation + + +def newest_migration_event_mentioning( + *, events: Iterable[MigrationEvent], relation: MigrationRelation +) -> MigrationEvent | None: + """Return the latest event naming the relation as origin or destination.""" + + newest: MigrationEvent | None = None + event: MigrationEvent + for event in events: + if not event.mentions(relation): + continue + if newest is None or (event.created_at, event.event_id) > ( + newest.created_at, + newest.event_id, + ): + newest = event + return newest diff --git a/src/sqlbuild/compiler/migrations/main/_read_events.py b/src/sqlbuild/compiler/migrations/main/_read_events.py new file mode 100644 index 0000000000..af847b5aff --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/_read_events.py @@ -0,0 +1,33 @@ +"""Read append-only model migration events from one schema.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.compiler.migrations._helpers.sql import build_read_sql, decode_event_row +from sqlbuild.compiler.migrations.exceptions import MigrationStateError +from sqlbuild.compiler.migrations.models import MigrationEvent + + +def read_migration_events( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], +) -> tuple[MigrationEvent, ...]: + """Read every migration event recorded in one schema's state table.""" + + sql: str = build_read_sql( + database=database, schema=schema, render_qualified_name=render_qualified_name + ) + try: + rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() + except Exception as error: + raise MigrationStateError( + f"Unable to read model migration events from schema '{schema}': {error}" + ) from error + return tuple(decode_event_row(tuple(row)) for row in rows) diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py new file mode 100644 index 0000000000..7c9ecf2f68 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/deterministic_event_id.py @@ -0,0 +1,27 @@ +"""Build stable identifiers for immutable model migration events.""" + +from __future__ import annotations + +import hashlib +import json + +from sqlbuild.compiler.migrations.models import MigrationRelation + + +def deterministic_migration_event_id( + *, + run_id: str, + target_name: str | None, + origin: MigrationRelation, + destination: MigrationRelation, +) -> str: + """Return the content-addressed ID for one migration performed by one run.""" + + identity: tuple[object, ...] = ( + run_id, + target_name, + origin.identity, + destination.identity, + ) + encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/migrations/main/write_event.py b/src/sqlbuild/compiler/migrations/main/write_event.py new file mode 100644 index 0000000000..4597617f8f --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/write_event.py @@ -0,0 +1,60 @@ +"""Append one model migration event idempotently.""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute, FrameworkType +from sqlbuild.compiler.migrations._helpers.sql import build_create_table_sql, build_insert_sql +from sqlbuild.compiler.migrations.constants import ( + MIGRATION_WRITE_ATTEMPTS, + MIGRATION_WRITE_RETRY_BASE_SECONDS, +) +from sqlbuild.compiler.migrations.exceptions import MigrationStateError +from sqlbuild.compiler.migrations.models import MigrationEvent + + +def write_migration_event( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + event: MigrationEvent, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool, +) -> None: + """Create the state table when missing and insert the event unless already present.""" + + if event.destination.schema is None: + raise MigrationStateError("model migration events require a destination schema") + statements: tuple[str, ...] = ( + build_create_table_sql( + database=event.destination.database, + schema=event.destination.schema, + render_qualified_name=render_qualified_name, + render_framework_type=render_framework_type, + transient=transient, + ), + build_insert_sql(event=event, render_qualified_name=render_qualified_name), + ) + attempt: int + for attempt in range(MIGRATION_WRITE_ATTEMPTS): + try: + statement: str + for statement in statements: + _ = execute(connection=connection, sql=statement) + return + except Exception as error: + if attempt + 1 == MIGRATION_WRITE_ATTEMPTS: + raise + logging.getLogger("sqlbuild.migrations").warning( + "model migration event write attempt %s/%s failed for '%s'; retrying: %s", + attempt + 1, + MIGRATION_WRITE_ATTEMPTS, + event.destination_model, + error, + ) + time.sleep(MIGRATION_WRITE_RETRY_BASE_SECONDS * (attempt + 1)) diff --git a/src/sqlbuild/compiler/migrations/models.py b/src/sqlbuild/compiler/migrations/models.py new file mode 100644 index 0000000000..cc83d5a4e3 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/models.py @@ -0,0 +1,58 @@ +"""Model migration state domain models.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery + + +@dataclass(frozen=True) +class MigrationRelation: + """One physical relation named by a migration event.""" + + database: str | None + schema: str | None + name: str + + @property + def identity(self) -> tuple[str | None, str, str]: + """Return the case-insensitive relation identity.""" + + return ( + None if self.database is None else self.database.lower(), + "" if self.schema is None else self.schema.lower(), + self.name.lower(), + ) + + def matches(self, other: MigrationRelation) -> bool: + """Compare relations, treating an unknown database as a wildcard.""" + + left: tuple[str | None, str, str] = self.identity + right: tuple[str | None, str, str] = other.identity + if left[1:] != right[1:]: + return False + return left[0] is None or right[0] is None or left[0] == right[0] + + +@dataclass(frozen=True) +class MigrationEvent: + """One immutable fact that a origin relation was cloned over a destination.""" + + event_id: str + target_name: str | None + origin_model: str | None + origin: MigrationRelation + destination_model: str + destination: MigrationRelation + origin_version_hash: str + discovery: MigrationDiscovery + decision: MigrationDecision + run_id: str + created_at: datetime + + def mentions(self, relation: MigrationRelation) -> bool: + """Return whether this event names the relation as origin or destination.""" + + return self.origin.matches(relation) or self.destination.matches(relation) diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py new file mode 100644 index 0000000000..6fad62c1c1 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/types.py @@ -0,0 +1,58 @@ +"""Stable model migration value types.""" + +from __future__ import annotations + +from enum import StrEnum + + +class MigrationDecision(StrEnum): + """Per-run decision for one declared or discovered model migration.""" + + DONE = "done" + MIGRATE = "migrate" + SUPERSEDED_REPLACE = "superseded_replace" + FORCED_REPLACE = "forced_replace" + CONFLICT = "conflict" + ORIGIN_MISSING = "origin_missing" + + @property + def moves_data(self) -> bool: + """Return whether this decision clones the origin over the destination.""" + + return self in _MOVING_DECISIONS + + @property + def checks_compatibility(self) -> bool: + """Return whether this decision evaluated the origin against the destination.""" + + return self.moves_data or self == MigrationDecision.CONFLICT + + @property + def label(self) -> str: + """Return the human-readable plan label for this decision.""" + + return self.value.replace("_", " ") + + +_MOVING_DECISIONS: frozenset[MigrationDecision] = frozenset( + { + MigrationDecision.MIGRATE, + MigrationDecision.SUPERSEDED_REPLACE, + MigrationDecision.FORCED_REPLACE, + } +) + + +class MigrationDiscovery(StrEnum): + """How a migration was requested.""" + + MANUAL = "manual" + AUTOMATIC = "automatic" + + +class MigrationCompatibility(StrEnum): + """Outcome of checking the origin relation against the destination model.""" + + COMPATIBLE = "compatible" + INCOMPATIBLE = "incompatible" + NOT_CHECKED = "not_checked" diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py b/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py new file mode 100644 index 0000000000..ccf1deae87 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py @@ -0,0 +1,175 @@ +"""Compatibility of a migration origin relation with its destination model.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import ColumnInfo, NormalizedType, RelationInfo +from sqlbuild.adapter.contract.types import TypeFamily +from sqlbuild.adapter.type_system.main.normalize_type import normalize_type +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.migrations.types import MigrationCompatibility +from sqlbuild.compiler.planner._helpers.changes.detect import detect_model_changes +from sqlbuild.compiler.planner.constants import VIEW_RELATION_TYPE_MARKER +from sqlbuild.compiler.planner.models import ( + ChangeDetectionResult, + MigrationCompatibilityResult, + SchemaFinding, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import ( + MaterializationType, + OnSchemaChange, + SchemaChangeKind, + SnapshotSchemaChangePolicy, +) +from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.models import SnapshotsConfig + +_DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS +_NUMERIC_FAMILIES: frozenset[TypeFamily] = frozenset( + {TypeFamily.INTEGER, TypeFamily.DECIMAL, TypeFamily.FLOAT} +) +_TEMPORAL_FAMILIES: frozenset[TypeFamily] = frozenset( + {TypeFamily.TIMESTAMP, TypeFamily.DATE, TypeFamily.DATETIME} +) + + +def check_migration_compatibility( + *, + model: CompiledModel, + origin_relation: RelationInfo, + origin_columns: tuple[ColumnInfo, ...], + sql_analysis_enabled: bool, + column_dialect: str | None, + snapshots_config: SnapshotsConfig, +) -> MigrationCompatibilityResult: + """Check the origin exactly as a normal build would check an existing destination.""" + + if VIEW_RELATION_TYPE_MARKER in origin_relation.relation_type.upper(): + return MigrationCompatibilityResult( + status=MigrationCompatibility.INCOMPATIBLE, + findings=("origin relation is a view; only tables carry migratable history",), + ) + change: ChangeDetectionResult = detect_model_changes( + model=model, + snapshot=WarehouseSnapshot( + existing_relations={model.name: origin_relation}, + existing_columns={model.name: origin_columns}, + column_dialect=column_dialect, + ), + sql_analysis_enabled=sql_analysis_enabled, + query_change_tracking=False, + full_refresh=False, + ) + if not _has_expected_columns(model=model): + return MigrationCompatibilityResult(status=MigrationCompatibility.NOT_CHECKED) + materialized: str | None = get_config_str(values=model.config.values, key="materialized") + blocking: tuple[str, ...] = ( + _snapshot_blocking_findings( + model=model, + findings=change.schema_findings, + dialect=column_dialect, + snapshots_config=snapshots_config, + ) + if materialized == MaterializationType.SNAPSHOT + else _incremental_blocking_findings( + model=model, findings=change.schema_findings, dialect=column_dialect + ) + ) + if blocking: + return MigrationCompatibilityResult( + status=MigrationCompatibility.INCOMPATIBLE, findings=blocking + ) + return MigrationCompatibilityResult(status=MigrationCompatibility.COMPATIBLE) + + +def _has_expected_columns(*, model: CompiledModel) -> bool: + if model.schema_entry is not None and any( + column.type is not None for column in model.schema_entry.columns + ): + return True + return bool(model.inferred_columns) + + +def _incremental_blocking_findings( + *, model: CompiledModel, findings: tuple[SchemaFinding, ...], dialect: str | None +) -> tuple[str, ...]: + raw_policy: str | None = get_config_str(values=model.config.values, key="on_schema_change") + policy: OnSchemaChange = OnSchemaChange(raw_policy) if raw_policy else _DEFAULT_ON_SCHEMA_CHANGE + blocking: list[str] = [] + finding: SchemaFinding + for finding in findings: + if policy == OnSchemaChange.FAIL: + blocking.append(f"{_describe(finding)} (on_schema_change fail)") + continue + if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED and not _types_convertible( + expected=finding.expected_type, actual=finding.actual_type, dialect=dialect + ): + blocking.append(f"{_describe(finding)} (incompatible types)") + return tuple(blocking) + + +def _snapshot_blocking_findings( + *, + model: CompiledModel, + findings: tuple[SchemaFinding, ...], + dialect: str | None, + snapshots_config: SnapshotsConfig, +) -> tuple[str, ...]: + raw_policy: str | None = get_config_str( + values=model.config.values, key="snapshot_schema_change" + ) + policy: SnapshotSchemaChangePolicy = SnapshotSchemaChangePolicy( + raw_policy or snapshots_config.schema_change + ) + blocking: list[str] = [] + finding: SchemaFinding + for finding in findings: + if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED and not _string_widening( + expected=finding.expected_type, actual=finding.actual_type, dialect=dialect + ): + blocking.append(f"{_describe(finding)} (snapshot type change)") + elif ( + finding.kind == SchemaChangeKind.COLUMN_ADDED + and policy == SnapshotSchemaChangePolicy.DENY + ): + blocking.append(f"{_describe(finding)} (snapshot_schema_change deny)") + return tuple(blocking) + + +def _types_convertible(*, expected: str | None, actual: str | None, dialect: str | None) -> bool: + if expected is None or actual is None: + return True + expected_family: TypeFamily = normalize_type(type_sql=expected, dialect=dialect).family + actual_family: TypeFamily = normalize_type(type_sql=actual, dialect=dialect).family + if TypeFamily.OTHER in {expected_family, actual_family}: + return True + if expected_family == actual_family: + return True + if {expected_family, actual_family} <= _NUMERIC_FAMILIES: + return True + return {expected_family, actual_family} <= _TEMPORAL_FAMILIES + + +def _string_widening(*, expected: str | None, actual: str | None, dialect: str | None) -> bool: + if expected is None or actual is None: + return True + target: NormalizedType = normalize_type(type_sql=actual, dialect=dialect) + incoming: NormalizedType = normalize_type(type_sql=expected, dialect=dialect) + if target == incoming: + return True + if target.family != TypeFamily.STRING or incoming.family != TypeFamily.STRING: + return False + if target.length is None: + return True + return incoming.length is not None and incoming.length <= target.length + + +def _describe(finding: SchemaFinding) -> str: + if finding.kind == SchemaChangeKind.COLUMN_ADDED: + return f"column {finding.column_name} is missing from the origin" + if finding.kind == SchemaChangeKind.COLUMN_REMOVED: + return f"origin column {finding.column_name} is not produced by the model" + return ( + f"column {finding.column_name} is {finding.actual_type} in the origin but the model " + f"produces {finding.expected_type}" + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py new file mode 100644 index 0000000000..de3a75bfa1 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -0,0 +1,573 @@ +"""Per-run decisions for direct-mode model migrations.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from datetime import UTC, datetime + +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY +from sqlbuild.compiler.compile.models import CompiledModel, CompiledRelationLocation +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import ( + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, +) +from sqlbuild.compiler.planner._helpers.migrations.compatibility import ( + MigrationCompatibilityResult, + check_migration_compatibility, +) +from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( + effectively_full_refreshed_model_names, +) +from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( + gather_redirected_cursor_snapshots, +) +from sqlbuild.compiler.planner.classes.migration_state_inspection import ( + MigrationStateInspection, +) +from sqlbuild.compiler.planner.constants import ( + MIGRATION_MODEL_NAME_METADATA_KEY, + QUALIFIED_RELATION_MAX_PARTS, +) +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.planner.models import ( + CursorSnapshotScope, + DeferralInputs, + ModelCursorSnapshot, + ModelMigrationPlanEntry, + ModelMigrationPlanning, + ModelMigrationRequest, + PlannerOverrides, + PlannerRuntime, + PlannerScope, + PlanWarning, + WarehouseFingerprints, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity +from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.models import SnapshotsConfig + + +def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: + """Collect selected models that declare migrate_from.""" + + requests: list[ModelMigrationRequest] = [] + key: object + for key in scope.execution_order: + if key not in scope.selected_keys or getattr(key, "resource_type", None) != ( + CompiledResourceType.MODEL + ): + continue + model: CompiledModel | None = scope.models_by_name.get(getattr(key, "name", "")) + if model is None: + continue + raw_origin: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) + if not isinstance(raw_origin, str): + continue + requests.append( + ModelMigrationRequest( + model=model, + discovery=MigrationDiscovery.MANUAL, + raw_origin=raw_origin.strip(), + force=model.config.values.get(MIGRATE_FORCE_CONFIG_KEY) is True, + ) + ) + return tuple(requests) + + +def plan_model_migrations( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + requests: tuple[ModelMigrationRequest, ...], + overrides: PlannerOverrides, + deferral: DeferralInputs, + inspection: MigrationStateInspection | None = None, +) -> ModelMigrationPlanning: + """Decide every requested migration and project the post-migration snapshot.""" + + if not requests: + return ModelMigrationPlanning(snapshot=snapshot) + state: MigrationStateInspection = inspection or MigrationStateInspection( + adapter=runtime.adapter, + connection=runtime.connection, + database=planning_database(runtime=runtime), + ) + state.inspect_schemas(schemas=project_schemas(runtime=runtime)) + resolved: list[tuple[ModelMigrationRequest, CompiledRelationLocation, str | None]] = [ + (request, *_resolve_origin(request=request, runtime=runtime, state=state)) + for request in requests + ] + origin: CompiledRelationLocation + for _, origin, _ in resolved: + _require_same_database(state=state, location=origin) + state.inspect_schemas( + schemas={location.schema for _, location, _ in resolved if location.schema is not None} + ) + state.inspect_relations(locations=tuple(location for _, location, _ in resolved)) + snapshots_config: SnapshotsConfig = ( + runtime.project_config.snapshots + if runtime.project_config is not None + else SnapshotsConfig() + ) + entries: list[ModelMigrationPlanEntry] = [] + handovers: dict[str, Fingerprint | None] = {} + request: ModelMigrationRequest + origin: CompiledRelationLocation + origin_model: str | None + for request, origin, origin_model in resolved: + entry: ModelMigrationPlanEntry + handover: Fingerprint | None + applies: bool + entry, handover, applies = _decide( + runtime=runtime, + request=request, + origin=origin, + origin_model=origin_model, + snapshot=snapshot, + state=state, + snapshots_config=snapshots_config, + ) + entries.append(entry) + if applies: + handovers[request.model.name] = handover + return ModelMigrationPlanning( + snapshot=_overlay_snapshot( + runtime=runtime, + scope=scope, + snapshot=snapshot, + entries=tuple(entries), + handovers=handovers, + state=state, + overrides=overrides, + deferral=deferral, + ), + entries=tuple(entries), + warnings=tuple( + warning for entry in entries if (warning := _entry_warning(entry)) is not None + ), + ) + + +def planning_database(*, runtime: PlannerRuntime) -> str | None: + """Return the single database that direct planning inspects for migration state.""" + + return next( + ( + model.destination.database + for model in runtime.project.models + if model.destination.database is not None + ), + runtime.project.effective_target_database, + ) + + +def project_schemas(*, runtime: PlannerRuntime) -> set[str]: + """Return every destination schema owned by project models.""" + + return { + model.destination.schema + for model in runtime.project.models + if model.destination.schema is not None + } + + +def _require_same_database( + *, state: MigrationStateInspection, location: CompiledRelationLocation +) -> None: + if ( + location.database is None + or state.database is None + or location.database.lower() == state.database.lower() + ): + return + raise PlannerInputError( + f"migration origin {location.qualified_name or location.name} is in database " + f"'{location.database}', but direct planning inspects '{state.database}'; " + "cross-database migrations are not supported" + ) + + +def _relation_key(location: CompiledRelationLocation) -> tuple[str, str]: + return ((location.schema or "").lower(), location.name.lower()) + + +def _migration_relation(location: CompiledRelationLocation) -> MigrationRelation: + return MigrationRelation(database=location.database, schema=location.schema, name=location.name) + + +def _resolve_origin( + *, request: ModelMigrationRequest, runtime: PlannerRuntime, state: MigrationStateInspection +) -> tuple[CompiledRelationLocation, str | None]: + if request.origin_location is not None: + return request.origin_location, request.origin_model + raw: str = request.raw_origin or "" + destination: CompiledRelationLocation = request.model.destination + parts: list[str] = [part.strip().strip('"`') for part in raw.split(".")] + if any(not part for part in parts) or len(parts) > QUALIFIED_RELATION_MAX_PARTS: + raise PlannerInputError( + f"model '{request.model.name}': migrate_from '{raw}' is not a model name or a " + "qualified relation" + ) + if len(parts) == 1: + return _resolve_named_origin( + name=parts[0], destination=destination, runtime=runtime, state=state + ) + database: str | None = ( + parts[0] if len(parts) == QUALIFIED_RELATION_MAX_PARTS else destination.database + ) + schema: str = parts[-2] + return ( + _location(runtime=runtime, database=database, schema=schema, name=parts[-1]), + None, + ) + + +def _resolve_named_origin( + *, + name: str, + destination: CompiledRelationLocation, + runtime: PlannerRuntime, + state: MigrationStateInspection, +) -> tuple[CompiledRelationLocation, str | None]: + project_model: CompiledModel | None = next( + (model for model in runtime.project.models if model.name == name), None + ) + if project_model is not None: + return project_model.destination, name + fingerprint: Fingerprint | None = state.named_fingerprint(model_name=name) + if fingerprint is not None and fingerprint.target_name is not None: + return ( + _location( + runtime=runtime, + database=fingerprint.target_database, + schema=fingerprint.target_schema, + name=fingerprint.target_name, + ), + name, + ) + return ( + _location( + runtime=runtime, database=destination.database, schema=destination.schema, name=name + ), + name, + ) + + +def _location( + *, runtime: PlannerRuntime, database: str | None, schema: str | None, name: str +) -> CompiledRelationLocation: + return CompiledRelationLocation( + database=database, + schema=schema, + name=name, + qualified_name=runtime.adapter.render_qualified_name( + database=database, schema=schema, name=name + ), + ) + + +def _decide( + *, + runtime: PlannerRuntime, + request: ModelMigrationRequest, + origin: CompiledRelationLocation, + origin_model: str | None, + snapshot: WarehouseSnapshot, + state: MigrationStateInspection, + snapshots_config: SnapshotsConfig, +) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: + model: CompiledModel = request.model + destination: CompiledRelationLocation = model.destination + if _relation_key(origin) == _relation_key(destination): + raise PlannerInputError( + f"model '{model.name}': migrate_from names the model's own relation" + ) + newest: MigrationEvent | None = newest_migration_event_mentioning( + events=state.events, relation=_migration_relation(destination) + ) + origin_relation: RelationInfo | None = state.relation(origin) + origin_fingerprint: Fingerprint | None = state.model_fingerprint( + location=origin, model_name=origin_model + ) + base: ModelMigrationPlanEntry = ModelMigrationPlanEntry( + model_name=model.name, + discovery=request.discovery, + decision=MigrationDecision.MIGRATE, + compatibility=MigrationCompatibility.NOT_CHECKED, + origin_model=origin_model, + origin=origin, + destination=destination, + target_name=runtime.project.effective_target_name, + origin_version_hash=( + origin_fingerprint.version_hash if origin_fingerprint is not None else "" + ), + origin_is_transient=bool(origin_relation is not None and origin_relation.is_transient), + ) + if ( + newest is not None + and newest.destination.matches(_migration_relation(destination)) + and newest.origin.matches(_migration_relation(origin)) + ): + return ( + replace( + base, + decision=MigrationDecision.DONE, + completed_at=newest.created_at, + origin_version_hash=newest.origin_version_hash, + target_name=newest.target_name, + ), + _done_handover( + model=model, + event=newest, + snapshot=snapshot, + origin_fingerprint=origin_fingerprint, + ), + _done_handover_applies(model=model, event=newest, snapshot=snapshot), + ) + if origin_relation is None: + return replace(base, decision=MigrationDecision.ORIGIN_MISSING), None, False + compatibility: MigrationCompatibilityResult = check_migration_compatibility( + model=model, + origin_relation=origin_relation, + origin_columns=state.relation_columns(origin), + sql_analysis_enabled=runtime.project.settings.sql_analysis, + column_dialect=snapshot.column_dialect, + snapshots_config=snapshots_config, + ) + decision: MigrationDecision + if model.name not in snapshot.existing_relations: + decision = MigrationDecision.MIGRATE + elif newest is not None and newest.origin.matches(_migration_relation(destination)): + decision = MigrationDecision.SUPERSEDED_REPLACE + elif request.force: + decision = MigrationDecision.FORCED_REPLACE + else: + decision = MigrationDecision.CONFLICT + entry: ModelMigrationPlanEntry = replace( + base, + decision=decision, + compatibility=compatibility.status, + compatibility_findings=compatibility.findings, + ) + if not decision.moves_data or entry.blocks_build: + return entry, None, False + return ( + replace( + entry, + statement=runtime.adapter.render_replace_with_clone( + origin=origin.qualified_name or origin.name, + destination=destination.qualified_name or destination.name, + origin_is_transient=entry.origin_is_transient, + ), + ), + _handover_fingerprint( + model=model, + fingerprint=origin_fingerprint, + version_hash=entry.origin_version_hash, + ), + True, + ) + + +def _done_handover_applies( + *, model: CompiledModel, event: MigrationEvent, snapshot: WarehouseSnapshot +) -> bool: + own: Fingerprint | None = snapshot.fingerprints.models.get(model.name) + return own is None or _utc(own.ts) < _utc(event.created_at) + + +def _utc(value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def _done_handover( + *, + model: CompiledModel, + event: MigrationEvent, + snapshot: WarehouseSnapshot, + origin_fingerprint: Fingerprint | None, +) -> Fingerprint | None: + if not _done_handover_applies(model=model, event=event, snapshot=snapshot): + return snapshot.fingerprints.models.get(model.name) + return _handover_fingerprint( + model=model, fingerprint=origin_fingerprint, version_hash=event.origin_version_hash + ) + + +def _handover_fingerprint( + *, model: CompiledModel, fingerprint: Fingerprint | None, version_hash: str +) -> Fingerprint | None: + if fingerprint is None: + return None + metadata_json: str = fingerprint.metadata_json + try: + payload: object = json.loads(metadata_json) + except json.JSONDecodeError: + payload = None + if isinstance(payload, dict) and MIGRATION_MODEL_NAME_METADATA_KEY in payload: + payload[MIGRATION_MODEL_NAME_METADATA_KEY] = model.name + metadata_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return replace( + fingerprint, + node_name=model.name, + target_database=model.destination.database, + target_schema=model.destination.schema, + target_name=model.destination.name, + version_hash=version_hash or fingerprint.version_hash, + metadata_json=metadata_json, + ) + + +def _overlay_snapshot( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + entries: tuple[ModelMigrationPlanEntry, ...], + handovers: dict[str, Fingerprint | None], + state: MigrationStateInspection, + overrides: PlannerOverrides, + deferral: DeferralInputs, +) -> WarehouseSnapshot: + if not handovers: + return snapshot + relations: dict[str, RelationInfo] = dict(snapshot.existing_relations) + columns: dict[str, tuple[ColumnInfo, ...]] = dict(snapshot.existing_columns) + model_fingerprints: dict[str, Fingerprint] = dict(snapshot.fingerprints.models) + redirected: dict[str, str] = {} + entry: ModelMigrationPlanEntry + for entry in entries: + if entry.model_name not in handovers: + continue + handover: Fingerprint | None = handovers[entry.model_name] + if handover is None: + _ = model_fingerprints.pop(entry.model_name, None) + else: + model_fingerprints[entry.model_name] = handover + if not entry.decision.moves_data: + continue + origin_relation: RelationInfo | None = state.relation(entry.origin) + if origin_relation is None: + continue + relations[entry.model_name] = replace( + origin_relation, + database=entry.destination.database, + schema=entry.destination.schema, + name=entry.destination.name, + ) + columns[entry.model_name] = state.relation_columns(entry.origin) + if entry.origin.qualified_name is not None and _has_cursor( + runtime=runtime, model_name=entry.model_name + ): + redirected[entry.model_name] = entry.origin.qualified_name + cursor_snapshots: dict[str, ModelCursorSnapshot] = { + **snapshot.cursor_snapshots, + **gather_redirected_cursor_snapshots( + project=runtime.project, + adapter=runtime.adapter, + connection=runtime.connection, + existing_relations=relations, + target_relations=redirected, + cursor_scope=CursorSnapshotScope( + model_keys=scope.selected_keys, + runtime_producer_keys=scope.selected_keys, + invocation_time=runtime.invocation_time, + start_cursor_config=( + runtime.project_config.cursors.start + if runtime.project_config is not None + else None + ), + cursor_overrides=overrides.cursor_overrides, + ), + full_refresh_model_names=effectively_full_refreshed_model_names( + project=runtime.project, cli_full_refresh=overrides.full_refresh + ), + deferred_locations=deferral.deferred_locations, + on_progress=runtime.on_progress, + ), + } + return replace( + snapshot, + existing_relations=relations, + existing_columns=columns, + fingerprints=WarehouseFingerprints( + models=model_fingerprints, + functions=snapshot.fingerprints.functions, + seeds=snapshot.fingerprints.seeds, + python_nodes=snapshot.fingerprints.python_nodes, + ), + cursor_snapshots=cursor_snapshots, + ) + + +def _has_cursor(*, runtime: PlannerRuntime, model_name: str) -> bool: + model: CompiledModel | None = next( + (model for model in runtime.project.models if model.name == model_name), None + ) + return ( + model is not None + and get_config_str(values=model.config.values, key="materialized") + == MaterializationType.INCREMENTAL + and get_config_str(values=model.config.values, key="cursor") is not None + ) + + +def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: + origin: str = entry.origin.qualified_name or entry.origin.name + destination: str = entry.destination.qualified_name or entry.destination.name + if entry.decision == MigrationDecision.DONE: + if entry.discovery != MigrationDiscovery.MANUAL: + return None + completed: str = entry.completed_at.isoformat() if entry.completed_at else "unknown time" + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.WARNING, + message=( + f"migration from {origin} completed at {completed} on target " + f"'{entry.target_name}'; migrate_from can be removed from '{entry.model_name}'" + ), + code="M101", + ) + if entry.decision == MigrationDecision.ORIGIN_MISSING: + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.WARNING, + message=( + f"migration origin {origin} does not exist; '{entry.model_name}' is built " + "without migrating history" + ), + code="M102", + ) + if entry.decision == MigrationDecision.CONFLICT: + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.ERROR, + message=( + f"migration conflict: {destination} already exists and has no recorded " + f"migration from {origin}; set migrate_force true to replace it (the replaced " + "table remains recoverable through warehouse time travel)" + ), + code="M103", + ) + if entry.blocks_build: + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.ERROR, + message=( + f"migration from {origin} is incompatible with '{entry.model_name}': " + + "; ".join(entry.compatibility_findings) + ), + code="M104", + ) + return None diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py index f179e64aca..4979a0c6ff 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py @@ -79,10 +79,12 @@ def with_storage_policies( warehouse: PlannerWarehouseState, scopes: PlannerScopeResolution, ) -> PlanOutput: - """Attach selected Snowflake table-type and retention work to the plan.""" + """Attach migrations and selected Snowflake table-type and retention work to the plan.""" return replace( plan_output, + migration_entries=warehouse.migration_entries, + warnings=(*warehouse.migration_warnings, *plan_output.warnings), table_type_entries=plan_table_types( runtime=runtime, warehouse=warehouse, scope=scopes.selected_scope ), diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index 86ac842e34..bf39e47c93 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -4,6 +4,10 @@ import time +from sqlbuild.compiler.planner._helpers.migrations.planning import ( + manual_migration_requests, + plan_model_migrations, +) from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, @@ -12,6 +16,7 @@ from sqlbuild.compiler.planner.models import ( CursorSnapshotScope, DeferralInputs, + ModelMigrationPlanning, PlannerOverrides, PlannerRelationsContext, PlannerRuntime, @@ -55,6 +60,14 @@ def gather_planner_warehouse_state( cursor_overrides=overrides.cursor_overrides, ), ) + migrations: ModelMigrationPlanning = plan_model_migrations( + runtime=runtime, + scope=scopes.selected_scope, + snapshot=snapshot, + requests=manual_migration_requests(scope=scopes.selected_scope), + overrides=overrides, + deferral=deferral, + ) inspection_relations: PlannerRelationsContext = build_planner_relations_context( project=runtime.project, adapter=runtime.adapter, @@ -69,4 +82,9 @@ def gather_planner_warehouse_state( f"Inspected warehouse state. ({time.monotonic() - warehouse_start:.2f}s)" ) runtime.on_progress("Generating plan...") - return PlannerWarehouseState(snapshot=snapshot, inspection_relations=inspection_relations) + return PlannerWarehouseState( + snapshot=migrations.snapshot, + inspection_relations=inspection_relations, + migration_entries=migrations.entries, + migration_warnings=migrations.warnings, + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index 45c5b2af52..c73c01f8dc 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -5,7 +5,7 @@ import logging import time from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime from typing import Any @@ -140,6 +140,7 @@ class _CursorGatherInputs: invocation_time: datetime | None start_cursor_config: StartCursorsConfig | None cursor_overrides: CursorOverrides | None + target_relation_overrides: dict[str, str] | None = None def build_warehouse_snapshot( @@ -279,6 +280,52 @@ def gather_warehouse_snapshot( ) +def gather_redirected_cursor_snapshots( + *, + project: CompiledProject, + adapter: BaseAdapter, + connection: Any, + existing_relations: dict[str, RelationInfo], + target_relations: dict[str, str], + cursor_scope: CursorSnapshotScope, + full_refresh_model_names: frozenset[str], + deferred_locations: dict[str, CompiledRelationLocation] | None = None, + on_progress: Callable[[str], None] | None = None, +) -> dict[str, ModelCursorSnapshot]: + """Gather cursor snapshots for models whose target history lives in another relation.""" + + if not target_relations: + return {} + model_keys: frozenset[CompiledObjectKey] = frozenset( + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) + for name in target_relations + ) + snapshots: dict[str, ModelCursorSnapshot] = _gather_cursor_snapshots( + project=project, + adapter=adapter, + connection=connection, + execute=adapter.execute, + existing_relations=existing_relations, + on_progress=on_progress, + inputs=_CursorGatherInputs( + selected_keys=model_keys, + full_refresh_model_names=full_refresh_model_names, + deferred_locations=deferred_locations, + runtime_producer_keys=cursor_scope.runtime_producer_keys, + invocation_time=cursor_scope.invocation_time, + start_cursor_config=cursor_scope.start_cursor_config, + cursor_overrides=cursor_scope.cursor_overrides, + target_relation_overrides=target_relations, + ), + ) + model_map: dict[str, CompiledModel] = {model.name: model for model in project.models} + return { + name: replace(snapshot, target_relation=model_map[name].destination.qualified_name) + for name, snapshot in snapshots.items() + if name in model_map + } + + def _relevant_state_keys( *, project: CompiledProject, @@ -704,7 +751,11 @@ def _collect_cursor_models( target_tag: str | None = None target_relation: str | None = None target_relation_info: RelationInfo | None = existing_relations.get(model.name) - if ( + redirected_target: str | None = (inputs.target_relation_overrides or {}).get(model.name) + if redirected_target is not None: + target_tag = f"{model.name}__target__max" + target_relation = redirected_target + elif ( model.destination.qualified_name is not None and target_relation_info is not None and target_relation_info.name == model.name diff --git a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py new file mode 100644 index 0000000000..dea9293152 --- /dev/null +++ b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py @@ -0,0 +1,205 @@ +"""Read-only warehouse evidence gathered once for all model migrations in one plan.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL +from sqlbuild.compiler.fingerprints.main.read import read_latest_fingerprints +from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet +from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME +from sqlbuild.compiler.migrations.main._read_events import read_migration_events +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation + + +class MigrationStateInspection: + """Relations, fingerprints, and migration events for one planning database.""" + + def __init__(self, *, adapter: BaseAdapter, connection: Any, database: str | None) -> None: + self._adapter: BaseAdapter = adapter + self._connection: Any = connection + self._database: str | None = database + self._read_schemas: set[str] = set() + self._fingerprints: dict[str, tuple[Fingerprint, ...]] = {} + self._events: list[MigrationEvent] = [] + self._relations: dict[tuple[str, str], RelationInfo] = {} + self._columns: dict[tuple[str, str], tuple[ColumnInfo, ...]] = {} + + @property + def database(self) -> str | None: + """Return the database whose namespaces this inspection reads.""" + + return self._database + + @property + def events(self) -> tuple[MigrationEvent, ...]: + """Return every migration event read so far.""" + + return tuple(self._events) + + def inspect_schemas(self, *, schemas: set[str]) -> None: + """Read state tables once for every schema not inspected yet.""" + + pending: tuple[str, ...] = tuple( + sorted(schema for schema in schemas if schema.lower() not in self._read_schemas) + ) + if not pending: + return + self._read_schemas.update(schema.lower() for schema in pending) + listed: tuple[RelationInfo, ...] = self._adapter.list_relations( + connection=self._connection, + database=self._database, + schemas=pending, + names=(MIGRATION_TABLE_NAME, FINGERPRINT_TABLE_NAME), + ) + state_tables: set[tuple[str, str]] = { + ((relation.schema or "").lower(), relation.name.lower()) for relation in listed + } + schema: str + for schema in pending: + if (schema.lower(), MIGRATION_TABLE_NAME) in state_tables: + self._events.extend( + read_migration_events( + connection=self._connection, + execute=self._adapter.execute, + database=self._database, + schema=schema, + render_qualified_name=self._adapter.render_qualified_name, + ) + ) + if (schema.lower(), FINGERPRINT_TABLE_NAME) in state_tables: + self._fingerprints[schema.lower()] = self._read_model_fingerprints(schema=schema) + + def _read_model_fingerprints(self, *, schema: str) -> tuple[Fingerprint, ...]: + fingerprint_set: FingerprintSet = read_latest_fingerprints( + connection=self._connection, + execute=self._adapter.execute, + table_exists=True, + database=self._database, + schema=schema, + render_qualified_name=self._adapter.render_qualified_name, + render_read_latest_sql=self._adapter.render_read_latest_fingerprints_sql, + ) + return tuple( + fingerprint + for fingerprint in (fingerprint_set.fingerprints_by_identity or {}).values() + if fingerprint.node_type == NODE_TYPE_MODEL + ) + + def inspect_relations(self, *, locations: tuple[CompiledRelationLocation, ...]) -> None: + """List candidate relations and their columns with one metadata read.""" + + pending: tuple[CompiledRelationLocation, ...] = tuple( + location + for location in locations + if location.schema is not None and _relation_key(location) not in self._relations + ) + if not pending: + return + listed: tuple[RelationInfo, ...] = self._adapter.list_relations( + connection=self._connection, + database=self._database, + schemas=tuple(sorted({location.schema or "" for location in pending})), + names=tuple(sorted({location.name for location in pending})), + ) + wanted: set[tuple[str, str]] = {_relation_key(location) for location in pending} + matched: tuple[RelationInfo, ...] = tuple( + relation for relation in listed if _listed_key(relation) in wanted + ) + columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + self._adapter.get_columns_for_relations(connection=self._connection, relations=matched) + if matched + else {} + ) + relation: RelationInfo + for relation in matched: + self._relations[_listed_key(relation)] = relation + self._columns[_listed_key(relation)] = columns.get(relation.identity, ()) + + def relation(self, location: CompiledRelationLocation) -> RelationInfo | None: + """Return the listed relation for a location.""" + + return self._relations.get(_relation_key(location)) + + def relation_columns(self, location: CompiledRelationLocation) -> tuple[ColumnInfo, ...]: + """Return listed columns for a location.""" + + return self._columns.get(_relation_key(location), ()) + + def model_fingerprint( + self, *, location: CompiledRelationLocation, model_name: str | None + ) -> Fingerprint | None: + """Return the latest model fingerprint that built the location or named model.""" + + candidates: tuple[Fingerprint, ...] = self._fingerprints.get( + (location.schema or "").lower(), () + ) + target: MigrationRelation = MigrationRelation( + database=location.database, schema=location.schema, name=location.name + ) + by_relation: tuple[Fingerprint, ...] = tuple( + fingerprint + for fingerprint in candidates + if _fingerprint_built(fingerprint=fingerprint, target=target) + ) + if by_relation: + return max(by_relation, key=_fingerprint_order) + if model_name is None: + return None + named: tuple[Fingerprint, ...] = tuple( + fingerprint for fingerprint in candidates if fingerprint.node_name == model_name + ) + return max(named, key=_fingerprint_order) if named else None + + def named_fingerprint(self, *, model_name: str) -> Fingerprint | None: + """Return the newest model fingerprint with this node name in any inspected schema.""" + + matches: list[Fingerprint] = [] + fingerprints: tuple[Fingerprint, ...] + for fingerprints in self._fingerprints.values(): + matches.extend( + fingerprint for fingerprint in fingerprints if fingerprint.node_name == model_name + ) + return max(matches, key=_fingerprint_order) if matches else None + + def orphan_fingerprints( + self, *, project_model_names: frozenset[str] + ) -> tuple[Fingerprint, ...]: + """Return latest model fingerprints whose names are no longer in the project.""" + + orphans: list[Fingerprint] = [] + fingerprints: tuple[Fingerprint, ...] + for fingerprints in self._fingerprints.values(): + orphans.extend( + fingerprint + for fingerprint in fingerprints + if fingerprint.node_name not in project_model_names + ) + return tuple(orphans) + + +def _relation_key(location: CompiledRelationLocation) -> tuple[str, str]: + return ((location.schema or "").lower(), location.name.lower()) + + +def _listed_key(relation: RelationInfo) -> tuple[str, str]: + return ((relation.schema or "").lower(), relation.name.lower()) + + +def _fingerprint_built(*, fingerprint: Fingerprint, target: MigrationRelation) -> bool: + if fingerprint.target_name is None: + return False + return target.matches( + MigrationRelation( + database=fingerprint.target_database, + schema=fingerprint.target_schema, + name=fingerprint.target_name, + ) + ) + + +def _fingerprint_order(fingerprint: Fingerprint) -> tuple[object, str]: + return (fingerprint.ts.isoformat(), fingerprint.run_id) diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 5f059d15dd..dcd17a65a7 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -59,3 +59,6 @@ SCENARIO_PLAN_INTERNAL: str = "S599" WHOLE_DAY_CURSOR_GRAINS: frozenset[str] = frozenset({"day", "month", "year"}) +VIEW_RELATION_TYPE_MARKER: str = "VIEW" +QUALIFIED_RELATION_MAX_PARTS: int = 3 +MIGRATION_MODEL_NAME_METADATA_KEY: str = "model_name" diff --git a/src/sqlbuild/compiler/planner/main/pre_build/__init__.py b/src/sqlbuild/compiler/planner/main/pre_build/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py new file mode 100644 index 0000000000..d35a1d0edf --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py @@ -0,0 +1,95 @@ +"""Plan text for storage and migration work that runs before model builds.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from sqlbuild.compiler.migrations.types import MigrationDiscovery +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput +from sqlbuild.presentation.main.append_overflow_line import append_overflow_line +from sqlbuild.presentation.main.visible_entries import visible_entries +from sqlbuild.presentation.models import DisplayOptions + + +def format_pre_build_work( + *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions +) -> list[str]: + """Format retention, table-type, and migration work that runs before model builds.""" + + if plan.table_type_entries: + lines.append("Table type conversions") + for table_type_entry in visible_entries( + entries=plan.table_type_entries, options=display_options + ): + actual_type: str = table_type_entry.actual_type or "unknown" + lines.append( + f" {table_type_entry.model_name} desired={table_type_entry.desired_type} " + f"actual={actual_type} source={table_type_entry.source}" + ) + if table_type_entry.irreversible_warning is not None: + lines.append(f" WARNING: {table_type_entry.irreversible_warning}") + lines = append_overflow_line( + lines=lines, + total_count=len(plan.table_type_entries), + visible_count=len( + visible_entries(entries=plan.table_type_entries, options=display_options) + ), + indent=" ", + options=display_options, + ) + if plan.retention_entries: + lines.append("Retention") + for entry in plan.retention_entries: + scope: str = ".".join( + part + for part in (entry.request.database, entry.request.schema, entry.request.name) + if part + ) + actual: str = "missing" if entry.actual_days is None else f"{entry.actual_days}d" + effective: str = "missing" if entry.effective_days is None else f"{entry.effective_days}d" + lines.append( + f" {scope} desired={entry.request.desired_days}d actual={actual} " + f"effective={effective} source={entry.source} direction={entry.direction.value} " + f"phase={entry.phase.value}" + ) + if entry.irreversible_warning is not None: + lines.append(f" WARNING: {entry.irreversible_warning}") + return _format_model_migrations(lines=lines, plan=plan, display_options=display_options) + + +def _format_model_migrations( + *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions +) -> list[str]: + if not plan.migration_entries: + return lines + lines.append(f"Migrations ({len(plan.migration_entries)})") + visible: Sequence[ModelMigrationPlanEntry] = visible_entries( + entries=plan.migration_entries, options=display_options + ) + entry: ModelMigrationPlanEntry + for entry in visible: + lines.extend(_migration_lines(entry)) + return append_overflow_line( + lines=lines, + total_count=len(plan.migration_entries), + visible_count=len(visible), + indent=" ", + options=display_options, + ) + + +def _migration_lines(entry: ModelMigrationPlanEntry) -> list[str]: + origin: str = entry.origin.qualified_name or entry.origin.name + destination: str = entry.destination.qualified_name or entry.destination.name + rows: list[str] = [f" {entry.model_name} {entry.decision.label} {origin} -> {destination}"] + if entry.decision.checks_compatibility: + rows.append(f" compatibility {entry.compatibility.value}") + if entry.discovery != MigrationDiscovery.MANUAL: + rows.append(f" discovery {entry.discovery.value}") + if entry.completed_at is not None: + rows.append( + f" completed {entry.completed_at.isoformat()} on target " + f"'{entry.target_name or 'default'}'" + ) + rows.extend(f" ! {finding}" for finding in entry.compatibility_findings) + return rows diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 58ff77e55e..412240b7d0 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -30,6 +30,11 @@ from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, FunctionLanguage from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.migrations.types import ( + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, +) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.types import ( BackfillAction, @@ -958,6 +963,34 @@ class TableTypePlanEntry: irreversible_warning: str | None = None +@dataclass(frozen=True) +class ModelMigrationPlanEntry: + """One declared or discovered model migration and its per-run decision.""" + + model_name: str + discovery: MigrationDiscovery + decision: MigrationDecision + compatibility: MigrationCompatibility + origin_model: str | None + origin: CompiledRelationLocation + destination: CompiledRelationLocation + target_name: str | None + origin_version_hash: str = "" + origin_is_transient: bool = False + statement: str | None = None + completed_at: datetime | None = None + compatibility_findings: tuple[str, ...] = () + message: str | None = None + + @property + def blocks_build(self) -> bool: + """Return whether this migration must stop a build before any execution.""" + + return self.decision == MigrationDecision.CONFLICT or ( + self.decision.moves_data and self.compatibility == MigrationCompatibility.INCOMPATIBLE + ) + + @dataclass(frozen=True) class PlanOutputExtras: """Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.""" @@ -1365,6 +1398,7 @@ class PlanOutput: warnings: tuple[PlanWarning, ...] = field(default_factory=tuple) retention_entries: tuple[RetentionPlanEntry, ...] = field(default_factory=tuple) table_type_entries: tuple[TableTypePlanEntry, ...] = field(default_factory=tuple) + migration_entries: tuple[ModelMigrationPlanEntry, ...] = field(default_factory=tuple) upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = field( default_factory=dict ) @@ -1451,6 +1485,37 @@ class PlannerWarehouseState: snapshot: WarehouseSnapshot inspection_relations: PlannerRelationsContext + migration_entries: tuple[ModelMigrationPlanEntry, ...] = () + migration_warnings: tuple[PlanWarning, ...] = () + + +@dataclass(frozen=True) +class ModelMigrationRequest: + """One destination model whose data should come from an earlier relation.""" + + model: CompiledModel + discovery: MigrationDiscovery + raw_origin: str | None = None + origin_location: CompiledRelationLocation | None = None + origin_model: str | None = None + force: bool = False + + +@dataclass(frozen=True) +class MigrationCompatibilityResult: + """Compatibility status plus human-readable blocking findings.""" + + status: MigrationCompatibility + findings: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ModelMigrationPlanning: + """Migration decisions plus the warehouse snapshot as it will look after moves.""" + + snapshot: WarehouseSnapshot + entries: tuple[ModelMigrationPlanEntry, ...] = () + warnings: tuple[PlanWarning, ...] = () @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/build/main/_apply_model_migrations.py b/src/sqlbuild/executor/build/main/_apply_model_migrations.py new file mode 100644 index 0000000000..b14901cd44 --- /dev/null +++ b/src/sqlbuild/executor/build/main/_apply_model_migrations.py @@ -0,0 +1,124 @@ +"""Execute planned direct-mode model migrations before any build node runs.""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.migrations.main.deterministic_event_id import ( + deterministic_migration_event_id, +) +from sqlbuild.compiler.migrations.main.write_event import write_migration_event +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput +from sqlbuild.errors.contracts.exceptions import ExecutorInputError + + +def apply_model_migrations( + *, + plan: PlanOutput, + adapter: BaseAdapter, + connection: Any, + run_id: str, + on_progress: Callable[[str], None] | None = None, +) -> None: + """Clone each pending migration origin over its destination, then record the event.""" + + blocked: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in plan.migration_entries if entry.blocks_build + ) + if blocked: + raise ExecutorInputError( + "model migrations are blocked for " + + ", ".join(f"'{entry.model_name}' ({entry.decision.value})" for entry in blocked), + code="M103", + ) + pending: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry + for entry in plan.migration_entries + if entry.decision.moves_data and entry.statement is not None + ) + entry: ModelMigrationPlanEntry + for entry in pending: + _ = _apply_one( + entry=entry, + adapter=adapter, + connection=connection, + run_id=run_id, + on_progress=on_progress, + ) + + +def _apply_one( + *, + entry: ModelMigrationPlanEntry, + adapter: BaseAdapter, + connection: Any, + run_id: str, + on_progress: Callable[[str], None] | None, +) -> None: + origin_label: str = entry.origin.qualified_name or entry.origin.name + destination_label: str = entry.destination.qualified_name or entry.destination.name + if on_progress is not None: + on_progress(f"Migrating {origin_label} -> {destination_label} ({entry.decision.value})...") + started: float = time.monotonic() + try: + _ = _clone_and_record(entry=entry, adapter=adapter, connection=connection, run_id=run_id) + except Exception as error: + raise ExecutorInputError( + f"model migration {origin_label} -> {destination_label} failed: {error}", + code="M106", + help="Re-run the build; migration decisions are re-evaluated from recorded events.", + ) from error + if on_progress is not None: + on_progress( + f"Migrated {origin_label} -> {destination_label}. ({time.monotonic() - started:.2f}s)" + ) + + +def _clone_and_record( + *, entry: ModelMigrationPlanEntry, adapter: BaseAdapter, connection: Any, run_id: str +) -> None: + adapter.ensure_schema( + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + statement_recorder=StatementRecorder(), + ) + _ = adapter.execute(connection=connection, sql=entry.statement or "") + origin: MigrationRelation = _relation(entry.origin) + destination: MigrationRelation = _relation(entry.destination) + write_migration_event( + connection=connection, + execute=adapter.execute, + event=MigrationEvent( + event_id=deterministic_migration_event_id( + run_id=run_id, + target_name=entry.target_name, + origin=origin, + destination=destination, + ), + target_name=entry.target_name, + origin_model=entry.origin_model, + origin=origin, + destination_model=entry.model_name, + destination=destination, + origin_version_hash=entry.origin_version_hash, + discovery=entry.discovery, + decision=entry.decision, + run_id=run_id, + created_at=datetime.now(tz=UTC), + ), + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + transient=adapter.state_tables_transient, + ) + + +def _relation(location: CompiledRelationLocation) -> MigrationRelation: + return MigrationRelation(database=location.database, schema=location.schema, name=location.name) diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 6ad0eb9052..3b96a8518c 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -21,6 +21,7 @@ reconcile_retention_after_build, ) from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler +from sqlbuild.executor.build.main._apply_model_migrations import apply_model_migrations from sqlbuild.executor.build.main.aggregate_result import aggregate_build_result from sqlbuild.executor.build.models import ( BuildCallbacks, @@ -72,6 +73,13 @@ def execute_build_plan( schema_prepared=schema_prepared, ) + _ = apply_model_migrations( + plan=plan, + adapter=adapter, + connection=scheduler_connection, + run_id=runtime.run_id, + on_progress=resolved_callbacks.on_migration_progress, + ) _ = apply_retention_phase( plan=plan, adapter=adapter, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 9f3f62dc27..1b412b8696 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -108,6 +108,7 @@ class BuildCallbacks: python_identity_recorder: PythonIdentityRecorder | None = None on_scheduler_state: Callable[[SchedulerState], None] | None = None on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None + on_migration_progress: Callable[[str], None] | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/virtual/executor/_helpers/build.py b/src/sqlbuild/virtual/executor/_helpers/build.py index cd893bc423..b4f5a96ae3 100644 --- a/src/sqlbuild/virtual/executor/_helpers/build.py +++ b/src/sqlbuild/virtual/executor/_helpers/build.py @@ -19,6 +19,7 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup +from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -533,6 +534,20 @@ def _execute_leased_virtual_build( return result, ingress_python_results, read_side_results +def _reject_model_migrations(*, project: CompiledProject) -> None: + declared: tuple[str, ...] = tuple( + model.name + for model in project.models + if model.config.values.get(MIGRATE_FROM_CONFIG_KEY) is not None + ) + if declared: + raise PlannerInputError( + "migrate_from is supported only in direct mode; virtual environments do not run " + f"model migrations (declared by: {', '.join(declared)})", + code="M105", + ) + + def _resolve_virtual_build( *, project_dir: Path, @@ -559,6 +574,7 @@ def _resolve_virtual_build( timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() if timing_tracker is not None: timing_tracker.compile_seconds = compile_seconds + _reject_model_migrations(project=graph.project) planning_start: float = time.monotonic() try: names: VirtualEnvironmentNames = _resolve_virtual_environment_names( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/__init__.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/__init__.py new file mode 100644 index 0000000000..5d734971d6 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/__init__.py @@ -0,0 +1 @@ +"""Integration tests for direct-mode model migrations.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py new file mode 100644 index 0000000000..754a29d044 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -0,0 +1,33 @@ +"""Test case types for model migration integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MigrationCompileErrorTestCase: + description: str + model_sql: str + expected_fragment: str + + +@dataclass(frozen=True) +class MigrationCompatibilityTestCase: + description: str + origin_setup_sql: str + destination_extra_config: str + destination_select_sql: str + expected_compatibility: str + expected_build_exit_code: int + expected_fragment: str + + +@dataclass(frozen=True) +class MigrationInterruptionTestCase: + description: str + failure_point: str + expected_first_exit_code: int + expected_decision_after_failure: str + rerun_with_force: bool + expected_final_decisions: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py new file mode 100644 index 0000000000..5f523eed96 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -0,0 +1,212 @@ +"""Project builders and CLI runners for model migration integration tests.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from textwrap import dedent +from typing import Any + +import duckdb +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main + +DATABASE_FILE: str = "orders.duckdb" +PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + + [connection] + database = "{DATABASE_FILE}" + """ +).lstrip() +TARGETS_PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + default_target = "dev" + + [connection] + database = "{DATABASE_FILE}" + + [targets.dev] + schema = "dev" + + [targets.prod] + schema = "prod" + """ +).lstrip() +RAW_SOURCES_YML: str = dedent( + """ + sources: + - name: raw_orders + schema: main + table: raw_orders + """ +).lstrip() + + +@dataclass(frozen=True) +class CliRun: + """Captured result of one in-process sqb invocation.""" + + exit_code: int + output: str + + +def incremental_orders_sql( + *, + migrate_from: str | None = None, + migrate_force: bool = False, + extra_config: str = "", + select_sql: str = 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")', +) -> str: + """Return an incremental orders model with optional migration header keys.""" + + migration_lines: str = "" + if migrate_from is not None: + migration_lines += f' migrate_from "{migrate_from}",\n' + if migrate_force: + migration_lines += " migrate_force true,\n" + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + f"{extra_config}" + f"{migration_lines}" + ");\n\n" + f"{select_sql}\n" + ) + + +def write_project( + *, project_dir: Path, models: dict[str, str], project_toml: str = PROJECT_TOML +) -> None: + """Write the project config, sources, and exactly the given model files.""" + + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "sqlbuild_project.toml").write_text(project_toml, encoding="utf-8") + sources_dir: Path = project_dir / "sources" + sources_dir.mkdir(exist_ok=True) + (sources_dir / "raw.yml").write_text(RAW_SOURCES_YML, encoding="utf-8") + models_dir: Path = project_dir / "models" + models_dir.mkdir(exist_ok=True) + existing: Path + for existing in models_dir.glob("*.sql"): + existing.unlink() + name: str + sql: str + for name, sql in models.items(): + (models_dir / f"{name}.sql").write_text(sql, encoding="utf-8") + + +def load_raw_orders(*, project_dir: Path, first_day: int, last_day: int) -> None: + """Replace raw orders with one order per day in the inclusive January day range.""" + + execute( + project_dir=project_dir, + sql=( + "CREATE OR REPLACE TABLE main.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) AS order_date, " + "100 + i AS amount_cents " + f"FROM range({first_day}, {last_day + 1}) AS t(i)" + ), + ) + + +def execute(*, project_dir: Path, sql: str) -> None: + """Run one mutating statement against the project database.""" + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / DATABASE_FILE)) + try: + _ = connection.execute(sql) + finally: + connection.close() + + +def query(*, project_dir: Path, sql: str) -> list[tuple[Any, ...]]: + """Read rows from the project database.""" + + connection: duckdb.DuckDBPyConnection = duckdb.connect( + str(project_dir / DATABASE_FILE), read_only=True + ) + try: + return connection.execute(sql).fetchall() + finally: + connection.close() + + +def order_ids(*, project_dir: Path, relation: str) -> list[int]: + """Return the sorted order IDs stored in one relation.""" + + return [ + int(row[0]) + for row in query(project_dir=project_dir, sql=f"SELECT order_id FROM {relation} ORDER BY 1") + ] + + +def migration_events(*, project_dir: Path, schema: str = "main") -> list[tuple[str, str, str]]: + """Return (origin, destination, decision) for every recorded event in creation order.""" + + return [ + (str(row[0]), str(row[1]), str(row[2])) + for row in query( + project_dir=project_dir, + sql=( + "SELECT origin_name, destination_name, decision " + f"FROM {schema}._sqlbuild_migrations ORDER BY created_at, event_id" + ), + ) + ] + + +def relation_names(*, project_dir: Path, schema: str) -> list[str]: + """Return every relation name in one schema.""" + + return [ + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' ORDER BY 1" + ), + ) + ] + + +def run_sqb( + *, project_dir: Path, args: tuple[str, ...], capsys: pytest.CaptureFixture[str] +) -> CliRun: + """Run sqb in-process and capture combined output.""" + + _ = capsys.readouterr() + exit_code: int = main(["--project-dir", str(project_dir), "--no-color", *args]) + captured: pytest.CaptureResult[str] = capsys.readouterr() + return CliRun(exit_code=exit_code, output=captured.out + captured.err) + + +def plan_json( + *, project_dir: Path, capsys: pytest.CaptureFixture[str], args: tuple[str, ...] = () +) -> dict[str, Any]: + """Run sqb plan --json and parse stdout.""" + + _ = capsys.readouterr() + exit_code: int = main(["--project-dir", str(project_dir), "--no-color", "plan", "--json", *args]) + captured: pytest.CaptureResult[str] = capsys.readouterr() + assert exit_code == 0, captured.out + captured.err + return json.loads(captured.out) + + +def model_entry(*, plan: dict[str, Any], name: str) -> dict[str, Any]: + """Return one serialized model entry from plan JSON.""" + + return next(model for model in plan["models"] if model["name"] == name) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py new file mode 100644 index 0000000000..0cbcb9d462 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -0,0 +1,511 @@ +"""Integration coverage for manual migrate_from migrations through the real CLI.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + MigrationCompatibilityTestCase, + MigrationCompileErrorTestCase, + MigrationInterruptionTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + CliRun, + execute, + incremental_orders_sql, + load_raw_orders, + migration_events, + model_entry, + order_ids, + plan_json, + query, + relation_names, + run_sqb, + write_project, +) + +_ORIGIN: str = "stg_orders" +_DESTINATION: str = "stg_customer_orders" + + +def _build(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: + try: + return run_sqb(project_dir=project_dir, args=("build",), capsys=capsys) + except Exception as error: + return CliRun(exit_code=1, output=str(error)) + + +def _build_ok(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: + result: CliRun = _build(project_dir=project_dir, capsys=capsys) + assert result.exit_code == 0, result.output + return result + + +def _build_origin(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: + write_project(project_dir=project_dir, models={_ORIGIN: incremental_orders_sql()}) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) + _ = _build_ok(project_dir=project_dir, capsys=capsys) + + +def _rename( + *, + project_dir: Path, + name: str, + migrate_from: str | None, + migrate_force: bool = False, + extra_config: str = "", +) -> None: + write_project( + project_dir=project_dir, + models={ + name: incremental_orders_sql( + migrate_from=migrate_from, migrate_force=migrate_force, extra_config=extra_config + ) + }, + ) + + +def _migration_decisions(plan: dict[str, Any]) -> list[str]: + return [migration["decision"] for migration in plan["migrations"]] + + +def test_given_renamed_incremental_when_building_then_history_is_cloned_and_recorded( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _build_origin(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=4, last_day=8) + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = _build_ok(project_dir=tmp_path, capsys=capsys) + + assert _migration_decisions(plan) == ["migrate"] + assert plan["migrations"][0]["compatibility"] == "compatible" + assert model_entry(plan=plan, name=_DESTINATION)["reason"] == "normal_incremental" + assert model_entry(plan=plan, name=_DESTINATION)["action"] == "incremental_delete_insert" + assert "Migrated main.stg_orders -> main.stg_customer_orders" in result.output + assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 9)) + assert order_ids(project_dir=tmp_path, relation=f"main.{_ORIGIN}") == list(range(1, 6)) + assert migration_events(project_dir=tmp_path) == [(_ORIGIN, _DESTINATION, "migrate")] + + +def test_given_rename_without_migrate_from_when_planning_then_destination_is_first_run( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _build_origin(project_dir=tmp_path, capsys=capsys) + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=None) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert plan["migrations"] == [] + assert model_entry(plan=plan, name=_DESTINATION)["reason"] == "first_run" + + +def test_given_completed_migration_when_rebuilding_then_warns_and_skips( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _build_origin(project_dir=tmp_path, capsys=capsys) + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN, migrate_force=True) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=7) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = _build_ok(project_dir=tmp_path, capsys=capsys) + + assert _migration_decisions(plan) == ["done"] + assert "migrate_from can be removed from 'stg_customer_orders'" in result.output + assert "Migrating" not in result.output + assert migration_events(project_dir=tmp_path) == [(_ORIGIN, _DESTINATION, "migrate")] + assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 8)) + + +def test_given_back_and_forth_renames_when_building_each_day_then_every_move_converges( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(project_dir=tmp_path, models={_ORIGIN: incremental_orders_sql()}) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=4) + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + day_one: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=6) + _rename(project_dir=tmp_path, name=_ORIGIN, migrate_from=_DESTINATION) + day_five: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + after_day_five: list[int] = order_ids(project_dir=tmp_path, relation=f"main.{_ORIGIN}") + + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=9) + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + day_nine: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + + assert _migration_decisions(day_one) == ["migrate"] + assert _migration_decisions(day_five) == ["superseded_replace"] + assert _migration_decisions(day_nine) == ["superseded_replace"] + assert after_day_five == list(range(1, 7)) + assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 10)) + assert migration_events(project_dir=tmp_path) == [ + (_ORIGIN, _DESTINATION, "migrate"), + (_DESTINATION, _ORIGIN, "superseded_replace"), + (_ORIGIN, _DESTINATION, "superseded_replace"), + ] + + +def test_given_unrecorded_existing_destination_when_building_then_conflict_until_forced( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _build_origin(project_dir=tmp_path, capsys=capsys) + execute( + project_dir=tmp_path, + sql=( + f"CREATE TABLE main.{_DESTINATION} AS SELECT 999 AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date, 1 AS amount_cents" + ), + ) + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + + conflict_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + conflict: CliRun = _build(project_dir=tmp_path, capsys=capsys) + preserved: list[int] = order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN, migrate_force=True) + forced_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + after_force: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert _migration_decisions(conflict_plan) == ["conflict"] + assert conflict.exit_code == 1 + assert "model migration conflict" in conflict.output + assert preserved == [999] + assert _migration_decisions(forced_plan) == ["forced_replace"] + assert _migration_decisions(after_force) == ["done"] + assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 6)) + assert migration_events(project_dir=tmp_path) == [(_ORIGIN, _DESTINATION, "forced_replace")] + + +def test_given_missing_origin_when_building_then_warns_and_builds_without_history( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={_DESTINATION: incremental_orders_sql(migrate_from="retired_orders")}, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = _build_ok(project_dir=tmp_path, capsys=capsys) + + assert _migration_decisions(plan) == ["origin_missing"] + assert "does not exist" in result.output + assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == [1, 2, 3] + assert "_sqlbuild_migrations" not in relation_names(project_dir=tmp_path, schema="main") + + +def test_given_replay_on_change_full_when_migrating_then_destination_is_not_replayed( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={_ORIGIN: incremental_orders_sql(extra_config=" replay_on_change full,\n")}, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=5, last_day=6) + _rename( + project_dir=tmp_path, + name=_DESTINATION, + migrate_from=_ORIGIN, + extra_config=" replay_on_change full,\n", + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + second_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert model_entry(plan=plan, name=_DESTINATION)["reason"] == "normal_incremental" + assert model_entry(plan=plan, name=_DESTINATION)["action"] != "create_table" + assert model_entry(plan=second_plan, name=_DESTINATION)["reason"] == "normal_incremental" + assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 7)) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationCompileErrorTestCase( + description="table materialization", + model_sql='MODEL (materialized table, migrate_from "stg_orders");\nSELECT 1 AS id\n', + expected_fragment="migrate_from is only valid for incremental and snapshot models", + ), + MigrationCompileErrorTestCase( + description="view materialization", + model_sql='MODEL (materialized view, migrate_from "stg_orders");\nSELECT 1 AS id\n', + expected_fragment="migrate_from is only valid for incremental and snapshot models", + ), + MigrationCompileErrorTestCase( + description="force without origin", + model_sql=incremental_orders_sql(extra_config=" migrate_force true,\n"), + expected_fragment="migrate_force requires migrate_from", + ), + MigrationCompileErrorTestCase( + description="origin names the model itself", + model_sql=incremental_orders_sql(migrate_from=_DESTINATION), + expected_fragment="migrate_from cannot name the model itself", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_migration_header_when_compiling_then_fails( + test_case: MigrationCompileErrorTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(project_dir=tmp_path, models={_DESTINATION: test_case.model_sql}) + + result: CliRun = run_sqb(project_dir=tmp_path, args=("compile",), capsys=capsys) + + assert result.exit_code == 1 + assert test_case.expected_fragment in result.output + + +def test_given_snapshot_model_when_migrating_then_history_rows_are_preserved( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + snapshot_sql: str = ( + "MODEL (\n" + " materialized snapshot,\n" + " unique_key [order_id],\n" + " snapshot_strategy timestamp,\n" + " updated_at order_date,\n" + "{migration}" + ");\n\n" + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")\n' + ) + write_project( + project_dir=tmp_path, models={"order_snapshot": snapshot_sql.format(migration="")} + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={ + "order_history": snapshot_sql.format(migration=' migrate_from "order_snapshot",\n') + }, + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + + assert _migration_decisions(plan) == ["migrate"] + assert order_ids(project_dir=tmp_path, relation="main.order_history") == [1, 2, 3] + assert migration_events(project_dir=tmp_path) == [ + ("order_snapshot", "order_history", "migrate") + ] + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationCompatibilityTestCase( + description="incompatible column type blocks the move", + origin_setup_sql=( + "CREATE TABLE main.legacy_orders AS SELECT 'A-1' AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date, 101 AS amount_cents" + ), + destination_extra_config="", + destination_select_sql=( + "SELECT CAST(order_id AS INTEGER) AS order_id, order_date, amount_cents " + 'FROM __source("raw_orders")' + ), + expected_compatibility="incompatible", + expected_build_exit_code=1, + expected_fragment="incompatible", + ), + MigrationCompatibilityTestCase( + description="on_schema_change fail rejects a missing column", + origin_setup_sql=( + "CREATE TABLE main.legacy_orders AS SELECT 1 AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date" + ), + destination_extra_config=" on_schema_change fail,\n", + destination_select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")' + ), + expected_compatibility="incompatible", + expected_build_exit_code=1, + expected_fragment="on_schema_change fail", + ), + MigrationCompatibilityTestCase( + description="append_new_columns accepts a new model column", + origin_setup_sql=( + "CREATE TABLE main.legacy_orders AS SELECT 1 AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date" + ), + destination_extra_config="", + destination_select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")' + ), + expected_compatibility="compatible", + expected_build_exit_code=0, + expected_fragment="Migrated main.legacy_orders -> main.stg_customer_orders", + ), + MigrationCompatibilityTestCase( + description="numeric widening is compatible under sync_all_columns", + origin_setup_sql=( + "CREATE TABLE main.legacy_orders AS SELECT CAST(1 AS INTEGER) AS order_id, " + "TIMESTAMP '2026-01-01' AS order_date, CAST(101 AS INTEGER) AS amount_cents" + ), + destination_extra_config=" on_schema_change sync_all_columns,\n", + destination_select_sql=( + "SELECT order_id, order_date, CAST(amount_cents AS BIGINT) AS amount_cents " + 'FROM __source("raw_orders")' + ), + expected_compatibility="compatible", + expected_build_exit_code=0, + expected_fragment="Migrated main.legacy_orders -> main.stg_customer_orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_origin_schema_when_migrating_then_normal_incremental_rules_decide( + test_case: MigrationCompatibilityTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project( + project_dir=tmp_path, + models={ + _DESTINATION: incremental_orders_sql( + migrate_from="main.legacy_orders", + extra_config=test_case.destination_extra_config, + select_sql=test_case.destination_select_sql, + ) + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + execute(project_dir=tmp_path, sql=test_case.origin_setup_sql) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = _build(project_dir=tmp_path, capsys=capsys) + + assert plan["migrations"][0]["compatibility"] == test_case.expected_compatibility + assert result.exit_code == test_case.expected_build_exit_code, result.output + assert test_case.expected_fragment in result.output + assert query(project_dir=tmp_path, sql="SELECT count(*) FROM main.legacy_orders") == [(1,)] + + +def _install_failure(*, monkeypatch: pytest.MonkeyPatch, failure_point: str) -> None: + def _raise(**_: object) -> None: + raise RuntimeError(f"simulated interruption at {failure_point}") + + if failure_point == "clone": + monkeypatch.setattr( + DuckDbAdapter, + "render_replace_with_clone", + lambda self, *, origin, destination, origin_is_transient=False: ( + "SELECT * FROM main.simulated_missing_relation" + ), + ) + elif failure_point == "record": + monkeypatch.setattr( + "sqlbuild.executor.build.main._apply_model_migrations.write_migration_event", _raise + ) + else: + monkeypatch.setattr("sqlbuild.executor.build.main._execute.apply_retention_phase", _raise) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationInterruptionTestCase( + description="failure while cloning redoes the move", + failure_point="clone", + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + rerun_with_force=False, + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="crash before recording leaves an unrecorded destination", + failure_point="record", + expected_first_exit_code=1, + expected_decision_after_failure="conflict", + rerun_with_force=True, + expected_final_decisions=("forced_replace",), + ), + MigrationInterruptionTestCase( + description="crash after recording continues from the migrated history", + failure_point="build", + expected_first_exit_code=1, + expected_decision_after_failure="done", + rerun_with_force=False, + expected_final_decisions=("migrate",), + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupted_first_migration_when_rerunning_then_converges( + test_case: MigrationInterruptionTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + _build_origin(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=4, last_day=7) + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + + with monkeypatch.context() as patch: + _install_failure(monkeypatch=patch, failure_point=test_case.failure_point) + interrupted: CliRun = _build(project_dir=tmp_path, capsys=capsys) + after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _rename( + project_dir=tmp_path, + name=_DESTINATION, + migrate_from=_ORIGIN, + migrate_force=test_case.rerun_with_force, + ) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + converged: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code == test_case.expected_first_exit_code, interrupted.output + assert _migration_decisions(after_failure) == [test_case.expected_decision_after_failure] + assert _migration_decisions(converged) == ["done"] + assert model_entry(plan=converged, name=_DESTINATION)["reason"] == "normal_incremental" + assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 8)) + assert order_ids(project_dir=tmp_path, relation=f"main.{_ORIGIN}") == list(range(1, 6)) + assert tuple(event[2] for event in migration_events(project_dir=tmp_path)) == ( + test_case.expected_final_decisions + ) + + +def test_given_interrupted_superseded_replace_when_rerunning_then_replaces_again( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, models={_ORIGIN: incremental_orders_sql()}) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _rename(project_dir=tmp_path, name=_ORIGIN, migrate_from=_DESTINATION) + + with monkeypatch.context() as patch: + _install_failure(monkeypatch=patch, failure_point="record") + interrupted: CliRun = _build(project_dir=tmp_path, capsys=capsys) + after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = _build_ok(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code == 1 + assert _migration_decisions(after_failure) == ["superseded_replace"] + assert order_ids(project_dir=tmp_path, relation=f"main.{_ORIGIN}") == list(range(1, 6)) + assert migration_events(project_dir=tmp_path) == [ + (_ORIGIN, _DESTINATION, "migrate"), + (_DESTINATION, _ORIGIN, "superseded_replace"), + ] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 18726d0de0..d9ce0a6db0 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -69,3 +69,12 @@ class RowDiffSampleSqlTestCase: description: str adapter: BaseAdapter expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class AdapterReplaceWithCloneTestCase: + description: str + adapter: BaseAdapter + origin_is_transient: bool + expected_statement: str | None + expected_error_fragment: str | None = None diff --git a/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py b/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py new file mode 100644 index 0000000000..f689d14af2 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py @@ -0,0 +1,108 @@ +"""Unit coverage for the single-statement migration clone capability.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from tests.unit.src.sqlbuild.adapters._test_types import AdapterReplaceWithCloneTestCase + +_SUPPORTED_CASES: tuple[AdapterReplaceWithCloneTestCase, ...] = ( + AdapterReplaceWithCloneTestCase( + description="snowflake replaces with a zero-copy clone", + adapter=SnowflakeAdapter(), + origin_is_transient=False, + expected_statement="CREATE OR REPLACE TABLE dev.stg_customer_orders CLONE dev.stg_orders", + ), + AdapterReplaceWithCloneTestCase( + description="snowflake preserves a transient origin table type", + adapter=SnowflakeAdapter(), + origin_is_transient=True, + expected_statement=( + "CREATE OR REPLACE TRANSIENT TABLE dev.stg_customer_orders CLONE dev.stg_orders" + ), + ), + AdapterReplaceWithCloneTestCase( + description="databricks uses an independent deep clone", + adapter=DatabricksAdapter(), + origin_is_transient=False, + expected_statement=( + "CREATE OR REPLACE TABLE dev.stg_customer_orders DEEP CLONE dev.stg_orders" + ), + ), + AdapterReplaceWithCloneTestCase( + description="bigquery replaces with a table clone", + adapter=BigQueryAdapter(), + origin_is_transient=False, + expected_statement=( + "CREATE OR REPLACE TABLE `dev.stg_customer_orders` CLONE `dev.stg_orders`" + ), + ), + AdapterReplaceWithCloneTestCase( + description="duckdb copies with one replacing CTAS", + adapter=DuckDbAdapter(), + origin_is_transient=False, + expected_statement=( + "CREATE OR REPLACE TABLE dev.stg_customer_orders AS SELECT * FROM dev.stg_orders" + ), + ), + AdapterReplaceWithCloneTestCase( + description="motherduck inherits the replacing CTAS", + adapter=MotherDuckAdapter(), + origin_is_transient=False, + expected_statement=( + "CREATE OR REPLACE TABLE dev.stg_customer_orders AS SELECT * FROM dev.stg_orders" + ), + ), +) + +_UNSUPPORTED_CASES: tuple[AdapterReplaceWithCloneTestCase, ...] = ( + AdapterReplaceWithCloneTestCase( + description="postgres rejects model migrations", + adapter=PostgresAdapter(), + origin_is_transient=False, + expected_statement=None, + expected_error_fragment="does not support model migrations", + ), + AdapterReplaceWithCloneTestCase( + description="sqlserver rejects model migrations", + adapter=SqlServerAdapter(), + origin_is_transient=False, + expected_statement=None, + expected_error_fragment="does not support model migrations", + ), +) + + +@pytest.mark.parametrize("test_case", _SUPPORTED_CASES, ids=lambda case: case.description) +def test_given_supported_adapter_when_rendering_replace_with_clone_then_returns_one_statement( + test_case: AdapterReplaceWithCloneTestCase, +) -> None: + statement: str = test_case.adapter.render_replace_with_clone( + origin="dev.stg_orders", + destination="dev.stg_customer_orders", + origin_is_transient=test_case.origin_is_transient, + ) + + assert statement == test_case.expected_statement + + +@pytest.mark.parametrize("test_case", _UNSUPPORTED_CASES, ids=lambda case: case.description) +def test_given_unsupported_adapter_when_rendering_replace_with_clone_then_raises( + test_case: AdapterReplaceWithCloneTestCase, +) -> None: + with pytest.raises(AdapterUserError, match=test_case.expected_error_fragment): + _ = test_case.adapter.render_replace_with_clone( + origin="dev.stg_orders", destination="dev.stg_customer_orders" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/__init__.py b/tests/unit/src/sqlbuild/compiler/migrations/__init__.py new file mode 100644 index 0000000000..bfa43ba22d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/__init__.py @@ -0,0 +1 @@ +"""Unit tests for model migration state.""" diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/__init__.py b/tests/unit/src/sqlbuild/compiler/migrations/main/__init__.py new file mode 100644 index 0000000000..6f9ea464f6 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/__init__.py @@ -0,0 +1 @@ +"""Unit tests for model migration state entrypoints.""" diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py new file mode 100644 index 0000000000..8bc13b6fc9 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -0,0 +1,13 @@ +"""Test case types for model migration state entrypoints.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NewestEventTestCase: + description: str + events: tuple[tuple[str, str], ...] + relation: str + expected: tuple[str, str] | None diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py new file mode 100644 index 0000000000..d4c28f4d02 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -0,0 +1,123 @@ +"""Unit coverage for append-only migration event projection and storage.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import duckdb +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning +from sqlbuild.compiler.migrations.main._read_events import read_migration_events +from sqlbuild.compiler.migrations.main.deterministic_event_id import ( + deterministic_migration_event_id, +) +from sqlbuild.compiler.migrations.main.write_event import write_migration_event +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import NewestEventTestCase + +_STARTED_AT: datetime = datetime(2026, 1, 1, tzinfo=UTC) + + +def _relation(name: str) -> MigrationRelation: + return MigrationRelation(database=None, schema="main", name=name) + + +def _event(*, origin: str, destination: str, day: int) -> MigrationEvent: + return MigrationEvent( + event_id=deterministic_migration_event_id( + run_id=f"run-{day}", + target_name="dev", + origin=_relation(origin), + destination=_relation(destination), + ), + target_name="dev", + origin_model=origin, + origin=_relation(origin), + destination_model=destination, + destination=_relation(destination), + origin_version_hash=f"hash-{origin}", + discovery=MigrationDiscovery.MANUAL, + decision=MigrationDecision.MIGRATE, + run_id=f"run-{day}", + created_at=_STARTED_AT + timedelta(days=day), + ) + + +@pytest.mark.parametrize( + "test_case", + [ + NewestEventTestCase( + description="day nine destination sees the day five event as origin", + events=(("stg_orders", "stg_customer_orders"), ("stg_customer_orders", "stg_orders")), + relation="stg_customer_orders", + expected=("stg_customer_orders", "stg_orders"), + ), + NewestEventTestCase( + description="day five destination sees the day one event as origin", + events=(("stg_orders", "stg_customer_orders"),), + relation="stg_orders", + expected=("stg_orders", "stg_customer_orders"), + ), + NewestEventTestCase( + description="unrelated relation has no event", + events=(("stg_orders", "stg_customer_orders"),), + relation="dim_customers", + expected=None, + ), + NewestEventTestCase( + description="relation identity is case insensitive", + events=(("STG_ORDERS", "Stg_Customer_Orders"),), + relation="stg_customer_orders", + expected=("STG_ORDERS", "Stg_Customer_Orders"), + ), + ], + ids=lambda case: case.description, +) +def test_given_event_history_when_projecting_newest_mention_then_returns_latest_event( + test_case: NewestEventTestCase, +) -> None: + events: tuple[MigrationEvent, ...] = tuple( + _event(origin=origin, destination=destination, day=index * 4 + 1) + for index, (origin, destination) in enumerate(test_case.events) + ) + + newest: MigrationEvent | None = newest_migration_event_mentioning( + events=reversed(events), relation=_relation(test_case.relation) + ) + + projected: tuple[str, str] | None = ( + None if newest is None else (newest.origin.name, newest.destination.name) + ) + assert projected == test_case.expected + + +def test_given_repeated_event_write_when_reading_then_event_is_stored_once() -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + event: MigrationEvent = _event(origin="stg_orders", destination="stg_customer_orders", day=1) + + for _ in range(2): + write_migration_event( + connection=connection, + execute=adapter.execute, + event=event, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + transient=False, + ) + stored: tuple[MigrationEvent, ...] = read_migration_events( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + ) + + assert stored == (event,) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From d1a82c59513b848364007ec84e40a24b48f85f02 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:39:53 +0100 Subject: [PATCH 02/20] feat(plan): preview another target with plan --as sqb plan --as resolves configuration and namespaces for another configured target and inspects them through the active target's connection, following the single-connection rule used by defer and diff. It is inspection-only and shows migration decisions for that target. --- .../cli/commands/_helpers/entry/dispatch.py | 1 + .../cli/commands/_helpers/entry/parser.py | 9 ++ .../cli/commands/_helpers/plan/invocation.py | 22 ++++ .../cli/commands/_helpers/plan/planning.py | 10 +- .../cli/commands/classes/cli_namespace.py | 2 + src/sqlbuild/cli/commands/models.py | 1 + .../model_migrations/test_plan_as_target.py | 113 ++++++++++++++++++ 7 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 1c5e10c7b6..aa1d7a50ea 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -110,6 +110,7 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) defer_to=args.defer_to, defer_sources_to=args.defer_sources_to, selected_target=args.target, + as_target=args.as_target, cursor_overrides=CursorOverrides( start_ts=args.start_cursor_ts, end_ts=args.end_cursor_ts, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index 37d29e7e7c..b0789d403a 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -179,6 +179,15 @@ def _add_plan_and_build_parsers( plan_parser.add_argument("--defer-to", default=None) plan_parser.add_argument("--defer-sources-to", default=None) plan_parser.add_argument("--target", default=None) + plan_parser.add_argument( + "--as", + dest="as_target", + default=None, + help=( + "Preview the plan as another configured target through the active target's " + "connection; inspection only" + ), + ) plan_parser.add_argument("--json", action="store_true", default=False) plan_parser.add_argument("--full-refresh", action="store_true", default=False) plan_parser.add_argument("--virtual-env", default=None) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index b4187a1732..2a1677f1d9 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -13,6 +13,7 @@ enforce_no_defer_to_in_virtual_mode, enforce_virtual_only_flags_in_virtual_mode, ) +from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( AdapterConnectionContext, PlanCommandRequest, @@ -46,6 +47,7 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: selected_target=request.selected_target, cli_changes_only=request.changes_only, ) + _validate_preview_target(discovered_inputs=discovered_inputs, as_target=request.as_target) enforce_no_defer_to_in_virtual_mode( discovered_inputs=discovered_inputs, command_name="plan", @@ -89,3 +91,23 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: should_load_sources=should_load_sources, virtual_mode=bool(discovered_inputs.project_config.settings.virtual_environments), ) + + +def _validate_preview_target( + *, discovered_inputs: DiscoveredProjectInputs, as_target: str | None +) -> None: + if as_target is None: + return + if discovered_inputs.project_config.settings.virtual_environments: + raise CliUserError( + "plan --as is supported only in direct mode", + help="Virtual environments preview other environments with --virtual-env.", + ) + configured: set[str] = set(discovered_inputs.project_config.targets) | set( + discovered_inputs.local_config.targets + ) + if as_target not in configured: + raise CliUserError( + f"unknown target '{as_target}' for plan --as", + help=f"Configured targets: {', '.join(sorted(configured)) or 'none'}.", + ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/planning.py b/src/sqlbuild/cli/commands/_helpers/plan/planning.py index b95c53a488..e760773b14 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/planning.py @@ -79,11 +79,19 @@ def compile_plan_pipeline( output_stream=StringIO(), ) return virtual_result + if request.as_target is not None: + invocation.progress_stream.write( + f"Previewing plan as target '{request.as_target}' through the active target's " + "connection (inspection only).\n" + ) + invocation.progress_stream.flush() result: CompilePipelineResult = run_compile_pipeline( discovered_inputs=invocation.discovered_inputs, adapter=invocation.adapter, options=CompilePipelineOptions( - selected_target=request.selected_target, + selected_target=( + request.as_target if request.as_target is not None else request.selected_target + ), no_sql_validation=request.no_sql_validation, no_cache=request.no_cache, defer_to=request.defer_to, diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index 38ca9e3714..c33697d784 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -51,6 +51,7 @@ "fail_fast": False, "full_refresh": False, "virtual_env": None, + "as_target": None, "skip_locked": False, "include_stale_upstreams": False, "changes_only": False, @@ -225,6 +226,7 @@ class CliNamespace: fail_fast: bool full_refresh: bool virtual_env: str | None + as_target: str | None skip_locked: bool include_stale_upstreams: bool changes_only: bool diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 98f8c3ec2d..4bebe0eff0 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -976,6 +976,7 @@ class PlanCommandRequest: defer_to: str | None = None defer_sources_to: str | None = None selected_target: str | None = None + as_target: str | None = None cursor_overrides: CursorOverrides | None = None json_output: bool = False full_refresh: bool = False diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py new file mode 100644 index 0000000000..83bfdb423c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py @@ -0,0 +1,113 @@ +"""Integration coverage for inspection-only plan --as previews of migrations.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + TARGETS_PROJECT_TOML, + CliRun, + incremental_orders_sql, + load_raw_orders, + plan_json, + query, + run_sqb, + write_project, +) + +_ORIGIN: str = "stg_orders" +_DESTINATION: str = "stg_customer_orders" + + +def _warehouse_state(*, project_dir: Path) -> list[tuple[Any, ...]]: + tables: list[tuple[Any, ...]] = query( + project_dir=project_dir, + sql=( + "SELECT table_schema, table_name FROM information_schema.tables " + "WHERE table_schema IN ('dev', 'prod') ORDER BY 1, 2" + ), + ) + counts: list[tuple[Any, ...]] = [ + ( + schema, + name, + query(project_dir=project_dir, sql=f"SELECT count(*) FROM {schema}.{name}")[0][0], + ) + for schema, name in tables + ] + return counts + + +def _prepare_prod_rename(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: + write_project( + project_dir=project_dir, + models={_ORIGIN: incremental_orders_sql()}, + project_toml=TARGETS_PROJECT_TOML, + ) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=4) + built: CliRun = run_sqb( + project_dir=project_dir, args=("build", "--target", "prod"), capsys=capsys + ) + assert built.exit_code == 0, built.output + write_project( + project_dir=project_dir, + models={_DESTINATION: incremental_orders_sql(migrate_from=_ORIGIN)}, + project_toml=TARGETS_PROJECT_TOML, + ) + + +def test_given_pending_prod_migration_when_planning_as_prod_then_previews_without_mutation( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + before: list[tuple[Any, ...]] = _warehouse_state(project_dir=tmp_path) + + preview: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys, args=("--as", "prod")) + active: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + text: CliRun = run_sqb(project_dir=tmp_path, args=("plan", "--as", "prod"), capsys=capsys) + after: list[tuple[Any, ...]] = _warehouse_state(project_dir=tmp_path) + + assert [migration["decision"] for migration in preview["migrations"]] == ["migrate"] + assert preview["migrations"][0]["destination"] == "prod.stg_customer_orders" + assert [migration["decision"] for migration in active["migrations"]] == ["origin_missing"] + assert text.exit_code == 0, text.output + assert "Previewing plan as target 'prod'" in text.output + assert "migrate prod.stg_orders -> prod.stg_customer_orders" in text.output + assert after == before + assert ("prod", "_sqlbuild_migrations", 0) not in after + assert all(name != _DESTINATION for _, name, _ in after) + + +def test_given_executed_prod_migration_when_planning_as_prod_then_reports_done( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + migrated: CliRun = run_sqb( + project_dir=tmp_path, args=("build", "--target", "prod"), capsys=capsys + ) + before: list[tuple[Any, ...]] = _warehouse_state(project_dir=tmp_path) + + preview: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys, args=("--as", "prod")) + + assert migrated.exit_code == 0, migrated.output + assert [migration["decision"] for migration in preview["migrations"]] == ["done"] + assert preview["migrations"][0]["target"] == "prod" + assert _warehouse_state(project_dir=tmp_path) == before + + +def test_given_unknown_target_when_planning_as_it_then_fails_clearly( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + + result: CliRun = run_sqb(project_dir=tmp_path, args=("plan", "--as", "staging"), capsys=capsys) + + assert result.exit_code == 1 + assert "unknown target 'staging' for plan --as" in result.output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From a67b8f5ae3a0a5f8fa3d133d733c9110d828b198 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:06:05 +0100 Subject: [PATCH 03/20] feat(migrations): discover renamed models by migration fingerprint Store a local migration fingerprint in each model fingerprint row and, when a model has no fingerprint on the target, match it against removed models whose relations still exist. Matching resolves renamed upstream refs bottom-up, ignores local CTE and alias names, comments, formatting and storage-only settings, and only accepts unique one-to-one matches; an explicit migrate_from always wins. Incremental and snapshot matches run the manual migration path and are recorded with discovery 'automatic'. Also bring the migration tests and helpers in line with the repository's dataclass-parameterized test conventions, and share the relation-location mapping used by planning, discovery and execution. --- .../migrations/main/relation_for_location.py | 12 + .../planner/_helpers/changes/metadata.py | 2 + .../planner/_helpers/migrations/discovery.py | 353 +++++++++++++ .../_helpers/migrations/fingerprint.py | 221 ++++++++ .../planner/_helpers/migrations/planning.py | 73 ++- .../_helpers/planning/output_assembly.py | 6 + .../_helpers/planning/warehouse_state.py | 6 +- src/sqlbuild/compiler/planner/constants.py | 19 + src/sqlbuild/compiler/planner/models.py | 10 + .../build/main/_apply_model_migrations.py | 10 +- .../_helpers/reuse/fingerprint_metadata.py | 15 + .../run/_helpers/reuse/fingerprinting.py | 4 + .../main/model_migrations/_test_types.py | 44 +- .../commands/main/model_migrations/helpers.py | 272 +++++++++- .../test_automatic_migrations.py | 250 +++++++++ .../test_manual_migrations.py | 491 ++++++++++-------- .../model_migrations/test_plan_as_target.py | 143 ++--- .../adapters/test_replace_with_clone.py | 130 ++--- .../compiler/migrations/main/_test_types.py | 9 +- .../compiler/migrations/main/helpers.py | 42 ++ .../migrations/main/test_migration_events.py | 84 ++- .../planner/_helpers/migrations/__init__.py | 0 .../_helpers/migrations/_test_types.py | 25 + .../planner/_helpers/migrations/helpers.py | 28 + .../migrations/test_migration_fingerprint.py | 148 ++++++ 25 files changed, 1960 insertions(+), 437 deletions(-) create mode 100644 src/sqlbuild/compiler/migrations/main/relation_for_location.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py diff --git a/src/sqlbuild/compiler/migrations/main/relation_for_location.py b/src/sqlbuild/compiler/migrations/main/relation_for_location.py new file mode 100644 index 0000000000..b298a89606 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/relation_for_location.py @@ -0,0 +1,12 @@ +"""Name compiled relation locations in migration events.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.migrations.models import MigrationRelation + + +def migration_relation_for_location(location: CompiledRelationLocation) -> MigrationRelation: + """Return the migration relation that names one compiled relation location.""" + + return MigrationRelation(database=location.database, schema=location.schema, name=location.name) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py index 216292d61f..f6952a740a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py @@ -5,6 +5,7 @@ import json from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY +from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY def version_identity_metadata_payload(metadata_json: str | None) -> object: @@ -20,4 +21,5 @@ def version_identity_metadata_payload(metadata_json: str | None) -> object: return None identity_payload: dict[str, object] = dict(payload) identity_payload.pop(AUDIT_GATE_METADATA_KEY, None) + identity_payload.pop(MIGRATION_FINGERPRINT_METADATA_KEY, None) return identity_payload diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py new file mode 100644 index 0000000000..6d9ae736a1 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -0,0 +1,353 @@ +"""Fingerprint-based discovery of renamed or relocated models.""" + +from __future__ import annotations + +import json +from collections import defaultdict +from typing import Any + +from sqlbuild.adapter.contract.constants import QUALIFIED_NAME_SEPARATOR +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledObjectKey, + CompiledRelationLocation, +) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning +from sqlbuild.compiler.migrations.main.relation_for_location import ( + migration_relation_for_location, +) +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDiscovery +from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint +from sqlbuild.compiler.planner.classes.migration_state_inspection import ( + MigrationStateInspection, +) +from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY +from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( + build_function_local_hashes, +) +from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( + build_model_version_identity_metadata_json, +) +from sqlbuild.compiler.planner.models import ( + ModelMigrationDiscovery, + ModelMigrationRequest, + PlannerRuntime, + PlannerScope, + PlanWarning, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity +from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities +from sqlbuild.spec.contracts.main.get_config_str import get_config_str + +_HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( + {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} +) +_CONFIG_KEY: str = "config" +_MATERIALIZED_KEY: str = "materialized" + + +def discover_model_migrations( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + manual_requests: tuple[ModelMigrationRequest, ...], + state: MigrationStateInspection, + project_schemas: set[str], +) -> ModelMigrationDiscovery: + """Add unique one-to-one fingerprint matches to the explicit migration requests.""" + + models: tuple[CompiledModel, ...] = _topological_models(runtime=runtime) + new_names: frozenset[str] = _new_model_names(models=models, scope=scope, snapshot=snapshot) + manual_renames: dict[str, str] = { + request.model.name: request.raw_origin + for request in manual_requests + if request.raw_origin is not None and QUALIFIED_NAME_SEPARATOR not in request.raw_origin + } + discoverable: frozenset[str] = new_names - frozenset(manual_renames) + if not discoverable and not manual_requests: + return ModelMigrationDiscovery() + metadata_jsons: dict[str, str] = _metadata_jsons(runtime=runtime, models=models) + dialect: str | None = runtime.adapter.sql_analysis_dialect() + if not discoverable: + return ModelMigrationDiscovery( + requests=manual_requests, + destination_fingerprints=_destination_fingerprints( + requests=manual_requests, + renames=manual_renames, + metadata_jsons=metadata_jsons, + dialect=dialect, + ), + ) + state.inspect_schemas(schemas=project_schemas) + candidates: tuple[Fingerprint, ...] = _origin_candidates( + runtime=runtime, state=state, models=models + ) + renames: dict[str, str] + matches: dict[str, Fingerprint] + ambiguous: dict[str, tuple[str, ...]] + renames, matches, ambiguous = _match( + models=models, + new_names=new_names, + manual_renames=manual_renames, + candidates=candidates, + metadata_jsons=metadata_jsons, + dialect=dialect, + ) + automatic: tuple[ModelMigrationRequest, ...] = tuple( + ModelMigrationRequest( + model=model, + discovery=MigrationDiscovery.AUTOMATIC, + origin_location=_fingerprint_location(runtime=runtime, fingerprint=matches[model.name]), + origin_model=matches[model.name].node_name, + ) + for model in models + if model.name in matches + and model.key in scope.selected_keys + and _moves_history(model=model, origin=matches[model.name]) + ) + requests: tuple[ModelMigrationRequest, ...] = (*manual_requests, *automatic) + return ModelMigrationDiscovery( + requests=requests, + warnings=tuple( + PlanWarning( + model_name=model_name, + severity=WarningSeverity.WARNING, + message=( + f"'{model_name}' matches several earlier models ({', '.join(origins)}); " + "no automatic migration was inferred. Add migrate_from to choose one." + ), + code="M107", + ) + for model_name, origins in sorted(ambiguous.items()) + ), + destination_fingerprints=_destination_fingerprints( + requests=requests, renames=renames, metadata_jsons=metadata_jsons, dialect=dialect + ), + ) + + +def _topological_models(*, runtime: PlannerRuntime) -> tuple[CompiledModel, ...]: + by_name: dict[str, CompiledModel] = {model.name: model for model in runtime.project.models} + ordered: list[CompiledModel] = [] + visited: set[str] = set() + pending: list[tuple[str, bool]] = [(name, False) for name in sorted(by_name, reverse=True)] + while pending: + name: str + expanded: bool + name, expanded = pending.pop() + if expanded: + ordered.append(by_name[name]) + continue + if name in visited: + continue + visited.add(name) + pending.append((name, True)) + pending.extend( + (dep.name, False) + for dep in sorted(by_name[name].deps, key=lambda key: key.name, reverse=True) + if dep.resource_type == CompiledResourceType.MODEL + and dep.name in by_name + and dep.name not in visited + ) + return tuple(ordered) + + +def _new_model_names( + *, models: tuple[CompiledModel, ...], scope: PlannerScope, snapshot: WarehouseSnapshot +) -> frozenset[str]: + relevant: set[CompiledObjectKey] = set(scope.selected_keys) + frontier: list[CompiledObjectKey] = list(scope.selected_keys) + while frontier: + key: CompiledObjectKey = frontier.pop() + upstream: CompiledObjectKey + for upstream in scope.upstream_deps.get(key, ()): + if upstream not in relevant: + relevant.add(upstream) + frontier.append(upstream) + return frozenset( + model.name + for model in models + if model.key in relevant and model.name not in snapshot.fingerprints.models + ) + + +def _metadata_jsons( + *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] +) -> dict[str, str]: + function_hashes: dict[str, str] = build_function_local_hashes( + functions=runtime.project.functions + ) + hook_hashes: dict[str, str] = { + name: identity.version_hash + for name, identity in build_hook_identities(runtime.project.hook_functions).items() + } + return { + model.name: build_model_version_identity_metadata_json( + model=model, function_local_hashes=function_hashes, hook_version_hashes=hook_hashes + ) + for model in models + } + + +def _origin_candidates( + *, + runtime: PlannerRuntime, + state: MigrationStateInspection, + models: tuple[CompiledModel, ...], +) -> tuple[Fingerprint, ...]: + project_model_names: frozenset[str] = frozenset(model.name for model in models) + orphans: tuple[Fingerprint, ...] = tuple( + fingerprint + for fingerprint in state.orphan_fingerprints(project_model_names=project_model_names) + if stored_migration_fingerprint(fingerprint) is not None + and fingerprint.target_name is not None + ) + locations: tuple[CompiledRelationLocation, ...] = tuple( + _fingerprint_location(runtime=runtime, fingerprint=fingerprint) for fingerprint in orphans + ) + state.inspect_relations(locations=locations) + return tuple( + fingerprint + for fingerprint, location in zip(orphans, locations, strict=True) + if state.relation(location) is not None + and not _superseded(events=state.events, location=location) + ) + + +def _superseded(*, events: tuple[MigrationEvent, ...], location: CompiledRelationLocation) -> bool: + relation: MigrationRelation = migration_relation_for_location(location) + newest: MigrationEvent | None = newest_migration_event_mentioning( + events=events, relation=relation + ) + return newest is not None and newest.origin.matches(relation) + + +def _match( + *, + models: tuple[CompiledModel, ...], + new_names: frozenset[str], + manual_renames: dict[str, str], + candidates: tuple[Fingerprint, ...], + metadata_jsons: dict[str, str], + dialect: str | None, +) -> tuple[dict[str, str], dict[str, Fingerprint], dict[str, tuple[str, ...]]]: + by_fingerprint: dict[str, list[Fingerprint]] = defaultdict(list) + candidate: Fingerprint + for candidate in candidates: + by_fingerprint[stored_migration_fingerprint(candidate) or ""].append(candidate) + claimed_by_manual: frozenset[str] = frozenset(manual_renames.values()) + blocked: set[str] = set() + ambiguous: dict[str, tuple[str, ...]] = {} + for _ in range(len(candidates) + 1): + renames: dict[str, str] = dict(manual_renames) + matches: dict[str, Fingerprint] = {} + claims: dict[str, list[str]] = defaultdict(list) + model: CompiledModel + for model in models: + if model.name not in new_names or model.name in manual_renames: + continue + fingerprint: str | None = build_migration_fingerprint( + query_sql=model.query_sql, + metadata_json=metadata_jsons[model.name], + ref_identities=renames, + dialect=dialect, + ) + options: list[Fingerprint] = [ + option + for option in by_fingerprint.get(fingerprint or "", []) + if option.node_name not in blocked and option.node_name not in claimed_by_manual + ] + if len(options) > 1: + ambiguous[model.name] = tuple(sorted(option.node_name for option in options)) + continue + if len(options) == 1: + renames[model.name] = options[0].node_name + matches[model.name] = options[0] + claims[options[0].node_name].append(model.name) + contested: dict[str, list[str]] = { + origin: names for origin, names in claims.items() if len(names) > 1 + } + if not contested: + return renames, matches, ambiguous + origin: str + names: list[str] + for origin, names in contested.items(): + blocked.add(origin) + name: str + for name in names: + ambiguous[name] = (*ambiguous.get(name, ()), origin) + return dict(manual_renames), {}, ambiguous + + +def _destination_fingerprints( + *, + requests: tuple[ModelMigrationRequest, ...], + renames: dict[str, str], + metadata_jsons: dict[str, str], + dialect: str | None, +) -> dict[str, str]: + fingerprints: dict[str, str] = {} + request: ModelMigrationRequest + for request in requests: + metadata_json: str | None = metadata_jsons.get(request.model.name) + if metadata_json is None: + continue + fingerprint: str | None = build_migration_fingerprint( + query_sql=request.model.query_sql, + metadata_json=metadata_json, + ref_identities=renames, + dialect=dialect, + ) + if fingerprint is not None: + fingerprints[request.model.name] = fingerprint + return fingerprints + + +def _moves_history(*, model: CompiledModel, origin: Fingerprint) -> bool: + return ( + get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) + in _HISTORY_MATERIALIZATIONS + and _stored_materialization(origin) in _HISTORY_MATERIALIZATIONS + ) + + +def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: + """Return the migration fingerprint recorded in a model fingerprint row, if any.""" + + payload: Any = _metadata_payload(fingerprint) + value: Any = payload.get(MIGRATION_FINGERPRINT_METADATA_KEY) if payload else None + return value if isinstance(value, str) else None + + +def _stored_materialization(fingerprint: Fingerprint) -> str | None: + payload: Any = _metadata_payload(fingerprint) + config: Any = payload.get(_CONFIG_KEY) if payload else None + value: Any = config.get(_MATERIALIZED_KEY) if isinstance(config, dict) else None + return value if isinstance(value, str) else None + + +def _metadata_payload(fingerprint: Fingerprint) -> dict[str, Any] | None: + try: + payload: Any = json.loads(fingerprint.metadata_json) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def _fingerprint_location( + *, runtime: PlannerRuntime, fingerprint: Fingerprint +) -> CompiledRelationLocation: + name: str = fingerprint.target_name or fingerprint.node_name + return CompiledRelationLocation( + database=fingerprint.target_database, + schema=fingerprint.target_schema, + name=name, + qualified_name=runtime.adapter.render_qualified_name( + database=fingerprint.target_database, schema=fingerprint.target_schema, name=name + ), + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py new file mode 100644 index 0000000000..91bf766e9b --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py @@ -0,0 +1,221 @@ +"""Local migration fingerprints that identify the same table across renames.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from dataclasses import replace +from typing import Any + +from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.planner._helpers.changes.metadata import version_identity_metadata_payload +from sqlbuild.compiler.planner.constants import ( + MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS, + MIGRATION_LOCAL_NAME_PREFIX, + MIGRATION_MODEL_NAME_METADATA_KEY, + MIGRATION_REF_PLACEHOLDER_PREFIX, +) +from sqlbuild.compiler.planner.models import ModelPlanEntry +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql + +_REF_MARKER: re.Pattern[str] = re.compile(r"""__ref\(\s*(?:"([^"]+)"|'([^']+)')\s*\)""") +_IDENTIFIER_NAME_KEY: str = "name" +_IDENTIFIER_QUOTED_KEY: str = "quoted" +_ALIAS_KEY: str = "alias" +_WITH_KEY: str = "with" +_CTES_KEY: str = "ctes" +_SPAN_KEY: str = "span" +_COMMENTS_SUFFIX: str = "comments" +_CONFIG_KEY: str = "config" +_GENERIC_DIALECT: str = "generic" +_ALIASED_RELATION_KEYS: frozenset[str] = frozenset({"table", "subquery"}) + + +def build_migration_fingerprint( + *, + query_sql: str, + metadata_json: str, + ref_identities: Mapping[str, str], + dialect: str | None, +) -> str | None: + """Hash normalized SQL and non-storage metadata with refs resolved through renames.""" + + normalized_sql: str | None = _normalized_query_sql( + query_sql=query_sql, ref_identities=ref_identities, dialect=dialect + ) + if normalized_sql is None: + return None + payload: str = json.dumps( + { + "metadata": _identity_metadata( + metadata_json=metadata_json, ref_identities=ref_identities + ), + "sql": normalized_sql, + }, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _identity_metadata(*, metadata_json: str, ref_identities: Mapping[str, str]) -> Any: + payload: Any = version_identity_metadata_payload(metadata_json) + if not isinstance(payload, dict): + return payload + identity: dict[str, Any] = {str(key): value for key, value in payload.items()} + _ = identity.pop(MIGRATION_MODEL_NAME_METADATA_KEY, None) + config: Any = identity.get(_CONFIG_KEY) + if isinstance(config, dict): + identity[_CONFIG_KEY] = { + key: _resolved_cursor_inputs(value=value, ref_identities=ref_identities) + if key == CURSOR_INPUTS_CONFIG_KEY + else value + for key, value in config.items() + if key not in MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS + } + return identity + + +def _resolved_cursor_inputs(*, value: Any, ref_identities: Mapping[str, str]) -> Any: + if not isinstance(value, dict): + return value + return {ref_identities.get(str(name), str(name)): column for name, column in value.items()} + + +def _normalized_query_sql( + *, query_sql: str, ref_identities: Mapping[str, str], dialect: str | None +) -> str | None: + placeholder_sql: str = _REF_MARKER.sub( + lambda match: _ref_placeholder( + identity=ref_identities.get( + match.group(1) or match.group(2), match.group(1) or match.group(2) + ) + ), + query_sql, + ) + polyglot: Any = import_polyglot_sql() + effective_dialect: str = dialect or _GENERIC_DIALECT + try: + parsed: dict[str, Any] = polyglot.parse_one( + placeholder_sql, dialect=effective_dialect + ).to_dict() + except polyglot.PolyglotError: + return None + stripped: Any = _strip_formatting(parsed) + local_names: dict[str, str] = { + name: f"{MIGRATION_LOCAL_NAME_PREFIX}{index}" + for index, name in enumerate(dict.fromkeys(_local_names(stripped)), start=1) + } + canonical: Any = _rename_identifiers(node=stripped, names=local_names) + generated: list[str] = polyglot.generate(canonical, dialect=effective_dialect) + return "\n".join(generated) + + +def _ref_placeholder(*, identity: str) -> str: + digest: str = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] + return f"{MIGRATION_REF_PLACEHOLDER_PREFIX}{digest}" + + +def _strip_formatting(node: Any) -> Any: + if isinstance(node, dict): + return { + key: [] if key.endswith(_COMMENTS_SUFFIX) else _strip_formatting(value) + for key, value in node.items() + if key != _SPAN_KEY + } + if isinstance(node, list): + return [_strip_formatting(value) for value in node] + return node + + +def _local_names(node: Any) -> list[str]: + """Return CTE and relation alias names in first-seen order.""" + + found: list[str] = [] + if isinstance(node, list): + item: Any + for item in node: + found.extend(_local_names(item)) + return found + if not isinstance(node, dict): + return found + with_clause: Any = node.get(_WITH_KEY) + if isinstance(with_clause, dict): + found.extend( + _identifier_names( + [ + cte.get(_ALIAS_KEY) + for cte in with_clause.get(_CTES_KEY) or () + if isinstance(cte, dict) + ] + ) + ) + key: str + value: Any + for key, value in node.items(): + if key in _ALIASED_RELATION_KEYS and isinstance(value, dict): + found.extend(_identifier_names([value.get(_ALIAS_KEY)])) + found.extend(_local_names(value)) + return found + + +def _identifier_names(identifiers: list[Any]) -> list[str]: + return [ + str(identifier[_IDENTIFIER_NAME_KEY]).lower() + for identifier in identifiers + if _is_identifier(identifier) + ] + + +def _rename_identifiers(*, node: Any, names: Mapping[str, str]) -> Any: + if isinstance(node, list): + return [_rename_identifiers(node=value, names=names) for value in node] + if not isinstance(node, dict): + return node + if _is_identifier(node): + canonical: str | None = names.get(str(node[_IDENTIFIER_NAME_KEY]).lower()) + if canonical is not None: + return {**node, _IDENTIFIER_NAME_KEY: canonical, _IDENTIFIER_QUOTED_KEY: False} + return node + return {key: _rename_identifiers(node=value, names=names) for key, value in node.items()} + + +def _is_identifier(node: Any) -> bool: + return ( + isinstance(node, dict) + and isinstance(node.get(_IDENTIFIER_NAME_KEY), str) + and _IDENTIFIER_QUOTED_KEY in node + ) + + +def with_migration_fingerprints( + *, + entries: tuple[ModelPlanEntry, ...], + models_by_name: Mapping[str, CompiledModel], + dialect: str | None, +) -> tuple[ModelPlanEntry, ...]: + """Attach the current-name migration fingerprint each build stores with its fingerprint.""" + + attached: list[ModelPlanEntry] = [] + entry: ModelPlanEntry + for entry in entries: + model: CompiledModel | None = models_by_name.get(entry.name) + if model is None or entry.fingerprint_metadata_json is None: + attached.append(entry) + continue + attached.append( + replace( + entry, + migration_fingerprint=build_migration_fingerprint( + query_sql=model.query_sql, + metadata_json=entry.fingerprint_metadata_json, + ref_identities={}, + dialect=dialect, + ), + ) + ) + return tuple(attached) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index de3a75bfa1..650c87dde4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -10,9 +10,11 @@ from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.models import CompiledModel, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location +from sqlbuild.compiler.migrations.models import MigrationEvent from sqlbuild.compiler.migrations.types import ( MigrationCompatibility, MigrationDecision, @@ -22,6 +24,10 @@ MigrationCompatibilityResult, check_migration_compatibility, ) +from sqlbuild.compiler.planner._helpers.migrations.discovery import ( + discover_model_migrations, + stored_migration_fingerprint, +) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) @@ -40,6 +46,7 @@ CursorSnapshotScope, DeferralInputs, ModelCursorSnapshot, + ModelMigrationDiscovery, ModelMigrationPlanEntry, ModelMigrationPlanning, ModelMigrationRequest, @@ -87,24 +94,33 @@ def plan_model_migrations( runtime: PlannerRuntime, scope: PlannerScope, snapshot: WarehouseSnapshot, - requests: tuple[ModelMigrationRequest, ...], overrides: PlannerOverrides, deferral: DeferralInputs, - inspection: MigrationStateInspection | None = None, ) -> ModelMigrationPlanning: - """Decide every requested migration and project the post-migration snapshot.""" + """Decide every declared or discovered migration and project the post-migration snapshot.""" - if not requests: - return ModelMigrationPlanning(snapshot=snapshot) - state: MigrationStateInspection = inspection or MigrationStateInspection( + manual: tuple[ModelMigrationRequest, ...] = manual_migration_requests(scope=scope) + schemas: set[str] = project_schemas(runtime=runtime) + state: MigrationStateInspection = MigrationStateInspection( adapter=runtime.adapter, connection=runtime.connection, database=planning_database(runtime=runtime), ) - state.inspect_schemas(schemas=project_schemas(runtime=runtime)) + if manual: + state.inspect_schemas(schemas=schemas) + discovery: ModelMigrationDiscovery = discover_model_migrations( + runtime=runtime, + scope=scope, + snapshot=snapshot, + manual_requests=manual, + state=state, + project_schemas=schemas, + ) + if not discovery.requests: + return ModelMigrationPlanning(snapshot=snapshot, warnings=discovery.warnings) resolved: list[tuple[ModelMigrationRequest, CompiledRelationLocation, str | None]] = [ (request, *_resolve_origin(request=request, runtime=runtime, state=state)) - for request in requests + for request in discovery.requests ] origin: CompiledRelationLocation for _, origin, _ in resolved: @@ -138,7 +154,11 @@ def plan_model_migrations( ) entries.append(entry) if applies: - handovers[request.model.name] = handover + handovers[request.model.name] = _equivalent_definition( + model=request.model, + handover=handover, + destination_fingerprint=discovery.destination_fingerprints.get(request.model.name), + ) return ModelMigrationPlanning( snapshot=_overlay_snapshot( runtime=runtime, @@ -151,12 +171,29 @@ def plan_model_migrations( deferral=deferral, ), entries=tuple(entries), - warnings=tuple( - warning for entry in entries if (warning := _entry_warning(entry)) is not None + warnings=( + *discovery.warnings, + *(warning for entry in entries if (warning := _entry_warning(entry)) is not None), ), ) +def _equivalent_definition( + *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprint: str | None +) -> Fingerprint | None: + if ( + handover is None + or destination_fingerprint is None + or stored_migration_fingerprint(handover) != destination_fingerprint + ): + return handover + return replace( + handover, + definition=model.query_sql, + definition_hash=compute_query_hash(model.query_sql), + ) + + def planning_database(*, runtime: PlannerRuntime) -> str | None: """Return the single database that direct planning inspects for migration state.""" @@ -200,10 +237,6 @@ def _relation_key(location: CompiledRelationLocation) -> tuple[str, str]: return ((location.schema or "").lower(), location.name.lower()) -def _migration_relation(location: CompiledRelationLocation) -> MigrationRelation: - return MigrationRelation(database=location.database, schema=location.schema, name=location.name) - - def _resolve_origin( *, request: ModelMigrationRequest, runtime: PlannerRuntime, state: MigrationStateInspection ) -> tuple[CompiledRelationLocation, str | None]: @@ -292,7 +325,7 @@ def _decide( f"model '{model.name}': migrate_from names the model's own relation" ) newest: MigrationEvent | None = newest_migration_event_mentioning( - events=state.events, relation=_migration_relation(destination) + events=state.events, relation=migration_relation_for_location(destination) ) origin_relation: RelationInfo | None = state.relation(origin) origin_fingerprint: Fingerprint | None = state.model_fingerprint( @@ -314,8 +347,8 @@ def _decide( ) if ( newest is not None - and newest.destination.matches(_migration_relation(destination)) - and newest.origin.matches(_migration_relation(origin)) + and newest.destination.matches(migration_relation_for_location(destination)) + and newest.origin.matches(migration_relation_for_location(origin)) ): return ( replace( @@ -346,7 +379,7 @@ def _decide( decision: MigrationDecision if model.name not in snapshot.existing_relations: decision = MigrationDecision.MIGRATE - elif newest is not None and newest.origin.matches(_migration_relation(destination)): + elif newest is not None and newest.origin.matches(migration_relation_for_location(destination)): decision = MigrationDecision.SUPERSEDED_REPLACE elif request.force: decision = MigrationDecision.FORCED_REPLACE diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py index 4979a0c6ff..fe0d02ed5e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py @@ -5,6 +5,7 @@ from dataclasses import replace from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.planner._helpers.migrations.fingerprint import with_migration_fingerprints from sqlbuild.compiler.planner._helpers.output.plan_output import build_plan_output from sqlbuild.compiler.planner._helpers.planning.retention import plan_retention, plan_table_types from sqlbuild.compiler.planner._helpers.pruning.selection_staleness import ( @@ -83,6 +84,11 @@ def with_storage_policies( return replace( plan_output, + model_entries=with_migration_fingerprints( + entries=plan_output.model_entries, + models_by_name={model.name: model for model in runtime.project.models}, + dialect=runtime.adapter.sql_analysis_dialect(), + ), migration_entries=warehouse.migration_entries, warnings=(*warehouse.migration_warnings, *plan_output.warnings), table_type_entries=plan_table_types( diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index bf39e47c93..0e809de25b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -4,10 +4,7 @@ import time -from sqlbuild.compiler.planner._helpers.migrations.planning import ( - manual_migration_requests, - plan_model_migrations, -) +from sqlbuild.compiler.planner._helpers.migrations.planning import plan_model_migrations from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, @@ -64,7 +61,6 @@ def gather_planner_warehouse_state( runtime=runtime, scope=scopes.selected_scope, snapshot=snapshot, - requests=manual_migration_requests(scope=scopes.selected_scope), overrides=overrides, deferral=deferral, ) diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index dcd17a65a7..56ec6c903f 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -62,3 +62,22 @@ VIEW_RELATION_TYPE_MARKER: str = "VIEW" QUALIFIED_RELATION_MAX_PARTS: int = 3 MIGRATION_MODEL_NAME_METADATA_KEY: str = "model_name" +MIGRATION_FINGERPRINT_METADATA_KEY: str = "migration_fingerprint" +MIGRATION_REF_PLACEHOLDER_PREFIX: str = "__sqb_migration_ref__" +MIGRATION_LOCAL_NAME_PREFIX: str = "__sqb_local_" +MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS: frozenset[str] = frozenset( + { + "batch_size", + "full_refresh", + "incremental_mode", + "incremental_strategy", + "lookback", + "materialized", + "max_microbatches", + "microbatch_limit", + "microbatch_strategy", + "on_schema_change", + "snapshot_full_refresh", + "snapshot_schema_change", + } +) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 412240b7d0..dceeb75433 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -1024,6 +1024,7 @@ class ModelPlanEntry: fingerprint_query_sql: str resolved_sql: str logical_ddl: str + migration_fingerprint: str | None = None incremental_strategy: str | None = None incremental_mode: str | None = None microbatch_strategy: str | None = None @@ -1501,6 +1502,15 @@ class ModelMigrationRequest: force: bool = False +@dataclass(frozen=True) +class ModelMigrationDiscovery: + """Manual and automatically discovered migration requests for one plan.""" + + requests: tuple[ModelMigrationRequest, ...] = () + warnings: tuple[PlanWarning, ...] = () + destination_fingerprints: dict[str, str] = field(default_factory=dict) + + @dataclass(frozen=True) class MigrationCompatibilityResult: """Compatibility status plus human-readable blocking findings.""" diff --git a/src/sqlbuild/executor/build/main/_apply_model_migrations.py b/src/sqlbuild/executor/build/main/_apply_model_migrations.py index b14901cd44..d63d071d17 100644 --- a/src/sqlbuild/executor/build/main/_apply_model_migrations.py +++ b/src/sqlbuild/executor/build/main/_apply_model_migrations.py @@ -9,10 +9,10 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.main.deterministic_event_id import ( deterministic_migration_event_id, ) +from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput @@ -91,8 +91,8 @@ def _clone_and_record( statement_recorder=StatementRecorder(), ) _ = adapter.execute(connection=connection, sql=entry.statement or "") - origin: MigrationRelation = _relation(entry.origin) - destination: MigrationRelation = _relation(entry.destination) + origin: MigrationRelation = migration_relation_for_location(entry.origin) + destination: MigrationRelation = migration_relation_for_location(entry.destination) write_migration_event( connection=connection, execute=adapter.execute, @@ -118,7 +118,3 @@ def _clone_and_record( render_framework_type=adapter.render_framework_type, transient=adapter.state_tables_transient, ) - - -def _relation(location: CompiledRelationLocation) -> MigrationRelation: - return MigrationRelation(database=location.database, schema=location.schema, name=location.name) diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py index 99183ae4fd..9cc0e2759b 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py @@ -11,6 +11,7 @@ from sqlbuild.compiler.auditing.models import AuditGateIdentity, AuditIdentity from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY +from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.models import ( @@ -84,6 +85,20 @@ def model_fingerprint_metadata_with_audit_gate( return json.dumps(metadata, sort_keys=True, separators=(",", ":"), default=str) +def model_fingerprint_metadata_with_migration_fingerprint( + *, metadata_json: str, migration_fingerprint: str | None +) -> str: + """Return model fingerprint metadata JSON carrying the rename-matching fingerprint.""" + + if migration_fingerprint is None: + return metadata_json + metadata: object = json.loads(metadata_json or "{}") + if not isinstance(metadata, dict): + return metadata_json + metadata[MIGRATION_FINGERPRINT_METADATA_KEY] = migration_fingerprint + return json.dumps(metadata, sort_keys=True, separators=(",", ":"), default=str) + + def same_target_audit_gate_reuse_decision( *, metadata_json: str | None, model_audits: tuple[AuditPlanEntry, ...] ) -> AuditGateReuseDecision: diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index 4712de9e75..b823a163d4 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -15,6 +15,7 @@ from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( model_fingerprint_metadata_with_audit_gate, + model_fingerprint_metadata_with_migration_fingerprint, ) @@ -46,6 +47,9 @@ def try_write_fingerprint( audit_results=audit_results, run_id=run_id, ) + metadata_json = model_fingerprint_metadata_with_migration_fingerprint( + metadata_json=metadata_json, migration_fingerprint=entry.migration_fingerprint + ) fingerprint: Fingerprint = Fingerprint( node_type=NODE_TYPE_MODEL, node_name=entry.name, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index 754a29d044..ea681edc5b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -2,8 +2,30 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass +import pytest + + +@dataclass(frozen=True) +class MigrationOutcomeTestCase: + description: str + expected_decisions: tuple[str, ...] + expected_events: tuple[tuple[str, str, str], ...] = () + expected_destination_ids: tuple[int, ...] = () + expected_output_fragment: str = "" + expected_reason: str = "" + + +@dataclass(frozen=True) +class BackAndForthMigrationTestCase: + description: str + expected_day_decisions: tuple[tuple[str, ...], ...] + expected_day_five_ids: tuple[int, ...] + expected_final_ids: tuple[int, ...] + expected_events: tuple[tuple[str, str, str], ...] + @dataclass(frozen=True) class MigrationCompileErrorTestCase: @@ -26,8 +48,26 @@ class MigrationCompatibilityTestCase: @dataclass(frozen=True) class MigrationInterruptionTestCase: description: str - failure_point: str + install_failure: Callable[[pytest.MonkeyPatch], None] + rerun_with_force: bool expected_first_exit_code: int expected_decision_after_failure: str - rerun_with_force: bool expected_final_decisions: tuple[str, ...] + + +@dataclass(frozen=True) +class AutomaticMigrationTestCase: + description: str + expected_migrations: tuple[tuple[str, str, str, str], ...] = () + expected_events: tuple[tuple[str, str, str], ...] = () + expected_reason: str = "" + expected_warning: str = "" + + +@dataclass(frozen=True) +class PlanAsTargetTestCase: + description: str + preview_target: str + expected_exit_code: int + expected_decisions: tuple[str, ...] = () + expected_fragment: str = "" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 5f523eed96..897f819ac2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -10,7 +10,9 @@ import duckdb import pytest +from _pytest.capture import CaptureResult +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main DATABASE_FILE: str = "orders.duckdb" @@ -49,6 +51,32 @@ ).lstrip() +_FORCE_LINES: dict[bool, str] = {True: " migrate_force true,\n", False: ""} +_DAILY_TOTALS_HEADER: str = ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_date,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + ");\n\n" +) +_SNAPSHOT_SQL: str = ( + "MODEL (\n" + " materialized snapshot,\n" + " unique_key [order_id],\n" + " snapshot_strategy timestamp,\n" + " updated_at order_date,\n" + "{migration}" + ");\n\n" + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")\n' +) +ORIGIN_MODEL: str = "stg_orders" +DESTINATION_MODEL: str = "stg_customer_orders" + + @dataclass(frozen=True) class CliRun: """Captured result of one in-process sqb invocation.""" @@ -66,11 +94,10 @@ def incremental_orders_sql( ) -> str: """Return an incremental orders model with optional migration header keys.""" - migration_lines: str = "" - if migrate_from is not None: - migration_lines += f' migrate_from "{migrate_from}",\n' - if migrate_force: - migration_lines += " migrate_force true,\n" + origin_lines: dict[str | None, str] = {None: ""} + migration_lines: str = origin_lines.get( + migrate_from, f' migrate_from "{migrate_from}",\n' + ) + _FORCE_LINES.get(migrate_force, "") return ( "MODEL (\n" " materialized incremental,\n" @@ -144,19 +171,21 @@ def query(*, project_dir: Path, sql: str) -> list[tuple[Any, ...]]: connection.close() -def order_ids(*, project_dir: Path, relation: str) -> list[int]: +def order_ids(*, project_dir: Path, relation: str) -> tuple[int, ...]: """Return the sorted order IDs stored in one relation.""" - return [ + return tuple( int(row[0]) for row in query(project_dir=project_dir, sql=f"SELECT order_id FROM {relation} ORDER BY 1") - ] + ) -def migration_events(*, project_dir: Path, schema: str = "main") -> list[tuple[str, str, str]]: +def migration_events( + *, project_dir: Path, schema: str = "main" +) -> tuple[tuple[str, str, str], ...]: """Return (origin, destination, decision) for every recorded event in creation order.""" - return [ + return tuple( (str(row[0]), str(row[1]), str(row[2])) for row in query( project_dir=project_dir, @@ -165,7 +194,7 @@ def migration_events(*, project_dir: Path, schema: str = "main") -> list[tuple[s f"FROM {schema}._sqlbuild_migrations ORDER BY created_at, event_id" ), ) - ] + ) def relation_names(*, project_dir: Path, schema: str) -> list[str]: @@ -190,18 +219,37 @@ def run_sqb( _ = capsys.readouterr() exit_code: int = main(["--project-dir", str(project_dir), "--no-color", *args]) - captured: pytest.CaptureResult[str] = capsys.readouterr() + captured: CaptureResult[str] = capsys.readouterr() return CliRun(exit_code=exit_code, output=captured.out + captured.err) +def build(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: + """Run sqb build, reporting an escaped exception as a failed run.""" + + try: + return run_sqb(project_dir=project_dir, args=("build",), capsys=capsys) + except Exception as error: + return CliRun(exit_code=1, output=str(error)) + + +def build_ok(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: + """Run sqb build and require success.""" + + result: CliRun = build(project_dir=project_dir, capsys=capsys) + assert result.exit_code == 0, result.output + return result + + def plan_json( *, project_dir: Path, capsys: pytest.CaptureFixture[str], args: tuple[str, ...] = () ) -> dict[str, Any]: """Run sqb plan --json and parse stdout.""" _ = capsys.readouterr() - exit_code: int = main(["--project-dir", str(project_dir), "--no-color", "plan", "--json", *args]) - captured: pytest.CaptureResult[str] = capsys.readouterr() + exit_code: int = main( + ["--project-dir", str(project_dir), "--no-color", "plan", "--json", *args] + ) + captured: CaptureResult[str] = capsys.readouterr() assert exit_code == 0, captured.out + captured.err return json.loads(captured.out) @@ -209,4 +257,198 @@ def plan_json( def model_entry(*, plan: dict[str, Any], name: str) -> dict[str, Any]: """Return one serialized model entry from plan JSON.""" - return next(model for model in plan["models"] if model["name"] == name) + models_by_name: dict[str, dict[str, Any]] = {model["name"]: model for model in plan["models"]} + return models_by_name[name] + + +def migration_decisions(plan: dict[str, Any]) -> tuple[str, ...]: + """Return every planned migration decision in plan order.""" + + return tuple(migration["decision"] for migration in plan["migrations"]) + + +def planned_migrations(plan: dict[str, Any]) -> tuple[tuple[str, str, str, str], ...]: + """Return sorted (origin, destination, discovery, decision) for every planned migration.""" + + return tuple( + sorted( + ( + migration["origin_model"], + migration["model"], + migration["discovery"], + migration["decision"], + ) + for migration in plan["migrations"] + ) + ) + + +def warning_messages(plan: dict[str, Any]) -> tuple[str, ...]: + """Return every plan warning message.""" + + return tuple(str(warning["message"]) for warning in plan["warnings"]) + + +def row_count(*, project_dir: Path, relation: str) -> int: + """Return the number of rows stored in one relation.""" + + return int(query(project_dir=project_dir, sql=f"SELECT count(*) FROM {relation}")[0][0]) + + +def build_origin(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Build the origin orders model over five days of raw orders.""" + + write_project(project_dir=project_dir, models={ORIGIN_MODEL: incremental_orders_sql()}) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) + _ = build_ok(project_dir=project_dir, capsys=capsys) + + +def rename_model( + *, + project_dir: Path, + name: str, + migrate_from: str | None, + migrate_force: bool = False, + extra_config: str = "", +) -> None: + """Replace the project with one incremental orders model under a new name.""" + + write_project( + project_dir=project_dir, + models={ + name: incremental_orders_sql( + migrate_from=migrate_from, migrate_force=migrate_force, extra_config=extra_config + ) + }, + ) + + +def snapshot_orders_sql(*, migration_lines: str) -> str: + """Return a timestamp snapshot of raw orders with optional migration header lines.""" + + return _SNAPSHOT_SQL.format(migration=migration_lines) + + +def _raise_interruption(**_: object) -> None: + raise RuntimeError("simulated interruption") + + +def fail_clone(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the migration clone statement fail.""" + + monkeypatch.setattr( + DuckDbAdapter, + "render_replace_with_clone", + lambda self, *, origin, destination, origin_is_transient=False: ( + "SELECT * FROM main.simulated_missing_relation" + ), + ) + + +def fail_record(monkeypatch: pytest.MonkeyPatch) -> None: + """Crash after the clone but before the migration event is recorded.""" + + monkeypatch.setattr( + "sqlbuild.executor.build.main._apply_model_migrations.write_migration_event", + _raise_interruption, + ) + + +def fail_after_migrations(monkeypatch: pytest.MonkeyPatch) -> None: + """Crash after migrations are recorded but before any model builds.""" + + monkeypatch.setattr( + "sqlbuild.executor.build.main._execute.apply_retention_phase", _raise_interruption + ) + + +def enriched_view_sql(*, upstream: str, alias: str) -> str: + """Return a view that selects every order column through a table alias.""" + + return ( + "MODEL (materialized view);\n\n" + f"SELECT {alias}.order_id, {alias}.order_date, {alias}.amount_cents " + f'FROM __ref("{upstream}") AS {alias}\n' + ) + + +def daily_totals_sql(*, upstream: str, cte: str, comment: str = "") -> str: + """Return an incremental daily revenue model that imports its upstream through a CTE.""" + + return ( + f"{_DAILY_TOTALS_HEADER}" + f"{comment}" + f'WITH {cte} AS (SELECT * FROM __ref("{upstream}"))\n' + f"SELECT\n {cte}.order_date,\n SUM({cte}.amount_cents) AS total_cents\n" + f"FROM {cte}\nGROUP BY {cte}.order_date\n" + ) + + +def original_order_models() -> dict[str, str]: + """Return the connected staging, view, and daily totals models before renaming.""" + + return { + "stg_orders": incremental_orders_sql(), + "orders_enriched": enriched_view_sql(upstream="stg_orders", alias="o"), + "daily_order_totals": daily_totals_sql(upstream="orders_enriched", cte="orders_enriched"), + } + + +def renamed_order_models() -> dict[str, str]: + """Return the same connected models renamed, reformatted, and with new storage settings.""" + + return { + "stg_customer_orders": incremental_orders_sql(extra_config=" lookback 1d,\n"), + "customer_orders_enriched": enriched_view_sql( + upstream="stg_customer_orders", alias="customer_order" + ), + "customer_daily_order_totals": daily_totals_sql( + upstream="customer_orders_enriched", + cte="customer_orders_enriched", + comment="-- Daily revenue per order date.\n", + ), + } + + +def build_original_order_models(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Build the connected models over five days of raw orders.""" + + write_project(project_dir=project_dir, models=original_order_models()) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) + _ = build_ok(project_dir=project_dir, capsys=capsys) + + +def warehouse_state(*, project_dir: Path) -> tuple[tuple[Any, ...], ...]: + """Return (schema, relation, row count) for every relation in the dev and prod schemas.""" + + tables: list[tuple[Any, ...]] = query( + project_dir=project_dir, + sql=( + "SELECT table_schema, table_name FROM information_schema.tables " + "WHERE table_schema IN ('dev', 'prod') ORDER BY 1, 2" + ), + ) + return tuple( + (schema, name, row_count(project_dir=project_dir, relation=f"{schema}.{name}")) + for schema, name in tables + ) + + +def prepare_prod_rename(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Build the origin on prod, then rename it with migrate_from in the project.""" + + write_project( + project_dir=project_dir, + models={ORIGIN_MODEL: incremental_orders_sql()}, + project_toml=TARGETS_PROJECT_TOML, + ) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=4) + built: CliRun = run_sqb( + project_dir=project_dir, args=("build", "--target", "prod"), capsys=capsys + ) + assert built.exit_code == 0, built.output + write_project( + project_dir=project_dir, + models={DESTINATION_MODEL: incremental_orders_sql(migrate_from=ORIGIN_MODEL)}, + project_toml=TARGETS_PROJECT_TOML, + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py new file mode 100644 index 0000000000..11cde7ba48 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py @@ -0,0 +1,250 @@ +"""Integration coverage for fingerprint-based automatic migrations through the real CLI.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + AutomaticMigrationTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + CliRun, + build_ok, + build_original_order_models, + daily_totals_sql, + execute, + incremental_orders_sql, + load_raw_orders, + migration_events, + model_entry, + order_ids, + original_order_models, + plan_json, + planned_migrations, + renamed_order_models, + row_count, + warning_messages, + write_project, +) + +_FIRST_RUN: str = "first_run" + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="incremental, view, and incremental renamed together", + expected_migrations=( + ("daily_order_totals", "customer_daily_order_totals", "automatic", "migrate"), + ("stg_orders", "stg_customer_orders", "automatic", "migrate"), + ), + expected_events=( + ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("stg_orders", "stg_customer_orders", "migrate"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_coordinated_renames_when_building_then_history_moves_automatically( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=3, last_day=8) + write_project(project_dir=tmp_path, models=renamed_order_models()) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + rebuild_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] != _FIRST_RUN + assert model_entry(plan=plan, name="customer_daily_order_totals")["reason"] != _FIRST_RUN + assert "Migrated main.stg_orders -> main.stg_customer_orders" in result.output + assert order_ids(project_dir=tmp_path, relation="main.stg_customer_orders") == tuple( + range(1, 9) + ) + assert row_count(project_dir=tmp_path, relation="main.customer_daily_order_totals") == 8 + assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events + assert planned_migrations(rebuild_plan) == () + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="downstream rename matches after an upstream logic change", + expected_migrations=( + ("daily_order_totals", "customer_daily_order_totals", "automatic", "migrate"), + ), + expected_events=(("daily_order_totals", "customer_daily_order_totals", "migrate"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_upstream_logic_change_when_renaming_then_downstream_still_matches( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + models: dict[str, str] = original_order_models() + models["stg_orders"] = incremental_orders_sql( + select_sql=( + "SELECT order_id, order_date, amount_cents * 2 AS amount_cents " + 'FROM __source("raw_orders")' + ) + ) + models["customer_daily_order_totals"] = daily_totals_sql( + upstream="orders_enriched", cte="enriched" + ) + del models["daily_order_totals"] + write_project(project_dir=tmp_path, models=models) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert model_entry(plan=plan, name="customer_daily_order_totals")["reason"] != _FIRST_RUN + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="renamed model with a new filter starts fresh", + expected_reason="first_run", + ) + ], + ids=lambda case: case.description, +) +def test_given_changed_logic_when_renaming_then_destination_is_first_run( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={ + "stg_customer_orders": incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE amount_cents > 0" + ) + ) + }, + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] == ( + test_case.expected_reason + ) + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="two identical removed models make the match ambiguous", + expected_reason="first_run", + expected_warning=( + "'stg_customer_orders' matches several earlier models (orders_copy, stg_orders); " + "no automatic migration was inferred. Add migrate_from to choose one." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_identical_orphans_when_renaming_then_warns_and_does_not_migrate( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={"stg_orders": incremental_orders_sql(), "orders_copy": incremental_orders_sql()}, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project(project_dir=tmp_path, models={"stg_customer_orders": incremental_orders_sql()}) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert test_case.expected_warning in warning_messages(plan) + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] == ( + test_case.expected_reason + ) + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="dropped removed relation cannot migrate", + expected_reason="first_run", + ) + ], + ids=lambda case: case.description, +) +def test_given_dropped_old_relation_when_renaming_then_destination_is_first_run( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(project_dir=tmp_path, models={"stg_orders": incremental_orders_sql()}) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + execute(project_dir=tmp_path, sql="DROP TABLE main.stg_orders") + write_project(project_dir=tmp_path, models={"stg_customer_orders": incremental_orders_sql()}) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] == ( + test_case.expected_reason + ) + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="migrate_from wins over a matching fingerprint", + expected_migrations=(("legacy_orders", "stg_customer_orders", "manual", "migrate"),), + expected_events=(("legacy_orders", "stg_customer_orders", "migrate"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_explicit_migrate_from_when_fingerprint_matches_another_then_manual_wins( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={ + "stg_orders": incremental_orders_sql(), + "legacy_orders": incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE order_id <= 2" + ) + ), + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=4) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={"stg_customer_orders": incremental_orders_sql(migrate_from="legacy_orders")}, + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index 0cbcb9d462..4dd0ff3191 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -7,231 +7,301 @@ import pytest -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + BackAndForthMigrationTestCase, MigrationCompatibilityTestCase, MigrationCompileErrorTestCase, MigrationInterruptionTestCase, + MigrationOutcomeTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + DESTINATION_MODEL, + ORIGIN_MODEL, CliRun, + build, + build_ok, + build_origin, execute, + fail_after_migrations, + fail_clone, + fail_record, incremental_orders_sql, load_raw_orders, + migration_decisions, migration_events, model_entry, order_ids, plan_json, query, relation_names, + rename_model, run_sqb, + snapshot_orders_sql, write_project, ) -_ORIGIN: str = "stg_orders" -_DESTINATION: str = "stg_customer_orders" - - -def _build(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: - try: - return run_sqb(project_dir=project_dir, args=("build",), capsys=capsys) - except Exception as error: - return CliRun(exit_code=1, output=str(error)) - - -def _build_ok(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: - result: CliRun = _build(project_dir=project_dir, capsys=capsys) - assert result.exit_code == 0, result.output - return result - - -def _build_origin(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: - write_project(project_dir=project_dir, models={_ORIGIN: incremental_orders_sql()}) - load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) - _ = _build_ok(project_dir=project_dir, capsys=capsys) - - -def _rename( - *, - project_dir: Path, - name: str, - migrate_from: str | None, - migrate_force: bool = False, - extra_config: str = "", -) -> None: - write_project( - project_dir=project_dir, - models={ - name: incremental_orders_sql( - migrate_from=migrate_from, migrate_force=migrate_force, extra_config=extra_config - ) - }, - ) - - -def _migration_decisions(plan: dict[str, Any]) -> list[str]: - return [migration["decision"] for migration in plan["migrations"]] - +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="first migration clones eight days of history", + expected_decisions=("migrate",), + expected_events=(("stg_orders", "stg_customer_orders", "migrate"),), + expected_destination_ids=tuple(range(1, 9)), + expected_output_fragment="Migrated main.stg_orders -> main.stg_customer_orders", + expected_reason="normal_incremental", + ) + ], + ids=lambda case: case.description, +) def test_given_renamed_incremental_when_building_then_history_is_cloned_and_recorded( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - _build_origin(project_dir=tmp_path, capsys=capsys) + build_origin(project_dir=tmp_path, capsys=capsys) load_raw_orders(project_dir=tmp_path, first_day=4, last_day=8) - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - result: CliRun = _build_ok(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) - assert _migration_decisions(plan) == ["migrate"] + assert migration_decisions(plan) == test_case.expected_decisions assert plan["migrations"][0]["compatibility"] == "compatible" - assert model_entry(plan=plan, name=_DESTINATION)["reason"] == "normal_incremental" - assert model_entry(plan=plan, name=_DESTINATION)["action"] == "incremental_delete_insert" - assert "Migrated main.stg_orders -> main.stg_customer_orders" in result.output - assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 9)) - assert order_ids(project_dir=tmp_path, relation=f"main.{_ORIGIN}") == list(range(1, 6)) - assert migration_events(project_dir=tmp_path) == [(_ORIGIN, _DESTINATION, "migrate")] - - -def test_given_rename_without_migrate_from_when_planning_then_destination_is_first_run( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - _build_origin(project_dir=tmp_path, capsys=capsys) - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=None) - - plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - - assert plan["migrations"] == [] - assert model_entry(plan=plan, name=_DESTINATION)["reason"] == "first_run" + assert model_entry(plan=plan, name=DESTINATION_MODEL)["reason"] == test_case.expected_reason + assert model_entry(plan=plan, name=DESTINATION_MODEL)["action"] == "incremental_delete_insert" + assert test_case.expected_output_fragment in result.output + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + assert order_ids(project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}") == tuple(range(1, 6)) + assert migration_events(project_dir=tmp_path) == test_case.expected_events +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="completed forced migration is skipped with a removal hint", + expected_decisions=("done",), + expected_events=(("stg_orders", "stg_customer_orders", "migrate"),), + expected_destination_ids=tuple(range(1, 8)), + expected_output_fragment="migrate_from can be removed from 'stg_customer_orders'", + ) + ], + ids=lambda case: case.description, +) def test_given_completed_migration_when_rebuilding_then_warns_and_skips( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - _build_origin(project_dir=tmp_path, capsys=capsys) - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN, migrate_force=True) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + build_origin(project_dir=tmp_path, capsys=capsys) + rename_model( + project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL, migrate_force=True + ) + _ = build_ok(project_dir=tmp_path, capsys=capsys) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=7) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - result: CliRun = _build_ok(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) - assert _migration_decisions(plan) == ["done"] - assert "migrate_from can be removed from 'stg_customer_orders'" in result.output + assert migration_decisions(plan) == test_case.expected_decisions + assert test_case.expected_output_fragment in result.output assert "Migrating" not in result.output - assert migration_events(project_dir=tmp_path) == [(_ORIGIN, _DESTINATION, "migrate")] - assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 8)) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) +@pytest.mark.parametrize( + "test_case", + [ + BackAndForthMigrationTestCase( + description="day one, five, and nine renames", + expected_day_decisions=( + ("migrate",), + ("superseded_replace",), + ("superseded_replace",), + ), + expected_day_five_ids=tuple(range(1, 7)), + expected_final_ids=tuple(range(1, 10)), + expected_events=( + ("stg_orders", "stg_customer_orders", "migrate"), + ("stg_customer_orders", "stg_orders", "superseded_replace"), + ("stg_orders", "stg_customer_orders", "superseded_replace"), + ), + ) + ], + ids=lambda case: case.description, +) def test_given_back_and_forth_renames_when_building_each_day_then_every_move_converges( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: BackAndForthMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - write_project(project_dir=tmp_path, models={_ORIGIN: incremental_orders_sql()}) + write_project(project_dir=tmp_path, models={ORIGIN_MODEL: incremental_orders_sql()}) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=4) - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) day_one: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=6) - _rename(project_dir=tmp_path, name=_ORIGIN, migrate_from=_DESTINATION) + rename_model(project_dir=tmp_path, name=ORIGIN_MODEL, migrate_from=DESTINATION_MODEL) day_five: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) - after_day_five: list[int] = order_ids(project_dir=tmp_path, relation=f"main.{_ORIGIN}") + _ = build_ok(project_dir=tmp_path, capsys=capsys) + after_day_five: tuple[int, ...] = order_ids( + project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}" + ) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=9) - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) day_nine: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) - - assert _migration_decisions(day_one) == ["migrate"] - assert _migration_decisions(day_five) == ["superseded_replace"] - assert _migration_decisions(day_nine) == ["superseded_replace"] - assert after_day_five == list(range(1, 7)) - assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 10)) - assert migration_events(project_dir=tmp_path) == [ - (_ORIGIN, _DESTINATION, "migrate"), - (_DESTINATION, _ORIGIN, "superseded_replace"), - (_ORIGIN, _DESTINATION, "superseded_replace"), - ] + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert ( + migration_decisions(day_one), + migration_decisions(day_five), + migration_decisions(day_nine), + ) == test_case.expected_day_decisions + assert after_day_five == test_case.expected_day_five_ids + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_final_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="unrecorded destination conflicts until migrate_force replaces it", + expected_decisions=("conflict", "forced_replace", "done"), + expected_events=(("stg_orders", "stg_customer_orders", "forced_replace"),), + expected_destination_ids=tuple(range(1, 6)), + expected_output_fragment="model migration conflict", + ) + ], + ids=lambda case: case.description, +) def test_given_unrecorded_existing_destination_when_building_then_conflict_until_forced( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - _build_origin(project_dir=tmp_path, capsys=capsys) + build_origin(project_dir=tmp_path, capsys=capsys) execute( project_dir=tmp_path, sql=( - f"CREATE TABLE main.{_DESTINATION} AS SELECT 999 AS order_id, " + f"CREATE TABLE main.{DESTINATION_MODEL} AS SELECT 999 AS order_id, " "TIMESTAMP '2026-01-01' AS order_date, 1 AS amount_cents" ), ) - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) conflict_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - conflict: CliRun = _build(project_dir=tmp_path, capsys=capsys) - preserved: list[int] = order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN, migrate_force=True) + conflict: CliRun = build(project_dir=tmp_path, capsys=capsys) + preserved: tuple[int, ...] = order_ids( + project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}" + ) + rename_model( + project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL, migrate_force=True + ) forced_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) after_force: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - assert _migration_decisions(conflict_plan) == ["conflict"] + assert ( + migration_decisions(conflict_plan) + + migration_decisions(forced_plan) + + migration_decisions(after_force) + ) == test_case.expected_decisions assert conflict.exit_code == 1 - assert "model migration conflict" in conflict.output - assert preserved == [999] - assert _migration_decisions(forced_plan) == ["forced_replace"] - assert _migration_decisions(after_force) == ["done"] - assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 6)) - assert migration_events(project_dir=tmp_path) == [(_ORIGIN, _DESTINATION, "forced_replace")] + assert test_case.expected_output_fragment in conflict.output + assert preserved == (999,) + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="missing origin builds without history and records nothing", + expected_decisions=("origin_missing",), + expected_destination_ids=(1, 2, 3), + expected_output_fragment="does not exist", + ) + ], + ids=lambda case: case.description, +) def test_given_missing_origin_when_building_then_warns_and_builds_without_history( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: write_project( project_dir=tmp_path, - models={_DESTINATION: incremental_orders_sql(migrate_from="retired_orders")}, + models={DESTINATION_MODEL: incremental_orders_sql(migrate_from="retired_orders")}, ) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - result: CliRun = _build_ok(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) - assert _migration_decisions(plan) == ["origin_missing"] - assert "does not exist" in result.output - assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == [1, 2, 3] + assert migration_decisions(plan) == test_case.expected_decisions + assert test_case.expected_output_fragment in result.output + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) assert "_sqlbuild_migrations" not in relation_names(project_dir=tmp_path, schema="main") +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="replay_on_change full does not replay after the handover", + expected_decisions=("migrate",), + expected_destination_ids=tuple(range(1, 7)), + expected_reason="normal_incremental", + ) + ], + ids=lambda case: case.description, +) def test_given_replay_on_change_full_when_migrating_then_destination_is_not_replayed( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: write_project( project_dir=tmp_path, - models={_ORIGIN: incremental_orders_sql(extra_config=" replay_on_change full,\n")}, + models={ORIGIN_MODEL: incremental_orders_sql(extra_config=" replay_on_change full,\n")}, ) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) load_raw_orders(project_dir=tmp_path, first_day=5, last_day=6) - _rename( + rename_model( project_dir=tmp_path, - name=_DESTINATION, - migrate_from=_ORIGIN, + name=DESTINATION_MODEL, + migrate_from=ORIGIN_MODEL, extra_config=" replay_on_change full,\n", ) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) second_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - assert model_entry(plan=plan, name=_DESTINATION)["reason"] == "normal_incremental" - assert model_entry(plan=plan, name=_DESTINATION)["action"] != "create_table" - assert model_entry(plan=second_plan, name=_DESTINATION)["reason"] == "normal_incremental" - assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 7)) + assert migration_decisions(plan) == test_case.expected_decisions + assert model_entry(plan=plan, name=DESTINATION_MODEL)["reason"] == test_case.expected_reason + assert model_entry(plan=plan, name=DESTINATION_MODEL)["action"] != "create_table" + assert ( + model_entry(plan=second_plan, name=DESTINATION_MODEL)["reason"] == test_case.expected_reason + ) + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) @pytest.mark.parametrize( @@ -254,7 +324,7 @@ def test_given_replay_on_change_full_when_migrating_then_destination_is_not_repl ), MigrationCompileErrorTestCase( description="origin names the model itself", - model_sql=incremental_orders_sql(migrate_from=_DESTINATION), + model_sql=incremental_orders_sql(migrate_from=DESTINATION_MODEL), expected_fragment="migrate_from cannot name the model itself", ), ], @@ -263,7 +333,7 @@ def test_given_replay_on_change_full_when_migrating_then_destination_is_not_repl def test_given_invalid_migration_header_when_compiling_then_fails( test_case: MigrationCompileErrorTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - write_project(project_dir=tmp_path, models={_DESTINATION: test_case.model_sql}) + write_project(project_dir=tmp_path, models={DESTINATION_MODEL: test_case.model_sql}) result: CliRun = run_sqb(project_dir=tmp_path, args=("compile",), capsys=capsys) @@ -271,39 +341,44 @@ def test_given_invalid_migration_header_when_compiling_then_fails( assert test_case.expected_fragment in result.output +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="snapshot history rows move to the renamed snapshot", + expected_decisions=("migrate",), + expected_events=(("order_snapshot", "order_history", "migrate"),), + expected_destination_ids=(1, 2, 3), + ) + ], + ids=lambda case: case.description, +) def test_given_snapshot_model_when_migrating_then_history_rows_are_preserved( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - snapshot_sql: str = ( - "MODEL (\n" - " materialized snapshot,\n" - " unique_key [order_id],\n" - " snapshot_strategy timestamp,\n" - " updated_at order_date,\n" - "{migration}" - ");\n\n" - 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")\n' - ) write_project( - project_dir=tmp_path, models={"order_snapshot": snapshot_sql.format(migration="")} + project_dir=tmp_path, models={"order_snapshot": snapshot_orders_sql(migration_lines="")} ) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) write_project( project_dir=tmp_path, models={ - "order_history": snapshot_sql.format(migration=' migrate_from "order_snapshot",\n') + "order_history": snapshot_orders_sql( + migration_lines=' migrate_from "order_snapshot",\n' + ) }, ) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) - assert _migration_decisions(plan) == ["migrate"] - assert order_ids(project_dir=tmp_path, relation="main.order_history") == [1, 2, 3] - assert migration_events(project_dir=tmp_path) == [ - ("order_snapshot", "order_history", "migrate") - ] + assert migration_decisions(plan) == test_case.expected_decisions + assert ( + order_ids(project_dir=tmp_path, relation="main.order_history") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events @pytest.mark.parametrize( @@ -378,7 +453,7 @@ def test_given_origin_schema_when_migrating_then_normal_incremental_rules_decide write_project( project_dir=tmp_path, models={ - _DESTINATION: incremental_orders_sql( + DESTINATION_MODEL: incremental_orders_sql( migrate_from="main.legacy_orders", extra_config=test_case.destination_extra_config, select_sql=test_case.destination_select_sql, @@ -389,7 +464,7 @@ def test_given_origin_schema_when_migrating_then_normal_incremental_rules_decide execute(project_dir=tmp_path, sql=test_case.origin_setup_sql) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - result: CliRun = _build(project_dir=tmp_path, capsys=capsys) + result: CliRun = build(project_dir=tmp_path, capsys=capsys) assert plan["migrations"][0]["compatibility"] == test_case.expected_compatibility assert result.exit_code == test_case.expected_build_exit_code, result.output @@ -397,51 +472,31 @@ def test_given_origin_schema_when_migrating_then_normal_incremental_rules_decide assert query(project_dir=tmp_path, sql="SELECT count(*) FROM main.legacy_orders") == [(1,)] -def _install_failure(*, monkeypatch: pytest.MonkeyPatch, failure_point: str) -> None: - def _raise(**_: object) -> None: - raise RuntimeError(f"simulated interruption at {failure_point}") - - if failure_point == "clone": - monkeypatch.setattr( - DuckDbAdapter, - "render_replace_with_clone", - lambda self, *, origin, destination, origin_is_transient=False: ( - "SELECT * FROM main.simulated_missing_relation" - ), - ) - elif failure_point == "record": - monkeypatch.setattr( - "sqlbuild.executor.build.main._apply_model_migrations.write_migration_event", _raise - ) - else: - monkeypatch.setattr("sqlbuild.executor.build.main._execute.apply_retention_phase", _raise) - - @pytest.mark.parametrize( "test_case", [ MigrationInterruptionTestCase( description="failure while cloning redoes the move", - failure_point="clone", + install_failure=fail_clone, + rerun_with_force=False, expected_first_exit_code=1, expected_decision_after_failure="migrate", - rerun_with_force=False, expected_final_decisions=("migrate",), ), MigrationInterruptionTestCase( description="crash before recording leaves an unrecorded destination", - failure_point="record", + install_failure=fail_record, + rerun_with_force=True, expected_first_exit_code=1, expected_decision_after_failure="conflict", - rerun_with_force=True, expected_final_decisions=("forced_replace",), ), MigrationInterruptionTestCase( description="crash after recording continues from the migrated history", - failure_point="build", + install_failure=fail_after_migrations, + rerun_with_force=False, expected_first_exit_code=1, expected_decision_after_failure="done", - rerun_with_force=False, expected_final_decisions=("migrate",), ), ], @@ -453,58 +508,78 @@ def test_given_interrupted_first_migration_when_rerunning_then_converges( capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - _build_origin(project_dir=tmp_path, capsys=capsys) + build_origin(project_dir=tmp_path, capsys=capsys) load_raw_orders(project_dir=tmp_path, first_day=4, last_day=7) - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) with monkeypatch.context() as patch: - _install_failure(monkeypatch=patch, failure_point=test_case.failure_point) - interrupted: CliRun = _build(project_dir=tmp_path, capsys=capsys) + test_case.install_failure(patch) + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _rename( + rename_model( project_dir=tmp_path, - name=_DESTINATION, - migrate_from=_ORIGIN, + name=DESTINATION_MODEL, + migrate_from=ORIGIN_MODEL, migrate_force=test_case.rerun_with_force, ) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) converged: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) assert interrupted.exit_code == test_case.expected_first_exit_code, interrupted.output - assert _migration_decisions(after_failure) == [test_case.expected_decision_after_failure] - assert _migration_decisions(converged) == ["done"] - assert model_entry(plan=converged, name=_DESTINATION)["reason"] == "normal_incremental" - assert order_ids(project_dir=tmp_path, relation=f"main.{_DESTINATION}") == list(range(1, 8)) - assert order_ids(project_dir=tmp_path, relation=f"main.{_ORIGIN}") == list(range(1, 6)) + assert migration_decisions(after_failure) == (test_case.expected_decision_after_failure,) + assert migration_decisions(converged) == ("done",) + assert model_entry(plan=converged, name=DESTINATION_MODEL)["reason"] == "normal_incremental" + assert order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") == tuple( + range(1, 8) + ) + assert order_ids(project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}") == tuple(range(1, 6)) assert tuple(event[2] for event in migration_events(project_dir=tmp_path)) == ( test_case.expected_final_decisions ) +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="crash before recording a superseded replace repeats the replace", + expected_decisions=("superseded_replace",), + expected_events=( + ("stg_orders", "stg_customer_orders", "migrate"), + ("stg_customer_orders", "stg_orders", "superseded_replace"), + ), + expected_destination_ids=tuple(range(1, 6)), + ) + ], + ids=lambda case: case.description, +) def test_given_interrupted_superseded_replace_when_rerunning_then_replaces_again( - tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch + test_case: MigrationOutcomeTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: - write_project(project_dir=tmp_path, models={_ORIGIN: incremental_orders_sql()}) + write_project(project_dir=tmp_path, models={ORIGIN_MODEL: incremental_orders_sql()}) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) - _rename(project_dir=tmp_path, name=_DESTINATION, migrate_from=_ORIGIN) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) - _rename(project_dir=tmp_path, name=_ORIGIN, migrate_from=_DESTINATION) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + rename_model(project_dir=tmp_path, name=ORIGIN_MODEL, migrate_from=DESTINATION_MODEL) with monkeypatch.context() as patch: - _install_failure(monkeypatch=patch, failure_point="record") - interrupted: CliRun = _build(project_dir=tmp_path, capsys=capsys) + fail_record(patch) + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _ = _build_ok(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) assert interrupted.exit_code == 1 - assert _migration_decisions(after_failure) == ["superseded_replace"] - assert order_ids(project_dir=tmp_path, relation=f"main.{_ORIGIN}") == list(range(1, 6)) - assert migration_events(project_dir=tmp_path) == [ - (_ORIGIN, _DESTINATION, "migrate"), - (_DESTINATION, _ORIGIN, "superseded_replace"), - ] + assert migration_decisions(after_failure) == test_case.expected_decisions + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py index 83bfdb423c..9e0e0add75 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_plan_as_target.py @@ -7,106 +7,113 @@ import pytest +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + PlanAsTargetTestCase, +) from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( - TARGETS_PROJECT_TOML, + DESTINATION_MODEL, CliRun, - incremental_orders_sql, - load_raw_orders, + migration_decisions, plan_json, - query, + prepare_prod_rename, run_sqb, - write_project, + warehouse_state, ) -_ORIGIN: str = "stg_orders" -_DESTINATION: str = "stg_customer_orders" - -def _warehouse_state(*, project_dir: Path) -> list[tuple[Any, ...]]: - tables: list[tuple[Any, ...]] = query( - project_dir=project_dir, - sql=( - "SELECT table_schema, table_name FROM information_schema.tables " - "WHERE table_schema IN ('dev', 'prod') ORDER BY 1, 2" - ), - ) - counts: list[tuple[Any, ...]] = [ - ( - schema, - name, - query(project_dir=project_dir, sql=f"SELECT count(*) FROM {schema}.{name}")[0][0], +@pytest.mark.parametrize( + "test_case", + [ + PlanAsTargetTestCase( + description="pending prod migration previews as migrate", + preview_target="prod", + expected_exit_code=0, + expected_decisions=("migrate",), + expected_fragment="migrate prod.stg_orders -> prod.stg_customer_orders", ) - for schema, name in tables - ] - return counts - - -def _prepare_prod_rename(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: - write_project( - project_dir=project_dir, - models={_ORIGIN: incremental_orders_sql()}, - project_toml=TARGETS_PROJECT_TOML, - ) - load_raw_orders(project_dir=project_dir, first_day=1, last_day=4) - built: CliRun = run_sqb( - project_dir=project_dir, args=("build", "--target", "prod"), capsys=capsys - ) - assert built.exit_code == 0, built.output - write_project( - project_dir=project_dir, - models={_DESTINATION: incremental_orders_sql(migrate_from=_ORIGIN)}, - project_toml=TARGETS_PROJECT_TOML, - ) - - + ], + ids=lambda case: case.description, +) def test_given_pending_prod_migration_when_planning_as_prod_then_previews_without_mutation( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: PlanAsTargetTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - _prepare_prod_rename(project_dir=tmp_path, capsys=capsys) - before: list[tuple[Any, ...]] = _warehouse_state(project_dir=tmp_path) + prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + before: tuple[tuple[Any, ...], ...] = warehouse_state(project_dir=tmp_path) - preview: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys, args=("--as", "prod")) + preview: dict[str, Any] = plan_json( + project_dir=tmp_path, capsys=capsys, args=("--as", test_case.preview_target) + ) active: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - text: CliRun = run_sqb(project_dir=tmp_path, args=("plan", "--as", "prod"), capsys=capsys) - after: list[tuple[Any, ...]] = _warehouse_state(project_dir=tmp_path) + text: CliRun = run_sqb( + project_dir=tmp_path, args=("plan", "--as", test_case.preview_target), capsys=capsys + ) + after: tuple[tuple[Any, ...], ...] = warehouse_state(project_dir=tmp_path) - assert [migration["decision"] for migration in preview["migrations"]] == ["migrate"] + assert migration_decisions(preview) == test_case.expected_decisions assert preview["migrations"][0]["destination"] == "prod.stg_customer_orders" - assert [migration["decision"] for migration in active["migrations"]] == ["origin_missing"] - assert text.exit_code == 0, text.output + assert migration_decisions(active) == ("origin_missing",) + assert text.exit_code == test_case.expected_exit_code, text.output assert "Previewing plan as target 'prod'" in text.output - assert "migrate prod.stg_orders -> prod.stg_customer_orders" in text.output + assert test_case.expected_fragment in text.output assert after == before assert ("prod", "_sqlbuild_migrations", 0) not in after - assert all(name != _DESTINATION for _, name, _ in after) + assert all(name != DESTINATION_MODEL for _, name, _ in after) +@pytest.mark.parametrize( + "test_case", + [ + PlanAsTargetTestCase( + description="executed prod migration previews as done", + preview_target="prod", + expected_exit_code=0, + expected_decisions=("done",), + ) + ], + ids=lambda case: case.description, +) def test_given_executed_prod_migration_when_planning_as_prod_then_reports_done( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: PlanAsTargetTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - _prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + prepare_prod_rename(project_dir=tmp_path, capsys=capsys) migrated: CliRun = run_sqb( project_dir=tmp_path, args=("build", "--target", "prod"), capsys=capsys ) - before: list[tuple[Any, ...]] = _warehouse_state(project_dir=tmp_path) + before: tuple[tuple[Any, ...], ...] = warehouse_state(project_dir=tmp_path) - preview: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys, args=("--as", "prod")) + preview: dict[str, Any] = plan_json( + project_dir=tmp_path, capsys=capsys, args=("--as", test_case.preview_target) + ) - assert migrated.exit_code == 0, migrated.output - assert [migration["decision"] for migration in preview["migrations"]] == ["done"] - assert preview["migrations"][0]["target"] == "prod" - assert _warehouse_state(project_dir=tmp_path) == before + assert migrated.exit_code == test_case.expected_exit_code, migrated.output + assert migration_decisions(preview) == test_case.expected_decisions + assert preview["migrations"][0]["target"] == test_case.preview_target + assert warehouse_state(project_dir=tmp_path) == before +@pytest.mark.parametrize( + "test_case", + [ + PlanAsTargetTestCase( + description="unknown target fails clearly", + preview_target="staging", + expected_exit_code=1, + expected_fragment="unknown target 'staging' for plan --as", + ) + ], + ids=lambda case: case.description, +) def test_given_unknown_target_when_planning_as_it_then_fails_clearly( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: PlanAsTargetTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - _prepare_prod_rename(project_dir=tmp_path, capsys=capsys) + prepare_prod_rename(project_dir=tmp_path, capsys=capsys) - result: CliRun = run_sqb(project_dir=tmp_path, args=("plan", "--as", "staging"), capsys=capsys) + result: CliRun = run_sqb( + project_dir=tmp_path, args=("plan", "--as", test_case.preview_target), capsys=capsys + ) - assert result.exit_code == 1 - assert "unknown target 'staging' for plan --as" in result.output + assert result.exit_code == test_case.expected_exit_code + assert test_case.expected_fragment in result.output if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py b/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py index f689d14af2..f9abf793f9 100644 --- a/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py +++ b/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py @@ -14,74 +14,59 @@ from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from tests.unit.src.sqlbuild.adapters._test_types import AdapterReplaceWithCloneTestCase -_SUPPORTED_CASES: tuple[AdapterReplaceWithCloneTestCase, ...] = ( - AdapterReplaceWithCloneTestCase( - description="snowflake replaces with a zero-copy clone", - adapter=SnowflakeAdapter(), - origin_is_transient=False, - expected_statement="CREATE OR REPLACE TABLE dev.stg_customer_orders CLONE dev.stg_orders", - ), - AdapterReplaceWithCloneTestCase( - description="snowflake preserves a transient origin table type", - adapter=SnowflakeAdapter(), - origin_is_transient=True, - expected_statement=( - "CREATE OR REPLACE TRANSIENT TABLE dev.stg_customer_orders CLONE dev.stg_orders" + +@pytest.mark.parametrize( + "test_case", + [ + AdapterReplaceWithCloneTestCase( + description="snowflake replaces with a zero-copy clone", + adapter=SnowflakeAdapter(), + origin_is_transient=False, + expected_statement="CREATE OR REPLACE TABLE dev.stg_customer_orders CLONE dev.stg_orders", ), - ), - AdapterReplaceWithCloneTestCase( - description="databricks uses an independent deep clone", - adapter=DatabricksAdapter(), - origin_is_transient=False, - expected_statement=( - "CREATE OR REPLACE TABLE dev.stg_customer_orders DEEP CLONE dev.stg_orders" + AdapterReplaceWithCloneTestCase( + description="snowflake preserves a transient origin table type", + adapter=SnowflakeAdapter(), + origin_is_transient=True, + expected_statement=( + "CREATE OR REPLACE TRANSIENT TABLE dev.stg_customer_orders CLONE dev.stg_orders" + ), ), - ), - AdapterReplaceWithCloneTestCase( - description="bigquery replaces with a table clone", - adapter=BigQueryAdapter(), - origin_is_transient=False, - expected_statement=( - "CREATE OR REPLACE TABLE `dev.stg_customer_orders` CLONE `dev.stg_orders`" + AdapterReplaceWithCloneTestCase( + description="databricks uses an independent deep clone", + adapter=DatabricksAdapter(), + origin_is_transient=False, + expected_statement=( + "CREATE OR REPLACE TABLE dev.stg_customer_orders DEEP CLONE dev.stg_orders" + ), ), - ), - AdapterReplaceWithCloneTestCase( - description="duckdb copies with one replacing CTAS", - adapter=DuckDbAdapter(), - origin_is_transient=False, - expected_statement=( - "CREATE OR REPLACE TABLE dev.stg_customer_orders AS SELECT * FROM dev.stg_orders" + AdapterReplaceWithCloneTestCase( + description="bigquery replaces with a table clone", + adapter=BigQueryAdapter(), + origin_is_transient=False, + expected_statement=( + "CREATE OR REPLACE TABLE `dev.stg_customer_orders` CLONE `dev.stg_orders`" + ), ), - ), - AdapterReplaceWithCloneTestCase( - description="motherduck inherits the replacing CTAS", - adapter=MotherDuckAdapter(), - origin_is_transient=False, - expected_statement=( - "CREATE OR REPLACE TABLE dev.stg_customer_orders AS SELECT * FROM dev.stg_orders" + AdapterReplaceWithCloneTestCase( + description="duckdb copies with one replacing CTAS", + adapter=DuckDbAdapter(), + origin_is_transient=False, + expected_statement=( + "CREATE OR REPLACE TABLE dev.stg_customer_orders AS SELECT * FROM dev.stg_orders" + ), ), - ), -) - -_UNSUPPORTED_CASES: tuple[AdapterReplaceWithCloneTestCase, ...] = ( - AdapterReplaceWithCloneTestCase( - description="postgres rejects model migrations", - adapter=PostgresAdapter(), - origin_is_transient=False, - expected_statement=None, - expected_error_fragment="does not support model migrations", - ), - AdapterReplaceWithCloneTestCase( - description="sqlserver rejects model migrations", - adapter=SqlServerAdapter(), - origin_is_transient=False, - expected_statement=None, - expected_error_fragment="does not support model migrations", - ), + AdapterReplaceWithCloneTestCase( + description="motherduck inherits the replacing CTAS", + adapter=MotherDuckAdapter(), + origin_is_transient=False, + expected_statement=( + "CREATE OR REPLACE TABLE dev.stg_customer_orders AS SELECT * FROM dev.stg_orders" + ), + ), + ], + ids=lambda case: case.description, ) - - -@pytest.mark.parametrize("test_case", _SUPPORTED_CASES, ids=lambda case: case.description) def test_given_supported_adapter_when_rendering_replace_with_clone_then_returns_one_statement( test_case: AdapterReplaceWithCloneTestCase, ) -> None: @@ -94,7 +79,26 @@ def test_given_supported_adapter_when_rendering_replace_with_clone_then_returns_ assert statement == test_case.expected_statement -@pytest.mark.parametrize("test_case", _UNSUPPORTED_CASES, ids=lambda case: case.description) +@pytest.mark.parametrize( + "test_case", + [ + AdapterReplaceWithCloneTestCase( + description="postgres rejects model migrations", + adapter=PostgresAdapter(), + origin_is_transient=False, + expected_statement=None, + expected_error_fragment="does not support model migrations", + ), + AdapterReplaceWithCloneTestCase( + description="sqlserver rejects model migrations", + adapter=SqlServerAdapter(), + origin_is_transient=False, + expected_statement=None, + expected_error_fragment="does not support model migrations", + ), + ], + ids=lambda case: case.description, +) def test_given_unsupported_adapter_when_rendering_replace_with_clone_then_raises( test_case: AdapterReplaceWithCloneTestCase, ) -> None: diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index 8bc13b6fc9..8621359dd7 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -10,4 +10,11 @@ class NewestEventTestCase: description: str events: tuple[tuple[str, str], ...] relation: str - expected: tuple[str, str] | None + expected_newest_index: int | None + + +@dataclass(frozen=True) +class RepeatedEventWriteTestCase: + description: str + write_count: int + expected_stored_count: int diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py new file mode 100644 index 0000000000..c39ce64147 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -0,0 +1,42 @@ +"""Event builders for model migration state unit tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from sqlbuild.compiler.migrations.main.deterministic_event_id import ( + deterministic_migration_event_id, +) +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery + +_STARTED_AT: datetime = datetime(2026, 1, 1, tzinfo=UTC) + + +def main_relation(name: str) -> MigrationRelation: + """Return a relation in the main schema with no database qualifier.""" + + return MigrationRelation(database=None, schema="main", name=name) + + +def migration_event(*, origin: str, destination: str, day: int) -> MigrationEvent: + """Return a manual migrate event recorded on the given day.""" + + return MigrationEvent( + event_id=deterministic_migration_event_id( + run_id=f"run-{day}", + target_name="dev", + origin=main_relation(origin), + destination=main_relation(destination), + ), + target_name="dev", + origin_model=origin, + origin=main_relation(origin), + destination_model=destination, + destination=main_relation(destination), + origin_version_hash=f"hash-{origin}", + discovery=MigrationDiscovery.MANUAL, + decision=MigrationDecision.MIGRATE, + run_id=f"run-{day}", + created_at=_STARTED_AT + timedelta(days=day), + ) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py index d4c28f4d02..5484ce2683 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -2,48 +2,22 @@ from __future__ import annotations -from datetime import UTC, datetime, timedelta - import duckdb import pytest from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.main.deterministic_event_id import ( - deterministic_migration_event_id, -) from sqlbuild.compiler.migrations.main.write_event import write_migration_event -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery -from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import NewestEventTestCase - -_STARTED_AT: datetime = datetime(2026, 1, 1, tzinfo=UTC) - - -def _relation(name: str) -> MigrationRelation: - return MigrationRelation(database=None, schema="main", name=name) - - -def _event(*, origin: str, destination: str, day: int) -> MigrationEvent: - return MigrationEvent( - event_id=deterministic_migration_event_id( - run_id=f"run-{day}", - target_name="dev", - origin=_relation(origin), - destination=_relation(destination), - ), - target_name="dev", - origin_model=origin, - origin=_relation(origin), - destination_model=destination, - destination=_relation(destination), - origin_version_hash=f"hash-{origin}", - discovery=MigrationDiscovery.MANUAL, - decision=MigrationDecision.MIGRATE, - run_id=f"run-{day}", - created_at=_STARTED_AT + timedelta(days=day), - ) +from sqlbuild.compiler.migrations.models import MigrationEvent +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( + NewestEventTestCase, + RepeatedEventWriteTestCase, +) +from tests.unit.src.sqlbuild.compiler.migrations.main.helpers import ( + main_relation, + migration_event, +) @pytest.mark.parametrize( @@ -53,25 +27,25 @@ def _event(*, origin: str, destination: str, day: int) -> MigrationEvent: description="day nine destination sees the day five event as origin", events=(("stg_orders", "stg_customer_orders"), ("stg_customer_orders", "stg_orders")), relation="stg_customer_orders", - expected=("stg_customer_orders", "stg_orders"), + expected_newest_index=1, ), NewestEventTestCase( description="day five destination sees the day one event as origin", events=(("stg_orders", "stg_customer_orders"),), relation="stg_orders", - expected=("stg_orders", "stg_customer_orders"), + expected_newest_index=0, ), NewestEventTestCase( description="unrelated relation has no event", events=(("stg_orders", "stg_customer_orders"),), relation="dim_customers", - expected=None, + expected_newest_index=None, ), NewestEventTestCase( description="relation identity is case insensitive", events=(("STG_ORDERS", "Stg_Customer_Orders"),), relation="stg_customer_orders", - expected=("STG_ORDERS", "Stg_Customer_Orders"), + expected_newest_index=0, ), ], ids=lambda case: case.description, @@ -80,26 +54,39 @@ def test_given_event_history_when_projecting_newest_mention_then_returns_latest_ test_case: NewestEventTestCase, ) -> None: events: tuple[MigrationEvent, ...] = tuple( - _event(origin=origin, destination=destination, day=index * 4 + 1) + migration_event(origin=origin, destination=destination, day=index * 4 + 1) for index, (origin, destination) in enumerate(test_case.events) ) + events_by_index: dict[int | None, MigrationEvent] = dict(enumerate(events)) newest: MigrationEvent | None = newest_migration_event_mentioning( - events=reversed(events), relation=_relation(test_case.relation) + events=reversed(events), relation=main_relation(test_case.relation) ) - projected: tuple[str, str] | None = ( - None if newest is None else (newest.origin.name, newest.destination.name) - ) - assert projected == test_case.expected + assert newest == events_by_index.get(test_case.expected_newest_index) -def test_given_repeated_event_write_when_reading_then_event_is_stored_once() -> None: +@pytest.mark.parametrize( + "test_case", + [ + RepeatedEventWriteTestCase( + description="writing the same event twice stores it once", + write_count=2, + expected_stored_count=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_repeated_event_write_when_reading_then_event_is_stored_once( + test_case: RepeatedEventWriteTestCase, +) -> None: adapter: DuckDbAdapter = DuckDbAdapter() connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") - event: MigrationEvent = _event(origin="stg_orders", destination="stg_customer_orders", day=1) + event: MigrationEvent = migration_event( + origin="stg_orders", destination="stg_customer_orders", day=1 + ) - for _ in range(2): + for _ in range(test_case.write_count): write_migration_event( connection=connection, execute=adapter.execute, @@ -116,6 +103,7 @@ def test_given_repeated_event_write_when_reading_then_event_is_stored_once() -> render_qualified_name=adapter.render_qualified_name, ) + assert len(stored) == test_case.expected_stored_count assert stored == (event,) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py new file mode 100644 index 0000000000..0446424ba6 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py @@ -0,0 +1,25 @@ +"""Test case types for migration fingerprint helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import BASE_CONFIG + + +@dataclass(frozen=True) +class MigrationFingerprintTestCase: + description: str + origin_sql: str + destination_sql: str + expected_match: bool + origin_config: dict[str, object] = field(default_factory=lambda: dict(BASE_CONFIG)) + destination_config: dict[str, object] = field(default_factory=lambda: dict(BASE_CONFIG)) + destination_ref_identities: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class UnparseableFingerprintTestCase: + description: str + query_sql: str + expected_fingerprint: str | None diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py new file mode 100644 index 0000000000..d5a94a4f24 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -0,0 +1,28 @@ +"""Fingerprint builders for migration fingerprint unit tests.""" + +from __future__ import annotations + +import json + +from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint + +BASE_CONFIG: dict[str, object] = { + "materialized": "incremental", + "incremental_strategy": "delete_insert", + "unique_key": "order_id", + "cursor": "order_date", +} +ORDERS_SQL: str = 'SELECT o.order_id, o.amount_cents FROM __ref("stg_orders") AS o' + + +def model_fingerprint( + *, model_name: str, query_sql: str, config: dict[str, object], ref_identities: dict[str, str] +) -> str | None: + """Return the DuckDB migration fingerprint for one model definition.""" + + return build_migration_fingerprint( + query_sql=query_sql, + metadata_json=json.dumps({"model_name": model_name, "config": config}), + ref_identities=ref_identities, + dialect="duckdb", + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py new file mode 100644 index 0000000000..45f75dc7eb --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py @@ -0,0 +1,148 @@ +"""Unit coverage for local migration fingerprints.""" + +from __future__ import annotations + +import pytest + +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( + MigrationFingerprintTestCase, + UnparseableFingerprintTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import ( + BASE_CONFIG, + ORDERS_SQL, + model_fingerprint, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationFingerprintTestCase( + expected_match=True, + description="table_alias_renamed", + origin_sql=ORDERS_SQL, + destination_sql=( + "SELECT customer_order.order_id, customer_order.amount_cents " + 'FROM __ref("stg_orders") AS customer_order' + ), + ), + MigrationFingerprintTestCase( + expected_match=True, + description="import_cte_renamed_with_upstream", + origin_sql=( + 'WITH stg_orders AS (SELECT * FROM __ref("stg_orders")) ' + "SELECT stg_orders.order_id FROM stg_orders" + ), + destination_sql=( + 'WITH stg_customer_orders AS (SELECT * FROM __ref("stg_customer_orders")) ' + "SELECT stg_customer_orders.order_id FROM stg_customer_orders" + ), + destination_ref_identities={"stg_customer_orders": "stg_orders"}, + ), + MigrationFingerprintTestCase( + expected_match=True, + description="comments_and_formatting", + origin_sql=ORDERS_SQL, + destination_sql=( + "-- orders with amounts\n" + "SELECT\n o.order_id,\n /* cents */ o.amount_cents\n" + 'FROM __ref("stg_orders") AS o\n' + ), + ), + MigrationFingerprintTestCase( + expected_match=True, + description="storage_config_changed", + origin_sql=ORDERS_SQL, + destination_sql=ORDERS_SQL, + destination_config={**BASE_CONFIG, "incremental_strategy": "merge", "lookback": "1d"}, + ), + MigrationFingerprintTestCase( + expected_match=True, + description="cursor_inputs_follow_renamed_upstream", + origin_sql=ORDERS_SQL, + destination_sql=ORDERS_SQL.replace("stg_orders", "stg_customer_orders"), + origin_config={**BASE_CONFIG, "cursor_inputs": {"stg_orders": "order_date"}}, + destination_config={ + **BASE_CONFIG, + "cursor_inputs": {"stg_customer_orders": "order_date"}, + }, + destination_ref_identities={"stg_customer_orders": "stg_orders"}, + ), + MigrationFingerprintTestCase( + expected_match=False, + description="filter_added", + origin_sql=ORDERS_SQL, + destination_sql=f"{ORDERS_SQL} WHERE o.amount_cents > 0", + ), + MigrationFingerprintTestCase( + expected_match=False, + description="unresolved_upstream_rename", + origin_sql=ORDERS_SQL, + destination_sql=ORDERS_SQL.replace("stg_orders", "stg_customer_orders"), + ), + MigrationFingerprintTestCase( + expected_match=False, + description="column_renamed", + origin_sql=ORDERS_SQL, + destination_sql=( + 'SELECT o.order_id, o.amount_cents AS total_cents FROM __ref("stg_orders") AS o' + ), + ), + MigrationFingerprintTestCase( + expected_match=False, + description="unique_key_changed", + origin_sql=ORDERS_SQL, + destination_sql=ORDERS_SQL, + destination_config={**BASE_CONFIG, "unique_key": "amount_cents"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_definition_when_fingerprinting_then_matches_only_equivalent_logic( + test_case: MigrationFingerprintTestCase, +) -> None: + origin: str | None = model_fingerprint( + model_name="stg_orders", + query_sql=test_case.origin_sql, + config=test_case.origin_config, + ref_identities={}, + ) + destination: str | None = model_fingerprint( + model_name="stg_customer_orders", + query_sql=test_case.destination_sql, + config=test_case.destination_config, + ref_identities=test_case.destination_ref_identities, + ) + + assert origin is not None + assert destination is not None + assert (origin == destination) is test_case.expected_match + + +@pytest.mark.parametrize( + "test_case", + [ + UnparseableFingerprintTestCase( + description="unparseable query has no fingerprint", + query_sql="SELECT FROM WHERE (", + expected_fingerprint=None, + ) + ], + ids=lambda case: case.description, +) +def test_given_unparseable_sql_when_fingerprinting_then_returns_none( + test_case: UnparseableFingerprintTestCase, +) -> None: + fingerprint: str | None = model_fingerprint( + model_name="stg_orders", + query_sql=test_case.query_sql, + config=BASE_CONFIG, + ref_identities={}, + ) + + assert fingerprint == test_case.expected_fingerprint + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 19f1a53e757d4dc73a02b42922d9f5f5a1232360 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:19:52 +0100 Subject: [PATCH 04/20] fix(migrations): redo interrupted clones and fail on missing origins A pending migration whose destination exists without any recorded event and without a fingerprint of its own is now a 'redo': the clone runs again and the event is recorded, so a crash between clone and record converges on a plain re-run. A destination with its own build history still conflicts (M103) until migrate_force is set. A migrate_from origin that does not exist, with no recorded migration into the destination, now blocks the build with M102 and tells the user to remove migrate_from if the migration already happened elsewhere or is no longer needed. A recorded migration still reports done. --- .../_helpers/build_planning/plan_policies.py | 20 ++- src/sqlbuild/compiler/migrations/types.py | 11 ++ .../planner/_helpers/migrations/planning.py | 16 ++- src/sqlbuild/compiler/planner/models.py | 2 +- .../test_manual_migrations.py | 119 +++++++++++++++--- 5 files changed, 144 insertions(+), 24 deletions(-) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py index de30fa3e8d..3fcabe3ac4 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -63,14 +63,30 @@ def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: ) if not blocked: return + missing: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry for entry in blocked if entry.decision == MigrationDecision.ORIGIN_MISSING + ) + if missing: + raise CliUserError( + "model migration origin does not exist for " + + ", ".join( + f"'{entry.model_name}' (migrate_from " + f"{entry.origin.qualified_name or entry.origin.name})" + for entry in missing + ) + + " and no recorded migration into it was found; if the migration already happened " + "elsewhere or is no longer needed, remove migrate_from from the model header", + code="M102", + help="Run sqb plan to see every model migration decision.", + ) conflicts: tuple[ModelMigrationPlanEntry, ...] = tuple( entry for entry in blocked if entry.decision == MigrationDecision.CONFLICT ) names: str = ", ".join(f"'{entry.model_name}'" for entry in blocked) if conflicts: raise CliUserError( - f"model migration conflict for {names}: the destination already exists without a " - "recorded migration", + f"model migration conflict for {names}: the destination already exists with its " + "own build history and no recorded migration", code="M103", help=( "Set migrate_force true on the model to replace the destination; the replaced " diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py index 6fad62c1c1..9c406ff181 100644 --- a/src/sqlbuild/compiler/migrations/types.py +++ b/src/sqlbuild/compiler/migrations/types.py @@ -10,6 +10,7 @@ class MigrationDecision(StrEnum): DONE = "done" MIGRATE = "migrate" + REDO = "redo" SUPERSEDED_REPLACE = "superseded_replace" FORCED_REPLACE = "forced_replace" CONFLICT = "conflict" @@ -21,6 +22,12 @@ def moves_data(self) -> bool: return self in _MOVING_DECISIONS + @property + def blocks_build(self) -> bool: + """Return whether this decision stops a build regardless of compatibility.""" + + return self in _BLOCKING_DECISIONS + @property def checks_compatibility(self) -> bool: """Return whether this decision evaluated the origin against the destination.""" @@ -37,10 +44,14 @@ def label(self) -> str: _MOVING_DECISIONS: frozenset[MigrationDecision] = frozenset( { MigrationDecision.MIGRATE, + MigrationDecision.REDO, MigrationDecision.SUPERSEDED_REPLACE, MigrationDecision.FORCED_REPLACE, } ) +_BLOCKING_DECISIONS: frozenset[MigrationDecision] = frozenset( + {MigrationDecision.CONFLICT, MigrationDecision.ORIGIN_MISSING} +) class MigrationDiscovery(StrEnum): diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 650c87dde4..93e90ae882 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -381,6 +381,8 @@ def _decide( decision = MigrationDecision.MIGRATE elif newest is not None and newest.origin.matches(migration_relation_for_location(destination)): decision = MigrationDecision.SUPERSEDED_REPLACE + elif newest is None and model.name not in snapshot.fingerprints.models: + decision = MigrationDecision.REDO elif request.force: decision = MigrationDecision.FORCED_REPLACE else: @@ -575,10 +577,11 @@ def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: if entry.decision == MigrationDecision.ORIGIN_MISSING: return PlanWarning( model_name=entry.model_name, - severity=WarningSeverity.WARNING, + severity=WarningSeverity.ERROR, message=( - f"migration origin {origin} does not exist; '{entry.model_name}' is built " - "without migrating history" + f"model '{entry.model_name}': migrate_from origin {origin} does not exist and " + "no recorded migration into it was found; if the migration already happened " + "elsewhere or is no longer needed, remove migrate_from from the model header" ), code="M102", ) @@ -587,9 +590,10 @@ def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: model_name=entry.model_name, severity=WarningSeverity.ERROR, message=( - f"migration conflict: {destination} already exists and has no recorded " - f"migration from {origin}; set migrate_force true to replace it (the replaced " - "table remains recoverable through warehouse time travel)" + f"migration conflict: {destination} already exists with its own build history " + f"and has no recorded migration from {origin}; set migrate_force true to " + "replace it (the replaced table remains recoverable through warehouse time " + "travel)" ), code="M103", ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index dceeb75433..00ab086a6e 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -986,7 +986,7 @@ class ModelMigrationPlanEntry: def blocks_build(self) -> bool: """Return whether this migration must stop a build before any execution.""" - return self.decision == MigrationDecision.CONFLICT or ( + return self.decision.blocks_build or ( self.decision.moves_data and self.compatibility == MigrationCompatibility.INCOMPATIBLE ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index 4dd0ff3191..d61e7219b0 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -177,16 +177,16 @@ def test_given_back_and_forth_renames_when_building_each_day_then_every_move_con "test_case", [ MigrationOutcomeTestCase( - description="unrecorded destination conflicts until migrate_force replaces it", - expected_decisions=("conflict", "forced_replace", "done"), - expected_events=(("stg_orders", "stg_customer_orders", "forced_replace"),), + description="destination without build history is replaced by a redo", + expected_decisions=("redo", "done"), + expected_events=(("stg_orders", "stg_customer_orders", "redo"),), expected_destination_ids=tuple(range(1, 6)), - expected_output_fragment="model migration conflict", + expected_output_fragment="Migrated main.stg_orders -> main.stg_customer_orders", ) ], ids=lambda case: case.description, ) -def test_given_unrecorded_existing_destination_when_building_then_conflict_until_forced( +def test_given_unrecorded_destination_without_history_when_building_then_redoes_the_clone( test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: build_origin(project_dir=tmp_path, capsys=capsys) @@ -199,6 +199,51 @@ def test_given_unrecorded_existing_destination_when_building_then_conflict_until ) rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + after: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) + migration_decisions(after) == test_case.expected_decisions + assert test_case.expected_output_fragment in result.output + assert ( + order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") + == test_case.expected_destination_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="destination with its own build history conflicts until forced", + expected_decisions=("conflict", "forced_replace", "done"), + expected_events=(("stg_orders", "stg_customer_orders", "forced_replace"),), + expected_destination_ids=tuple(range(1, 6)), + expected_output_fragment="model migration conflict", + ) + ], + ids=lambda case: case.description, +) +def test_given_destination_with_build_history_when_building_then_conflict_until_forced( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={ + ORIGIN_MODEL: incremental_orders_sql(), + DESTINATION_MODEL: incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE order_id <= 2" + ) + ), + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + conflict_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) conflict: CliRun = build(project_dir=tmp_path, capsys=capsys) preserved: tuple[int, ...] = order_ids( @@ -218,7 +263,8 @@ def test_given_unrecorded_existing_destination_when_building_then_conflict_until ) == test_case.expected_decisions assert conflict.exit_code == 1 assert test_case.expected_output_fragment in conflict.output - assert preserved == (999,) + assert "M103" in conflict.output + assert preserved == (1, 2) assert ( order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") == test_case.expected_destination_ids @@ -230,15 +276,18 @@ def test_given_unrecorded_existing_destination_when_building_then_conflict_until "test_case", [ MigrationOutcomeTestCase( - description="missing origin builds without history and records nothing", + description="missing origin without a recorded migration fails the build", expected_decisions=("origin_missing",), - expected_destination_ids=(1, 2, 3), - expected_output_fragment="does not exist", + expected_output_fragment=( + "'stg_customer_orders' (migrate_from main.retired_orders) and no recorded " + "migration into it was found; if the migration already happened elsewhere or " + "is no longer needed, remove migrate_from from the model header" + ), ) ], ids=lambda case: case.description, ) -def test_given_missing_origin_when_building_then_warns_and_builds_without_history( +def test_given_missing_origin_without_event_when_building_then_fails_before_building( test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: write_project( @@ -247,6 +296,46 @@ def test_given_missing_origin_when_building_then_warns_and_builds_without_histor ) load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + text: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + result: CliRun = build(project_dir=tmp_path, capsys=capsys) + + assert migration_decisions(plan) == test_case.expected_decisions + assert tuple(warning["severity"] for warning in plan["warnings"]) == ("error",) + assert plan["warnings"][0]["message"] == ( + "model 'stg_customer_orders': migrate_from origin main.retired_orders does not exist " + "and no recorded migration into it was found; if the migration already happened " + "elsewhere or is no longer needed, remove migrate_from from the model header" + ) + assert "origin missing main.retired_orders -> main.stg_customer_orders" in text.output + assert result.exit_code == 1 + assert "M102" in result.output + assert test_case.expected_output_fragment in result.output + assert relation_names(project_dir=tmp_path, schema="main") == ["raw_orders"] + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationOutcomeTestCase( + description="recorded migration keeps a leftover migrate_from harmless", + expected_decisions=("done",), + expected_events=(("stg_orders", "stg_customer_orders", "migrate"),), + expected_destination_ids=tuple(range(1, 8)), + expected_output_fragment="migrate_from can be removed from 'stg_customer_orders'", + ) + ], + ids=lambda case: case.description, +) +def test_given_missing_origin_with_recorded_migration_when_building_then_reports_done( + test_case: MigrationOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_origin(project_dir=tmp_path, capsys=capsys) + rename_model(project_dir=tmp_path, name=DESTINATION_MODEL, migrate_from=ORIGIN_MODEL) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + execute(project_dir=tmp_path, sql=f"DROP TABLE main.{ORIGIN_MODEL}") + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=7) + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) @@ -256,7 +345,7 @@ def test_given_missing_origin_when_building_then_warns_and_builds_without_histor order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") == test_case.expected_destination_ids ) - assert "_sqlbuild_migrations" not in relation_names(project_dir=tmp_path, schema="main") + assert migration_events(project_dir=tmp_path) == test_case.expected_events @pytest.mark.parametrize( @@ -484,12 +573,12 @@ def test_given_origin_schema_when_migrating_then_normal_incremental_rules_decide expected_final_decisions=("migrate",), ), MigrationInterruptionTestCase( - description="crash before recording leaves an unrecorded destination", + description="crash before recording redoes the clone on a plain re-run", install_failure=fail_record, - rerun_with_force=True, + rerun_with_force=False, expected_first_exit_code=1, - expected_decision_after_failure="conflict", - expected_final_decisions=("forced_replace",), + expected_decision_after_failure="redo", + expected_final_decisions=("redo",), ), MigrationInterruptionTestCase( description="crash after recording continues from the migrated history", From d8dab52760f1c067a61e592545fa7e7865760130 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:35:36 +0100 Subject: [PATCH 05/20] fix(migrations): normalize only CTE and alias positions in fingerprints Local-name canonicalization now rewrites CTE definitions, unqualified CTE references, relation alias definitions, and alias qualifiers of columns and stars. Column names and other identifiers are never rewritten, so a.a and b.b no longer fingerprint alike. --- .../_helpers/migrations/fingerprint.py | 101 +++++++++++++----- .../migrations/test_migration_fingerprint.py | 38 +++++++ 2 files changed, 114 insertions(+), 25 deletions(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py index 91bf766e9b..1e8086de6f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py @@ -31,7 +31,10 @@ _COMMENTS_SUFFIX: str = "comments" _CONFIG_KEY: str = "config" _GENERIC_DIALECT: str = "generic" -_ALIASED_RELATION_KEYS: frozenset[str] = frozenset({"table", "subquery"}) +_TABLE_KEY: str = "table" +_ALIASED_RELATION_KEYS: frozenset[str] = frozenset({_TABLE_KEY, "subquery"}) +_QUALIFIED_REFERENCE_KEYS: frozenset[str] = frozenset({"column", "star"}) +_RELATION_QUALIFIER_KEYS: tuple[str, ...] = ("schema", "catalog") def build_migration_fingerprint( @@ -106,11 +109,14 @@ def _normalized_query_sql( except polyglot.PolyglotError: return None stripped: Any = _strip_formatting(parsed) + definitions: list[tuple[str, bool]] = _local_names(stripped) + ordered_names: list[str] = list(dict.fromkeys(name for name, _ in definitions)) local_names: dict[str, str] = { name: f"{MIGRATION_LOCAL_NAME_PREFIX}{index}" - for index, name in enumerate(dict.fromkeys(_local_names(stripped)), start=1) + for index, name in enumerate(ordered_names, start=1) } - canonical: Any = _rename_identifiers(node=stripped, names=local_names) + cte_names: dict[str, str] = {name: local_names[name] for name, is_cte in definitions if is_cte} + canonical: Any = _canonical_local_names(node=stripped, names=local_names, cte_names=cte_names) generated: list[str] = polyglot.generate(canonical, dialect=effective_dialect) return "\n".join(generated) @@ -132,10 +138,10 @@ def _strip_formatting(node: Any) -> Any: return node -def _local_names(node: Any) -> list[str]: - """Return CTE and relation alias names in first-seen order.""" +def _local_names(node: Any) -> list[tuple[str, bool]]: + """Return (name, is_cte) for CTE and relation alias definitions in first-seen order.""" - found: list[str] = [] + found: list[tuple[str, bool]] = [] if isinstance(node, list): item: Any for item in node: @@ -145,20 +151,15 @@ def _local_names(node: Any) -> list[str]: return found with_clause: Any = node.get(_WITH_KEY) if isinstance(with_clause, dict): - found.extend( - _identifier_names( - [ - cte.get(_ALIAS_KEY) - for cte in with_clause.get(_CTES_KEY) or () - if isinstance(cte, dict) - ] - ) - ) + cte_aliases: list[Any] = [ + cte.get(_ALIAS_KEY) for cte in with_clause.get(_CTES_KEY) or () if isinstance(cte, dict) + ] + found.extend((name, True) for name in _identifier_names(cte_aliases)) key: str value: Any for key, value in node.items(): - if key in _ALIASED_RELATION_KEYS and isinstance(value, dict): - found.extend(_identifier_names([value.get(_ALIAS_KEY)])) + if key in _ALIASED_RELATION_KEYS and _is_relation(value): + found.extend((name, False) for name in _identifier_names([value.get(_ALIAS_KEY)])) found.extend(_local_names(value)) return found @@ -171,17 +172,67 @@ def _identifier_names(identifiers: list[Any]) -> list[str]: ] -def _rename_identifiers(*, node: Any, names: Mapping[str, str]) -> Any: +def _canonical_local_names( + *, node: Any, names: Mapping[str, str], cte_names: Mapping[str, str] +) -> Any: + """Rename CTE and alias names only where they define or qualify a relation.""" + if isinstance(node, list): - return [_rename_identifiers(node=value, names=names) for value in node] + return [ + _canonical_local_names(node=value, names=names, cte_names=cte_names) for value in node + ] if not isinstance(node, dict): return node - if _is_identifier(node): - canonical: str | None = names.get(str(node[_IDENTIFIER_NAME_KEY]).lower()) - if canonical is not None: - return {**node, _IDENTIFIER_NAME_KEY: canonical, _IDENTIFIER_QUOTED_KEY: False} - return node - return {key: _rename_identifiers(node=value, names=names) for key, value in node.items()} + rewritten: dict[str, Any] = { + key: _canonical_local_names(node=value, names=names, cte_names=cte_names) + for key, value in node.items() + } + key: str + value: Any + for key, value in node.items(): + if key in _QUALIFIED_REFERENCE_KEYS and isinstance(value, dict): + rewritten[key] = { + **rewritten[key], + _TABLE_KEY: _renamed(identifier=value.get(_TABLE_KEY), names=names), + } + elif key in _ALIASED_RELATION_KEYS and _is_relation(value): + rewritten[key] = { + **rewritten[key], + _ALIAS_KEY: _renamed(identifier=value.get(_ALIAS_KEY), names=names), + **_cte_reference(relation=value, cte_names=cte_names), + } + elif key == _WITH_KEY and isinstance(value, dict): + rewritten[key] = { + **rewritten[key], + _CTES_KEY: [ + {**cte, _ALIAS_KEY: _renamed(identifier=cte.get(_ALIAS_KEY), names=names)} + for cte in rewritten[key].get(_CTES_KEY) or () + ], + } + return rewritten + + +def _cte_reference(*, relation: dict[str, Any], cte_names: Mapping[str, str]) -> dict[str, Any]: + """Rename an unqualified relation name that refers to a CTE.""" + + qualifiers: tuple[Any, ...] = tuple(relation.get(key) for key in _RELATION_QUALIFIER_KEYS) + name: Any = relation.get(_IDENTIFIER_NAME_KEY) + if any(qualifier is not None for qualifier in qualifiers) or not _is_identifier(name): + return {} + return {_IDENTIFIER_NAME_KEY: _renamed(identifier=name, names=cte_names)} + + +def _renamed(*, identifier: Any, names: Mapping[str, str]) -> Any: + if not _is_identifier(identifier): + return identifier + canonical: str | None = names.get(str(identifier[_IDENTIFIER_NAME_KEY]).lower()) + if canonical is None: + return identifier + return {**identifier, _IDENTIFIER_NAME_KEY: canonical, _IDENTIFIER_QUOTED_KEY: False} + + +def _is_relation(node: Any) -> bool: + return isinstance(node, dict) and not _is_identifier(node) def _is_identifier(node: Any) -> bool: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py index 45f75dc7eb..e6297ac28e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py @@ -18,6 +18,44 @@ @pytest.mark.parametrize( "test_case", [ + MigrationFingerprintTestCase( + expected_match=False, + description="alias rename does not rewrite the qualified column name", + origin_sql='SELECT a.a AS value FROM __ref("stg_orders") a', + destination_sql='SELECT b.b AS value FROM __ref("stg_orders") b', + ), + MigrationFingerprintTestCase( + expected_match=True, + description="cte and alias renames keep columns named like the cte", + origin_sql=( + 'WITH orders AS (SELECT orders FROM __ref("stg_orders")) ' + "SELECT orders.orders, totals.total FROM orders " + "JOIN (SELECT 1 AS total) AS totals ON TRUE" + ), + destination_sql=( + 'WITH customer_orders AS (SELECT orders FROM __ref("stg_orders")) ' + "SELECT customer_orders.orders, sums.total FROM customer_orders " + "JOIN (SELECT 1 AS total) AS sums ON TRUE" + ), + ), + MigrationFingerprintTestCase( + expected_match=False, + description="column named like the renamed cte is still a column", + origin_sql=( + 'WITH orders AS (SELECT orders FROM __ref("stg_orders")) ' + "SELECT orders.orders FROM orders" + ), + destination_sql=( + 'WITH customer_orders AS (SELECT customer_orders FROM __ref("stg_orders")) ' + "SELECT customer_orders.customer_orders FROM customer_orders" + ), + ), + MigrationFingerprintTestCase( + expected_match=True, + description="qualified star follows the alias rename", + origin_sql='SELECT o.* FROM __ref("stg_orders") AS o', + destination_sql='SELECT customer_order.* FROM __ref("stg_orders") AS customer_order', + ), MigrationFingerprintTestCase( expected_match=True, description="table_alias_renamed", From 452e7c41c8b4fe8d00a0a1579d1ed302eb2bc090 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:35:37 +0100 Subject: [PATCH 06/20] fix(migrations): match automatic migrations across the whole project Every model that declares migrate_from, qualified or not and selected or not, is excluded from automatic matching and its resolved origin leaves the candidate pool. Recorded moves into models not yet built resume as done and seed the rename map, so an interrupted coordinated rename continues downstream. Matching and ambiguity use all unbuilt project models; only selected destinations migrate. --- .../planner/_helpers/migrations/discovery.py | 266 ++++++++++++++---- .../planner/_helpers/migrations/planning.py | 53 +++- src/sqlbuild/compiler/planner/models.py | 9 + .../main/model_migrations/_test_types.py | 9 +- .../commands/main/model_migrations/helpers.py | 46 ++- .../test_automatic_migrations.py | 134 +++++++++ 6 files changed, 446 insertions(+), 71 deletions(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index 6d9ae736a1..04420c9593 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -1,4 +1,4 @@ -"""Fingerprint-based discovery of renamed or relocated models.""" +"""Fingerprint-based discovery of renamed models within the project's schemas.""" from __future__ import annotations @@ -6,7 +6,6 @@ from collections import defaultdict from typing import Any -from sqlbuild.adapter.contract.constants import QUALIFIED_NAME_SEPARATOR from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -32,6 +31,7 @@ build_model_version_identity_metadata_json, ) from sqlbuild.compiler.planner.models import ( + ModelMigrationDeclaration, ModelMigrationDiscovery, ModelMigrationRequest, PlannerRuntime, @@ -56,81 +56,217 @@ def discover_model_migrations( scope: PlannerScope, snapshot: WarehouseSnapshot, manual_requests: tuple[ModelMigrationRequest, ...], + declarations: tuple[ModelMigrationDeclaration, ...], state: MigrationStateInspection, project_schemas: set[str], ) -> ModelMigrationDiscovery: """Add unique one-to-one fingerprint matches to the explicit migration requests.""" models: tuple[CompiledModel, ...] = _topological_models(runtime=runtime) - new_names: frozenset[str] = _new_model_names(models=models, scope=scope, snapshot=snapshot) - manual_renames: dict[str, str] = { - request.model.name: request.raw_origin - for request in manual_requests - if request.raw_origin is not None and QUALIFIED_NAME_SEPARATOR not in request.raw_origin - } - discoverable: frozenset[str] = new_names - frozenset(manual_renames) - if not discoverable and not manual_requests: + declared: frozenset[str] = frozenset(item.model_name for item in declarations) + triggered: bool = bool( + _new_model_names(models=models, scope=scope, snapshot=snapshot) - declared + ) + if not triggered and not manual_requests: return ModelMigrationDiscovery() metadata_jsons: dict[str, str] = _metadata_jsons(runtime=runtime, models=models) dialect: str | None = runtime.adapter.sql_analysis_dialect() - if not discoverable: + if not triggered: return ModelMigrationDiscovery( requests=manual_requests, destination_fingerprints=_destination_fingerprints( requests=manual_requests, - renames=manual_renames, + renames=_declared_renames(declarations=declarations, state=state), metadata_jsons=metadata_jsons, dialect=dialect, ), ) state.inspect_schemas(schemas=project_schemas) + unbuilt: frozenset[str] = frozenset( + model.name + for model in models + if model.name not in declared and state.named_fingerprint(model_name=model.name) is None + ) + resumed: dict[str, MigrationEvent] = _resumed_events( + runtime=runtime, models=models, unbuilt=unbuilt, state=state + ) + seed_renames: dict[str, str] = _declared_renames(declarations=declarations, state=state) + seed_renames.update( + {name: event.origin_model for name, event in resumed.items() if event.origin_model} + ) candidates: tuple[Fingerprint, ...] = _origin_candidates( - runtime=runtime, state=state, models=models + runtime=runtime, + state=state, + models=models, + excluded=_claimed_origins(declarations=declarations, resumed=resumed), ) renames: dict[str, str] matches: dict[str, Fingerprint] - ambiguous: dict[str, tuple[str, ...]] + ambiguous: dict[str, str] renames, matches, ambiguous = _match( models=models, - new_names=new_names, - manual_renames=manual_renames, + new_names=unbuilt - frozenset(resumed), + seed_renames=seed_renames, candidates=candidates, metadata_jsons=metadata_jsons, dialect=dialect, ) - automatic: tuple[ModelMigrationRequest, ...] = tuple( - ModelMigrationRequest( - model=model, - discovery=MigrationDiscovery.AUTOMATIC, - origin_location=_fingerprint_location(runtime=runtime, fingerprint=matches[model.name]), - origin_model=matches[model.name].node_name, - ) - for model in models - if model.name in matches - and model.key in scope.selected_keys - and _moves_history(model=model, origin=matches[model.name]) + requests: tuple[ModelMigrationRequest, ...] = ( + *manual_requests, + *_automatic_requests( + runtime=runtime, scope=scope, models=models, matches=matches, resumed=resumed + ), ) - requests: tuple[ModelMigrationRequest, ...] = (*manual_requests, *automatic) return ModelMigrationDiscovery( requests=requests, - warnings=tuple( - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - message=( - f"'{model_name}' matches several earlier models ({', '.join(origins)}); " - "no automatic migration was inferred. Add migrate_from to choose one." - ), - code="M107", - ) - for model_name, origins in sorted(ambiguous.items()) - ), + warnings=_ambiguity_warnings(models=models, scope=scope, ambiguous=ambiguous), destination_fingerprints=_destination_fingerprints( requests=requests, renames=renames, metadata_jsons=metadata_jsons, dialect=dialect ), ) +def _declared_renames( + *, declarations: tuple[ModelMigrationDeclaration, ...], state: MigrationStateInspection +) -> dict[str, str]: + """Map each explicitly migrated model to the model name its origin was built as.""" + + renames: dict[str, str] = {} + declaration: ModelMigrationDeclaration + for declaration in declarations: + origin_fingerprint: Fingerprint | None = state.model_fingerprint( + location=declaration.origin_location, model_name=declaration.origin_model + ) + origin_model: str | None = declaration.origin_model or ( + origin_fingerprint.node_name if origin_fingerprint is not None else None + ) + if origin_model is not None: + renames[declaration.model_name] = origin_model + return renames + + +def _resumed_events( + *, + runtime: PlannerRuntime, + models: tuple[CompiledModel, ...], + unbuilt: frozenset[str], + state: MigrationStateInspection, +) -> dict[str, MigrationEvent]: + """Return recorded moves into models that have not been built since the move.""" + + target_name: str | None = runtime.project.effective_target_name + resumed: dict[str, MigrationEvent] = {} + model: CompiledModel + for model in models: + if model.name not in unbuilt or not _stores_history(model): + continue + destination: MigrationRelation = migration_relation_for_location(model.destination) + newest: MigrationEvent | None = newest_migration_event_mentioning( + events=state.events, relation=destination + ) + if ( + newest is not None + and newest.destination.matches(destination) + and newest.target_name in (None, target_name) + ): + resumed[model.name] = newest + return resumed + + +def _claimed_origins( + *, + declarations: tuple[ModelMigrationDeclaration, ...], + resumed: dict[str, MigrationEvent], +) -> frozenset[tuple[str, str]]: + """Return relation keys already claimed by explicit or recorded migrations.""" + + declared: set[tuple[str, str]] = { + _relation_key(schema=item.origin_location.schema, name=item.origin_location.name) + for item in declarations + } + recorded: set[tuple[str, str]] = { + _relation_key(schema=event.origin.schema, name=event.origin.name) + for event in resumed.values() + } + return frozenset(declared | recorded) + + +def _relation_key(*, schema: str | None, name: str) -> tuple[str, str]: + return ((schema or "").lower(), name.lower()) + + +def _automatic_requests( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + models: tuple[CompiledModel, ...], + matches: dict[str, Fingerprint], + resumed: dict[str, MigrationEvent], +) -> tuple[ModelMigrationRequest, ...]: + """Return automatic requests for selected destinations only.""" + + requests: list[ModelMigrationRequest] = [] + model: CompiledModel + for model in models: + if model.key not in scope.selected_keys: + continue + event: MigrationEvent | None = resumed.get(model.name) + match: Fingerprint | None = matches.get(model.name) + if event is not None: + requests.append( + ModelMigrationRequest( + model=model, + discovery=MigrationDiscovery.AUTOMATIC, + origin_location=_event_origin_location(runtime=runtime, event=event), + origin_model=event.origin_model, + ) + ) + elif match is not None and _moves_history(model=model, origin=match): + requests.append( + ModelMigrationRequest( + model=model, + discovery=MigrationDiscovery.AUTOMATIC, + origin_location=_fingerprint_location(runtime=runtime, fingerprint=match), + origin_model=match.node_name, + ) + ) + return tuple(requests) + + +def _event_origin_location( + *, runtime: PlannerRuntime, event: MigrationEvent +) -> CompiledRelationLocation: + return CompiledRelationLocation( + database=event.origin.database, + schema=event.origin.schema, + name=event.origin.name, + qualified_name=runtime.adapter.render_qualified_name( + database=event.origin.database, schema=event.origin.schema, name=event.origin.name + ), + ) + + +def _ambiguity_warnings( + *, models: tuple[CompiledModel, ...], scope: PlannerScope, ambiguous: dict[str, str] +) -> tuple[PlanWarning, ...]: + selected: frozenset[str] = frozenset( + model.name for model in models if model.key in scope.selected_keys + ) + return tuple( + PlanWarning( + model_name=model_name, + severity=WarningSeverity.WARNING, + message=( + f"'{model_name}' matches {detail}; no automatic migration was inferred. " + "Add migrate_from to choose one." + ), + code="M107", + ) + for model_name, detail in sorted(ambiguous.items()) + if model_name in selected + ) + + def _topological_models(*, runtime: PlannerRuntime) -> tuple[CompiledModel, ...]: by_name: dict[str, CompiledModel] = {model.name: model for model in runtime.project.models} ordered: list[CompiledModel] = [] @@ -199,6 +335,7 @@ def _origin_candidates( runtime: PlannerRuntime, state: MigrationStateInspection, models: tuple[CompiledModel, ...], + excluded: frozenset[tuple[str, str]], ) -> tuple[Fingerprint, ...]: project_model_names: frozenset[str] = frozenset(model.name for model in models) orphans: tuple[Fingerprint, ...] = tuple( @@ -215,6 +352,7 @@ def _origin_candidates( fingerprint for fingerprint, location in zip(orphans, locations, strict=True) if state.relation(location) is not None + and _relation_key(schema=location.schema, name=location.name) not in excluded and not _superseded(events=state.events, location=location) ) @@ -231,25 +369,24 @@ def _match( *, models: tuple[CompiledModel, ...], new_names: frozenset[str], - manual_renames: dict[str, str], + seed_renames: dict[str, str], candidates: tuple[Fingerprint, ...], metadata_jsons: dict[str, str], dialect: str | None, -) -> tuple[dict[str, str], dict[str, Fingerprint], dict[str, tuple[str, ...]]]: +) -> tuple[dict[str, str], dict[str, Fingerprint], dict[str, str]]: by_fingerprint: dict[str, list[Fingerprint]] = defaultdict(list) candidate: Fingerprint for candidate in candidates: by_fingerprint[stored_migration_fingerprint(candidate) or ""].append(candidate) - claimed_by_manual: frozenset[str] = frozenset(manual_renames.values()) blocked: set[str] = set() - ambiguous: dict[str, tuple[str, ...]] = {} + ambiguous: dict[str, str] = {} for _ in range(len(candidates) + 1): - renames: dict[str, str] = dict(manual_renames) + renames: dict[str, str] = dict(seed_renames) matches: dict[str, Fingerprint] = {} claims: dict[str, list[str]] = defaultdict(list) model: CompiledModel for model in models: - if model.name not in new_names or model.name in manual_renames: + if model.name not in new_names: continue fingerprint: str | None = build_migration_fingerprint( query_sql=model.query_sql, @@ -260,10 +397,13 @@ def _match( options: list[Fingerprint] = [ option for option in by_fingerprint.get(fingerprint or "", []) - if option.node_name not in blocked and option.node_name not in claimed_by_manual + if option.node_name not in blocked ] if len(options) > 1: - ambiguous[model.name] = tuple(sorted(option.node_name for option in options)) + ambiguous[model.name] = ( + "several earlier models " + f"({', '.join(sorted(option.node_name for option in options))})" + ) continue if len(options) == 1: renames[model.name] = options[0].node_name @@ -274,14 +414,23 @@ def _match( } if not contested: return renames, matches, ambiguous - origin: str - names: list[str] - for origin, names in contested.items(): - blocked.add(origin) - name: str - for name in names: - ambiguous[name] = (*ambiguous.get(name, ()), origin) - return dict(manual_renames), {}, ambiguous + blocked.update(contested) + ambiguous.update(_contested_details(contested)) + return dict(seed_renames), {}, ambiguous + + +def _contested_details(contested: dict[str, list[str]]) -> dict[str, str]: + """Describe each model that shares its only matching earlier model with others.""" + + details: dict[str, str] = {} + origin: str + names: list[str] + for origin, names in contested.items(): + name: str + for name in names: + others: list[str] = [f"'{other}'" for other in sorted(names) if other != name] + details[name] = f"earlier model '{origin}', which also matches {', '.join(others)}" + return details def _destination_fingerprints( @@ -309,10 +458,13 @@ def _destination_fingerprints( def _moves_history(*, model: CompiledModel, origin: Fingerprint) -> bool: + return _stores_history(model) and _stored_materialization(origin) in _HISTORY_MATERIALIZATIONS + + +def _stores_history(model: CompiledModel) -> bool: return ( get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) in _HISTORY_MATERIALIZATIONS - and _stored_materialization(origin) in _HISTORY_MATERIALIZATIONS ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 93e90ae882..655df61bb1 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -46,6 +46,7 @@ CursorSnapshotScope, DeferralInputs, ModelCursorSnapshot, + ModelMigrationDeclaration, ModelMigrationDiscovery, ModelMigrationPlanEntry, ModelMigrationPlanning, @@ -73,20 +74,44 @@ def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationReq ): continue model: CompiledModel | None = scope.models_by_name.get(getattr(key, "name", "")) - if model is None: - continue - raw_origin: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) - if not isinstance(raw_origin, str): + request: ModelMigrationRequest | None = None if model is None else _declared_request(model) + if request is not None: + requests.append(request) + return tuple(requests) + + +def _declared_request(model: CompiledModel) -> ModelMigrationRequest | None: + raw_origin: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) + if not isinstance(raw_origin, str): + return None + return ModelMigrationRequest( + model=model, + discovery=MigrationDiscovery.MANUAL, + raw_origin=raw_origin.strip(), + force=model.config.values.get(MIGRATE_FORCE_CONFIG_KEY) is True, + ) + + +def _project_declarations( + *, runtime: PlannerRuntime, state: MigrationStateInspection +) -> tuple[ModelMigrationDeclaration, ...]: + """Resolve every project model's migrate_from, selected or not.""" + + declarations: list[ModelMigrationDeclaration] = [] + model: CompiledModel + for model in runtime.project.models: + request: ModelMigrationRequest | None = _declared_request(model) + if request is None: continue - requests.append( - ModelMigrationRequest( - model=model, - discovery=MigrationDiscovery.MANUAL, - raw_origin=raw_origin.strip(), - force=model.config.values.get(MIGRATE_FORCE_CONFIG_KEY) is True, + origin: CompiledRelationLocation + origin_model: str | None + origin, origin_model = _resolve_origin(request=request, runtime=runtime, state=state) + declarations.append( + ModelMigrationDeclaration( + model_name=model.name, origin_location=origin, origin_model=origin_model ) ) - return tuple(requests) + return tuple(declarations) def plan_model_migrations( @@ -106,13 +131,17 @@ def plan_model_migrations( connection=runtime.connection, database=planning_database(runtime=runtime), ) - if manual: + declares_migrations: bool = any( + _declared_request(model) is not None for model in runtime.project.models + ) + if declares_migrations: state.inspect_schemas(schemas=schemas) discovery: ModelMigrationDiscovery = discover_model_migrations( runtime=runtime, scope=scope, snapshot=snapshot, manual_requests=manual, + declarations=_project_declarations(runtime=runtime, state=state), state=state, project_schemas=schemas, ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 00ab086a6e..2a25abc56e 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -1502,6 +1502,15 @@ class ModelMigrationRequest: force: bool = False +@dataclass(frozen=True) +class ModelMigrationDeclaration: + """One project model's explicit migrate_from, resolved to a relation.""" + + model_name: str + origin_location: CompiledRelationLocation + origin_model: str | None + + @dataclass(frozen=True) class ModelMigrationDiscovery: """Manual and automatically discovered migration requests for one plan.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index ea681edc5b..4f50695d2a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -58,7 +58,7 @@ class MigrationInterruptionTestCase: @dataclass(frozen=True) class AutomaticMigrationTestCase: description: str - expected_migrations: tuple[tuple[str, str, str, str], ...] = () + expected_migrations: tuple[tuple[str | None, str, str, str], ...] = () expected_events: tuple[tuple[str, str, str], ...] = () expected_reason: str = "" expected_warning: str = "" @@ -71,3 +71,10 @@ class PlanAsTargetTestCase: expected_exit_code: int expected_decisions: tuple[str, ...] = () expected_fragment: str = "" + + +@dataclass(frozen=True) +class VirtualModeMigrationTestCase: + description: str + expected_exit_code: int + expected_fragment: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 897f819ac2..ae949f909a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from textwrap import dedent @@ -41,6 +42,29 @@ schema = "prod" """ ).lstrip() +VIRTUAL_PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + default_target = "dev" + + [settings] + virtual_environments = true + + [connection] + database = "{DATABASE_FILE}" + + [targets.dev] + schema = "dev" + + [targets.dev.state] + backend = "duckdb" + schema = "sqlbuild_state" + + [targets.dev.state.connection] + database = "state.duckdb" + """ +).lstrip() RAW_SOURCES_YML: str = dedent( """ sources: @@ -267,7 +291,7 @@ def migration_decisions(plan: dict[str, Any]) -> tuple[str, ...]: return tuple(migration["decision"] for migration in plan["migrations"]) -def planned_migrations(plan: dict[str, Any]) -> tuple[tuple[str, str, str, str], ...]: +def planned_migrations(plan: dict[str, Any]) -> tuple[tuple[str | None, str, str, str], ...]: """Return sorted (origin, destination, discovery, decision) for every planned migration.""" return tuple( @@ -345,6 +369,26 @@ def fail_clone(monkeypatch: pytest.MonkeyPatch) -> None: ) +def fail_clone_into(*, monkeypatch: pytest.MonkeyPatch, destination: str) -> None: + """Make only the clone into one destination fail; every other clone runs normally.""" + + original: Callable[..., str] = DuckDbAdapter.render_replace_with_clone + failing: dict[str, str] = {destination: "SELECT * FROM main.simulated_missing_relation"} + monkeypatch.setattr( + DuckDbAdapter, + "render_replace_with_clone", + lambda self, *, origin, destination, origin_is_transient=False: failing.get( + destination, + original( + self, + origin=origin, + destination=destination, + origin_is_transient=origin_is_transient, + ), + ), + ) + + def fail_record(monkeypatch: pytest.MonkeyPatch) -> None: """Crash after the clone but before the migration event is recorded.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py index 11cde7ba48..466e7313e3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py @@ -12,10 +12,12 @@ ) from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( CliRun, + build, build_ok, build_original_order_models, daily_totals_sql, execute, + fail_clone_into, incremental_orders_sql, load_raw_orders, migration_events, @@ -24,8 +26,10 @@ original_order_models, plan_json, planned_migrations, + relation_names, renamed_order_models, row_count, + run_sqb, warning_messages, write_project, ) @@ -246,5 +250,135 @@ def test_given_explicit_migrate_from_when_fingerprint_matches_another_then_manua assert migration_events(project_dir=tmp_path) == test_case.expected_events +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="qualified migrate_from removes its origin and destination from matching", + expected_migrations=((None, "stg_customer_orders", "manual", "migrate"),), + expected_events=(("legacy_orders", "stg_customer_orders", "migrate"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_qualified_migrate_from_when_another_orphan_matches_then_only_declared_moves( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={ + "stg_orders": incremental_orders_sql(), + "legacy_orders": incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE order_id <= 2" + ) + ), + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=4) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={"stg_customer_orders": incremental_orders_sql(migrate_from="main.legacy_orders")}, + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_migrations(plan) == test_case.expected_migrations + assert migration_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="recorded upstream move resolves the downstream rename on retry", + expected_migrations=( + ("daily_order_totals", "customer_daily_order_totals", "automatic", "migrate"), + ("stg_orders", "stg_customer_orders", "automatic", "done"), + ), + expected_events=( + ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("stg_orders", "stg_customer_orders", "migrate"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_interrupted_coordinated_renames_when_rerunning_then_downstream_resumes( + test_case: AutomaticMigrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=3, last_day=8) + write_project(project_dir=tmp_path, models=renamed_order_models()) + + with monkeypatch.context() as patch: + fail_clone_into(monkeypatch=patch, destination="main.customer_daily_order_totals") + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) + events_after_crash: tuple[tuple[str, str, str], ...] = migration_events(project_dir=tmp_path) + retry_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code == 1, interrupted.output + assert events_after_crash == (("stg_orders", "stg_customer_orders", "migrate"),) + assert planned_migrations(retry_plan) == test_case.expected_migrations + assert model_entry(plan=retry_plan, name="stg_customer_orders")["reason"] != _FIRST_RUN + assert order_ids(project_dir=tmp_path, relation="main.stg_customer_orders") == tuple( + range(1, 9) + ) + assert row_count(project_dir=tmp_path, relation="main.customer_daily_order_totals") == 8 + assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="selecting one of two new models still sees the whole-project ambiguity", + expected_reason="first_run", + expected_warning=( + "'stg_customer_orders' matches earlier model 'stg_orders', which also matches " + "'stg_web_orders'; no automatic migration was inferred. " + "Add migrate_from to choose one." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_one_orphan_matching_two_new_models_when_selecting_one_then_warns_ambiguous( + test_case: AutomaticMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(project_dir=tmp_path, models={"stg_orders": incremental_orders_sql()}) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={ + "stg_customer_orders": incremental_orders_sql(), + "stg_web_orders": incremental_orders_sql(), + }, + ) + + plan: dict[str, Any] = plan_json( + project_dir=tmp_path, capsys=capsys, args=("--select", "stg_customer_orders") + ) + built: CliRun = run_sqb( + project_dir=tmp_path, args=("build", "--select", "stg_customer_orders"), capsys=capsys + ) + + assert planned_migrations(plan) == test_case.expected_migrations + assert warning_messages(plan) == (test_case.expected_warning,) + assert model_entry(plan=plan, name="stg_customer_orders")["reason"] == ( + test_case.expected_reason + ) + assert built.exit_code == 0, built.output + assert "_sqlbuild_migrations" not in relation_names(project_dir=tmp_path, schema="main") + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From d326bf650ab5bb2a3101cd0f74cf801d707024bb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:35:37 +0100 Subject: [PATCH 07/20] fix(bigquery): reject model migrations BigQuery has no CREATE OR REPLACE TABLE ... CLONE, so it now raises the same unsupported-migrations error as Postgres and SQL Server. --- .../adapters/bigquery/classes/bigquery_adapter.py | 7 +++---- .../sqlbuild/adapters/test_replace_with_clone.py | 15 +++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index ae65cd3c58..6422253d77 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1856,10 +1856,9 @@ def render_durable_clone( def render_replace_with_clone( self, *, origin: str, destination: str, origin_is_transient: bool = False ) -> str: - del origin_is_transient - return ( - f"CREATE OR REPLACE TABLE {self._quote_identifier_path(destination)} " - f"CLONE {self._quote_identifier_path(origin)}" + del origin, destination, origin_is_transient + raise AdapterUserError( + message=f"adapter '{self.adapter_name}' does not support model migrations" ) def render_query_with_cursor_bounds( diff --git a/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py b/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py index f9abf793f9..bdd932cfc7 100644 --- a/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py +++ b/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py @@ -40,14 +40,6 @@ "CREATE OR REPLACE TABLE dev.stg_customer_orders DEEP CLONE dev.stg_orders" ), ), - AdapterReplaceWithCloneTestCase( - description="bigquery replaces with a table clone", - adapter=BigQueryAdapter(), - origin_is_transient=False, - expected_statement=( - "CREATE OR REPLACE TABLE `dev.stg_customer_orders` CLONE `dev.stg_orders`" - ), - ), AdapterReplaceWithCloneTestCase( description="duckdb copies with one replacing CTAS", adapter=DuckDbAdapter(), @@ -89,6 +81,13 @@ def test_given_supported_adapter_when_rendering_replace_with_clone_then_returns_ expected_statement=None, expected_error_fragment="does not support model migrations", ), + AdapterReplaceWithCloneTestCase( + description="bigquery rejects model migrations", + adapter=BigQueryAdapter(), + origin_is_transient=False, + expected_statement=None, + expected_error_fragment="does not support model migrations", + ), AdapterReplaceWithCloneTestCase( description="sqlserver rejects model migrations", adapter=SqlServerAdapter(), From 36d9e87e659268bd66e67ca6f6e0aad898cb256d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:35:37 +0100 Subject: [PATCH 08/20] fix(virtual): reject migrate_from in the virtual planning phase Move the M105 check from inline graph resolution into _plan_virtual_build, and cover it with a virtual build integration test. --- .../virtual/executor/_helpers/build.py | 2 +- .../model_migrations/test_virtual_mode.py | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py diff --git a/src/sqlbuild/virtual/executor/_helpers/build.py b/src/sqlbuild/virtual/executor/_helpers/build.py index b4f5a96ae3..3a61334f74 100644 --- a/src/sqlbuild/virtual/executor/_helpers/build.py +++ b/src/sqlbuild/virtual/executor/_helpers/build.py @@ -574,7 +574,6 @@ def _resolve_virtual_build( timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() if timing_tracker is not None: timing_tracker.compile_seconds = compile_seconds - _reject_model_migrations(project=graph.project) planning_start: float = time.monotonic() try: names: VirtualEnvironmentNames = _resolve_virtual_environment_names( @@ -902,6 +901,7 @@ def _plan_virtual_build( reads: _VirtualBuildStateReads, deferred_relations: dict[str, RelationInfo], ) -> _VirtualBuildPlan: + _reject_model_migrations(project=graph.project) adapter: BaseAdapter = runtime.adapter options: VirtualBuildOptions = runtime.options hooks: VirtualBuildHooks = runtime.hooks diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py new file mode 100644 index 0000000000..149f232b6c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py @@ -0,0 +1,54 @@ +"""Integration coverage for rejecting model migrations in virtual environments.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + VirtualModeMigrationTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + VIRTUAL_PROJECT_TOML, + CliRun, + incremental_orders_sql, + load_raw_orders, + run_sqb, + write_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + VirtualModeMigrationTestCase( + description="virtual build rejects migrate_from", + expected_exit_code=1, + expected_fragment="migrate_from is supported only in direct mode", + ) + ], + ids=lambda case: case.description, +) +def test_given_migrate_from_when_building_virtually_then_rejects_before_building( + test_case: VirtualModeMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + project_dir=tmp_path, + models={"stg_customer_orders": incremental_orders_sql(migrate_from="stg_orders")}, + project_toml=VIRTUAL_PROJECT_TOML, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) + initialized: CliRun = run_sqb(project_dir=tmp_path, args=("state", "init"), capsys=capsys) + + result: CliRun = run_sqb( + project_dir=tmp_path, args=("build", "--virtual-env", "feature"), capsys=capsys + ) + + assert initialized.exit_code == 0, initialized.output + assert result.exit_code == test_case.expected_exit_code, result.output + assert test_case.expected_fragment in result.output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 339b1418d17cbd063faa8c03b0af41f873c6a046 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:27:04 +0100 Subject: [PATCH 09/20] fix(plan): list error-severity diagnostics under an Errors heading Plan text now renders error-severity diagnostics, such as M102 missing origins and M103 conflicts, in an 'Errors (n)' section ahead of 'Warnings (n)'. JSON output already carried the severity and is unchanged. Integration tests also pin that build prints each M102/M103 error exactly once. --- src/sqlbuild/cli/output/_helpers/plan_text.py | 56 ++++++++++++++----- .../test_manual_migrations.py | 9 ++- .../cli/output/main/plan/test_plan_text.py | 24 ++++++++ 3 files changed, 73 insertions(+), 16 deletions(-) diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index 4bb16a23fd..3c5a2909d0 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -1398,7 +1398,7 @@ def _format_warnings( include_direct_freshness_diagnostics: bool, display_options: DisplayOptions, ) -> list[str]: - """Append the warnings section.""" + """Append the errors and warnings sections.""" selection_diagnostics: bool | None = direct_selection_diagnostics_enabled(plan) include_stale_input_warnings: bool = ( @@ -1406,7 +1406,7 @@ def _format_warnings( if selection_diagnostics is not None else include_direct_freshness_diagnostics ) - warning_entries: list[PlanWarning] = [ + diagnostics: list[PlanWarning] = [ warning for warning in plan.warnings if warning.severity != WarningSeverity.INFO @@ -1415,18 +1415,46 @@ def _format_warnings( or not warning.message.startswith(_STALE_INPUT_WARNING_TITLE) ) ] - if not warning_entries: + style: CliStyle = CliStyle(use_color=True) + lines = _format_diagnostic_section( + lines=lines, + entries=[entry for entry in diagnostics if entry.severity == WarningSeverity.ERROR], + title="Errors", + heading_style=style.error_strong, + line_style=style.error, + display_options=display_options, + ) + return _format_diagnostic_section( + lines=lines, + entries=[entry for entry in diagnostics if entry.severity != WarningSeverity.ERROR], + title="Warnings", + heading_style=style.warning_strong, + line_style=style.warning, + display_options=display_options, + ) + + +def _format_diagnostic_section( + *, + lines: list[str], + entries: list[PlanWarning], + title: str, + heading_style: Callable[[str], str], + line_style: Callable[[str], str], + display_options: DisplayOptions, +) -> list[str]: + """Append one severity-grouped plan diagnostic section.""" + + if not entries: return lines style: CliStyle = CliStyle(use_color=True) lines.append("") - lines.append(style.warning_strong(f"Warnings ({len(warning_entries)})")) - shown_warnings: Sequence[PlanWarning] = visible_entries( - entries=warning_entries, options=display_options - ) + lines.append(heading_style(f"{title} ({len(entries)})")) + shown: Sequence[PlanWarning] = visible_entries(entries=entries, options=display_options) warning: PlanWarning warning_index: int - for warning_index, warning in enumerate(shown_warnings): - connector: str = tree_connector(style=style, last=warning_index == len(shown_warnings) - 1) + for warning_index, warning in enumerate(shown): + connector: str = tree_connector(style=style, last=warning_index == len(shown) - 1) message_lines: list[str] = warning.message.split("\n") if warning.model_name is not None: lines.append(f"{connector} {style.object_name(warning.model_name)}") @@ -1436,16 +1464,16 @@ def _format_warnings( style=style, last=message_index == len(message_lines) - 1, ) - lines.append(f" {child_connector} {style.warning(message_line)}") + lines.append(f" {child_connector} {line_style(message_line)}") else: - lines.append(f"{connector} {style.warning(message_lines[0])}") + lines.append(f"{connector} {line_style(message_lines[0])}") continuation: str for continuation in message_lines[1:]: - lines.append(f" {style.warning(continuation)}") + lines.append(f" {line_style(continuation)}") return append_overflow_line( lines=lines, - total_count=len(warning_entries), - visible_count=len(shown_warnings), + total_count=len(entries), + visible_count=len(shown), indent=" ", options=display_options, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index d61e7219b0..9791230cc4 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -263,7 +263,8 @@ def test_given_destination_with_build_history_when_building_then_conflict_until_ ) == test_case.expected_decisions assert conflict.exit_code == 1 assert test_case.expected_output_fragment in conflict.output - assert "M103" in conflict.output + assert conflict.output.count("error[M103]") == 1 + assert "Errors (1)\n└── stg_customer_orders\n └── migration conflict:" in conflict.output assert preserved == (1, 2) assert ( order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") @@ -309,7 +310,11 @@ def test_given_missing_origin_without_event_when_building_then_fails_before_buil ) assert "origin missing main.retired_orders -> main.stg_customer_orders" in text.output assert result.exit_code == 1 - assert "M102" in result.output + assert result.output.count("error[M102]") == 1 + assert result.output.count("= help:") == 1 + assert "Errors (1)" in text.output + assert "Warnings (" not in text.output + assert "Errors (1)" in result.output assert test_case.expected_output_fragment in result.output assert relation_names(project_dir=tmp_path, schema="main") == ["raw_orders"] diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index c15779d1a8..070f29c2e0 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -897,6 +897,30 @@ "type change detected", ), ), + FormatPlanTestCase( + description="error-severity diagnostics render under an errors heading", + plan_output=build_plan_output( + warnings=( + build_warning( + model_name="stg_customer_orders", + message="migrate_from origin main.stg_orders does not exist", + severity=WarningSeverity.ERROR, + code="M102", + ), + build_warning( + model_name="stg_customers", + message="type change detected", + severity=WarningSeverity.WARNING, + ), + ), + ), + expected_fragments=( + "Errors (1)\n└── stg_customer_orders\n" + " └── migrate_from origin main.stg_orders does not exist", + "Warnings (1)\n└── stg_customers\n └── type change detected", + ), + unexpected_fragments=("Warnings (2)",), + ), FormatPlanTestCase( description="multiline warning messages terminate their child tree", plan_output=build_plan_output( From 6969711994c49819958160558bc71b208bfac6a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 02:23:55 +0100 Subject: [PATCH 10/20] fix(migrations): reject ambiguous CTE scopes and keep recorded renames Migration fingerprints are no longer produced for queries whose local names could resolve differently by position: nested or non-top-level WITH, recursive WITH, CTE bodies that reference their own or a later CTE name, and aliases that reuse a physical relation name. Such models are not eligible for automatic matching. Discovery now seeds the rename map from the latest recorded migration event per destination model on the current target, ignoring origins that are still project models, even after the destination has been built. It fingerprints each candidate with and without these recorded renames, so a retried partial chain keeps downstream history and later renames of models rebuilt under their new names still match. --- .../planner/_helpers/migrations/discovery.py | 133 +++++++++++++----- .../_helpers/migrations/fingerprint.py | 75 +++++++++- .../planner/_helpers/migrations/planning.py | 12 +- src/sqlbuild/compiler/planner/models.py | 2 +- .../commands/main/model_migrations/helpers.py | 29 +++- .../test_automatic_migrations.py | 69 +++++++++ .../_helpers/migrations/_test_types.py | 2 +- .../migrations/test_migration_fingerprint.py | 58 +++++++- 8 files changed, 325 insertions(+), 55 deletions(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index 04420c9593..6228ac8f97 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -71,17 +71,23 @@ def discover_model_migrations( return ModelMigrationDiscovery() metadata_jsons: dict[str, str] = _metadata_jsons(runtime=runtime, models=models) dialect: str | None = runtime.adapter.sql_analysis_dialect() + state.inspect_schemas(schemas=project_schemas) + declared_renames: dict[str, str] = _declared_renames(declarations=declarations, state=state) + event_renames: dict[str, str] = _event_renames(runtime=runtime, models=models, state=state) + seed_variants: tuple[dict[str, str], ...] = ( + {**event_renames, **declared_renames}, + dict(declared_renames), + ) if not triggered: return ModelMigrationDiscovery( requests=manual_requests, destination_fingerprints=_destination_fingerprints( requests=manual_requests, - renames=_declared_renames(declarations=declarations, state=state), + variants=seed_variants, metadata_jsons=metadata_jsons, dialect=dialect, ), ) - state.inspect_schemas(schemas=project_schemas) unbuilt: frozenset[str] = frozenset( model.name for model in models @@ -90,23 +96,19 @@ def discover_model_migrations( resumed: dict[str, MigrationEvent] = _resumed_events( runtime=runtime, models=models, unbuilt=unbuilt, state=state ) - seed_renames: dict[str, str] = _declared_renames(declarations=declarations, state=state) - seed_renames.update( - {name: event.origin_model for name, event in resumed.items() if event.origin_model} - ) candidates: tuple[Fingerprint, ...] = _origin_candidates( runtime=runtime, state=state, models=models, excluded=_claimed_origins(declarations=declarations, resumed=resumed), ) - renames: dict[str, str] + variants: tuple[dict[str, str], ...] matches: dict[str, Fingerprint] ambiguous: dict[str, str] - renames, matches, ambiguous = _match( + variants, matches, ambiguous = _match( models=models, new_names=unbuilt - frozenset(resumed), - seed_renames=seed_renames, + seed_variants=seed_variants, candidates=candidates, metadata_jsons=metadata_jsons, dialect=dialect, @@ -121,7 +123,7 @@ def discover_model_migrations( requests=requests, warnings=_ambiguity_warnings(models=models, scope=scope, ambiguous=ambiguous), destination_fingerprints=_destination_fingerprints( - requests=requests, renames=renames, metadata_jsons=metadata_jsons, dialect=dialect + requests=requests, variants=variants, metadata_jsons=metadata_jsons, dialect=dialect ), ) @@ -145,6 +147,27 @@ def _declared_renames( return renames +def _event_renames( + *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...], state: MigrationStateInspection +) -> dict[str, str]: + """Map project models to the removed model their latest recorded move came from.""" + + project_names: frozenset[str] = frozenset(model.name for model in models) + target_name: str | None = runtime.project.effective_target_name + latest: dict[str, MigrationEvent] = {} + event: MigrationEvent + for event in sorted(state.events, key=lambda item: (item.created_at, item.event_id)): + if event.target_name in (None, target_name): + latest[event.destination_model] = event + return { + name: event.origin_model + for name, event in latest.items() + if name in project_names + and event.origin_model is not None + and event.origin_model not in project_names + } + + def _resumed_events( *, runtime: PlannerRuntime, @@ -369,11 +392,13 @@ def _match( *, models: tuple[CompiledModel, ...], new_names: frozenset[str], - seed_renames: dict[str, str], + seed_variants: tuple[dict[str, str], ...], candidates: tuple[Fingerprint, ...], metadata_jsons: dict[str, str], dialect: str | None, -) -> tuple[dict[str, str], dict[str, Fingerprint], dict[str, str]]: +) -> tuple[tuple[dict[str, str], ...], dict[str, Fingerprint], dict[str, str]]: + """Match unbuilt models to candidates under every rename-map variant.""" + by_fingerprint: dict[str, list[Fingerprint]] = defaultdict(list) candidate: Fingerprint for candidate in candidates: @@ -381,24 +406,23 @@ def _match( blocked: set[str] = set() ambiguous: dict[str, str] = {} for _ in range(len(candidates) + 1): - renames: dict[str, str] = dict(seed_renames) + variants: tuple[dict[str, str], ...] = tuple(dict(seed) for seed in seed_variants) matches: dict[str, Fingerprint] = {} claims: dict[str, list[str]] = defaultdict(list) model: CompiledModel for model in models: if model.name not in new_names: continue - fingerprint: str | None = build_migration_fingerprint( - query_sql=model.query_sql, - metadata_json=metadata_jsons[model.name], - ref_identities=renames, - dialect=dialect, + options: list[Fingerprint] = _match_options( + fingerprints=_variant_fingerprints( + model=model, + metadata_json=metadata_jsons[model.name], + variants=variants, + dialect=dialect, + ), + by_fingerprint=by_fingerprint, + blocked=blocked, ) - options: list[Fingerprint] = [ - option - for option in by_fingerprint.get(fingerprint or "", []) - if option.node_name not in blocked - ] if len(options) > 1: ambiguous[model.name] = ( "several earlier models " @@ -406,17 +430,59 @@ def _match( ) continue if len(options) == 1: - renames[model.name] = options[0].node_name + variant: dict[str, str] + for variant in variants: + variant[model.name] = options[0].node_name matches[model.name] = options[0] claims[options[0].node_name].append(model.name) contested: dict[str, list[str]] = { origin: names for origin, names in claims.items() if len(names) > 1 } if not contested: - return renames, matches, ambiguous + return variants, matches, ambiguous blocked.update(contested) ambiguous.update(_contested_details(contested)) - return dict(seed_renames), {}, ambiguous + return tuple(dict(seed) for seed in seed_variants), {}, ambiguous + + +def _variant_fingerprints( + *, + model: CompiledModel, + metadata_json: str, + variants: tuple[dict[str, str], ...], + dialect: str | None, +) -> tuple[str, ...]: + """Return the distinct migration fingerprints of one model under each rename map.""" + + fingerprints: list[str | None] = [ + build_migration_fingerprint( + query_sql=model.query_sql, + metadata_json=metadata_json, + ref_identities=variant, + dialect=dialect, + ) + for variant in variants + ] + return tuple(dict.fromkeys(item for item in fingerprints if item is not None)) + + +def _match_options( + *, + fingerprints: tuple[str, ...], + by_fingerprint: dict[str, list[Fingerprint]], + blocked: set[str], +) -> list[Fingerprint]: + options: dict[str, Fingerprint] = {} + fingerprint: str + for fingerprint in fingerprints: + options.update( + { + option.node_name: option + for option in by_fingerprint.get(fingerprint, []) + if option.node_name not in blocked + } + ) + return list(options.values()) def _contested_details(contested: dict[str, list[str]]) -> dict[str, str]: @@ -436,24 +502,19 @@ def _contested_details(contested: dict[str, list[str]]) -> dict[str, str]: def _destination_fingerprints( *, requests: tuple[ModelMigrationRequest, ...], - renames: dict[str, str], + variants: tuple[dict[str, str], ...], metadata_jsons: dict[str, str], dialect: str | None, -) -> dict[str, str]: - fingerprints: dict[str, str] = {} +) -> dict[str, tuple[str, ...]]: + fingerprints: dict[str, tuple[str, ...]] = {} request: ModelMigrationRequest for request in requests: metadata_json: str | None = metadata_jsons.get(request.model.name) if metadata_json is None: continue - fingerprint: str | None = build_migration_fingerprint( - query_sql=request.model.query_sql, - metadata_json=metadata_json, - ref_identities=renames, - dialect=dialect, + fingerprints[request.model.name] = _variant_fingerprints( + model=request.model, metadata_json=metadata_json, variants=variants, dialect=dialect ) - if fingerprint is not None: - fingerprints[request.model.name] = fingerprint return fingerprints diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py index 1e8086de6f..423a009235 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py @@ -32,6 +32,8 @@ _CONFIG_KEY: str = "config" _GENERIC_DIALECT: str = "generic" _TABLE_KEY: str = "table" +_RECURSIVE_KEY: str = "recursive" +_CTE_BODY_KEY: str = "this" _ALIASED_RELATION_KEYS: frozenset[str] = frozenset({_TABLE_KEY, "subquery"}) _QUALIFIED_REFERENCE_KEYS: frozenset[str] = frozenset({"column", "star"}) _RELATION_QUALIFIER_KEYS: tuple[str, ...] = ("schema", "catalog") @@ -109,6 +111,8 @@ def _normalized_query_sql( except polyglot.PolyglotError: return None stripped: Any = _strip_formatting(parsed) + if not _has_unambiguous_local_scopes(stripped): + return None definitions: list[tuple[str, bool]] = _local_names(stripped) ordered_names: list[str] = list(dict.fromkeys(name for name, _ in definitions)) local_names: dict[str, str] = { @@ -138,6 +142,74 @@ def _strip_formatting(node: Any) -> Any: return node +def _has_unambiguous_local_scopes(root: Any) -> bool: + """Accept one top-level WITH whose names never shadow physical relations.""" + + statements: list[Any] = list(root.values()) if isinstance(root, dict) else [] + top_level: Any = statements[0].get(_WITH_KEY) if len(statements) == 1 else None + clauses: list[Any] = _with_clauses(root) + if clauses and (not isinstance(top_level, dict) or len(clauses) > 1): + return False + if isinstance(top_level, dict) and top_level.get(_RECURSIVE_KEY): + return False + ctes: list[Any] = [ + cte for cte in (top_level or {}).get(_CTES_KEY) or () if isinstance(cte, dict) + ] + cte_names: list[str] = _identifier_names([cte.get(_ALIAS_KEY) for cte in ctes]) + if len(cte_names) != len(ctes) or len(set(cte_names)) != len(cte_names): + return False + index: int + cte: Any + for index, cte in enumerate(ctes): + if set(_relation_names(cte.get(_CTE_BODY_KEY))) & set(cte_names[index:]): + return False + physical: set[str] = set(_relation_names(root)) - set(cte_names) + aliases: set[str] = {name for name, is_cte in _local_names(root) if not is_cte} + return not physical & aliases + + +def _with_clauses(node: Any) -> list[Any]: + found: list[Any] = [] + if isinstance(node, list): + item: Any + for item in node: + found.extend(_with_clauses(item)) + return found + if not isinstance(node, dict): + return found + key: str + value: Any + for key, value in node.items(): + if key == _WITH_KEY and isinstance(value, dict): + found.append(value) + found.extend(_with_clauses(value)) + return found + + +def _relation_names(node: Any) -> list[str]: + """Return lower-cased unqualified relation names referenced under a node.""" + + found: list[str] = [] + if isinstance(node, list): + item: Any + for item in node: + found.extend(_relation_names(item)) + return found + if not isinstance(node, dict): + return found + key: str + value: Any + for key, value in node.items(): + if key == _TABLE_KEY and _is_relation(value) and _unqualified(value): + found.extend(_identifier_names([value.get(_IDENTIFIER_NAME_KEY)])) + found.extend(_relation_names(value)) + return found + + +def _unqualified(relation: dict[str, Any]) -> bool: + return all(relation.get(key) is None for key in _RELATION_QUALIFIER_KEYS) + + def _local_names(node: Any) -> list[tuple[str, bool]]: """Return (name, is_cte) for CTE and relation alias definitions in first-seen order.""" @@ -215,9 +287,8 @@ def _canonical_local_names( def _cte_reference(*, relation: dict[str, Any], cte_names: Mapping[str, str]) -> dict[str, Any]: """Rename an unqualified relation name that refers to a CTE.""" - qualifiers: tuple[Any, ...] = tuple(relation.get(key) for key in _RELATION_QUALIFIER_KEYS) name: Any = relation.get(_IDENTIFIER_NAME_KEY) - if any(qualifier is not None for qualifier in qualifiers) or not _is_identifier(name): + if not _unqualified(relation) or not _is_identifier(name): return {} return {_IDENTIFIER_NAME_KEY: _renamed(identifier=name, names=cte_names)} diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 655df61bb1..dfc1f1803c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -186,7 +186,9 @@ def plan_model_migrations( handovers[request.model.name] = _equivalent_definition( model=request.model, handover=handover, - destination_fingerprint=discovery.destination_fingerprints.get(request.model.name), + destination_fingerprints=discovery.destination_fingerprints.get( + request.model.name, () + ), ) return ModelMigrationPlanning( snapshot=_overlay_snapshot( @@ -208,13 +210,9 @@ def plan_model_migrations( def _equivalent_definition( - *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprint: str | None + *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprints: tuple[str, ...] ) -> Fingerprint | None: - if ( - handover is None - or destination_fingerprint is None - or stored_migration_fingerprint(handover) != destination_fingerprint - ): + if handover is None or stored_migration_fingerprint(handover) not in destination_fingerprints: return handover return replace( handover, diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 2a25abc56e..9daba01e15 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -1517,7 +1517,7 @@ class ModelMigrationDiscovery: requests: tuple[ModelMigrationRequest, ...] = () warnings: tuple[PlanWarning, ...] = () - destination_fingerprints: dict[str, str] = field(default_factory=dict) + destination_fingerprints: dict[str, tuple[str, ...]] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index ae949f909a..0a90de0a3f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -15,6 +15,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.executor.build._helpers import scheduler as scheduler_module DATABASE_FILE: str = "orders.duckdb" PROJECT_TOML: str = dedent( @@ -85,6 +86,7 @@ " cursor_type timestamp,\n" " cursor_grain day,\n" ' cursor_start "2026-01-01",\n' + "{extra_config}" ");\n\n" ) _SNAPSHOT_SQL: str = ( @@ -389,6 +391,24 @@ def fail_clone_into(*, monkeypatch: pytest.MonkeyPatch, destination: str) -> Non ) +def fail_model_build(*, monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: + """Make one incremental model's build raise after migrations have run.""" + + failing: dict[str, Callable[..., Any]] = {model_name: _raise_interruption} + incremental: Callable[..., Any] = scheduler_module.execute_incremental_entry + table: Callable[..., Any] = scheduler_module.execute_table_entry + monkeypatch.setattr( + scheduler_module, + "execute_incremental_entry", + lambda **kwargs: failing.get(kwargs["context"].entry.name, incremental)(**kwargs), + ) + monkeypatch.setattr( + scheduler_module, + "execute_table_entry", + lambda **kwargs: failing.get(kwargs["context"].entry.name, table)(**kwargs), + ) + + def fail_record(monkeypatch: pytest.MonkeyPatch) -> None: """Crash after the clone but before the migration event is recorded.""" @@ -416,11 +436,11 @@ def enriched_view_sql(*, upstream: str, alias: str) -> str: ) -def daily_totals_sql(*, upstream: str, cte: str, comment: str = "") -> str: +def daily_totals_sql(*, upstream: str, cte: str, comment: str = "", extra_config: str = "") -> str: """Return an incremental daily revenue model that imports its upstream through a CTE.""" return ( - f"{_DAILY_TOTALS_HEADER}" + f"{_DAILY_TOTALS_HEADER.format(extra_config=extra_config)}" f"{comment}" f'WITH {cte} AS (SELECT * FROM __ref("{upstream}"))\n' f"SELECT\n {cte}.order_date,\n SUM({cte}.amount_cents) AS total_cents\n" @@ -457,7 +477,10 @@ def renamed_order_models() -> dict[str, str]: def build_original_order_models(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> None: """Build the connected models over five days of raw orders.""" - write_project(project_dir=project_dir, models=original_order_models()) + write_project( + project_dir=project_dir, + models=original_order_models(), + ) load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) _ = build_ok(project_dir=project_dir, capsys=capsys) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py index 466e7313e3..8908b27cbd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py @@ -18,6 +18,7 @@ daily_totals_sql, execute, fail_clone_into, + fail_model_build, incremental_orders_sql, load_raw_orders, migration_events, @@ -380,5 +381,73 @@ def test_given_one_orphan_matching_two_new_models_when_selecting_one_then_warns_ assert "_sqlbuild_migrations" not in relation_names(project_dir=tmp_path, schema="main") +@pytest.mark.parametrize( + "test_case", + [ + AutomaticMigrationTestCase( + description="downstream failed after its clone while the upstream was rebuilt", + expected_migrations=( + ("daily_order_totals", "customer_daily_order_totals", "automatic", "done"), + ), + expected_events=( + ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("stg_orders", "stg_customer_orders", "migrate"), + ), + expected_reason="normal_incremental", + ) + ], + ids=lambda case: case.description, +) +def test_given_fingerprinted_upstream_when_retrying_failed_downstream_then_history_is_kept( + test_case: AutomaticMigrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + replay_full: str = " replay_on_change full,\n" + write_project( + project_dir=tmp_path, + models={ + "stg_orders": incremental_orders_sql(), + "daily_order_totals": daily_totals_sql( + upstream="stg_orders", cte="stg_orders", extra_config=replay_full + ), + }, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + load_raw_orders(project_dir=tmp_path, first_day=3, last_day=8) + write_project( + project_dir=tmp_path, + models={ + "stg_customer_orders": incremental_orders_sql(), + "customer_daily_order_totals": daily_totals_sql( + upstream="stg_customer_orders", + cte="stg_customer_orders", + extra_config=replay_full, + ), + }, + ) + + with monkeypatch.context() as patch: + fail_model_build(monkeypatch=patch, model_name="customer_daily_order_totals") + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) + upstream_fingerprinted: dict[str, Any] = plan_json( + project_dir=tmp_path, capsys=capsys, args=("--select", "stg_customer_orders") + ) + retry_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code == 1, interrupted.output + assert planned_migrations(upstream_fingerprinted) == () + assert planned_migrations(retry_plan) == test_case.expected_migrations + assert model_entry(plan=retry_plan, name="stg_customer_orders")["reason"] != _FIRST_RUN + assert model_entry(plan=retry_plan, name="customer_daily_order_totals")["reason"] == ( + test_case.expected_reason + ) + assert row_count(project_dir=tmp_path, relation="main.customer_daily_order_totals") == 8 + assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py index 0446424ba6..5d3d916a71 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py @@ -19,7 +19,7 @@ class MigrationFingerprintTestCase: @dataclass(frozen=True) -class UnparseableFingerprintTestCase: +class IneligibleFingerprintTestCase: description: str query_sql: str expected_fingerprint: str | None diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py index e6297ac28e..211a7bb3f6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_migration_fingerprint.py @@ -5,8 +5,8 @@ import pytest from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( + IneligibleFingerprintTestCase, MigrationFingerprintTestCase, - UnparseableFingerprintTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import ( BASE_CONFIG, @@ -38,6 +38,18 @@ "JOIN (SELECT 1 AS total) AS sums ON TRUE" ), ), + MigrationFingerprintTestCase( + expected_match=True, + description="later cte reading an earlier renamed cte still matches", + origin_sql=( + 'WITH src AS (SELECT * FROM __ref("stg_orders")), totals AS ' + "(SELECT src.order_id FROM src) SELECT totals.order_id FROM totals" + ), + destination_sql=( + 'WITH base AS (SELECT * FROM __ref("stg_orders")), sums AS ' + "(SELECT base.order_id FROM base) SELECT sums.order_id FROM sums" + ), + ), MigrationFingerprintTestCase( expected_match=False, description="column named like the renamed cte is still a column", @@ -161,16 +173,52 @@ def test_given_renamed_definition_when_fingerprinting_then_matches_only_equivale @pytest.mark.parametrize( "test_case", [ - UnparseableFingerprintTestCase( + IneligibleFingerprintTestCase( description="unparseable query has no fingerprint", query_sql="SELECT FROM WHERE (", expected_fingerprint=None, - ) + ), + IneligibleFingerprintTestCase( + description="nested with shadows an outer physical relation", + query_sql=( + "WITH wrapper AS (WITH a AS (SELECT 1 AS value) SELECT value FROM a) " + "SELECT value FROM a" + ), + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="with nested in a subquery", + query_sql="SELECT x FROM (WITH q AS (SELECT 1 AS x) SELECT x FROM q) AS s", + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="cte body reads the physical relation it shadows", + query_sql="WITH orders AS (SELECT * FROM orders) SELECT * FROM orders", + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="cte body reads a later cte name", + query_sql=( + "WITH totals AS (SELECT * FROM orders), orders AS (SELECT 1 AS x) " + "SELECT * FROM totals" + ), + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="recursive with", + query_sql="WITH RECURSIVE n AS (SELECT 1 AS x) SELECT x FROM n", + expected_fingerprint=None, + ), + IneligibleFingerprintTestCase( + description="alias reuses a physical relation name", + query_sql="SELECT o.x FROM orders AS o JOIN o ON TRUE", + expected_fingerprint=None, + ), ], ids=lambda case: case.description, ) -def test_given_unparseable_sql_when_fingerprinting_then_returns_none( - test_case: UnparseableFingerprintTestCase, +def test_given_unparseable_or_ambiguous_sql_when_fingerprinting_then_returns_none( + test_case: IneligibleFingerprintTestCase, ) -> None: fingerprint: str | None = model_fingerprint( model_name="stg_orders", From bc50fd75fa1b9ce939643d13ad414d945231084d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:13:32 +0100 Subject: [PATCH 11/20] refactor(executor): extract staged-relation promotion from full refresh --- .../_helpers/materializations/full_refresh.py | 92 ++++++++----- .../run/main/promote_staged_relation.py | 34 +++++ .../executor/run/_helpers/_test_types.py | 16 +++ .../sqlbuild/executor/run/_helpers/helpers.py | 35 +++++ .../run/_helpers/test_staged_promotion.py | 128 ++++++++++++++++++ 5 files changed, 271 insertions(+), 34 deletions(-) create mode 100644 src/sqlbuild/executor/run/main/promote_staged_relation.py create mode 100644 tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py diff --git a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py index 8edc02b125..97abb0c853 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py @@ -95,49 +95,73 @@ def promote_full_refresh_rebuild( target_kind="relation", ), ) as lifecycle: - if not target_exists: - adapter.rename( - connection=connection, - origin=relations.rebuild_qualified, - destination=relations.target_qualified, - statement_recorder=statement_recorder, - ) - else: + if target_exists: adapter.drop( connection=connection, destination=relations.previous_qualified, if_exists=True, statement_recorder=statement_recorder, ) - if adapter.adapter_name == BuiltinAdapter.SNOWFLAKE: - adapter.swap( - connection=connection, - left=relations.target_qualified, - right=relations.rebuild_qualified, - statement_recorder=statement_recorder, - ) - adapter.rename( - connection=connection, - origin=relations.rebuild_qualified, - destination=relations.previous_qualified, - statement_recorder=statement_recorder, - ) - else: - adapter.rename( - connection=connection, - origin=relations.target_qualified, - destination=relations.previous_qualified, - statement_recorder=statement_recorder, - ) - adapter.rename( - connection=connection, - origin=relations.rebuild_qualified, - destination=relations.target_qualified, - statement_recorder=statement_recorder, - ) + promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified=relations.target_qualified, + staged_qualified=relations.rebuild_qualified, + displaced_qualified=relations.previous_qualified, + target_exists=target_exists, + statement_recorder=statement_recorder, + ) lifecycle.completed(metadata={"changed_count": 1}) +def promote_staged_relation( + *, + adapter: BaseAdapter, + connection: Any, + target_qualified: str, + staged_qualified: str, + displaced_qualified: str, + target_exists: bool, + statement_recorder: StatementRecorder, +) -> None: + """Promote a staged relation without dropping anything; a live target moves aside.""" + + if not target_exists: + adapter.rename( + connection=connection, + origin=staged_qualified, + destination=target_qualified, + statement_recorder=statement_recorder, + ) + return + if adapter.adapter_name == BuiltinAdapter.SNOWFLAKE: + adapter.swap( + connection=connection, + left=target_qualified, + right=staged_qualified, + statement_recorder=statement_recorder, + ) + adapter.rename( + connection=connection, + origin=staged_qualified, + destination=displaced_qualified, + statement_recorder=statement_recorder, + ) + return + adapter.rename( + connection=connection, + origin=target_qualified, + destination=displaced_qualified, + statement_recorder=statement_recorder, + ) + adapter.rename( + connection=connection, + origin=staged_qualified, + destination=target_qualified, + statement_recorder=statement_recorder, + ) + + def _artifact_name(*, logical_name: str, fixed_prefix: str, identifier_limit: int) -> str: available: int = identifier_limit - len(fixed_prefix) if len(logical_name) <= available: diff --git a/src/sqlbuild/executor/run/main/promote_staged_relation.py b/src/sqlbuild/executor/run/main/promote_staged_relation.py new file mode 100644 index 0000000000..02465892ae --- /dev/null +++ b/src/sqlbuild/executor/run/main/promote_staged_relation.py @@ -0,0 +1,34 @@ +"""Public staged-relation promotion shared by build-aside rebuilds and model migrations.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.executor.run._helpers.materializations.full_refresh import ( + promote_staged_relation as _promote_staged_relation, +) + + +def promote_staged_relation( + *, + adapter: BaseAdapter, + connection: Any, + target_qualified: str, + staged_qualified: str, + displaced_qualified: str, + target_exists: bool, + statement_recorder: StatementRecorder, +) -> None: + """Promote a staged relation into its target, moving a live target to the displaced name.""" + + _ = _promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified=target_qualified, + staged_qualified=staged_qualified, + displaced_qualified=displaced_qualified, + target_exists=target_exists, + statement_recorder=statement_recorder, + ) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index 6abdb62087..f7c9f78ce3 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -499,3 +499,19 @@ class RuntimeContractValidationTestCase: contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = () expected_error_fragment: str | None = None expected_error_code: str | None = None + + +@dataclass(frozen=True) +class StagedPromotionTestCase: + description: str + adapter_name: str + target_exists: bool + expected_statements: tuple[str, ...] + + +@dataclass(frozen=True) +class FullRefreshPromotionTestCase: + description: str + adapter_name: str + target_exists: bool + expected_statements: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index a0a7713b2e..e2ce01e86b 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -6,6 +6,8 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -263,3 +265,36 @@ def create_table_as( del connection, destination, config, statement_recorder self.calls.append("create_table_as") self.sql = sql + + +class RecordingConnection: + def __init__(self) -> None: + self.executed: list[str] = [] + + def execute(self, sql: str) -> None: + self.executed.append(sql) + + +class RecordingSnowflakeAdapter(SnowflakeAdapter): + def _execute(self, *, connection: Any, sql: str) -> Any: + return connection.execute(sql) + + +class RecordingPostgresAdapter(PostgresAdapter): + def _execute(self, *, connection: Any, sql: str) -> Any: + return connection.execute(sql) + + +class RecordingDuckDbAdapter(DuckDbAdapter): + def _execute(self, *, connection: Any, sql: str) -> Any: + return connection.execute(sql) + + +def build_recording_adapter( + adapter_name: str, +) -> RecordingSnowflakeAdapter | RecordingPostgresAdapter | RecordingDuckDbAdapter: + return { + "snowflake": RecordingSnowflakeAdapter, + "postgres": RecordingPostgresAdapter, + "duckdb": RecordingDuckDbAdapter, + }[adapter_name]() diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py new file mode 100644 index 0000000000..bd65b3d594 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py @@ -0,0 +1,128 @@ +"""Tests for staged-relation promotion shared by full refresh and model migrations.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.executor.run._helpers.materializations.full_refresh import ( + promote_full_refresh_rebuild, + resolve_full_refresh_relations, +) +from sqlbuild.executor.run.main.promote_staged_relation import promote_staged_relation +from sqlbuild.executor.run.models import FullRefreshRelations +from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( + FullRefreshPromotionTestCase, + StagedPromotionTestCase, +) +from tests.unit.src.sqlbuild.executor.run._helpers.helpers import ( + RecordingConnection, + build_recording_adapter, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StagedPromotionTestCase( + description="missing target is promoted by renaming the stage", + adapter_name="duckdb", + target_exists=False, + expected_statements=("ALTER TABLE main.orders_stage RENAME TO orders",), + ), + StagedPromotionTestCase( + description="rename adapters move the live target aside before renaming the stage", + adapter_name="postgres", + target_exists=True, + expected_statements=( + "ALTER TABLE main.orders RENAME TO orders_previous", + "ALTER TABLE main.orders_stage RENAME TO orders", + ), + ), + StagedPromotionTestCase( + description="snowflake swaps then keeps the displaced target under the displaced name", + adapter_name="snowflake", + target_exists=True, + expected_statements=( + "ALTER TABLE main.orders SWAP WITH main.orders_stage", + "ALTER TABLE main.orders_stage RENAME TO main.orders_previous", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_staged_relation_when_promoting_then_nothing_is_dropped( + test_case: StagedPromotionTestCase, +) -> None: + """Promotion never drops the live target or the stage.""" + + adapter: BaseAdapter = build_recording_adapter(test_case.adapter_name) + connection: RecordingConnection = RecordingConnection() + + promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified="main.orders", + staged_qualified="main.orders_stage", + displaced_qualified="main.orders_previous", + target_exists=test_case.target_exists, + statement_recorder=StatementRecorder(), + ) + + assert tuple(connection.executed) == test_case.expected_statements + + +@pytest.mark.parametrize( + "test_case", + [ + FullRefreshPromotionTestCase( + description="first full refresh renames the rebuild into place", + adapter_name="duckdb", + target_exists=False, + expected_statements=("ALTER TABLE main.__sqb_rebuild__orders RENAME TO orders",), + ), + FullRefreshPromotionTestCase( + description="rename adapters drop the previous copy then rename twice", + adapter_name="duckdb", + target_exists=True, + expected_statements=( + "DROP TABLE IF EXISTS main.__sqb_prev__orders", + "ALTER TABLE main.orders RENAME TO __sqb_prev__orders", + "ALTER TABLE main.__sqb_rebuild__orders RENAME TO orders", + ), + ), + FullRefreshPromotionTestCase( + description="snowflake drops the previous copy then swaps", + adapter_name="snowflake", + target_exists=True, + expected_statements=( + "DROP TABLE IF EXISTS main.__sqb_prev__orders", + "DROP VIEW IF EXISTS main.__sqb_prev__orders", + "ALTER TABLE main.orders SWAP WITH main.__sqb_rebuild__orders", + "ALTER TABLE main.__sqb_rebuild__orders RENAME TO main.__sqb_prev__orders", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_full_refresh_rebuild_when_promoting_then_statements_are_unchanged( + test_case: FullRefreshPromotionTestCase, +) -> None: + """Full refresh keeps its drop-previous-then-promote sequence.""" + + adapter: BaseAdapter = build_recording_adapter(test_case.adapter_name) + connection: RecordingConnection = RecordingConnection() + relations: FullRefreshRelations = resolve_full_refresh_relations( + adapter=adapter, database=None, schema="main", target_name="orders" + ) + + promote_full_refresh_rebuild( + adapter=adapter, + connection=connection, + relations=relations, + target_exists=test_case.target_exists, + statement_recorder=StatementRecorder(), + ) + + assert tuple(connection.executed) == test_case.expected_statements From a4b56c173aca72b6aaa9799f279e88b39497e5bb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:16:11 +0100 Subject: [PATCH 12/20] feat(adapters): stage migrations by clone or copy per adapter --- .../adapter/contract/classes/base_adapter.py | 19 ++ .../contract/classes/duckdb_backed_adapter.py | 19 ++ .../contract/classes/strict_adapter.py | 18 ++ src/sqlbuild/adapter/contract/models.py | 10 + src/sqlbuild/adapter/contract/types.py | 7 + .../bigquery/classes/bigquery_adapter.py | 22 ++ .../databricks/classes/databricks_adapter.py | 19 ++ .../postgres/classes/postgres_adapter.py | 19 ++ .../snowflake/classes/snowflake_adapter.py | 28 +++ .../sqlserver/classes/sqlserver_adapter.py | 19 ++ .../unit/src/sqlbuild/adapters/_test_types.py | 19 ++ .../sqlbuild/adapters/test_migration_stage.py | 195 ++++++++++++++++++ 12 files changed, 394 insertions(+) create mode 100644 tests/unit/src/sqlbuild/adapters/test_migration_stage.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 8d0cf69e01..8e44a448e3 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -36,6 +36,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RowDiffCoverage, @@ -57,6 +58,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, @@ -710,6 +712,23 @@ def render_replace_with_clone( message=f"adapter '{self.adapter_name}' does not support model migrations" ) + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + ) + + def supports_transactional_ddl(self) -> bool: + return False + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 44c2748292..9df7667b4c 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -41,6 +41,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RowDiffColumnResult, @@ -61,6 +62,7 @@ CursorKind, FrameworkType, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, TablePromotionMode, ) @@ -1530,6 +1532,23 @@ def render_replace_with_clone( del origin_is_transient return f"CREATE OR REPLACE TABLE {destination} AS SELECT * FROM {origin}" + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + ) + + def supports_transactional_ddl(self) -> bool: + return True + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 0db902cad5..b1e24572a2 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -15,6 +15,7 @@ ColumnInfo, CursorValue, ExpressionInferenceProfile, + MigrationStagePlan, RowDiffTolerance, RowDiffTolerances, SnapshotChangeTarget, @@ -348,6 +349,23 @@ def render_replace_with_clone( """Render one statement that replaces the destination with a durable source copy.""" ... + @abstractmethod + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + """Render statements that create a fresh, independent stage holding the origin's data.""" + ... + + @abstractmethod + def supports_transactional_ddl(self) -> bool: + """Return whether renames and state inserts can commit or roll back together.""" + ... + @abstractmethod def render_query_with_cursor_bounds( self, diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index d376cbebc0..b89dbc85eb 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -15,6 +15,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LifeCycleEventKind, + MigrationTransfer, RetentionChangePhase, RetentionScope, SnapshotLatestVersionStyle, @@ -184,6 +185,15 @@ class RenderedRetentionChange: statements: tuple[str, ...] +@dataclass(frozen=True) +class MigrationStagePlan: + """Statements that create a fresh migration stage, plus a copy fallback if clone is refused.""" + + transfer: MigrationTransfer + statements: tuple[str, ...] + fallback_statements: tuple[str, ...] = () + + @dataclass(frozen=True) class FunctionInfo: """Metadata for one discovered warehouse function or routine.""" diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 56b0f4de32..32b8a9f5a1 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -36,6 +36,13 @@ class PromotionStrategy(StrEnum): CREATE_NEW = "create_new" +class MigrationTransfer(StrEnum): + """How a model migration stage receives the origin's data.""" + + CLONE = "clone" + COPY = "copy" + + class TablePromotionMode(StrEnum): IMMEDIATE = "immediate" STAGED = "staged" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 9414b13e74..aface50fab 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -46,6 +46,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RenderedRetentionChange, @@ -73,6 +74,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, RetentionChangePhase, RetentionScope, @@ -1720,6 +1722,26 @@ def render_replace_with_clone( message=f"adapter '{self.adapter_name}' does not support model migrations" ) + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + quoted_stage: str = self._quote_identifier_path(stage) + quoted_origin: str = self._quote_identifier_path(origin) + return MigrationStagePlan( + transfer=MigrationTransfer.CLONE, + statements=(f"CREATE TABLE {quoted_stage} CLONE {quoted_origin}",), + fallback_statements=(f"CREATE TABLE {quoted_stage} COPY {quoted_origin}",), + ) + + def supports_transactional_ddl(self) -> bool: + return False + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 54247e99e0..0039a5e88e 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -44,6 +44,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RenderedRetentionChange, @@ -71,6 +72,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, RelationType, RetentionChangePhase, @@ -1757,6 +1759,23 @@ def render_replace_with_clone( del origin_is_transient return f"CREATE OR REPLACE TABLE {destination} DEEP CLONE {origin}" + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} DEEP CLONE {origin}",), + ) + + def supports_transactional_ddl(self) -> bool: + return False + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index b83239d971..30c0638355 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -46,6 +46,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RowDiffColumnResult, @@ -70,6 +71,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, @@ -585,6 +587,23 @@ def render_replace_with_clone( message=f"adapter '{self.adapter_name}' does not support model migrations" ) + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + ) + + def supports_transactional_ddl(self) -> bool: + return True + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 143f0aca94..23a19be7ed 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -41,6 +41,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RenderedRetentionChange, @@ -68,6 +69,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, RetentionChangePhase, RetentionScope, @@ -2237,6 +2239,32 @@ def render_replace_with_clone( origin=origin, destination=destination, origin_is_transient=origin_is_transient )[0] + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + transient: bool = origin_is_transient if stage_is_transient is None else stage_is_transient + if origin_is_transient and not transient: + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=( + f"CREATE TABLE {stage} LIKE {origin} COPY GRANTS", + f"INSERT INTO {stage} SELECT * FROM {origin}", + ), + ) + table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" + return MigrationStagePlan( + transfer=MigrationTransfer.CLONE, + statements=(f"CREATE {table_kind} {stage} CLONE {origin} COPY GRANTS",), + ) + + def supports_transactional_ddl(self) -> bool: + return False + def render_query_with_cursor_bounds( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index ad21630ac0..2abe4ce1cc 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -48,6 +48,7 @@ ExpressionInferenceProfile, FunctionDefinition, FunctionInfo, + MigrationStagePlan, QueryResult, RelationInfo, RowDiffColumnResult, @@ -72,6 +73,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MigrationTransfer, PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, @@ -1780,6 +1782,23 @@ def render_replace_with_clone( message=f"adapter '{self.adapter_name}' does not support model migrations" ) + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"SELECT * INTO {stage} FROM {origin}",), + ) + + def supports_transactional_ddl(self) -> bool: + return True + def render_query_with_cursor_bounds( self, *, diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 977bb55db8..2c02ce3089 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -2,6 +2,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter +from sqlbuild.adapter.contract.types import MigrationTransfer @dataclass(frozen=True) @@ -85,3 +86,21 @@ class AdapterRelationAgeMetadataCapabilityTestCase: description: str adapter: StrictAdapter expected_supported: bool + + +@dataclass(frozen=True) +class AdapterMigrationStageTestCase: + description: str + adapter: BaseAdapter + origin_is_transient: bool + stage_is_transient: bool | None + expected_transfer: MigrationTransfer + expected_statements: tuple[str, ...] + expected_fallback_statements: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AdapterTransactionalDdlTestCase: + description: str + adapter: BaseAdapter + expected_transactional: bool diff --git a/tests/unit/src/sqlbuild/adapters/test_migration_stage.py b/tests/unit/src/sqlbuild/adapters/test_migration_stage.py new file mode 100644 index 0000000000..b3ab9e6955 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/test_migration_stage.py @@ -0,0 +1,195 @@ +"""Unit coverage for per-adapter model migration staging statements.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.models import MigrationStagePlan +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from tests.unit.src.sqlbuild.adapters._test_types import ( + AdapterMigrationStageTestCase, + AdapterTransactionalDdlTestCase, +) + +_STAGE: str = "dev._sqb_archive__20260102t030405z__migration_stage__stg_customer_orders" + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterMigrationStageTestCase( + description="snowflake permanent to permanent is a zero-copy clone with grants", + adapter=SnowflakeAdapter(), + origin_is_transient=False, + stage_is_transient=False, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=(f"CREATE TABLE {_STAGE} CLONE dev.stg_orders COPY GRANTS",), + ), + AdapterMigrationStageTestCase( + description="snowflake permanent to transient is a zero-copy transient clone", + adapter=SnowflakeAdapter(), + origin_is_transient=False, + stage_is_transient=True, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=( + f"CREATE TRANSIENT TABLE {_STAGE} CLONE dev.stg_orders COPY GRANTS", + ), + ), + AdapterMigrationStageTestCase( + description="snowflake transient to transient is a zero-copy transient clone", + adapter=SnowflakeAdapter(), + origin_is_transient=True, + stage_is_transient=True, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=( + f"CREATE TRANSIENT TABLE {_STAGE} CLONE dev.stg_orders COPY GRANTS", + ), + ), + AdapterMigrationStageTestCase( + description="snowflake transient to permanent is a physical copy into a permanent table", + adapter=SnowflakeAdapter(), + origin_is_transient=True, + stage_is_transient=False, + expected_transfer=MigrationTransfer.COPY, + expected_statements=( + f"CREATE TABLE {_STAGE} LIKE dev.stg_orders COPY GRANTS", + f"INSERT INTO {_STAGE} SELECT * FROM dev.stg_orders", + ), + ), + AdapterMigrationStageTestCase( + description="snowflake without a destination type keeps the origin type", + adapter=SnowflakeAdapter(), + origin_is_transient=True, + stage_is_transient=None, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=( + f"CREATE TRANSIENT TABLE {_STAGE} CLONE dev.stg_orders COPY GRANTS", + ), + ), + AdapterMigrationStageTestCase( + description="bigquery clones and falls back to a table copy when clone is refused", + adapter=BigQueryAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.CLONE, + expected_statements=(f"CREATE TABLE `{_STAGE}` CLONE `dev.stg_orders`",), + expected_fallback_statements=(f"CREATE TABLE `{_STAGE}` COPY `dev.stg_orders`",), + ), + AdapterMigrationStageTestCase( + description="databricks copies through an independent deep clone", + adapter=DatabricksAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"CREATE TABLE {_STAGE} DEEP CLONE dev.stg_orders",), + ), + AdapterMigrationStageTestCase( + description="postgres copies into a new table without replacing anything", + adapter=PostgresAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"CREATE TABLE {_STAGE} AS SELECT * FROM dev.stg_orders",), + ), + AdapterMigrationStageTestCase( + description="sqlserver copies with select into", + adapter=SqlServerAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"SELECT * INTO {_STAGE} FROM dev.stg_orders",), + ), + AdapterMigrationStageTestCase( + description="duckdb copies into a new table without replacing anything", + adapter=DuckDbAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"CREATE TABLE {_STAGE} AS SELECT * FROM dev.stg_orders",), + ), + AdapterMigrationStageTestCase( + description="motherduck copies like duckdb", + adapter=MotherDuckAdapter(), + origin_is_transient=False, + stage_is_transient=None, + expected_transfer=MigrationTransfer.COPY, + expected_statements=(f"CREATE TABLE {_STAGE} AS SELECT * FROM dev.stg_orders",), + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_when_rendering_migration_stage_then_creates_a_fresh_stage( + test_case: AdapterMigrationStageTestCase, +) -> None: + """Stage statements never replace or drop an existing relation.""" + + plan: MigrationStagePlan = test_case.adapter.render_migration_stage( + origin="dev.stg_orders", + stage=_STAGE, + origin_is_transient=test_case.origin_is_transient, + stage_is_transient=test_case.stage_is_transient, + ) + + assert plan.transfer == test_case.expected_transfer + assert plan.statements == test_case.expected_statements + assert plan.fallback_statements == test_case.expected_fallback_statements + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterTransactionalDdlTestCase( + description="snowflake ddl commits implicitly", + adapter=SnowflakeAdapter(), + expected_transactional=False, + ), + AdapterTransactionalDdlTestCase( + description="bigquery has no transactional ddl", + adapter=BigQueryAdapter(), + expected_transactional=False, + ), + AdapterTransactionalDdlTestCase( + description="databricks has no transactional ddl", + adapter=DatabricksAdapter(), + expected_transactional=False, + ), + AdapterTransactionalDdlTestCase( + description="postgres renames roll back with the transaction", + adapter=PostgresAdapter(), + expected_transactional=True, + ), + AdapterTransactionalDdlTestCase( + description="sqlserver renames roll back with the transaction", + adapter=SqlServerAdapter(), + expected_transactional=True, + ), + AdapterTransactionalDdlTestCase( + description="duckdb renames roll back with the transaction", + adapter=DuckDbAdapter(), + expected_transactional=True, + ), + AdapterTransactionalDdlTestCase( + description="motherduck follows duckdb transactions", + adapter=MotherDuckAdapter(), + expected_transactional=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_when_checking_transactional_ddl_then_reports_capability( + test_case: AdapterTransactionalDdlTestCase, +) -> None: + """Transactional promotion is only claimed where renames roll back.""" + + assert test_case.adapter.supports_transactional_ddl() is test_case.expected_transactional + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 5a06c71665ecaac3a633b8a195af50057300a752 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:36:32 +0100 Subject: [PATCH 13/20] feat(migrations): execute migrations through staged promotion with archived displacement Each moving migration now stages the origin into a fresh _sqb_archive____migration_stage__ relation with the destination's table type, verifies it (existence, table type, columns, row count for copies), and promotes it with the shared staged-relation promotion. A live destination is kept as _sqb_archive____migration_previous__ so janitor expires it by name. Transactional adapters commit the renames, PostgreSQL dependent-view rebinds, and the event together. --- .../adapter/contract/classes/base_adapter.py | 14 +- .../contract/classes/duckdb_backed_adapter.py | 12 +- .../contract/classes/strict_adapter.py | 14 +- src/sqlbuild/adapter/contract/types.py | 6 + .../bigquery/classes/bigquery_adapter.py | 14 +- .../databricks/classes/databricks_adapter.py | 12 +- .../postgres/classes/postgres_adapter.py | 38 +++- .../snowflake/classes/snowflake_adapter.py | 13 +- .../sqlserver/classes/sqlserver_adapter.py | 14 +- src/sqlbuild/cli/output/_helpers/plan_json.py | 4 +- .../compiler/migrations/main/ensure_table.py | 33 +++ .../compiler/migrations/main/write_event.py | 29 ++- src/sqlbuild/compiler/migrations/types.py | 14 ++ .../planner/_helpers/migrations/planning.py | 57 ++++- .../main/pre_build/pre_build_work_display.py | 7 + src/sqlbuild/compiler/planner/models.py | 15 +- src/sqlbuild/executor/build/main/_execute.py | 2 +- .../janitor/_helpers/archive_names.py | 8 +- .../janitor/main/_build_archive_name.py | 20 ++ src/sqlbuild/executor/migrations/__init__.py | 1 + .../executor/migrations/_helpers/__init__.py | 1 + .../executor/migrations/_helpers/naming.py | 86 ++++++++ .../executor/migrations/_helpers/promotion.py | 122 +++++++++++ .../executor/migrations/_helpers/staging.py | 150 +++++++++++++ src/sqlbuild/executor/migrations/constants.py | 7 + .../executor/migrations/main/__init__.py | 1 + .../main/_apply.py} | 68 +++--- src/sqlbuild/executor/migrations/models.py | 17 ++ .../main/model_migrations/_test_types.py | 29 +++ .../commands/main/model_migrations/helpers.py | 190 +++++++++++++++-- .../test_automatic_migrations.py | 2 +- .../test_manual_migrations.py | 40 +++- .../model_migrations/test_staged_execution.py | 198 ++++++++++++++++++ .../sqlbuild/executor/migrations/__init__.py | 0 .../executor/migrations/_helpers/__init__.py | 0 .../migrations/_helpers/_test_types.py | 16 ++ .../executor/migrations/_helpers/helpers.py | 33 +++ .../migrations/_helpers/test_naming.py | 76 +++++++ .../unit/src/sqlbuild/adapters/_test_types.py | 9 - .../adapters/test_replace_with_clone.py | 111 ---------- .../executor/janitor/_helpers/_test_types.py | 1 + .../janitor/_helpers/test_archive_names.py | 21 ++ 42 files changed, 1241 insertions(+), 264 deletions(-) create mode 100644 src/sqlbuild/compiler/migrations/main/ensure_table.py create mode 100644 src/sqlbuild/executor/janitor/main/_build_archive_name.py create mode 100644 src/sqlbuild/executor/migrations/__init__.py create mode 100644 src/sqlbuild/executor/migrations/_helpers/__init__.py create mode 100644 src/sqlbuild/executor/migrations/_helpers/naming.py create mode 100644 src/sqlbuild/executor/migrations/_helpers/promotion.py create mode 100644 src/sqlbuild/executor/migrations/_helpers/staging.py create mode 100644 src/sqlbuild/executor/migrations/constants.py create mode 100644 src/sqlbuild/executor/migrations/main/__init__.py rename src/sqlbuild/executor/{build/main/_apply_model_migrations.py => migrations/main/_apply.py} (57%) create mode 100644 src/sqlbuild/executor/migrations/models.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py create mode 100644 tests/integration/src/sqlbuild/executor/migrations/__init__.py create mode 100644 tests/integration/src/sqlbuild/executor/migrations/_helpers/__init__.py create mode 100644 tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py create mode 100644 tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py create mode 100644 tests/integration/src/sqlbuild/executor/migrations/_helpers/test_naming.py delete mode 100644 tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 8e44a448e3..c37f4f9806 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -704,14 +704,6 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") - def render_replace_with_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> str: - del origin, destination, origin_is_transient - raise AdapterUserError( - message=f"adapter '{self.adapter_name}' does not support model migrations" - ) - def render_migration_stage( self, *, @@ -726,6 +718,12 @@ def render_migration_stage( statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), ) + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + def supports_transactional_ddl(self) -> bool: return False diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 9df7667b4c..73e76b65c1 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1526,12 +1526,6 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") - def render_replace_with_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> str: - del origin_is_transient - return f"CREATE OR REPLACE TABLE {destination} AS SELECT * FROM {origin}" - def render_migration_stage( self, *, @@ -1546,6 +1540,12 @@ def render_migration_stage( statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), ) + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + def supports_transactional_ddl(self) -> bool: return True diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index b1e24572a2..1b1a19a1c0 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -342,13 +342,6 @@ def render_durable_clone( """Render SQL statements that clone/copy into a durable independent destination.""" ... - @abstractmethod - def render_replace_with_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> str: - """Render one statement that replaces the destination with a durable source copy.""" - ... - @abstractmethod def render_migration_stage( self, @@ -361,6 +354,13 @@ def render_migration_stage( """Render statements that create a fresh, independent stage holding the origin's data.""" ... + @abstractmethod + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + """Capture statements that re-point identity-bound dependent views at this table name.""" + ... + @abstractmethod def supports_transactional_ddl(self) -> bool: """Return whether renames and state inserts can commit or roll back together.""" diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 32b8a9f5a1..68e116ffea 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -42,6 +42,12 @@ class MigrationTransfer(StrEnum): CLONE = "clone" COPY = "copy" + @property + def label(self) -> str: + """Return the human-readable plan label for this transfer.""" + + return "zero-copy clone" if self == MigrationTransfer.CLONE else "physical copy" + class TablePromotionMode(StrEnum): IMMEDIATE = "immediate" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index aface50fab..e6a862d6a7 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1714,14 +1714,6 @@ def render_durable_clone( f"CLONE {self._quote_identifier_path(origin)}", ) - def render_replace_with_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> str: - del origin, destination, origin_is_transient - raise AdapterUserError( - message=f"adapter '{self.adapter_name}' does not support model migrations" - ) - def render_migration_stage( self, *, @@ -1739,6 +1731,12 @@ def render_migration_stage( fallback_statements=(f"CREATE TABLE {quoted_stage} COPY {quoted_origin}",), ) + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + def supports_transactional_ddl(self) -> bool: return False diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 0039a5e88e..ac4f765d59 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -1753,12 +1753,6 @@ def render_durable_clone( del origin_is_transient return (f"CREATE TABLE {destination} DEEP CLONE {origin}",) - def render_replace_with_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> str: - del origin_is_transient - return f"CREATE OR REPLACE TABLE {destination} DEEP CLONE {origin}" - def render_migration_stage( self, *, @@ -1773,6 +1767,12 @@ def render_migration_stage( statements=(f"CREATE TABLE {stage} DEEP CLONE {origin}",), ) + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + def supports_transactional_ddl(self) -> bool: return False diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 30c0638355..7165813ff0 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -579,14 +579,6 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") - def render_replace_with_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> str: - del origin, destination, origin_is_transient - raise AdapterUserError( - message=f"adapter '{self.adapter_name}' does not support model migrations" - ) - def render_migration_stage( self, *, @@ -601,6 +593,36 @@ def render_migration_stage( statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), ) + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del database + query: str = ( + "SELECT DISTINCT dependent_namespace.nspname, dependent.relname, " + "pg_get_viewdef(dependent.oid) " + "FROM pg_depend AS dependency " + "JOIN pg_rewrite AS rewrite ON rewrite.oid = dependency.objid " + "JOIN pg_class AS dependent ON dependent.oid = rewrite.ev_class " + "JOIN pg_namespace AS dependent_namespace " + "ON dependent_namespace.oid = dependent.relnamespace " + "JOIN pg_class AS referenced ON referenced.oid = dependency.refobjid " + "JOIN pg_namespace AS referenced_namespace " + "ON referenced_namespace.oid = referenced.relnamespace " + "WHERE dependency.classid = 'pg_rewrite'::regclass " + "AND dependency.refclassid = 'pg_class'::regclass " + "AND dependent.relkind = 'v' " + "AND dependent.oid <> referenced.oid " + f"AND referenced_namespace.nspname = {_quote_sql_string(schema)} " + f"AND referenced.relname = {_quote_sql_string(name)} " + "ORDER BY 1, 2" + ) + cursor: Any = connection.execute(query) + return tuple( + f"CREATE OR REPLACE VIEW {self.render_identifier(str(row[0]))}." + f"{self.render_identifier(str(row[1]))} AS {str(row[2]).strip().rstrip(';')}" + for row in cursor.fetchall() + ) + def supports_transactional_ddl(self) -> bool: return True diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 23a19be7ed..73427c911b 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -2232,13 +2232,6 @@ def render_durable_clone( table_kind: str = "TRANSIENT TABLE" if origin_is_transient else "TABLE" return (f"CREATE OR REPLACE {table_kind} {destination} CLONE {origin}",) - def render_replace_with_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> str: - return self.render_durable_clone( - origin=origin, destination=destination, origin_is_transient=origin_is_transient - )[0] - def render_migration_stage( self, *, @@ -2262,6 +2255,12 @@ def render_migration_stage( statements=(f"CREATE {table_kind} {stage} CLONE {origin} COPY GRANTS",), ) + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + def supports_transactional_ddl(self) -> bool: return False diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 2abe4ce1cc..d1bd64c594 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -1774,14 +1774,6 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") - def render_replace_with_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> str: - del origin, destination, origin_is_transient - raise AdapterUserError( - message=f"adapter '{self.adapter_name}' does not support model migrations" - ) - def render_migration_stage( self, *, @@ -1796,6 +1788,12 @@ def render_migration_stage( statements=(f"SELECT * INTO {stage} FROM {origin}",), ) + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + def supports_transactional_ddl(self) -> bool: return True diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 89ed08db32..7720e21e1b 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -486,7 +486,9 @@ def _serialize_model_migrations(plan: PlanOutput) -> list[dict[str, object]]: "destination": entry.destination.qualified_name or entry.destination.name, "target": entry.target_name, "origin_version_hash": entry.origin_version_hash, - "statement": entry.statement, + "transfer": entry.transfer.value if entry.transfer is not None else None, + "storage_transition": entry.storage_transition, + "promotion": entry.promotion.value if entry.promotion is not None else None, "completed_at": ( entry.completed_at.isoformat() if entry.completed_at is not None else None ), diff --git a/src/sqlbuild/compiler/migrations/main/ensure_table.py b/src/sqlbuild/compiler/migrations/main/ensure_table.py new file mode 100644 index 0000000000..637cb44f70 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/ensure_table.py @@ -0,0 +1,33 @@ +"""Create the append-only model migration table before a transactional promotion.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute, FrameworkType +from sqlbuild.compiler.migrations._helpers.sql import build_create_table_sql + + +def ensure_migration_table( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool, +) -> None: + """Create the migration state table in one schema when it is missing.""" + + _ = execute( + connection=connection, + sql=build_create_table_sql( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + render_framework_type=render_framework_type, + transient=transient, + ), + ) diff --git a/src/sqlbuild/compiler/migrations/main/write_event.py b/src/sqlbuild/compiler/migrations/main/write_event.py index 4597617f8f..e7649beb5f 100644 --- a/src/sqlbuild/compiler/migrations/main/write_event.py +++ b/src/sqlbuild/compiler/migrations/main/write_event.py @@ -25,35 +25,44 @@ def write_migration_event( render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], transient: bool, + create_table: bool = True, + attempts: int = MIGRATION_WRITE_ATTEMPTS, ) -> None: """Create the state table when missing and insert the event unless already present.""" if event.destination.schema is None: raise MigrationStateError("model migration events require a destination schema") + create_statements: tuple[str, ...] = ( + ( + build_create_table_sql( + database=event.destination.database, + schema=event.destination.schema, + render_qualified_name=render_qualified_name, + render_framework_type=render_framework_type, + transient=transient, + ), + ) + if create_table + else () + ) statements: tuple[str, ...] = ( - build_create_table_sql( - database=event.destination.database, - schema=event.destination.schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - transient=transient, - ), + *create_statements, build_insert_sql(event=event, render_qualified_name=render_qualified_name), ) attempt: int - for attempt in range(MIGRATION_WRITE_ATTEMPTS): + for attempt in range(attempts): try: statement: str for statement in statements: _ = execute(connection=connection, sql=statement) return except Exception as error: - if attempt + 1 == MIGRATION_WRITE_ATTEMPTS: + if attempt + 1 == attempts: raise logging.getLogger("sqlbuild.migrations").warning( "model migration event write attempt %s/%s failed for '%s'; retrying: %s", attempt + 1, - MIGRATION_WRITE_ATTEMPTS, + attempts, event.destination_model, error, ) diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py index 9c406ff181..0389346ab8 100644 --- a/src/sqlbuild/compiler/migrations/types.py +++ b/src/sqlbuild/compiler/migrations/types.py @@ -61,6 +61,20 @@ class MigrationDiscovery(StrEnum): AUTOMATIC = "automatic" +class MigrationPromotion(StrEnum): + """How a completed migration stage replaces or creates the destination.""" + + SWAP = "swap" + RENAME = "rename" + TRANSACTIONAL_RENAME = "transactional_rename" + + @property + def label(self) -> str: + """Return the human-readable plan label for this promotion.""" + + return self.value.replace("_", " ") + + class MigrationCompatibility(StrEnum): """Outcome of checking the origin relation against the destination model.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index dfc1f1803c..4674a228d3 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -6,7 +6,8 @@ from dataclasses import replace from datetime import UTC, datetime -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo +from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.models import CompiledModel, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType @@ -19,6 +20,7 @@ MigrationCompatibility, MigrationDecision, MigrationDiscovery, + MigrationPromotion, ) from sqlbuild.compiler.planner._helpers.migrations.compatibility import ( MigrationCompatibilityResult, @@ -61,6 +63,7 @@ from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.models import SnapshotsConfig +from sqlbuild.spec.contracts.types import TableType def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: @@ -423,14 +426,7 @@ def _decide( if not decision.moves_data or entry.blocks_build: return entry, None, False return ( - replace( - entry, - statement=runtime.adapter.render_replace_with_clone( - origin=origin.qualified_name or origin.name, - destination=destination.qualified_name or destination.name, - origin_is_transient=entry.origin_is_transient, - ), - ), + _with_execution(runtime=runtime, model=model, entry=entry), _handover_fingerprint( model=model, fingerprint=origin_fingerprint, @@ -440,6 +436,40 @@ def _decide( ) +def _with_execution( + *, runtime: PlannerRuntime, model: CompiledModel, entry: ModelMigrationPlanEntry +) -> ModelMigrationPlanEntry: + """Describe how the executor will stage and promote this migration.""" + + stage_is_transient: bool | None = ( + model.config.table_type.value == TableType.TRANSIENT + if runtime.adapter.adapter_name == BuiltinAdapter.SNOWFLAKE + else None + ) + stage: MigrationStagePlan = runtime.adapter.render_migration_stage( + origin=entry.origin.qualified_name or entry.origin.name, + stage=entry.destination.qualified_name or entry.destination.name, + origin_is_transient=entry.origin_is_transient, + stage_is_transient=stage_is_transient, + ) + return replace( + entry, + stage_is_transient=stage_is_transient, + transfer=stage.transfer, + promotion=_promotion(runtime=runtime, decision=entry.decision), + ) + + +def _promotion(*, runtime: PlannerRuntime, decision: MigrationDecision) -> MigrationPromotion: + if runtime.adapter.supports_transactional_ddl(): + return MigrationPromotion.TRANSACTIONAL_RENAME + if decision != MigrationDecision.MIGRATE and ( + runtime.adapter.adapter_name == BuiltinAdapter.SNOWFLAKE + ): + return MigrationPromotion.SWAP + return MigrationPromotion.RENAME + + def _done_handover_applies( *, model: CompiledModel, event: MigrationEvent, snapshot: WarehouseSnapshot ) -> bool: @@ -527,6 +557,11 @@ def _overlay_snapshot( database=entry.destination.database, schema=entry.destination.schema, name=entry.destination.name, + is_transient=( + origin_relation.is_transient + if entry.stage_is_transient is None + else entry.stage_is_transient + ), ) columns[entry.model_name] = state.relation_columns(entry.origin) if entry.origin.qualified_name is not None and _has_cursor( @@ -619,8 +654,8 @@ def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: message=( f"migration conflict: {destination} already exists with its own build history " f"and has no recorded migration from {origin}; set migrate_force true to " - "replace it (the replaced table remains recoverable through warehouse time " - "travel)" + "replace it (the replaced table is kept under a _sqb_archive__ name until " + "janitor expires it)" ), code="M103", ) diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py index d35a1d0edf..dca3503599 100644 --- a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py +++ b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py @@ -84,6 +84,13 @@ def _migration_lines(entry: ModelMigrationPlanEntry) -> list[str]: rows: list[str] = [f" {entry.model_name} {entry.decision.label} {origin} -> {destination}"] if entry.decision.checks_compatibility: rows.append(f" compatibility {entry.compatibility.value}") + if entry.transfer is not None: + details: tuple[str | None, ...] = ( + entry.transfer.label, + entry.storage_transition, + None if entry.promotion is None else f"promote by {entry.promotion.label}", + ) + rows.append(f" transfer {', '.join(detail for detail in details if detail)}") if entry.discovery != MigrationDiscovery.MANUAL: rows.append(f" discovery {entry.discovery.value}") if entry.completed_at is not None: diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 9daba01e15..c9470158da 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -11,6 +11,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RetentionRequest +from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.auditing.models import MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -34,6 +35,7 @@ MigrationCompatibility, MigrationDecision, MigrationDiscovery, + MigrationPromotion, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.types import ( @@ -977,11 +979,22 @@ class ModelMigrationPlanEntry: target_name: str | None origin_version_hash: str = "" origin_is_transient: bool = False - statement: str | None = None + stage_is_transient: bool | None = None + transfer: MigrationTransfer | None = None + promotion: MigrationPromotion | None = None completed_at: datetime | None = None compatibility_findings: tuple[str, ...] = () message: str | None = None + @property + def storage_transition(self) -> str | None: + """Return the Snowflake table-type transition label, when both types are known.""" + + if self.stage_is_transient is None: + return None + labels: tuple[str, str] = ("permanent", "transient") + return f"{labels[self.origin_is_transient]} -> {labels[self.stage_is_transient]}" + @property def blocks_build(self) -> bool: """Return whether this migration must stop a build before any execution.""" diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 3b96a8518c..1d49045db1 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -21,7 +21,6 @@ reconcile_retention_after_build, ) from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler -from sqlbuild.executor.build.main._apply_model_migrations import apply_model_migrations from sqlbuild.executor.build.main.aggregate_result import aggregate_build_result from sqlbuild.executor.build.models import ( BuildCallbacks, @@ -34,6 +33,7 @@ ) from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.migrations.main._apply import apply_model_migrations from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_names.py b/src/sqlbuild/executor/janitor/_helpers/archive_names.py index b9382bfa27..af806b6cdb 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_names.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_names.py @@ -31,13 +31,17 @@ def archive_timestamp(value: datetime) -> datetime: return aware.replace(microsecond=0) -def build_archive_name(*, original_name: str, archived_at: datetime, identifier_limit: int) -> str: - """Build the fitted, case-folded archive name, keeping the prefix and timestamp intact.""" +def build_archive_name( + *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str | None = None +) -> str: + """Build the fitted, case-folded archive name, keeping prefix, timestamp, and kind intact.""" + kind_segment: str = "" if kind is None else f"{kind}{ARCHIVE_NAME_SEPARATOR}" fixed_prefix: str = ( f"{ARCHIVE_NAME_PREFIX}" f"{archive_timestamp(archived_at).strftime(ARCHIVE_TIMESTAMP_FORMAT)}" f"{ARCHIVE_NAME_SEPARATOR}" + f"{kind_segment}" ) logical_part: str = fit_artifact_logical_name( logical_name=original_name, diff --git a/src/sqlbuild/executor/janitor/main/_build_archive_name.py b/src/sqlbuild/executor/janitor/main/_build_archive_name.py new file mode 100644 index 0000000000..e7f55b8153 --- /dev/null +++ b/src/sqlbuild/executor/janitor/main/_build_archive_name.py @@ -0,0 +1,20 @@ +"""Public janitor archive naming for relations archived outside the janitor.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlbuild.executor.janitor._helpers.archive_names import build_archive_name + + +def build_janitor_archive_name( + *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str +) -> str: + """Build a strict archive name that janitor expires by its embedded timestamp.""" + + return build_archive_name( + original_name=original_name, + archived_at=archived_at, + identifier_limit=identifier_limit, + kind=kind, + ) diff --git a/src/sqlbuild/executor/migrations/__init__.py b/src/sqlbuild/executor/migrations/__init__.py new file mode 100644 index 0000000000..3a4ddf9cb2 --- /dev/null +++ b/src/sqlbuild/executor/migrations/__init__.py @@ -0,0 +1 @@ +"""Model migration executor package.""" diff --git a/src/sqlbuild/executor/migrations/_helpers/__init__.py b/src/sqlbuild/executor/migrations/_helpers/__init__.py new file mode 100644 index 0000000000..2c57a3bbe3 --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/__init__.py @@ -0,0 +1 @@ +"""Model migration executor helpers.""" diff --git a/src/sqlbuild/executor/migrations/_helpers/naming.py b/src/sqlbuild/executor/migrations/_helpers/naming.py new file mode 100644 index 0000000000..7c4a2ae6c0 --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/naming.py @@ -0,0 +1,86 @@ +"""Fresh stage and displaced-destination names for one migration attempt.""" + +from __future__ import annotations + +import itertools +from datetime import datetime, timedelta +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( + resolve_qualified_name_parts, +) +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.janitor.main._build_archive_name import build_janitor_archive_name +from sqlbuild.executor.migrations.constants import ( + MIGRATION_NAME_ATTEMPTS, + MIGRATION_PREVIOUS_ARCHIVE_KIND, + MIGRATION_STAGE_ARCHIVE_KIND, +) +from sqlbuild.executor.migrations.models import MigrationArtifactNames + + +def resolve_artifact_names( + *, adapter: BaseAdapter, connection: Any, entry: ModelMigrationPlanEntry, now: datetime +) -> MigrationArtifactNames: + """Pick the latest second at or before now, never later, whose archive names are unused.""" + + schema: str | None = entry.destination.schema + if schema is None: + raise ExecutorInputError( + f"model '{entry.model_name}': model migrations require a destination schema" + ) + database: str | None = entry.destination.database + identifier_limit: int = adapter.maximum_identifier_length() + candidates: tuple[tuple[str, str], ...] = tuple( + ( + build_janitor_archive_name( + original_name=entry.destination.name, + archived_at=now - timedelta(seconds=offset), + identifier_limit=identifier_limit, + kind=MIGRATION_STAGE_ARCHIVE_KIND, + ), + build_janitor_archive_name( + original_name=entry.destination.name, + archived_at=now - timedelta(seconds=offset), + identifier_limit=identifier_limit, + kind=MIGRATION_PREVIOUS_ARCHIVE_KIND, + ), + ) + for offset in range(MIGRATION_NAME_ATTEMPTS) + ) + listed: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=database, + schemas=(schema,), + names=(entry.destination.name, *itertools.chain.from_iterable(candidates)), + ) + existing: frozenset[str] = frozenset( + relation.name.lower() + for relation in listed + if (relation.schema or "").lower() == schema.lower() + ) + stage_name: str + displaced_name: str + for stage_name, displaced_name in candidates: + if stage_name not in existing and displaced_name not in existing: + return MigrationArtifactNames( + stage_name=stage_name, + stage_qualified=resolve_qualified_name_parts( + adapter=adapter, database=database, schema=schema, name=stage_name + ), + displaced_name=displaced_name, + displaced_qualified=resolve_qualified_name_parts( + adapter=adapter, database=database, schema=schema, name=displaced_name + ), + destination_qualified=resolve_qualified_name_parts( + adapter=adapter, database=database, schema=schema, name=entry.destination.name + ), + destination_exists=entry.destination.name.lower() in existing, + ) + raise ExecutorInputError( + f"model '{entry.model_name}': no unused migration stage name was found in the last " + f"{MIGRATION_NAME_ATTEMPTS} seconds" + ) diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py new file mode 100644 index 0000000000..f4d4d13601 --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -0,0 +1,122 @@ +"""Promote a verified migration stage and record the completed migration.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS +from sqlbuild.compiler.migrations.main.deterministic_event_id import ( + deterministic_migration_event_id, +) +from sqlbuild.compiler.migrations.main.ensure_table import ensure_migration_table +from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location +from sqlbuild.compiler.migrations.main.write_event import write_migration_event +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.executor.migrations.models import MigrationArtifactNames +from sqlbuild.executor.run.main.promote_staged_relation import promote_staged_relation + + +def promote_and_record( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelMigrationPlanEntry, + names: MigrationArtifactNames, + event: MigrationEvent, +) -> None: + """Promote the stage, keeping a displaced destination, then append the migration event.""" + + if not adapter.supports_transactional_ddl(): + _promote(adapter=adapter, connection=connection, names=names) + record_event(adapter=adapter, connection=connection, event=event) + return + ensure_migration_table( + connection=connection, + execute=adapter.execute, + database=event.destination.database, + schema=event.destination.schema or "", + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + transient=adapter.state_tables_transient, + ) + with adapter.transaction(connection): + rebinds: tuple[str, ...] = ( + adapter.capture_dependent_view_rebinds( + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema or "", + name=entry.destination.name, + ) + if names.destination_exists + else () + ) + _promote(adapter=adapter, connection=connection, names=names) + statement: str + for statement in rebinds: + _ = adapter.execute(connection=connection, sql=statement) + record_event( + adapter=adapter, connection=connection, event=event, create_table=False, attempts=1 + ) + + +def record_event( + *, + adapter: BaseAdapter, + connection: Any, + event: MigrationEvent, + create_table: bool = True, + attempts: int = MIGRATION_WRITE_ATTEMPTS, +) -> None: + """Append one migration event through the adapter's state-table rendering.""" + + write_migration_event( + connection=connection, + execute=adapter.execute, + event=event, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + transient=adapter.state_tables_transient, + create_table=create_table, + attempts=attempts, + ) + + +def _promote(*, adapter: BaseAdapter, connection: Any, names: MigrationArtifactNames) -> None: + promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified=names.destination_qualified, + staged_qualified=names.stage_qualified, + displaced_qualified=names.displaced_qualified, + target_exists=names.destination_exists, + statement_recorder=StatementRecorder(), + ) + + +def migration_event(*, entry: ModelMigrationPlanEntry, run_id: str) -> MigrationEvent: + """Build the append-only event recording one completed migration.""" + + origin: MigrationRelation = migration_relation_for_location(entry.origin) + destination: MigrationRelation = migration_relation_for_location(entry.destination) + return MigrationEvent( + event_id=deterministic_migration_event_id( + run_id=run_id, + target_name=entry.target_name, + origin=origin, + destination=destination, + ), + target_name=entry.target_name, + origin_model=entry.origin_model, + origin=origin, + destination_model=entry.model_name, + destination=destination, + origin_version_hash=entry.origin_version_hash, + discovery=entry.discovery, + decision=entry.decision, + run_id=run_id, + created_at=datetime.now(tz=UTC), + ) diff --git a/src/sqlbuild/executor/migrations/_helpers/staging.py b/src/sqlbuild/executor/migrations/_helpers/staging.py new file mode 100644 index 0000000000..5e1693f536 --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/staging.py @@ -0,0 +1,150 @@ +"""Create and verify an isolated migration stage without touching the live destination.""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.migrations.models import MigrationArtifactNames + + +def create_stage( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelMigrationPlanEntry, + names: MigrationArtifactNames, +) -> MigrationTransfer: + """Clone or copy the origin into the fresh stage, falling back to a copy if clone is refused.""" + + plan: MigrationStagePlan = adapter.render_migration_stage( + origin=entry.origin.qualified_name or entry.origin.name, + stage=names.stage_qualified, + origin_is_transient=entry.origin_is_transient, + stage_is_transient=entry.stage_is_transient, + ) + try: + _execute_all(adapter=adapter, connection=connection, statements=plan.statements) + except Exception as error: + if not plan.fallback_statements: + raise + logging.getLogger("sqlbuild.migrations").warning( + "migration clone into '%s' was refused; copying instead: %s", + names.stage_name, + error, + ) + _execute_all(adapter=adapter, connection=connection, statements=plan.fallback_statements) + return MigrationTransfer.COPY + return plan.transfer + + +def verify_stage( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelMigrationPlanEntry, + names: MigrationArtifactNames, + transfer: MigrationTransfer, +) -> None: + """Refuse promotion unless the stage is complete, typed as planned, and matches the origin.""" + + stage: RelationInfo = _listed( + adapter=adapter, + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + name=names.stage_name, + missing=f"migration stage {names.stage_qualified} was not created", + ) + origin: RelationInfo = _listed( + adapter=adapter, + connection=connection, + database=entry.origin.database or entry.destination.database, + schema=entry.origin.schema, + name=entry.origin.name, + missing=f"migration origin {entry.origin.qualified_name} no longer exists", + ) + if ( + entry.stage_is_transient is not None + and stage.is_transient is not None + and stage.is_transient != entry.stage_is_transient + ): + raise ExecutorInputError( + f"migration stage {names.stage_qualified} was not created with the destination's " + "table type; refusing promotion" + ) + columns: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + adapter.get_columns_for_relations(connection=connection, relations=(stage, origin)) + ) + if _column_names(columns.get(stage.identity, ())) != _column_names( + columns.get(origin.identity, ()) + ): + raise ExecutorInputError( + f"migration stage {names.stage_qualified} columns differ from the origin; " + "refusing promotion" + ) + if transfer != MigrationTransfer.COPY: + return + stage_rows: int | None = _row_count( + adapter=adapter, connection=connection, relation=names.stage_qualified + ) + origin_rows: int | None = _row_count( + adapter=adapter, + connection=connection, + relation=entry.origin.qualified_name or entry.origin.name, + ) + if stage_rows != origin_rows: + raise ExecutorInputError( + f"migration stage {names.stage_qualified} holds {stage_rows} rows but the origin " + f"holds {origin_rows}; refusing to promote an incomplete copy" + ) + + +def _execute_all(*, adapter: BaseAdapter, connection: Any, statements: tuple[str, ...]) -> None: + statement: str + for statement in statements: + _ = adapter.execute(connection=connection, sql=statement) + + +def _listed( + *, + adapter: BaseAdapter, + connection: Any, + database: str | None, + schema: str | None, + name: str, + missing: str, +) -> RelationInfo: + listed: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=database, + schemas=(schema,) if schema is not None else None, + names=(name,), + ) + relation: RelationInfo | None = next( + ( + relation + for relation in listed + if relation.name.lower() == name.lower() + and (relation.schema or "").lower() == (schema or "").lower() + ), + None, + ) + if relation is None: + raise ExecutorInputError(missing) + return relation + + +def _column_names(columns: tuple[ColumnInfo, ...]) -> tuple[str, ...]: + return tuple(column.name.lower() for column in columns) + + +def _row_count(*, adapter: BaseAdapter, connection: Any, relation: str) -> int | None: + cursor: Any = adapter.execute(connection=connection, sql=f"SELECT COUNT(*) FROM {relation}") + row: Any = cursor.fetchone() + return None if row is None or row[0] is None else int(row[0]) diff --git a/src/sqlbuild/executor/migrations/constants.py b/src/sqlbuild/executor/migrations/constants.py new file mode 100644 index 0000000000..2798088f67 --- /dev/null +++ b/src/sqlbuild/executor/migrations/constants.py @@ -0,0 +1,7 @@ +"""Model migration executor constants.""" + +from __future__ import annotations + +MIGRATION_STAGE_ARCHIVE_KIND: str = "migration_stage" +MIGRATION_PREVIOUS_ARCHIVE_KIND: str = "migration_previous" +MIGRATION_NAME_ATTEMPTS: int = 30 diff --git a/src/sqlbuild/executor/migrations/main/__init__.py b/src/sqlbuild/executor/migrations/main/__init__.py new file mode 100644 index 0000000000..5c0117eccc --- /dev/null +++ b/src/sqlbuild/executor/migrations/main/__init__.py @@ -0,0 +1 @@ +"""Model migration executor entrypoints.""" diff --git a/src/sqlbuild/executor/build/main/_apply_model_migrations.py b/src/sqlbuild/executor/migrations/main/_apply.py similarity index 57% rename from src/sqlbuild/executor/build/main/_apply_model_migrations.py rename to src/sqlbuild/executor/migrations/main/_apply.py index d63d071d17..d6d6b7a224 100644 --- a/src/sqlbuild/executor/build/main/_apply_model_migrations.py +++ b/src/sqlbuild/executor/migrations/main/_apply.py @@ -9,14 +9,13 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.migrations.main.deterministic_event_id import ( - deterministic_migration_event_id, -) -from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location -from sqlbuild.compiler.migrations.main.write_event import write_migration_event -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.migrations._helpers.naming import resolve_artifact_names +from sqlbuild.executor.migrations._helpers.promotion import migration_event, promote_and_record +from sqlbuild.executor.migrations._helpers.staging import create_stage, verify_stage +from sqlbuild.executor.migrations.models import MigrationArtifactNames def apply_model_migrations( @@ -27,7 +26,7 @@ def apply_model_migrations( run_id: str, on_progress: Callable[[str], None] | None = None, ) -> None: - """Clone each pending migration origin over its destination, then record the event.""" + """Stage, verify, and promote each pending migration, then record its event.""" blocked: tuple[ModelMigrationPlanEntry, ...] = tuple( entry for entry in plan.migration_entries if entry.blocks_build @@ -39,9 +38,7 @@ def apply_model_migrations( code="M103", ) pending: tuple[ModelMigrationPlanEntry, ...] = tuple( - entry - for entry in plan.migration_entries - if entry.decision.moves_data and entry.statement is not None + entry for entry in plan.migration_entries if entry.decision.moves_data ) entry: ModelMigrationPlanEntry for entry in pending: @@ -68,12 +65,17 @@ def _apply_one( on_progress(f"Migrating {origin_label} -> {destination_label} ({entry.decision.value})...") started: float = time.monotonic() try: - _ = _clone_and_record(entry=entry, adapter=adapter, connection=connection, run_id=run_id) + _ = _stage_promote_and_record( + entry=entry, adapter=adapter, connection=connection, run_id=run_id + ) except Exception as error: raise ExecutorInputError( f"model migration {origin_label} -> {destination_label} failed: {error}", code="M106", - help="Re-run the build; migration decisions are re-evaluated from recorded events.", + help=( + "Re-run the build; migration decisions are re-evaluated from recorded events. " + "Any abandoned _sqb_archive__ stage is left for janitor to expire." + ), ) from error if on_progress is not None: on_progress( @@ -81,7 +83,7 @@ def _apply_one( ) -def _clone_and_record( +def _stage_promote_and_record( *, entry: ModelMigrationPlanEntry, adapter: BaseAdapter, connection: Any, run_id: str ) -> None: adapter.ensure_schema( @@ -90,31 +92,19 @@ def _clone_and_record( schema=entry.destination.schema, statement_recorder=StatementRecorder(), ) - _ = adapter.execute(connection=connection, sql=entry.statement or "") - origin: MigrationRelation = migration_relation_for_location(entry.origin) - destination: MigrationRelation = migration_relation_for_location(entry.destination) - write_migration_event( + names: MigrationArtifactNames = resolve_artifact_names( + adapter=adapter, connection=connection, entry=entry, now=datetime.now(tz=UTC) + ) + transfer: MigrationTransfer = create_stage( + adapter=adapter, connection=connection, entry=entry, names=names + ) + _ = verify_stage( + adapter=adapter, connection=connection, entry=entry, names=names, transfer=transfer + ) + _ = promote_and_record( + adapter=adapter, connection=connection, - execute=adapter.execute, - event=MigrationEvent( - event_id=deterministic_migration_event_id( - run_id=run_id, - target_name=entry.target_name, - origin=origin, - destination=destination, - ), - target_name=entry.target_name, - origin_model=entry.origin_model, - origin=origin, - destination_model=entry.model_name, - destination=destination, - origin_version_hash=entry.origin_version_hash, - discovery=entry.discovery, - decision=entry.decision, - run_id=run_id, - created_at=datetime.now(tz=UTC), - ), - render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - transient=adapter.state_tables_transient, + entry=entry, + names=names, + event=migration_event(entry=entry, run_id=run_id), ) diff --git a/src/sqlbuild/executor/migrations/models.py b/src/sqlbuild/executor/migrations/models.py new file mode 100644 index 0000000000..7a6994abee --- /dev/null +++ b/src/sqlbuild/executor/migrations/models.py @@ -0,0 +1,17 @@ +"""Model migration executor models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MigrationArtifactNames: + """Fresh janitor-archive names for one migration attempt's stage and displaced destination.""" + + stage_name: str + stage_qualified: str + displaced_name: str + displaced_qualified: str + destination_qualified: str + destination_exists: bool diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index 4f50695d2a..8184f356ed 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -78,3 +78,32 @@ class VirtualModeMigrationTestCase: description: str expected_exit_code: int expected_fragment: str + + +@dataclass(frozen=True) +class StagedReplaceTestCase: + description: str + install_failure: Callable[[pytest.MonkeyPatch], None] + expected_first_exit_code: int + expected_relations_after_failure: tuple[str, ...] + expected_previous_ids_after_failure: tuple[tuple[int, ...], ...] + expected_decision_after_failure: str + expected_final_previous_ids: tuple[tuple[int, ...], ...] + expected_final_abandoned_stages: int + expected_final_events: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class StagedPlanOutputTestCase: + description: str + expected_transfer: str + expected_promotion: str + expected_storage_transition: str | None + expected_text_fragment: str + + +@dataclass(frozen=True) +class MigrationArchiveExpiryTestCase: + description: str + expected_archives_before: int + expected_relations_after: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 0a90de0a3f..5b499a204c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -13,6 +13,8 @@ import pytest from _pytest.capture import CaptureResult +from sqlbuild.adapter.contract.models import MigrationStagePlan +from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.executor.build._helpers import scheduler as scheduler_module @@ -66,6 +68,24 @@ database = "state.duckdb" """ ).lstrip() +JANITOR_PROJECT_TOML: str = dedent( + f""" + name = "orders_project" + adapter = "duckdb" + + default_target = "dev" + + [connection] + database = "{DATABASE_FILE}" + + [targets.dev] + schema = "dev" + + [janitor] + enabled = true + archive_retention_days = 0 + """ +).lstrip() RAW_SOURCES_YML: str = dedent( """ sources: @@ -355,42 +375,100 @@ def snapshot_orders_sql(*, migration_lines: str) -> str: return _SNAPSHOT_SQL.format(migration=migration_lines) -def _raise_interruption(**_: object) -> None: +def _raise_interruption(*_: object, **__: object) -> None: raise RuntimeError("simulated interruption") def fail_clone(monkeypatch: pytest.MonkeyPatch) -> None: - """Make the migration clone statement fail.""" + """Make the migration stage statement fail before any stage exists.""" monkeypatch.setattr( DuckDbAdapter, - "render_replace_with_clone", - lambda self, *, origin, destination, origin_is_transient=False: ( - "SELECT * FROM main.simulated_missing_relation" + "render_migration_stage", + lambda self, **_: MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=("SELECT * FROM main.simulated_missing_relation",), ), ) -def fail_clone_into(*, monkeypatch: pytest.MonkeyPatch, destination: str) -> None: - """Make only the clone into one destination fail; every other clone runs normally.""" +def fail_partial_stage(monkeypatch: pytest.MonkeyPatch) -> None: + """Leave an empty, incomplete stage behind and then fail the copy.""" - original: Callable[..., str] = DuckDbAdapter.render_replace_with_clone - failing: dict[str, str] = {destination: "SELECT * FROM main.simulated_missing_relation"} monkeypatch.setattr( DuckDbAdapter, - "render_replace_with_clone", - lambda self, *, origin, destination, origin_is_transient=False: failing.get( - destination, - original( - self, - origin=origin, - destination=destination, - origin_is_transient=origin_is_transient, + "render_migration_stage", + lambda self, *, origin, stage, **_: MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=( + f"CREATE TABLE {stage} AS SELECT * FROM {origin} WHERE false", + "SELECT * FROM main.simulated_missing_relation", ), ), ) +def fail_verification(monkeypatch: pytest.MonkeyPatch) -> None: + """Complete the stage, then refuse promotion.""" + + monkeypatch.setattr( + "sqlbuild.executor.migrations.main._apply.verify_stage", _raise_interruption + ) + + +def fail_clone_into(*, monkeypatch: pytest.MonkeyPatch, origin: str) -> None: + """Make only the stage copied from one origin fail; every other stage runs normally.""" + + original: Callable[..., MigrationStagePlan] = DuckDbAdapter.render_migration_stage + failing: dict[str, MigrationStagePlan] = { + origin: MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=("SELECT * FROM main.simulated_missing_relation",), + ) + } + monkeypatch.setattr( + DuckDbAdapter, + "render_migration_stage", + lambda self, *, origin, **kwargs: ( + failing.get(origin) or original(self, origin=origin, **kwargs) + ), + ) + + +def disable_transactional_ddl(monkeypatch: pytest.MonkeyPatch) -> None: + """Promote without a transaction, as Snowflake, BigQuery, and Databricks do.""" + + monkeypatch.setattr(DuckDbAdapter, "supports_transactional_ddl", lambda self: False) + + +def fail_promotion_rename(monkeypatch: pytest.MonkeyPatch) -> None: + """Fail when the stage is renamed into place, after any live destination moved aside.""" + + original: Callable[..., None] = DuckDbAdapter.rename + failing: dict[bool, Callable[..., None]] = {True: _raise_interruption} + monkeypatch.setattr( + DuckDbAdapter, + "rename", + lambda self, *, origin, **kwargs: failing.get("migration_stage" in origin, original)( + self, origin=origin, **kwargs + ), + ) + + +def fail_non_transactional_promotion_rename(monkeypatch: pytest.MonkeyPatch) -> None: + """Leave the destination moved aside and missing, as a non-transactional crash would.""" + + disable_transactional_ddl(monkeypatch) + fail_promotion_rename(monkeypatch) + + +def fail_non_transactional_record(monkeypatch: pytest.MonkeyPatch) -> None: + """Crash after a committed promotion but before the event, as non-transactional adapters can.""" + + disable_transactional_ddl(monkeypatch) + fail_record(monkeypatch) + + def fail_model_build(*, monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: """Make one incremental model's build raise after migrations have run.""" @@ -413,7 +491,7 @@ def fail_record(monkeypatch: pytest.MonkeyPatch) -> None: """Crash after the clone but before the migration event is recorded.""" monkeypatch.setattr( - "sqlbuild.executor.build.main._apply_model_migrations.write_migration_event", + "sqlbuild.executor.migrations._helpers.promotion.write_migration_event", _raise_interruption, ) @@ -519,3 +597,79 @@ def prepare_prod_rename(*, project_dir: Path, capsys: pytest.CaptureFixture[str] models={DESTINATION_MODEL: incremental_orders_sql(migrate_from=ORIGIN_MODEL)}, project_toml=TARGETS_PROJECT_TOML, ) + + +def archived_relations(*, project_dir: Path, kind: str, schema: str = "main") -> tuple[str, ...]: + """Return every janitor-archive relation of one migration kind in one schema.""" + + return tuple( + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' " + "AND starts_with(table_name, '_sqb_archive__') " + f"AND contains(table_name, '__{kind}__') ORDER BY 1" + ), + ) + ) + + +def archived_order_ids(*, project_dir: Path, kind: str) -> tuple[tuple[int, ...], ...]: + """Return the order IDs held by each archive of one migration kind.""" + + return tuple( + order_ids(project_dir=project_dir, relation=f"main.{name}") + for name in archived_relations(project_dir=project_dir, kind=kind) + ) + + +def prepare_forced_replace( + *, project_dir: Path, capsys: pytest.CaptureFixture[str], project_toml: str = PROJECT_TOML +) -> None: + """Build an origin and an independent destination, then force the origin over it.""" + + write_project( + project_dir=project_dir, + models={ + ORIGIN_MODEL: incremental_orders_sql(), + DESTINATION_MODEL: incremental_orders_sql( + select_sql=( + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders") ' + "WHERE order_id <= 2" + ) + ), + }, + project_toml=project_toml, + ) + load_raw_orders(project_dir=project_dir, first_day=1, last_day=5) + _ = build_ok(project_dir=project_dir, capsys=capsys) + write_project( + project_dir=project_dir, + models={ + DESTINATION_MODEL: incremental_orders_sql(migrate_from=ORIGIN_MODEL, migrate_force=True) + }, + project_toml=project_toml, + ) + + +def no_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """Install no failure.""" + + del monkeypatch + + +def live_relations(*, project_dir: Path, schema: str = "main") -> tuple[str, ...]: + """Return relations in one schema that are neither state tables nor archives.""" + + return tuple( + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' AND NOT starts_with(table_name, '_') ORDER BY 1" + ), + ) + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py index 8908b27cbd..e9a866eb27 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py @@ -319,7 +319,7 @@ def test_given_interrupted_coordinated_renames_when_rerunning_then_downstream_re write_project(project_dir=tmp_path, models=renamed_order_models()) with monkeypatch.context() as patch: - fail_clone_into(monkeypatch=patch, destination="main.customer_daily_order_totals") + fail_clone_into(monkeypatch=patch, origin="main.daily_order_totals") interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) events_after_crash: tuple[tuple[str, str, str], ...] = migration_events(project_dir=tmp_path) retry_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index 9791230cc4..14423a36bd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -24,7 +24,11 @@ execute, fail_after_migrations, fail_clone, + fail_non_transactional_promotion_rename, + fail_non_transactional_record, + fail_partial_stage, fail_record, + fail_verification, incremental_orders_sql, load_raw_orders, migration_decisions, @@ -578,13 +582,45 @@ def test_given_origin_schema_when_migrating_then_normal_incremental_rules_decide expected_final_decisions=("migrate",), ), MigrationInterruptionTestCase( - description="crash before recording redoes the clone on a plain re-run", + description="partial stage is abandoned and a fresh stage is used on re-run", + install_failure=fail_partial_stage, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="failed stage verification never promotes the stage", + install_failure=fail_verification, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="transactional crash before recording rolls the promotion back", install_failure=fail_record, rerun_with_force=False, expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), + MigrationInterruptionTestCase( + description="non-transactional crash before recording redoes the move", + install_failure=fail_non_transactional_record, + rerun_with_force=False, + expected_first_exit_code=1, expected_decision_after_failure="redo", expected_final_decisions=("redo",), ), + MigrationInterruptionTestCase( + description="non-transactional crash while renaming the stage migrates again", + install_failure=fail_non_transactional_promotion_rename, + rerun_with_force=False, + expected_first_exit_code=1, + expected_decision_after_failure="migrate", + expected_final_decisions=("migrate",), + ), MigrationInterruptionTestCase( description="crash after recording continues from the migrated history", install_failure=fail_after_migrations, @@ -662,7 +698,7 @@ def test_given_interrupted_superseded_replace_when_rerunning_then_replaces_again rename_model(project_dir=tmp_path, name=ORIGIN_MODEL, migrate_from=DESTINATION_MODEL) with monkeypatch.context() as patch: - fail_record(patch) + fail_non_transactional_record(patch) interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) _ = build_ok(project_dir=tmp_path, capsys=capsys) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py new file mode 100644 index 0000000000..5e08ab5ecc --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py @@ -0,0 +1,198 @@ +"""Integration coverage for staged migration execution, archived displacement, and recovery.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + MigrationArchiveExpiryTestCase, + StagedPlanOutputTestCase, + StagedReplaceTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + DESTINATION_MODEL, + JANITOR_PROJECT_TOML, + ORIGIN_MODEL, + CliRun, + archived_order_ids, + archived_relations, + build, + build_ok, + fail_non_transactional_promotion_rename, + fail_partial_stage, + fail_promotion_rename, + live_relations, + migration_decisions, + migration_events, + no_failure, + order_ids, + plan_json, + prepare_forced_replace, + run_sqb, +) + +_STAGE: str = "migration_stage" +_PREVIOUS: str = "migration_previous" +_FORCED_EVENT: tuple[str, str, str] = (ORIGIN_MODEL, DESTINATION_MODEL, "forced_replace") + + +@pytest.mark.parametrize( + "test_case", + [ + StagedReplaceTestCase( + description="successful forced replace keeps the displaced destination archived", + install_failure=no_failure, + expected_first_exit_code=0, + expected_relations_after_failure=("raw_orders", "stg_customer_orders", "stg_orders"), + expected_previous_ids_after_failure=((1, 2),), + expected_decision_after_failure="done", + expected_final_previous_ids=((1, 2),), + expected_final_abandoned_stages=0, + expected_final_events=(_FORCED_EVENT,), + ), + StagedReplaceTestCase( + description="transactional crash while promoting leaves the live destination intact", + install_failure=fail_promotion_rename, + expected_first_exit_code=1, + expected_relations_after_failure=("raw_orders", "stg_customer_orders", "stg_orders"), + expected_previous_ids_after_failure=(), + expected_decision_after_failure="forced_replace", + expected_final_previous_ids=((1, 2),), + expected_final_abandoned_stages=1, + expected_final_events=(_FORCED_EVENT,), + ), + StagedReplaceTestCase( + description="non-transactional crash between renames keeps the old data archived", + install_failure=fail_non_transactional_promotion_rename, + expected_first_exit_code=1, + expected_relations_after_failure=("raw_orders", "stg_orders"), + expected_previous_ids_after_failure=((1, 2),), + expected_decision_after_failure="migrate", + expected_final_previous_ids=((1, 2),), + expected_final_abandoned_stages=1, + expected_final_events=((ORIGIN_MODEL, DESTINATION_MODEL, "migrate"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_forced_replace_when_interrupted_then_retry_converges_without_losing_data( + test_case: StagedReplaceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The destination is never dropped and a retry never promotes an old stage.""" + + prepare_forced_replace(project_dir=tmp_path, capsys=capsys) + + with monkeypatch.context() as patch: + test_case.install_failure(patch) + first: CliRun = build(project_dir=tmp_path, capsys=capsys) + relations_after_failure: tuple[str, ...] = live_relations(project_dir=tmp_path) + previous_after_failure: tuple[tuple[int, ...], ...] = archived_order_ids( + project_dir=tmp_path, kind=_PREVIOUS + ) + after_failure: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert first.exit_code == test_case.expected_first_exit_code, first.output + assert relations_after_failure == test_case.expected_relations_after_failure + assert previous_after_failure == test_case.expected_previous_ids_after_failure + assert migration_decisions(after_failure) == (test_case.expected_decision_after_failure,) + assert order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") == tuple( + range(1, 6) + ) + assert order_ids(project_dir=tmp_path, relation=f"main.{ORIGIN_MODEL}") == tuple(range(1, 6)) + assert len(archived_relations(project_dir=tmp_path, kind=_STAGE)) == ( + test_case.expected_final_abandoned_stages + ) + assert archived_order_ids(project_dir=tmp_path, kind=_PREVIOUS) == ( + test_case.expected_final_previous_ids + ) + assert migration_events(project_dir=tmp_path) == test_case.expected_final_events + + +@pytest.mark.parametrize( + "test_case", + [ + StagedPlanOutputTestCase( + description="duckdb plans a physical copy promoted in one transaction", + expected_transfer="copy", + expected_promotion="transactional_rename", + expected_storage_transition=None, + expected_text_fragment="└── transfer physical copy, promote by transactional rename", + ) + ], + ids=lambda case: case.description, +) +def test_given_forced_replace_when_planning_then_reports_transfer_and_promotion( + test_case: StagedPlanOutputTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Plan text and JSON state how the migration is staged and promoted.""" + + prepare_forced_replace(project_dir=tmp_path, capsys=capsys) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + text: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + + migration: dict[str, Any] = plan["migrations"][0] + assert migration["transfer"] == test_case.expected_transfer + assert migration["promotion"] == test_case.expected_promotion + assert migration["storage_transition"] == test_case.expected_storage_transition + assert "statement" not in migration + assert test_case.expected_text_fragment in text.output + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationArchiveExpiryTestCase( + description="janitor expires abandoned stages and displaced destinations only", + expected_archives_before=2, + expected_relations_after=("stg_customer_orders", "stg_orders"), + ) + ], + ids=lambda case: case.description, +) +def test_given_migration_archives_when_janitor_runs_then_expires_them_by_name( + test_case: MigrationArchiveExpiryTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Archive names alone let janitor remove migration leftovers without touching live data.""" + + prepare_forced_replace(project_dir=tmp_path, capsys=capsys, project_toml=JANITOR_PROJECT_TOML) + with monkeypatch.context() as patch: + fail_partial_stage(patch) + _ = build(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + archives_before: int = len( + archived_relations(project_dir=tmp_path, kind=_STAGE, schema="dev") + ) + len(archived_relations(project_dir=tmp_path, kind=_PREVIOUS, schema="dev")) + + janitor: CliRun = run_sqb( + project_dir=tmp_path, + args=("janitor", "--auto-approve", "--retention-days", "30"), + capsys=capsys, + ) + + assert janitor.exit_code == 0, janitor.output + assert archives_before == test_case.expected_archives_before + assert archived_relations(project_dir=tmp_path, kind=_STAGE, schema="dev") == () + assert archived_relations(project_dir=tmp_path, kind=_PREVIOUS, schema="dev") == (), ( + janitor.output + ) + assert live_relations(project_dir=tmp_path, schema="dev") == ( + test_case.expected_relations_after + ) + assert order_ids(project_dir=tmp_path, relation=f"dev.{DESTINATION_MODEL}") == tuple( + range(1, 6) + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/migrations/__init__.py b/tests/integration/src/sqlbuild/executor/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/__init__.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py new file mode 100644 index 0000000000..c0eccdb248 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py @@ -0,0 +1,16 @@ +"""Test case types for migration executor helper integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class ArtifactNamingTestCase: + description: str + existing_relations: tuple[str, ...] + now: datetime + expected_stage_name: str + expected_displaced_name: str + expected_destination_exists: bool diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py new file mode 100644 index 0000000000..bb46c82e98 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -0,0 +1,33 @@ +"""Builders for migration executor helper integration tests.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.migrations.types import ( + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, +) +from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry + + +def orders_migration_entry() -> ModelMigrationPlanEntry: + """Return a planned forced replace of main.orders_v2 from main.orders.""" + + return ModelMigrationPlanEntry( + model_name="orders_v2", + discovery=MigrationDiscovery.MANUAL, + decision=MigrationDecision.FORCED_REPLACE, + compatibility=MigrationCompatibility.COMPATIBLE, + origin_model="orders", + origin=CompiledRelationLocation( + database=None, schema="main", name="orders", qualified_name="main.orders" + ), + destination=CompiledRelationLocation( + database=None, + schema="main", + name="orders_v2", + qualified_name="main.orders_v2", + ), + target_name=None, + ) diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_naming.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_naming.py new file mode 100644 index 0000000000..322c9ca1c8 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_naming.py @@ -0,0 +1,76 @@ +"""Integration coverage for fresh migration stage and displaced-destination names.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.migrations._helpers.naming import resolve_artifact_names +from sqlbuild.executor.migrations.models import MigrationArtifactNames +from tests.integration.src.sqlbuild.executor.migrations._helpers._test_types import ( + ArtifactNamingTestCase, +) +from tests.integration.src.sqlbuild.executor.migrations._helpers.helpers import ( + orders_migration_entry, +) + +_NOW: datetime = datetime(2026, 1, 2, 3, 4, 5, 678000, tzinfo=UTC) + + +@pytest.mark.parametrize( + "test_case", + [ + ArtifactNamingTestCase( + description="unused names take the current second and see a missing destination", + existing_relations=(), + now=_NOW, + expected_stage_name="_sqb_archive__20260102t030405z__migration_stage__orders_v2", + expected_displaced_name=( + "_sqb_archive__20260102t030405z__migration_previous__orders_v2" + ), + expected_destination_exists=False, + ), + ArtifactNamingTestCase( + description="an abandoned stage from the same second steps back one second", + existing_relations=( + "orders_v2", + "_sqb_archive__20260102t030405z__migration_stage__orders_v2", + ), + now=_NOW, + expected_stage_name="_sqb_archive__20260102t030404z__migration_stage__orders_v2", + expected_displaced_name=( + "_sqb_archive__20260102t030404z__migration_previous__orders_v2" + ), + expected_destination_exists=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_existing_relations_when_resolving_names_then_never_reuses_an_existing_stage( + test_case: ArtifactNamingTestCase, +) -> None: + """Every attempt stages under a name that did not exist when the attempt began.""" + + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + name: str + for name in test_case.existing_relations: + _ = adapter.execute( + connection=connection, sql=f"CREATE TABLE main.{name} AS SELECT 1 AS id" + ) + + names: MigrationArtifactNames = resolve_artifact_names( + adapter=adapter, connection=connection, entry=orders_migration_entry(), now=test_case.now + ) + + assert names.stage_name == test_case.expected_stage_name + assert names.displaced_name == test_case.expected_displaced_name + assert names.destination_exists is test_case.expected_destination_exists + assert names.stage_qualified == f"main.{test_case.expected_stage_name}" + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 2c02ce3089..3cc1f0296f 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -72,15 +72,6 @@ class RowDiffSampleSqlTestCase: expected_fragments: tuple[str, ...] -@dataclass(frozen=True) -class AdapterReplaceWithCloneTestCase: - description: str - adapter: BaseAdapter - origin_is_transient: bool - expected_statement: str | None - expected_error_fragment: str | None = None - - @dataclass(frozen=True) class AdapterRelationAgeMetadataCapabilityTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py b/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py deleted file mode 100644 index bdd932cfc7..0000000000 --- a/tests/unit/src/sqlbuild/adapters/test_replace_with_clone.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Unit coverage for the single-statement migration clone capability.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter -from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter -from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from tests.unit.src.sqlbuild.adapters._test_types import AdapterReplaceWithCloneTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - AdapterReplaceWithCloneTestCase( - description="snowflake replaces with a zero-copy clone", - adapter=SnowflakeAdapter(), - origin_is_transient=False, - expected_statement="CREATE OR REPLACE TABLE dev.stg_customer_orders CLONE dev.stg_orders", - ), - AdapterReplaceWithCloneTestCase( - description="snowflake preserves a transient origin table type", - adapter=SnowflakeAdapter(), - origin_is_transient=True, - expected_statement=( - "CREATE OR REPLACE TRANSIENT TABLE dev.stg_customer_orders CLONE dev.stg_orders" - ), - ), - AdapterReplaceWithCloneTestCase( - description="databricks uses an independent deep clone", - adapter=DatabricksAdapter(), - origin_is_transient=False, - expected_statement=( - "CREATE OR REPLACE TABLE dev.stg_customer_orders DEEP CLONE dev.stg_orders" - ), - ), - AdapterReplaceWithCloneTestCase( - description="duckdb copies with one replacing CTAS", - adapter=DuckDbAdapter(), - origin_is_transient=False, - expected_statement=( - "CREATE OR REPLACE TABLE dev.stg_customer_orders AS SELECT * FROM dev.stg_orders" - ), - ), - AdapterReplaceWithCloneTestCase( - description="motherduck inherits the replacing CTAS", - adapter=MotherDuckAdapter(), - origin_is_transient=False, - expected_statement=( - "CREATE OR REPLACE TABLE dev.stg_customer_orders AS SELECT * FROM dev.stg_orders" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_supported_adapter_when_rendering_replace_with_clone_then_returns_one_statement( - test_case: AdapterReplaceWithCloneTestCase, -) -> None: - statement: str = test_case.adapter.render_replace_with_clone( - origin="dev.stg_orders", - destination="dev.stg_customer_orders", - origin_is_transient=test_case.origin_is_transient, - ) - - assert statement == test_case.expected_statement - - -@pytest.mark.parametrize( - "test_case", - [ - AdapterReplaceWithCloneTestCase( - description="postgres rejects model migrations", - adapter=PostgresAdapter(), - origin_is_transient=False, - expected_statement=None, - expected_error_fragment="does not support model migrations", - ), - AdapterReplaceWithCloneTestCase( - description="bigquery rejects model migrations", - adapter=BigQueryAdapter(), - origin_is_transient=False, - expected_statement=None, - expected_error_fragment="does not support model migrations", - ), - AdapterReplaceWithCloneTestCase( - description="sqlserver rejects model migrations", - adapter=SqlServerAdapter(), - origin_is_transient=False, - expected_statement=None, - expected_error_fragment="does not support model migrations", - ), - ], - ids=lambda case: case.description, -) -def test_given_unsupported_adapter_when_rendering_replace_with_clone_then_raises( - test_case: AdapterReplaceWithCloneTestCase, -) -> None: - with pytest.raises(AdapterUserError, match=test_case.expected_error_fragment): - _ = test_case.adapter.render_replace_with_clone( - origin="dev.stg_orders", destination="dev.stg_customer_orders" - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py index 6a11a18d0d..db317f90d9 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/_test_types.py @@ -28,6 +28,7 @@ class ArchiveNameBuildTestCase: expected_prefix: str expected_length: int expected_logical_name_is_original: bool + kind: str | None = None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py index 464e7bd297..a552c637d7 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py +++ b/tests/unit/src/sqlbuild/executor/janitor/_helpers/test_archive_names.py @@ -111,6 +111,26 @@ def test_given_non_strict_name_when_parsing_archive_name_then_returns_none( expected_length=63, expected_logical_name_is_original=False, ), + ArchiveNameBuildTestCase( + description="kind segment follows the timestamp and precedes the original name", + original_name="orders", + archived_at=ARCHIVED_AT, + identifier_limit=255, + expected_prefix="_sqb_archive__20260924t101500z__migration_stage__orders", + expected_length=len("_sqb_archive__20260924t101500z__migration_stage__orders"), + expected_logical_name_is_original=False, + kind="migration_stage", + ), + ArchiveNameBuildTestCase( + description="long name is fitted after an intact kind at the postgres limit", + original_name=LONG_NAME, + archived_at=ARCHIVED_AT, + identifier_limit=63, + expected_prefix="_sqb_archive__20260924t101500z__migration_previous__fu_", + expected_length=63, + expected_logical_name_is_original=False, + kind="migration_previous", + ), ], ids=lambda case: case.description, ) @@ -121,6 +141,7 @@ def test_given_original_name_when_building_archive_name_then_round_trips_through original_name=test_case.original_name, archived_at=test_case.archived_at, identifier_limit=test_case.identifier_limit, + kind=test_case.kind, ) parsed: JanitorParsedArchiveName | None = parse_archive_name(archive_name) From b97a8ad877e5def694b6378555db4b4fa0e93ddd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:44:31 +0100 Subject: [PATCH 14/20] feat(migrations): hand over identity for renamed tables and views A table or view matched by fingerprint to a removed table or view is still rebuilt, but inherits the old fingerprint and plans as `renamed` with forward-only backfill, so downstream replay_on_change does not rebuild over migrated history. Each handover is recorded as a `renamed` event so a retry after the renamed model was built keeps resolving the rename. Plans list renames in their own Renamed section. --- src/sqlbuild/cli/output/_helpers/plan_text.py | 4 + src/sqlbuild/compiler/migrations/types.py | 1 + .../planner/_helpers/changes/detect.py | 8 + .../planner/_helpers/migrations/discovery.py | 31 ++- .../planner/_helpers/migrations/planning.py | 95 +++++++++- .../planner/_helpers/output/strategy.py | 5 + .../main/pre_build/pre_build_work_display.py | 45 +++-- src/sqlbuild/compiler/planner/models.py | 2 + src/sqlbuild/compiler/planner/types.py | 2 + .../executor/migrations/_helpers/promotion.py | 27 +++ .../executor/migrations/main/_apply.py | 9 +- .../main/model_migrations/_test_types.py | 20 ++ .../commands/main/model_migrations/helpers.py | 43 ++++- .../test_automatic_migrations.py | 4 + .../test_identity_handover.py | 177 ++++++++++++++++++ 15 files changed, 439 insertions(+), 34 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index 3c5a2909d0..60f240c2ea 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -74,6 +74,7 @@ PlanReason.SCHEMA_CHANGED, PlanReason.RUN_DESPITE_UNCHANGED, PlanReason.FIRST_RUN, + PlanReason.RENAMED, ) _REASON_GROUP_LABELS: dict[PlanReason, str] = { @@ -83,6 +84,7 @@ PlanReason.SCHEMA_CHANGED: "Schema changed", PlanReason.RUN_DESPITE_UNCHANGED: "Runs despite unchanged", PlanReason.FIRST_RUN: "First run", + PlanReason.RENAMED: "Renamed", } _ANSI_ESCAPE_PATTERN: re.Pattern[str] = re.compile(r"\033\[[0-9;]*m") @@ -1145,6 +1147,8 @@ def _plan_reason_text(reason: PlanReason) -> str: return "schema changed" if reason == PlanReason.FIRST_RUN: return "first run" + if reason == PlanReason.RENAMED: + return "renamed" if reason == PlanReason.FULL_REFRESH: return "full refresh" if reason == PlanReason.RUN_DESPITE_UNCHANGED: diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py index 0389346ab8..6fc0791987 100644 --- a/src/sqlbuild/compiler/migrations/types.py +++ b/src/sqlbuild/compiler/migrations/types.py @@ -15,6 +15,7 @@ class MigrationDecision(StrEnum): FORCED_REPLACE = "forced_replace" CONFLICT = "conflict" ORIGIN_MISSING = "origin_missing" + RENAMED = "renamed" @property def moves_data(self) -> bool: diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 57cda287d5..8554b8ca5c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -183,6 +183,14 @@ def detect_model_changes( fingerprint_metadata_json=metadata_json, fingerprint_version_hash=expected_version_hash, ) + if not relation_exists and model_name in snapshot.renamed_models: + return ChangeDetectionResult( + model_name=model_name, + change_kind=ChangeKind.RENAMED, + backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + fingerprint_metadata_json=metadata_json, + fingerprint_version_hash=expected_version_hash, + ) query_changed: bool = False config_changed: bool = ( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index 6228ac8f97..8b69f97752 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -18,7 +18,7 @@ migration_relation_for_location, ) from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDiscovery +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, @@ -46,6 +46,9 @@ _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} ) +_IDENTITY_MATERIALIZATIONS: frozenset[str] = frozenset( + {MaterializationType.TABLE, MaterializationType.VIEW} +) _CONFIG_KEY: str = "config" _MATERIALIZED_KEY: str = "materialized" @@ -175,13 +178,13 @@ def _resumed_events( unbuilt: frozenset[str], state: MigrationStateInspection, ) -> dict[str, MigrationEvent]: - """Return recorded moves into models that have not been built since the move.""" + """Return recorded moves or renames into models that have not been built since.""" target_name: str | None = runtime.project.effective_target_name resumed: dict[str, MigrationEvent] = {} model: CompiledModel for model in models: - if model.name not in unbuilt or not _stores_history(model): + if model.name not in unbuilt or not (_stores_history(model) or _hands_over_identity(model)): continue destination: MigrationRelation = migration_relation_for_location(model.destination) newest: MigrationEvent | None = newest_migration_event_mentioning( @@ -191,6 +194,7 @@ def _resumed_events( newest is not None and newest.destination.matches(destination) and newest.target_name in (None, target_name) + and (newest.decision == MigrationDecision.RENAMED) is _hands_over_identity(model) ): resumed[model.name] = newest return resumed @@ -242,15 +246,20 @@ def _automatic_requests( discovery=MigrationDiscovery.AUTOMATIC, origin_location=_event_origin_location(runtime=runtime, event=event), origin_model=event.origin_model, + identity_only=_hands_over_identity(model), ) ) - elif match is not None and _moves_history(model=model, origin=match): + elif match is not None and ( + _moves_history(model=model, origin=match) + or _renames_identity(model=model, origin=match) + ): requests.append( ModelMigrationRequest( model=model, discovery=MigrationDiscovery.AUTOMATIC, origin_location=_fingerprint_location(runtime=runtime, fingerprint=match), origin_model=match.node_name, + identity_only=_hands_over_identity(model), ) ) return tuple(requests) @@ -522,6 +531,20 @@ def _moves_history(*, model: CompiledModel, origin: Fingerprint) -> bool: return _stores_history(model) and _stored_materialization(origin) in _HISTORY_MATERIALIZATIONS +def _renames_identity(*, model: CompiledModel, origin: Fingerprint) -> bool: + return ( + _hands_over_identity(model) + and _stored_materialization(origin) in _IDENTITY_MATERIALIZATIONS + ) + + +def _hands_over_identity(model: CompiledModel) -> bool: + return ( + get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) + in _IDENTITY_MATERIALIZATIONS + ) + + def _stores_history(model: CompiledModel) -> bool: return ( get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 4674a228d3..657e73f757 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -193,16 +193,24 @@ def plan_model_migrations( request.model.name, () ), ) + renamed: frozenset[str] = _renamed_models( + runtime=runtime, entries=tuple(entries), handovers=handovers + ) return ModelMigrationPlanning( - snapshot=_overlay_snapshot( - runtime=runtime, - scope=scope, - snapshot=snapshot, - entries=tuple(entries), - handovers=handovers, - state=state, - overrides=overrides, - deferral=deferral, + snapshot=replace( + _overlay_snapshot( + runtime=runtime, + scope=scope, + snapshot=snapshot, + entries=tuple(entries), + handovers=_effective_handovers( + entries=tuple(entries), handovers=handovers, renamed=renamed + ), + state=state, + overrides=overrides, + deferral=deferral, + ), + renamed_models=renamed, ), entries=tuple(entries), warnings=( @@ -212,6 +220,40 @@ def plan_model_migrations( ) +def _renamed_models( + *, + runtime: PlannerRuntime, + entries: tuple[ModelMigrationPlanEntry, ...], + handovers: dict[str, Fingerprint | None], +) -> frozenset[str]: + """Return renamed tables and views whose handed-over definition matches their own.""" + + query_sql: dict[str, str] = {model.name: model.query_sql for model in runtime.project.models} + return frozenset( + entry.model_name + for entry in entries + if entry.decision == MigrationDecision.RENAMED + and (handover := handovers.get(entry.model_name)) is not None + and handover.definition == query_sql.get(entry.model_name) + ) + + +def _effective_handovers( + *, + entries: tuple[ModelMigrationPlanEntry, ...], + handovers: dict[str, Fingerprint | None], + renamed: frozenset[str], +) -> dict[str, Fingerprint | None]: + """Drop the handover of any rename whose definition no longer matches.""" + + unmatched: frozenset[str] = frozenset( + entry.model_name + for entry in entries + if entry.decision == MigrationDecision.RENAMED and entry.model_name not in renamed + ) + return {name: None if name in unmatched else handover for name, handover in handovers.items()} + + def _equivalent_definition( *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprints: tuple[str, ...] ) -> Fingerprint | None: @@ -375,6 +417,14 @@ def _decide( ), origin_is_transient=bool(origin_relation is not None and origin_relation.is_transient), ) + if request.identity_only: + return _decide_rename( + model=model, + base=base, + newest=newest, + origin=origin, + origin_fingerprint=origin_fingerprint, + ) if ( newest is not None and newest.destination.matches(migration_relation_for_location(destination)) @@ -436,6 +486,33 @@ def _decide( ) +def _decide_rename( + *, + model: CompiledModel, + base: ModelMigrationPlanEntry, + newest: MigrationEvent | None, + origin: CompiledRelationLocation, + origin_fingerprint: Fingerprint | None, +) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: + """Hand a renamed table or view's identity to its unbuilt successor; no data moves.""" + + recorded: bool = ( + newest is not None + and newest.decision == MigrationDecision.RENAMED + and newest.destination.matches(migration_relation_for_location(model.destination)) + and newest.origin.matches(migration_relation_for_location(origin)) + ) + return ( + replace( + base, + decision=MigrationDecision.RENAMED, + completed_at=newest.created_at if recorded and newest is not None else None, + ), + _handover_fingerprint(model=model, fingerprint=origin_fingerprint, version_hash=""), + True, + ) + + def _with_execution( *, runtime: PlannerRuntime, model: CompiledModel, entry: ModelMigrationPlanEntry ) -> ModelMigrationPlanEntry: diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index d9c5ab26f3..7acdf31041 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -67,6 +67,9 @@ def resolve_model_plan_action( if change_result.change_kind == ChangeKind.FIRST_RUN: return PlanAction.CREATE_TABLE, PlanReason.FIRST_RUN + if change_result.change_kind == ChangeKind.RENAMED: + return PlanAction.CREATE_TABLE, PlanReason.RENAMED + if change_result.backfill.action == BackfillAction.FULL: reason: PlanReason = _backfill_reason(change_result) return PlanAction.CREATE_TABLE, reason @@ -296,6 +299,8 @@ def _view_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> return PlanReason.FULL_REFRESH if change_result.change_kind == ChangeKind.FIRST_RUN: return PlanReason.FIRST_RUN + if change_result.change_kind == ChangeKind.RENAMED: + return PlanReason.RENAMED if change_result.change_kind == ChangeKind.QUERY_CHANGED: return PlanReason.QUERY_CHANGED if change_result.change_kind == ChangeKind.CONFIG_CHANGED: diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py index dca3503599..3f35eaa104 100644 --- a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py +++ b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py @@ -4,7 +4,7 @@ from collections.abc import Sequence -from sqlbuild.compiler.migrations.types import MigrationDiscovery +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput from sqlbuild.presentation.main.append_overflow_line import append_overflow_line from sqlbuild.presentation.main.visible_entries import visible_entries @@ -60,27 +60,38 @@ def format_pre_build_work( def _format_model_migrations( *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions ) -> list[str]: - if not plan.migration_entries: - return lines - lines.append(f"Migrations ({len(plan.migration_entries)})") - visible: Sequence[ModelMigrationPlanEntry] = visible_entries( - entries=plan.migration_entries, options=display_options - ) - entry: ModelMigrationPlanEntry - for entry in visible: - lines.extend(_migration_lines(entry)) - return append_overflow_line( - lines=lines, - total_count=len(plan.migration_entries), - visible_count=len(visible), - indent=" ", - options=display_options, - ) + title: str + renamed: bool + for title, renamed in (("Migrations", False), ("Renamed", True)): + entries: tuple[ModelMigrationPlanEntry, ...] = tuple( + entry + for entry in plan.migration_entries + if (entry.decision == MigrationDecision.RENAMED) is renamed + ) + if not entries: + continue + lines.append(f"{title} ({len(entries)})") + visible: Sequence[ModelMigrationPlanEntry] = visible_entries( + entries=entries, options=display_options + ) + entry: ModelMigrationPlanEntry + for entry in visible: + lines.extend(_migration_lines(entry)) + lines = append_overflow_line( + lines=lines, + total_count=len(entries), + visible_count=len(visible), + indent=" ", + options=display_options, + ) + return lines def _migration_lines(entry: ModelMigrationPlanEntry) -> list[str]: origin: str = entry.origin.qualified_name or entry.origin.name destination: str = entry.destination.qualified_name or entry.destination.name + if entry.decision == MigrationDecision.RENAMED: + return [f" {entry.model_name} {origin} -> {destination} (identity handed over)"] rows: list[str] = [f" {entry.model_name} {entry.decision.label} {origin} -> {destination}"] if entry.decision.checks_compatibility: rows.append(f" compatibility {entry.compatibility.value}") diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index c9470158da..6e2084e298 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -646,6 +646,7 @@ class WarehouseSnapshot: cursor_snapshots: dict[str, ModelCursorSnapshot] = field(default_factory=dict) source_freshness_state_schemas: frozenset[str] = field(default_factory=frozenset) column_dialect: str | None = None + renamed_models: frozenset[str] = field(default_factory=frozenset) @dataclass(frozen=True) @@ -1513,6 +1514,7 @@ class ModelMigrationRequest: origin_location: CompiledRelationLocation | None = None origin_model: str | None = None force: bool = False + identity_only: bool = False @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index f7ba982083..080c37f6ab 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -75,6 +75,7 @@ class GraphResourceKind(StrEnum): class ChangeKind(StrEnum): FIRST_RUN = "first_run" + RENAMED = "renamed" QUERY_CHANGED = "query_changed" CONFIG_CHANGED = "config_changed" SCHEMA_CHANGED = "schema_changed" @@ -145,6 +146,7 @@ class PlanAction(StrEnum): class PlanReason(StrEnum): FIRST_RUN = "first_run" + RENAMED = "renamed" FULL_REFRESH = "full_refresh" QUERY_CHANGED = "query_changed" FUNCTION_CHANGED = "function_changed" diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index f4d4d13601..f33acf56f6 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -15,6 +15,7 @@ from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.executor.migrations.models import MigrationArtifactNames from sqlbuild.executor.run.main.promote_staged_relation import promote_staged_relation @@ -120,3 +121,29 @@ def migration_event(*, entry: ModelMigrationPlanEntry, run_id: str) -> Migration run_id=run_id, created_at=datetime.now(tz=UTC), ) + + +def record_renames( + *, + adapter: BaseAdapter, + connection: Any, + entries: tuple[ModelMigrationPlanEntry, ...], + run_id: str, +) -> None: + """Record each newly handed-over table or view rename so retries keep the identity.""" + + entry: ModelMigrationPlanEntry + for entry in entries: + if entry.decision != MigrationDecision.RENAMED or entry.completed_at is not None: + continue + adapter.ensure_schema( + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + statement_recorder=StatementRecorder(), + ) + record_event( + adapter=adapter, + connection=connection, + event=migration_event(entry=entry, run_id=run_id), + ) diff --git a/src/sqlbuild/executor/migrations/main/_apply.py b/src/sqlbuild/executor/migrations/main/_apply.py index d6d6b7a224..4887be3402 100644 --- a/src/sqlbuild/executor/migrations/main/_apply.py +++ b/src/sqlbuild/executor/migrations/main/_apply.py @@ -13,7 +13,11 @@ from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.naming import resolve_artifact_names -from sqlbuild.executor.migrations._helpers.promotion import migration_event, promote_and_record +from sqlbuild.executor.migrations._helpers.promotion import ( + migration_event, + promote_and_record, + record_renames, +) from sqlbuild.executor.migrations._helpers.staging import create_stage, verify_stage from sqlbuild.executor.migrations.models import MigrationArtifactNames @@ -49,6 +53,9 @@ def apply_model_migrations( run_id=run_id, on_progress=on_progress, ) + _ = record_renames( + adapter=adapter, connection=connection, entries=plan.migration_entries, run_id=run_id + ) def _apply_one( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index 8184f356ed..49b9a87808 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -107,3 +107,23 @@ class MigrationArchiveExpiryTestCase: description: str expected_archives_before: int expected_relations_after: tuple[str, ...] + + +@dataclass(frozen=True) +class IdentityHandoverTestCase: + description: str + middle_materialization: str + install_failure: Callable[..., None] + failing_model: str + expected_first_exit_code: int + expected_first_view_reason: str + expected_retry_totals_reason: str + expected_totals_rows: int + expected_marked_total: int + expected_events: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class RenamedPlanTextTestCase: + description: str + expected_fragment: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 5b499a204c..37cf35f2f6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -504,11 +504,11 @@ def fail_after_migrations(monkeypatch: pytest.MonkeyPatch) -> None: ) -def enriched_view_sql(*, upstream: str, alias: str) -> str: - """Return a view that selects every order column through a table alias.""" +def enriched_view_sql(*, upstream: str, alias: str, materialized: str = "view") -> str: + """Return a view or table that selects every order column through a table alias.""" return ( - "MODEL (materialized view);\n\n" + f"MODEL (materialized {materialized});\n\n" f"SELECT {alias}.order_id, {alias}.order_date, {alias}.amount_cents " f'FROM __ref("{upstream}") AS {alias}\n' ) @@ -673,3 +673,40 @@ def live_relations(*, project_dir: Path, schema: str = "main") -> tuple[str, ... ), ) ) + + +_REPLAY_FULL: str = " replay_on_change full,\n" + + +def original_replay_order_models(*, middle: str) -> dict[str, str]: + """Return the staging, enriched, and daily totals chain with full replay on the totals.""" + + return { + "stg_orders": incremental_orders_sql(), + "orders_enriched": enriched_view_sql(upstream="stg_orders", alias="o", materialized=middle), + "daily_order_totals": daily_totals_sql( + upstream="orders_enriched", cte="orders_enriched", extra_config=_REPLAY_FULL + ), + } + + +def renamed_replay_order_models(*, middle: str) -> dict[str, str]: + """Return the same full-replay chain with every model renamed.""" + + return { + "stg_customer_orders": incremental_orders_sql(), + "customer_orders_enriched": enriched_view_sql( + upstream="stg_customer_orders", alias="customer_order", materialized=middle + ), + "customer_daily_order_totals": daily_totals_sql( + upstream="customer_orders_enriched", + cte="customer_orders_enriched", + extra_config=_REPLAY_FULL, + ), + } + + +def fail_nothing_in_build(*, monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: + """Install no build failure.""" + + del monkeypatch, model_name diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py index e9a866eb27..912a5e1ab5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py @@ -45,10 +45,12 @@ description="incremental, view, and incremental renamed together", expected_migrations=( ("daily_order_totals", "customer_daily_order_totals", "automatic", "migrate"), + ("orders_enriched", "customer_orders_enriched", "automatic", "renamed"), ("stg_orders", "stg_customer_orders", "automatic", "migrate"), ), expected_events=( ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("orders_enriched", "customer_orders_enriched", "renamed"), ("stg_orders", "stg_customer_orders", "migrate"), ), ) @@ -298,10 +300,12 @@ def test_given_qualified_migrate_from_when_another_orphan_matches_then_only_decl description="recorded upstream move resolves the downstream rename on retry", expected_migrations=( ("daily_order_totals", "customer_daily_order_totals", "automatic", "migrate"), + ("orders_enriched", "customer_orders_enriched", "automatic", "renamed"), ("stg_orders", "stg_customer_orders", "automatic", "done"), ), expected_events=( ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("orders_enriched", "customer_orders_enriched", "renamed"), ("stg_orders", "stg_customer_orders", "migrate"), ), ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py new file mode 100644 index 0000000000..2d0a1d4eef --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py @@ -0,0 +1,177 @@ +"""Integration coverage for identity handover of renamed views and tables.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + IdentityHandoverTestCase, + RenamedPlanTextTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + CliRun, + build, + build_ok, + execute, + fail_model_build, + fail_nothing_in_build, + load_raw_orders, + migration_events, + model_entry, + original_replay_order_models, + plan_json, + query, + renamed_replay_order_models, + row_count, + run_sqb, + write_project, +) + +_MARK_HISTORY: str = ( + "UPDATE main.daily_order_totals SET total_cents = -1 WHERE order_date = TIMESTAMP '2026-01-01'" +) +_READ_MARK: str = ( + "SELECT total_cents FROM main.customer_daily_order_totals " + "WHERE order_date = TIMESTAMP '2026-01-01'" +) +_EVENTS: tuple[tuple[str, str, str], ...] = ( + ("daily_order_totals", "customer_daily_order_totals", "migrate"), + ("orders_enriched", "customer_orders_enriched", "renamed"), + ("stg_orders", "stg_customer_orders", "migrate"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + IdentityHandoverTestCase( + middle_materialization="view", + description="renamed view between incrementals does not replay downstream history", + install_failure=fail_nothing_in_build, + failing_model="", + expected_first_exit_code=0, + expected_first_view_reason="renamed", + expected_retry_totals_reason="normal_incremental", + expected_totals_rows=8, + expected_marked_total=-1, + expected_events=_EVENTS, + ), + IdentityHandoverTestCase( + middle_materialization="view", + description="retry after the renamed view was built keeps downstream history", + install_failure=fail_model_build, + failing_model="customer_daily_order_totals", + expected_first_exit_code=1, + expected_first_view_reason="renamed", + expected_retry_totals_reason="normal_incremental", + expected_totals_rows=8, + expected_marked_total=-1, + expected_events=_EVENTS, + ), + IdentityHandoverTestCase( + middle_materialization="view", + description="retry before the renamed view was built keeps downstream history", + install_failure=fail_model_build, + failing_model="stg_customer_orders", + expected_first_exit_code=1, + expected_first_view_reason="renamed", + expected_retry_totals_reason="normal_incremental", + expected_totals_rows=8, + expected_marked_total=-1, + expected_events=_EVENTS, + ), + IdentityHandoverTestCase( + middle_materialization="table", + description="renamed table between incrementals is rebuilt without replaying history", + install_failure=fail_model_build, + failing_model="customer_daily_order_totals", + expected_first_exit_code=1, + expected_first_view_reason="renamed", + expected_retry_totals_reason="normal_incremental", + expected_totals_rows=8, + expected_marked_total=-1, + expected_events=_EVENTS, + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_chain_with_full_replay_when_building_then_migrated_history_survives( + test_case: IdentityHandoverTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A renamed view hands its identity downstream so replay_on_change keeps marked history.""" + + write_project( + project_dir=tmp_path, + models=original_replay_order_models(middle=test_case.middle_materialization), + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + execute(project_dir=tmp_path, sql=_MARK_HISTORY) + load_raw_orders(project_dir=tmp_path, first_day=3, last_day=8) + write_project( + project_dir=tmp_path, + models=renamed_replay_order_models(middle=test_case.middle_materialization), + ) + + first_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + with monkeypatch.context() as patch: + test_case.install_failure(monkeypatch=patch, model_name=test_case.failing_model) + first: CliRun = build(project_dir=tmp_path, capsys=capsys) + retry_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert first.exit_code == test_case.expected_first_exit_code, first.output + assert model_entry(plan=first_plan, name="customer_orders_enriched")["reason"] == ( + test_case.expected_first_view_reason + ) + assert model_entry(plan=first_plan, name="customer_daily_order_totals")["reason"] != ( + "upstream_changed" + ) + assert model_entry(plan=retry_plan, name="customer_daily_order_totals")["reason"] == ( + test_case.expected_retry_totals_reason + ) + assert ( + row_count(project_dir=tmp_path, relation="main.customer_daily_order_totals") + == test_case.expected_totals_rows + ) + assert query(project_dir=tmp_path, sql=_READ_MARK) == [(test_case.expected_marked_total,)] + assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + RenamedPlanTextTestCase( + description="plan lists renamed views separately from data migrations", + expected_fragment=( + "Renamed (1)\n└── customer_orders_enriched main.orders_enriched -> " + "main.customer_orders_enriched (identity handed over)" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_view_when_planning_then_text_lists_it_under_renamed( + test_case: RenamedPlanTextTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Renamed tables and views move no data and appear in their own plan section.""" + + write_project(project_dir=tmp_path, models=original_replay_order_models(middle="view")) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project(project_dir=tmp_path, models=renamed_replay_order_models(middle="view")) + + text: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + + assert text.exit_code == 0, text.output + assert test_case.expected_fragment in text.output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 009d96d5603a55281e3c8573fb739d1ebd6f4b40 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:49:55 +0100 Subject: [PATCH 15/20] test(migrations): cover staged migrations on PostgreSQL and through the CLI --- .../main/model_migrations/__init__.py | 0 .../main/model_migrations/_test_types.py | 14 ++ .../commands/main/model_migrations/helpers.py | 127 ++++++++++ .../test_model_migration_lifecycle.py | 76 ++++++ .../cli/commands/main/postgres/_test_types.py | 18 ++ .../cli/commands/main/postgres/helpers.py | 131 ++++++++++ .../main/postgres/test_model_migrations.py | 228 ++++++++++++++++++ 7 files changed, 594 insertions(+) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/__init__.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_model_migration_lifecycle.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py new file mode 100644 index 0000000000..9a8644d8fb --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -0,0 +1,14 @@ +"""Test case types for model migration CLI e2e tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class MigrationLifecycleE2ETestCase: + description: str + expected_step_decisions: tuple[str, ...] + expected_final_ids: tuple[int, ...] + expected_previous_archive_ids: tuple[tuple[int, ...], ...] + expected_events: tuple[tuple[str, str, str], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py new file mode 100644 index 0000000000..070c2dfd3b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -0,0 +1,127 @@ +"""Project builders for model migration CLI e2e tests.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +DATABASE_FILE: str = "orders.duckdb" +_PROJECT_TOML: str = ( + f'name = "orders_project"\nadapter = "duckdb"\n\n[connection]\ndatabase = "{DATABASE_FILE}"\n' +) +_SOURCES_YML: str = "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + + +def orders_sql(*, migrate_from: str) -> str: + """Return an incremental orders model with an optional migrate_from header.""" + + migration: str = {"": ""}.get(migrate_from, f' migrate_from "{migrate_from}",\n') + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + f"{migration}" + ");\n\n" + 'SELECT order_id, order_date, amount_cents FROM __source("raw_orders")\n' + ) + + +def write_orders_project(*, tmp_path: Path, name: str, migrate_from: str) -> Path: + """Write a project containing exactly one incremental orders model.""" + + project_dir: Path = tmp_path / "orders_project" + stale: Path + for stale in (project_dir / "models").glob("*.sql"): + stale.unlink() + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": _SOURCES_YML, + f"models/{name}.sql": orders_sql(migrate_from=migrate_from), + }, + ) + + +def load_raw_orders(*, project_dir: Path, last_day: int) -> None: + """Replace raw orders with one order per January day up to the given day.""" + + execute_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "CREATE OR REPLACE TABLE main.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) AS order_date, " + f"100 + i AS amount_cents FROM range(1, {last_day + 1}) AS t(i)" + ), + ) + + +def plan_decisions(*, project_dir: Path) -> tuple[str, ...]: + """Run sqb plan --json in a subprocess and return the migration decisions.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--json"), project_dir=project_dir + ) + payload: dict[str, Any] = json.loads(result.stdout) + return tuple(str(migration["decision"]) for migration in payload["migrations"]) + + +def build(*, project_dir: Path) -> subprocess.CompletedProcess[str]: + """Run sqb build in a subprocess.""" + + return run_sqb(command=("--no-color", "build"), project_dir=project_dir) + + +def order_ids(*, project_dir: Path, relation: str) -> tuple[int, ...]: + """Return the sorted order IDs stored in one relation.""" + + return tuple( + int(row[0]) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=f"SELECT order_id FROM {relation} ORDER BY 1", + ) + ) + + +def previous_archive_ids(*, project_dir: Path) -> tuple[tuple[int, ...], ...]: + """Return the order IDs held by each displaced-destination archive.""" + + names: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' " + "AND contains(table_name, '__migration_previous__') ORDER BY 1" + ), + ) + return tuple(order_ids(project_dir=project_dir, relation=f"main.{row[0]}") for row in names) + + +def migration_events(*, project_dir: Path) -> tuple[tuple[str, str, str], ...]: + """Return (origin, destination, decision) for every recorded migration event in order.""" + + return tuple( + (str(row[0]), str(row[1]), str(row[2])) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT origin_name, destination_name, decision FROM main._sqlbuild_migrations " + "ORDER BY created_at, event_id" + ), + ) + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_model_migration_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_model_migration_lifecycle.py new file mode 100644 index 0000000000..a8a3f9d51a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_model_migration_lifecycle.py @@ -0,0 +1,76 @@ +"""CLI e2e coverage for a model renamed away and back through staged migrations.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + MigrationLifecycleE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + build, + load_raw_orders, + migration_events, + order_ids, + plan_decisions, + previous_archive_ids, + write_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MigrationLifecycleE2ETestCase( + description="rename away then back keeps history and archives the stale original", + expected_step_decisions=("migrate", "superseded_replace", "done"), + expected_final_ids=(1, 2, 3, 4, 5), + expected_previous_archive_ids=((1, 2, 3),), + expected_events=( + ("stg_orders", "stg_customer_orders", "migrate"), + ("stg_customer_orders", "stg_orders", "superseded_replace"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_model_renamed_away_and_back_when_building_then_history_follows_the_model( + tmp_path: Path, test_case: MigrationLifecycleE2ETestCase +) -> None: + """The real CLI stages, promotes, archives, and records each move.""" + + project_dir: Path = write_orders_project(tmp_path=tmp_path, name="stg_orders", migrate_from="") + load_raw_orders(project_dir=project_dir, last_day=3) + initial: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + _ = write_orders_project( + tmp_path=tmp_path, name="stg_customer_orders", migrate_from="stg_orders" + ) + load_raw_orders(project_dir=project_dir, last_day=4) + away_decisions: tuple[str, ...] = plan_decisions(project_dir=project_dir) + away: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + _ = write_orders_project( + tmp_path=tmp_path, name="stg_orders", migrate_from="stg_customer_orders" + ) + load_raw_orders(project_dir=project_dir, last_day=5) + back_decisions: tuple[str, ...] = plan_decisions(project_dir=project_dir) + back: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + done_decisions: tuple[str, ...] = plan_decisions(project_dir=project_dir) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert away.returncode == 0, away.stdout + away.stderr + assert back.returncode == 0, back.stdout + back.stderr + assert away_decisions + back_decisions + done_decisions == test_case.expected_step_decisions + assert order_ids(project_dir=project_dir, relation="main.stg_orders") == ( + test_case.expected_final_ids + ) + assert previous_archive_ids(project_dir=project_dir) == ( + test_case.expected_previous_archive_ids + ) + assert migration_events(project_dir=project_dir) == test_case.expected_events + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index 91164c51e3..c6a43b39bf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -260,3 +260,21 @@ class PostgresDbtSeedChangeE2ETestCase: expected_changed_total: int expected_changed_fragments: tuple[str, ...] expected_rerun_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PostgresModelMigrationE2ETestCase: + description: str + expected_destination_ids: tuple[tuple[object, ...], ...] + expected_view_ids: tuple[tuple[object, ...], ...] + expected_archive_ids: tuple[tuple[object, ...], ...] + expected_events: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class PostgresMigrationRollbackE2ETestCase: + description: str + expected_failure_fragment: str + expected_destination_ids_after_failure: tuple[tuple[object, ...], ...] + expected_previous_archives_after_failure: int + expected_final_destination_ids: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py index 85659c4ec4..5e4a799669 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py @@ -13,6 +13,7 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, prepare_source_loader_strategies, prepare_waffle_shop, stringify_warehouse_rows, @@ -631,3 +632,133 @@ def append_postgres_dbt_seed_change_order(*, project_dir: Path, order_id: int, a seed_path.read_text(encoding="utf-8") + f"{order_id},{amount}\n", encoding="utf-8", ) + + +ORIGIN_MODEL: str = "stg_orders" +DESTINATION_MODEL: str = "stg_customer_orders" +_JANITOR_CONFIG: str = "\n[janitor]\nenabled = true\narchive_retention_days = 0\n" +_MIGRATION_TEXT_COLUMNS: tuple[str, ...] = ( + "event_id", + "target_name", + "origin_model", + "origin_database", + "origin_schema", + "origin_name", + "destination_model", + "destination_database", + "destination_schema", + "destination_name", + "origin_version_hash", + "discovery", + "decision", + "run_id", +) + + +def orders_sql(*, migrate_from: str = "", where: str = "") -> str: + """Return an incremental orders model with an optional forced migration header.""" + + migration: str = {"": ""}.get( + migrate_from, f' migrate_from "{migrate_from}",\n migrate_force true,\n' + ) + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + f"{migration}" + ");\n\n" + f'SELECT order_id, order_date, amount_cents FROM __source("raw_orders"){where}\n' + ) + + +def report_view_sql() -> str: + """Return a project view that reads the destination by name.""" + + return f'MODEL (materialized view);\n\nSELECT order_id FROM __ref("{DESTINATION_MODEL}")\n' + + +def write_migration_project( + *, + tmp_path: Path, + schema_name: str, + raw_schema_name: str, + config: dict[str, object], + models: dict[str, str], +) -> Path: + """Write a Postgres project whose sources live outside the managed schema.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": build_postgres_project_toml( + project_name="postgres_model_migrations", schema_name=schema_name, config=config + ) + + _JANITOR_CONFIG, + "sources/raw.yml": ( + f"sources:\n - name: raw_orders\n schema: {raw_schema_name}\n" + " table: raw_orders\n" + ), + } + project_dir: Path = tmp_path / "postgres_model_migrations" + models_dir: Path = project_dir / "models" + stale: Path + for stale in models_dir.glob("*.sql"): + stale.unlink() + files.update({f"models/{name}.sql": sql for name, sql in models.items()}) + return prepare_inline_project( + tmp_path=tmp_path, project_name="postgres_model_migrations", repo_files=files + ) + + +def load_raw_orders(*, raw_schema_name: str, config: dict[str, object]) -> None: + """Create five raw orders, one per January day, in the source schema.""" + + execute_postgres_sql(sql=f"CREATE SCHEMA IF NOT EXISTS {raw_schema_name}", config=config) + execute_postgres_sql( + sql=( + f"CREATE TABLE {raw_schema_name}.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + (i - 1) * INTERVAL '1 day' AS order_date, " + "100 + i AS amount_cents FROM generate_series(1, 5) AS t(i)" + ), + config=config, + ) + + +def create_unwritable_migration_table(*, schema_name: str, config: dict[str, object]) -> None: + """Occupy the migration state name with a read-only view so the event insert fails.""" + + columns: str = ", ".join( + f"CAST(NULL AS TEXT) AS {column}" for column in _MIGRATION_TEXT_COLUMNS + ) + execute_postgres_sql( + sql=( + f"CREATE VIEW {schema_name}._sqlbuild_migrations AS SELECT {columns}, " + "CAST(NULL AS TIMESTAMP) AS created_at WHERE false" + ), + config=config, + ) + + +def archive_names( + *, schema_name: str, kind: str, config: dict[str, object] +) -> tuple[tuple[object, ...], ...]: + """Return every migration archive of one kind in the managed schema.""" + + return fetch_postgres_rows( + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema_name}' " + "AND table_name LIKE '\\_sqb\\_archive\\_\\_%' " + f"AND position('__{kind}__' IN table_name) > 0 ORDER BY 1" + ), + config=config, + ) + + +def ordered_ids(*, relation: str, config: dict[str, object]) -> tuple[tuple[object, ...], ...]: + """Return the sorted order IDs readable through one relation.""" + + return fetch_postgres_rows(sql=f"SELECT order_id FROM {relation} ORDER BY 1", config=config) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py new file mode 100644 index 0000000000..b89c99ebdc --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py @@ -0,0 +1,228 @@ +"""PostgreSQL e2e coverage for staged model migrations and identity-bound dependent views.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.postgres._test_types import ( + PostgresMigrationRollbackE2ETestCase, + PostgresModelMigrationE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.postgres.helpers import ( + DESTINATION_MODEL, + ORIGIN_MODEL, + archive_names, + build_unique_schema_name, + cleanup_postgres_schema, + create_unwritable_migration_table, + ensure_postgres_schema_ready, + execute_postgres_sql, + fetch_postgres_rows, + load_raw_orders, + ordered_ids, + orders_sql, + report_view_sql, + write_migration_project, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +_TWO_ORDERS: str = " WHERE order_id <= 2" +_ALL_IDS: tuple[tuple[object, ...], ...] = ((1,), (2,), (3,), (4,), (5,)) + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresModelMigrationE2ETestCase( + description="forced replace rebinds identity-bound views and archives the old table", + expected_destination_ids=_ALL_IDS, + expected_view_ids=_ALL_IDS, + expected_archive_ids=((1,), (2,)), + expected_events=((ORIGIN_MODEL, DESTINATION_MODEL, "forced_replace"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_views_on_destination_when_forcing_migration_then_views_follow_the_new_table( + tmp_path: Path, + test_case: PostgresModelMigrationE2ETestCase, + postgres_e2e_config: dict[str, object], +) -> None: + """Views bound by OID read the promoted table, so janitor can drop the archive.""" + + schema_name: str = build_unique_schema_name(prefix="sqlbuild_migration") + raw_schema_name: str = f"{schema_name}_raw" + ensure_postgres_schema_ready(schema_name=schema_name, config=postgres_e2e_config) + load_raw_orders(raw_schema_name=raw_schema_name, config=postgres_e2e_config) + project_dir: Path = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={ + ORIGIN_MODEL: orders_sql(), + DESTINATION_MODEL: orders_sql(where=_TWO_ORDERS), + "orders_report": report_view_sql(), + }, + ) + try: + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + execute_postgres_sql( + sql=( + f"CREATE VIEW {raw_schema_name}.orders_dashboard AS " + f"SELECT order_id FROM {schema_name}.{DESTINATION_MODEL}" + ), + config=postgres_e2e_config, + ) + project_dir = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={ + DESTINATION_MODEL: orders_sql(migrate_from=ORIGIN_MODEL), + "orders_report": report_view_sql(), + }, + ) + migrated: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + archives: tuple[tuple[object, ...], ...] = archive_names( + schema_name=schema_name, kind="migration_previous", config=postgres_e2e_config + ) + archive_ids: tuple[tuple[object, ...], ...] = ordered_ids( + relation=f"{schema_name}.{archives[0][0]}", config=postgres_e2e_config + ) + janitor: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "janitor", "--auto-approve", "--retention-days", "30"), + project_dir=project_dir, + ) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert migrated.returncode == 0, migrated.stdout + migrated.stderr + assert len(archives) == 1 + assert archive_ids == test_case.expected_archive_ids + assert janitor.returncode == 0, janitor.stdout + janitor.stderr + assert ( + archive_names( + schema_name=schema_name, kind="migration_previous", config=postgres_e2e_config + ) + == () + ) + assert ( + ordered_ids(relation=f"{schema_name}.{DESTINATION_MODEL}", config=postgres_e2e_config) + == test_case.expected_destination_ids + ) + assert ( + ordered_ids(relation=f"{raw_schema_name}.orders_dashboard", config=postgres_e2e_config) + == test_case.expected_view_ids + ) + assert ( + ordered_ids(relation=f"{schema_name}.orders_report", config=postgres_e2e_config) + == test_case.expected_view_ids + ) + assert ( + ordered_ids(relation=f"{schema_name}.{ORIGIN_MODEL}", config=postgres_e2e_config) + == _ALL_IDS + ) + assert ( + fetch_postgres_rows( + sql=( + "SELECT origin_name, destination_name, decision " + f"FROM {schema_name}._sqlbuild_migrations" + ), + config=postgres_e2e_config, + ) + == test_case.expected_events + ) + finally: + cleanup_postgres_schema(schema_name=raw_schema_name, config=postgres_e2e_config) + cleanup_postgres_schema(schema_name=schema_name, config=postgres_e2e_config) + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresMigrationRollbackE2ETestCase( + description="failed event insert rolls the promotion back and keeps the destination", + expected_failure_fragment="M106", + expected_destination_ids_after_failure=((1,), (2,)), + expected_previous_archives_after_failure=0, + expected_final_destination_ids=_ALL_IDS, + ) + ], + ids=lambda case: case.description, +) +def test_given_unwritable_event_table_when_migrating_then_promotion_rolls_back( + tmp_path: Path, + test_case: PostgresMigrationRollbackE2ETestCase, + postgres_e2e_config: dict[str, object], +) -> None: + """Promotion and the migration event commit or roll back together on PostgreSQL.""" + + schema_name: str = build_unique_schema_name(prefix="sqlbuild_migration_rollback") + raw_schema_name: str = f"{schema_name}_raw" + ensure_postgres_schema_ready(schema_name=schema_name, config=postgres_e2e_config) + load_raw_orders(raw_schema_name=raw_schema_name, config=postgres_e2e_config) + project_dir: Path = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={ + ORIGIN_MODEL: orders_sql(), + DESTINATION_MODEL: orders_sql(where=_TWO_ORDERS), + }, + ) + try: + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + create_unwritable_migration_table(schema_name=schema_name, config=postgres_e2e_config) + project_dir = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={DESTINATION_MODEL: orders_sql(migrate_from=ORIGIN_MODEL)}, + ) + failed: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + ids_after_failure: tuple[tuple[object, ...], ...] = ordered_ids( + relation=f"{schema_name}.{DESTINATION_MODEL}", config=postgres_e2e_config + ) + previous_after_failure: int = len( + archive_names( + schema_name=schema_name, kind="migration_previous", config=postgres_e2e_config + ) + ) + execute_postgres_sql( + sql=f"DROP VIEW {schema_name}._sqlbuild_migrations", config=postgres_e2e_config + ) + retried: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert failed.returncode != 0 + assert test_case.expected_failure_fragment in failed.stdout + failed.stderr + assert ids_after_failure == test_case.expected_destination_ids_after_failure + assert previous_after_failure == test_case.expected_previous_archives_after_failure + assert retried.returncode == 0, retried.stdout + retried.stderr + assert ( + ordered_ids(relation=f"{schema_name}.{DESTINATION_MODEL}", config=postgres_e2e_config) + == test_case.expected_final_destination_ids + ) + finally: + cleanup_postgres_schema(schema_name=raw_schema_name, config=postgres_e2e_config) + cleanup_postgres_schema(schema_name=schema_name, config=postgres_e2e_config) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From fbac92554e31119187715b3fa4ded1abef909e9f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:51:40 +0100 Subject: [PATCH 16/20] fix(janitor): never archive the model migration state table --- src/sqlbuild/executor/janitor/constants.py | 2 ++ .../main/model_migrations/_test_types.py | 2 ++ .../commands/main/model_migrations/helpers.py | 17 +++++++++++++++++ .../model_migrations/test_staged_execution.py | 17 +++++++++++++++-- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 5cd2663418..7fd9271740 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -3,6 +3,7 @@ from __future__ import annotations from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME +from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.audit_results.constants import AUDIT_RESULTS_TABLE_NAME from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME @@ -16,6 +17,7 @@ NODE_RESULTS_TABLE_NAME, AUDIT_RESULTS_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, + MIGRATION_TABLE_NAME, ) DIRECT_DEFAULT_RETENTION_DAYS: int = 14 diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index 49b9a87808..c490b22af0 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -105,8 +105,10 @@ class StagedPlanOutputTestCase: @dataclass(frozen=True) class MigrationArchiveExpiryTestCase: description: str + janitor_retention_days: str expected_archives_before: int expected_relations_after: tuple[str, ...] + expected_migration_state_kept: bool @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 37cf35f2f6..fec375de75 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -84,6 +84,7 @@ [janitor] enabled = true archive_retention_days = 0 + delete_tracked_only = false """ ).lstrip() RAW_SOURCES_YML: str = dedent( @@ -710,3 +711,19 @@ def fail_nothing_in_build(*, monkeypatch: pytest.MonkeyPatch, model_name: str) - """Install no build failure.""" del monkeypatch, model_name + + +def state_tables(*, project_dir: Path, schema: str) -> tuple[str, ...]: + """Return the SQLBuild state tables present in one schema.""" + + return tuple( + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' AND starts_with(table_name, '_sqlbuild_') " + "ORDER BY 1" + ), + ) + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py index 5e08ab5ecc..a3b6d2e9ad 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py @@ -32,6 +32,7 @@ plan_json, prepare_forced_replace, run_sqb, + state_tables, ) _STAGE: str = "migration_stage" @@ -151,9 +152,18 @@ def test_given_forced_replace_when_planning_then_reports_transfer_and_promotion( [ MigrationArchiveExpiryTestCase( description="janitor expires abandoned stages and displaced destinations only", + janitor_retention_days="30", expected_archives_before=2, expected_relations_after=("stg_customer_orders", "stg_orders"), - ) + expected_migration_state_kept=True, + ), + MigrationArchiveExpiryTestCase( + description="janitor removing stale relations still keeps the migration state table", + janitor_retention_days="0", + expected_archives_before=2, + expected_relations_after=("stg_customer_orders",), + expected_migration_state_kept=True, + ), ], ids=lambda case: case.description, ) @@ -176,7 +186,7 @@ def test_given_migration_archives_when_janitor_runs_then_expires_them_by_name( janitor: CliRun = run_sqb( project_dir=tmp_path, - args=("janitor", "--auto-approve", "--retention-days", "30"), + args=("janitor", "--auto-approve", "--retention-days", test_case.janitor_retention_days), capsys=capsys, ) @@ -192,6 +202,9 @@ def test_given_migration_archives_when_janitor_runs_then_expires_them_by_name( assert order_ids(project_dir=tmp_path, relation=f"dev.{DESTINATION_MODEL}") == tuple( range(1, 6) ) + assert ("_sqlbuild_migrations" in state_tables(project_dir=tmp_path, schema="dev")) is ( + test_case.expected_migration_state_kept + ) if __name__ == "__main__": From 59e77843134d4fda8ebf175ed49bde50c291736a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:57:38 +0100 Subject: [PATCH 17/20] test(migrations): run the model migration lifecycle e2e in the CLI shard --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 1043a3ee74..124fe03891 100644 --- a/Makefile +++ b/Makefile @@ -130,6 +130,7 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/init \ tests/e2e/src/sqlbuild/cli/commands/main/rules \ tests/e2e/src/sqlbuild/cli/commands/main/lineage \ + tests/e2e/src/sqlbuild/cli/commands/main/model_migrations \ tests/e2e/src/sqlbuild/cli/commands/main/motherduck \ tests/e2e/src/sqlbuild/cli/commands/main/playground \ tests/e2e/src/sqlbuild/cli/commands/main/postgres \ From 619fd37c40e17289ae5a23390cd4a57e37b0e30c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:01:04 +0100 Subject: [PATCH 18/20] fix(postgres): keep view options when re-pointing dependent views after a migration --- .../adapters/postgres/_helpers/__init__.py | 1 + .../adapters/postgres/_helpers/view_rebind.py | 32 +++++++ .../postgres/classes/postgres_adapter.py | 11 ++- .../cli/commands/main/postgres/_test_types.py | 1 + .../cli/commands/main/postgres/helpers.py | 42 +++++++++ .../main/postgres/test_model_migrations.py | 27 ++++-- .../sqlbuild/adapters/postgres/_test_types.py | 7 ++ .../postgres/test_dependent_view_rebinds.py | 89 +++++++++++++++++++ 8 files changed, 201 insertions(+), 9 deletions(-) create mode 100644 src/sqlbuild/adapters/postgres/_helpers/__init__.py create mode 100644 src/sqlbuild/adapters/postgres/_helpers/view_rebind.py create mode 100644 tests/unit/src/sqlbuild/adapters/postgres/test_dependent_view_rebinds.py diff --git a/src/sqlbuild/adapters/postgres/_helpers/__init__.py b/src/sqlbuild/adapters/postgres/_helpers/__init__.py new file mode 100644 index 0000000000..47b31f3a24 --- /dev/null +++ b/src/sqlbuild/adapters/postgres/_helpers/__init__.py @@ -0,0 +1 @@ +"""PostgreSQL adapter helpers.""" diff --git a/src/sqlbuild/adapters/postgres/_helpers/view_rebind.py b/src/sqlbuild/adapters/postgres/_helpers/view_rebind.py new file mode 100644 index 0000000000..0ae6a65481 --- /dev/null +++ b/src/sqlbuild/adapters/postgres/_helpers/view_rebind.py @@ -0,0 +1,32 @@ +"""Re-create a dependent PostgreSQL view with its original options.""" + +from __future__ import annotations + +import re + +_CHECK_OPTION_KEY: str = "check_option" +_PLAIN_OPTION_VALUE: re.Pattern[str] = re.compile(r"[A-Za-z0-9_.]+") + + +def render_postgres_view_rebind(*, view: str, definition: str, reloptions: tuple[str, ...]) -> str: + """Render CREATE OR REPLACE VIEW that keeps security and check options from reloptions.""" + + options: list[str] = [] + check_option: str | None = None + option: str + for option in reloptions: + key, _, value = option.partition("=") + if key == _CHECK_OPTION_KEY: + check_option = value.upper() + continue + options.append(f"{key}={_option_value(value)}") + with_clause: str = f" WITH ({', '.join(options)})" if options else "" + check_clause: str = f" WITH {check_option} CHECK OPTION" if check_option else "" + body: str = definition.strip().rstrip(";").rstrip() + return f"CREATE OR REPLACE VIEW {view}{with_clause} AS {body}{check_clause}" + + +def _option_value(value: str) -> str: + if _PLAIN_OPTION_VALUE.fullmatch(value): + return value + return "'" + value.replace("'", "''") + "'" diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 7165813ff0..6456d06baf 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -85,6 +85,7 @@ ) from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.postgres._helpers.view_rebind import render_postgres_view_rebind from sqlbuild.adapters.postgres.classes.postgres_connection import _PostgresConnection from sqlbuild.adapters.postgres.constants import TABLE_FUNCTION_RETURN_TYPE from sqlbuild.compiler.compile.types import FunctionLanguage @@ -599,7 +600,8 @@ def capture_dependent_view_rebinds( del database query: str = ( "SELECT DISTINCT dependent_namespace.nspname, dependent.relname, " - "pg_get_viewdef(dependent.oid) " + "pg_get_viewdef(dependent.oid), " + "array_to_string(dependent.reloptions, chr(31)) " "FROM pg_depend AS dependency " "JOIN pg_rewrite AS rewrite ON rewrite.oid = dependency.objid " "JOIN pg_class AS dependent ON dependent.oid = rewrite.ev_class " @@ -618,8 +620,11 @@ def capture_dependent_view_rebinds( ) cursor: Any = connection.execute(query) return tuple( - f"CREATE OR REPLACE VIEW {self.render_identifier(str(row[0]))}." - f"{self.render_identifier(str(row[1]))} AS {str(row[2]).strip().rstrip(';')}" + render_postgres_view_rebind( + view=f"{self.render_identifier(str(row[0]))}.{self.render_identifier(str(row[1]))}", + definition=str(row[2]), + reloptions=tuple(str(row[3]).split(chr(31))) if row[3] else (), + ) for row in cursor.fetchall() ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index c6a43b39bf..de8b43bd2f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -268,6 +268,7 @@ class PostgresModelMigrationE2ETestCase: expected_destination_ids: tuple[tuple[object, ...], ...] expected_view_ids: tuple[tuple[object, ...], ...] expected_archive_ids: tuple[tuple[object, ...], ...] + expected_view_options: tuple[tuple[object, ...], ...] expected_events: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py index 5e4a799669..dae8b3c63a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py @@ -762,3 +762,45 @@ def ordered_ids(*, relation: str, config: dict[str, object]) -> tuple[tuple[obje """Return the sorted order IDs readable through one relation.""" return fetch_postgres_rows(sql=f"SELECT order_id FROM {relation} ORDER BY 1", config=config) + + +def create_external_dependent_views( + *, raw_schema_name: str, schema_name: str, config: dict[str, object] +) -> None: + """Create plain, security-option, and check-option views on the destination.""" + + destination: str = f"{schema_name}.{DESTINATION_MODEL}" + statements: tuple[str, ...] = ( + f"CREATE VIEW {raw_schema_name}.orders_dashboard AS SELECT order_id FROM {destination}", + f"GRANT SELECT ON {raw_schema_name}.orders_dashboard TO PUBLIC", + ( + f"CREATE VIEW {raw_schema_name}.orders_invoker " + "WITH (security_barrier=true, security_invoker=true) AS " + f"SELECT order_id FROM {destination}" + ), + ( + f"CREATE VIEW {raw_schema_name}.orders_checked AS " + f"SELECT order_id, order_date, amount_cents FROM {destination} " + "WHERE order_id > 0 WITH CASCADED CHECK OPTION" + ), + ) + statement: str + for statement in statements: + execute_postgres_sql(sql=statement, config=config) + + +def view_catalog( + *, raw_schema_name: str, config: dict[str, object] +) -> tuple[tuple[object, ...], ...]: + """Return (view, options, owner, privileges) for every view in the source schema.""" + + return fetch_postgres_rows( + sql=( + "SELECT view.relname, coalesce(array_to_string(view.reloptions, ','), ''), " + "pg_get_userbyid(view.relowner), coalesce(view.relacl::text, '') " + "FROM pg_class AS view JOIN pg_namespace AS namespace " + "ON namespace.oid = view.relnamespace " + f"WHERE namespace.nspname = '{raw_schema_name}' AND view.relkind = 'v' ORDER BY 1" + ), + config=config, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py index b89c99ebdc..5b615b11b3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_model_migrations.py @@ -17,6 +17,7 @@ archive_names, build_unique_schema_name, cleanup_postgres_schema, + create_external_dependent_views, create_unwritable_migration_table, ensure_postgres_schema_ready, execute_postgres_sql, @@ -25,6 +26,7 @@ ordered_ids, orders_sql, report_view_sql, + view_catalog, write_migration_project, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb @@ -41,6 +43,11 @@ expected_destination_ids=_ALL_IDS, expected_view_ids=_ALL_IDS, expected_archive_ids=((1,), (2,)), + expected_view_options=( + ("orders_checked", "check_option=cascaded"), + ("orders_dashboard", ""), + ("orders_invoker", "security_barrier=true,security_invoker=true"), + ), expected_events=((ORIGIN_MODEL, DESTINATION_MODEL, "forced_replace"),), ) ], @@ -72,12 +79,11 @@ def test_given_views_on_destination_when_forcing_migration_then_views_follow_the initial: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build"), project_dir=project_dir ) - execute_postgres_sql( - sql=( - f"CREATE VIEW {raw_schema_name}.orders_dashboard AS " - f"SELECT order_id FROM {schema_name}.{DESTINATION_MODEL}" - ), - config=postgres_e2e_config, + create_external_dependent_views( + raw_schema_name=raw_schema_name, schema_name=schema_name, config=postgres_e2e_config + ) + views_before: tuple[tuple[object, ...], ...] = view_catalog( + raw_schema_name=raw_schema_name, config=postgres_e2e_config ) project_dir = write_migration_project( tmp_path=tmp_path, @@ -92,6 +98,9 @@ def test_given_views_on_destination_when_forcing_migration_then_views_follow_the migrated: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build"), project_dir=project_dir ) + views_after: tuple[tuple[object, ...], ...] = view_catalog( + raw_schema_name=raw_schema_name, config=postgres_e2e_config + ) archives: tuple[tuple[object, ...], ...] = archive_names( schema_name=schema_name, kind="migration_previous", config=postgres_e2e_config ) @@ -105,6 +114,12 @@ def test_given_views_on_destination_when_forcing_migration_then_views_follow_the assert initial.returncode == 0, initial.stdout + initial.stderr assert migrated.returncode == 0, migrated.stdout + migrated.stderr + assert views_after == views_before + assert tuple((row[0], row[1]) for row in views_after) == test_case.expected_view_options + assert ( + ordered_ids(relation=f"{raw_schema_name}.orders_checked", config=postgres_e2e_config) + == test_case.expected_view_ids + ) assert len(archives) == 1 assert archive_ids == test_case.expected_archive_ids assert janitor.returncode == 0, janitor.stdout + janitor.stderr diff --git a/tests/unit/src/sqlbuild/adapters/postgres/_test_types.py b/tests/unit/src/sqlbuild/adapters/postgres/_test_types.py index 4872190836..13ae7d8cae 100644 --- a/tests/unit/src/sqlbuild/adapters/postgres/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/postgres/_test_types.py @@ -118,3 +118,10 @@ class PostgresRenderSourceFreshnessQueryTestCase: source_is_subquery: bool where_sql: str expected_sql: str + + +@dataclass(frozen=True) +class PostgresDependentViewRebindTestCase: + description: str + catalog_rows: tuple[tuple[object, ...], ...] + expected_statements: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/adapters/postgres/test_dependent_view_rebinds.py b/tests/unit/src/sqlbuild/adapters/postgres/test_dependent_view_rebinds.py new file mode 100644 index 0000000000..8338541221 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/postgres/test_dependent_view_rebinds.py @@ -0,0 +1,89 @@ +"""Unit coverage for re-pointing PostgreSQL views bound to a promoted table.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from tests.unit.src.sqlbuild.adapters.postgres._test_types import ( + PostgresDependentViewRebindTestCase, +) +from tests.unit.src.sqlbuild.adapters.postgres.helpers import ( + FakePostgresConnection, + FakePostgresCursor, +) + +_DEFINITION: str = " SELECT orders.order_id\n FROM dev.orders;" +_BODY: str = "SELECT orders.order_id\n FROM dev.orders" + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresDependentViewRebindTestCase( + description="view without options is re-created plainly", + catalog_rows=(("reporting", "orders_dashboard", _DEFINITION, None),), + expected_statements=( + f'CREATE OR REPLACE VIEW "reporting"."orders_dashboard" AS {_BODY}', + ), + ), + PostgresDependentViewRebindTestCase( + description="security options are re-emitted in the with clause", + catalog_rows=( + ( + "reporting", + "orders_secure", + _DEFINITION, + "security_barrier=true\x1fsecurity_invoker=true", + ), + ), + expected_statements=( + 'CREATE OR REPLACE VIEW "reporting"."orders_secure" ' + f"WITH (security_barrier=true, security_invoker=true) AS {_BODY}", + ), + ), + PostgresDependentViewRebindTestCase( + description="check option is re-emitted after the query", + catalog_rows=( + ( + "reporting", + "orders_checked", + _DEFINITION, + "check_option=cascaded\x1fsecurity_barrier=false", + ), + ), + expected_statements=( + 'CREATE OR REPLACE VIEW "reporting"."orders_checked" ' + f"WITH (security_barrier=false) AS {_BODY} WITH CASCADED CHECK OPTION", + ), + ), + PostgresDependentViewRebindTestCase( + description="local check option alone adds no with clause", + catalog_rows=(("reporting", "orders_local", _DEFINITION, "check_option=local"),), + expected_statements=( + 'CREATE OR REPLACE VIEW "reporting"."orders_local" ' + f"AS {_BODY} WITH LOCAL CHECK OPTION", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_dependent_views_when_capturing_rebinds_then_view_options_survive( + test_case: PostgresDependentViewRebindTestCase, +) -> None: + """CREATE OR REPLACE VIEW resets omitted options, so every captured option is re-emitted.""" + + connection: FakePostgresConnection = FakePostgresConnection( + FakePostgresCursor(rows=test_case.catalog_rows) + ) + + statements: tuple[str, ...] = PostgresAdapter().capture_dependent_view_rebinds( + connection=connection, database=None, schema="dev", name="orders" + ) + + assert statements == test_case.expected_statements + assert "reloptions" in connection.executed_sql[0] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 6acd8e320b6e49b5964f30df4d6580033b97a5cb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:06:56 +0100 Subject: [PATCH 19/20] fix(migrations): create and write migration state through adapter-rendered SQL Migration state-table DDL now comes from each adapter, so SQL Server uses its guarded create-if-missing instead of an invalid CREATE TABLE IF NOT EXISTS. Event writes follow the janitor-event pattern: an existence check then INSERT ... VALUES with text-timestamp literals, avoiding ANSI TIMESTAMP casts that SQL Server rejects. --- .../adapter/contract/classes/base_adapter.py | 15 ++ .../contract/classes/duckdb_backed_adapter.py | 12 ++ .../contract/classes/strict_adapter.py | 5 + .../bigquery/classes/bigquery_adapter.py | 12 ++ .../databricks/classes/databricks_adapter.py | 12 ++ .../postgres/classes/postgres_adapter.py | 12 ++ .../snowflake/classes/snowflake_adapter.py | 13 ++ .../sqlserver/classes/sqlserver_adapter.py | 19 ++ .../compiler/migrations/_helpers/sql.py | 59 +++--- src/sqlbuild/compiler/migrations/constants.py | 2 +- .../migrations/main/create_table_sql.py | 27 +++ .../compiler/migrations/main/ensure_table.py | 33 ---- .../compiler/migrations/main/write_event.py | 39 ++-- .../executor/migrations/_helpers/promotion.py | 22 +-- .../adapters/sqlserver/_test_types.py | 22 +++ .../sqlbuild/adapters/sqlserver/helpers.py | 20 +++ .../sqlserver/test_migration_state_sql.py | 168 ++++++++++++++++++ .../migrations/main/test_migration_events.py | 5 +- 18 files changed, 395 insertions(+), 102 deletions(-) create mode 100644 src/sqlbuild/compiler/migrations/main/create_table_sql.py delete mode 100644 src/sqlbuild/compiler/migrations/main/ensure_table.py create mode 100644 tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index c37f4f9806..afca25076e 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2220,6 +2220,21 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: transient=self.state_tables_transient, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the model migration event table when it is missing.""" + + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 73e76b65c1..35690b7107 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -853,6 +853,18 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 1b1a19a1c0..f7524f205f 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -910,6 +910,11 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: """Render DDL that creates the janitor audit event table when it is missing.""" ... + @abstractmethod + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the model migration event table when it is missing.""" + ... + @abstractmethod def render_prune_fingerprint_history_sql( self, diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index e6a862d6a7..93f7f9cd9e 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -323,6 +323,18 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index ac4f765d59..245a4f7a29 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -417,6 +417,18 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 6456d06baf..e4760b67bc 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1584,6 +1584,18 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_read_latest_source_freshness_sql( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 73427c911b..b62a05ef61 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -499,6 +499,19 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: transient=self.state_tables_transient, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + return build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index d1bd64c594..36d8af3132 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -633,6 +633,25 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: table_name=JANITOR_EVENTS_TABLE_NAME, ) + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME + from sqlbuild.compiler.migrations.main.create_table_sql import ( + build_migration_state_create_table_sql, + ) + + create_sql: str = build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ).replace("CREATE TABLE IF NOT EXISTS", "CREATE TABLE", 1) + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=MIGRATION_TABLE_NAME, + ) + def render_read_latest_source_freshness_sql( self, *, diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py index 8b1b9e9e39..8e4a05257f 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/sql.py @@ -7,6 +7,9 @@ from typing import Any from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) from sqlbuild.compiler.migrations.constants import ( MIGRATION_COLUMN_TYPES, MIGRATION_COLUMNS, @@ -38,45 +41,47 @@ def build_create_table_sql( render_framework_type: Callable[[FrameworkType], str], transient: bool, ) -> str: - table: str = qualified_migration_table( - database=database, schema=schema, render_qualified_name=render_qualified_name + return render_state_table_create_sql( + qualified_name=qualified_migration_table( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=MIGRATION_COLUMNS, + column_types=MIGRATION_COLUMN_TYPES, + required_columns=frozenset(), + render_framework_type=render_framework_type, + transient=transient, ) - string_type: str = render_framework_type(FrameworkType.STRING) - timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) - definitions: str = ", ".join( - f"{column} " - + ( - timestamp_type - if MIGRATION_COLUMN_TYPES[column] == StateSqlValueType.TIMESTAMP - else string_type - ) - for column in MIGRATION_COLUMNS + + +def build_existing_event_sql( + *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] +) -> str: + table: str = _event_table(event=event, render_qualified_name=render_qualified_name) + event_id_literal: str = render_state_sql_literal( + value=event.event_id, declared_type=StateSqlValueType.STRING ) - table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" - return f"CREATE {table_kind} IF NOT EXISTS {table} ({definitions})" + return f"SELECT event_id FROM {table} WHERE event_id = {event_id_literal}" def build_insert_sql( *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] ) -> str: - if event.destination.schema is None: - raise MigrationStateError("model migration events require a destination schema") - table: str = qualified_migration_table( - database=event.destination.database, - schema=event.destination.schema, - render_qualified_name=render_qualified_name, - ) + table: str = _event_table(event=event, render_qualified_name=render_qualified_name) values: tuple[object | None, ...] = _event_values(event) literals: str = ", ".join( render_state_sql_literal(value=value, declared_type=MIGRATION_COLUMN_TYPES[column]) for column, value in zip(MIGRATION_COLUMNS, values, strict=True) ) - event_id_literal: str = render_state_sql_literal( - value=event.event_id, declared_type=StateSqlValueType.STRING - ) - return ( - f"INSERT INTO {table} ({', '.join(MIGRATION_COLUMNS)}) SELECT {literals} " - f"WHERE NOT EXISTS (SELECT 1 FROM {table} WHERE event_id = {event_id_literal})" + return f"INSERT INTO {table} ({', '.join(MIGRATION_COLUMNS)}) VALUES ({literals})" + + +def _event_table(*, event: MigrationEvent, render_qualified_name: Callable[..., str | None]) -> str: + if event.destination.schema is None: + raise MigrationStateError("model migration events require a destination schema") + return qualified_migration_table( + database=event.destination.database, + schema=event.destination.schema, + render_qualified_name=render_qualified_name, ) diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py index 5d6efbbfc9..c08a9dcc3e 100644 --- a/src/sqlbuild/compiler/migrations/constants.py +++ b/src/sqlbuild/compiler/migrations/constants.py @@ -28,5 +28,5 @@ MIGRATION_COLUMN_TYPES: dict[str, StateSqlValueType] = { **{column: StateSqlValueType.STRING for column in MIGRATION_COLUMNS}, - "created_at": StateSqlValueType.TIMESTAMP, + "created_at": StateSqlValueType.TEXT_TIMESTAMP, } diff --git a/src/sqlbuild/compiler/migrations/main/create_table_sql.py b/src/sqlbuild/compiler/migrations/main/create_table_sql.py new file mode 100644 index 0000000000..c2464b188c --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/create_table_sql.py @@ -0,0 +1,27 @@ +"""Model migration state table SQL entrypoint.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.compiler.migrations._helpers.sql import build_create_table_sql + + +def build_migration_state_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + """Build DDL that creates the model migration event table when it is missing.""" + + return build_create_table_sql( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + render_framework_type=render_framework_type, + transient=transient, + ) diff --git a/src/sqlbuild/compiler/migrations/main/ensure_table.py b/src/sqlbuild/compiler/migrations/main/ensure_table.py deleted file mode 100644 index 637cb44f70..0000000000 --- a/src/sqlbuild/compiler/migrations/main/ensure_table.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Create the append-only model migration table before a transactional promotion.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute, FrameworkType -from sqlbuild.compiler.migrations._helpers.sql import build_create_table_sql - - -def ensure_migration_table( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], - transient: bool, -) -> None: - """Create the migration state table in one schema when it is missing.""" - - _ = execute( - connection=connection, - sql=build_create_table_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - transient=transient, - ), - ) diff --git a/src/sqlbuild/compiler/migrations/main/write_event.py b/src/sqlbuild/compiler/migrations/main/write_event.py index e7649beb5f..fd0c04edb3 100644 --- a/src/sqlbuild/compiler/migrations/main/write_event.py +++ b/src/sqlbuild/compiler/migrations/main/write_event.py @@ -7,13 +7,12 @@ from collections.abc import Callable from typing import Any -from sqlbuild.adapter.contract.types import AdapterExecute, FrameworkType -from sqlbuild.compiler.migrations._helpers.sql import build_create_table_sql, build_insert_sql +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.compiler.migrations._helpers.sql import build_existing_event_sql, build_insert_sql from sqlbuild.compiler.migrations.constants import ( MIGRATION_WRITE_ATTEMPTS, MIGRATION_WRITE_RETRY_BASE_SECONDS, ) -from sqlbuild.compiler.migrations.exceptions import MigrationStateError from sqlbuild.compiler.migrations.models import MigrationEvent @@ -23,38 +22,22 @@ def write_migration_event( execute: AdapterExecute[Any, Any], event: MigrationEvent, render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], - transient: bool, - create_table: bool = True, + create_table_sql: str | None, attempts: int = MIGRATION_WRITE_ATTEMPTS, ) -> None: - """Create the state table when missing and insert the event unless already present.""" + """Create the state table from adapter DDL when given, then insert the event if absent.""" - if event.destination.schema is None: - raise MigrationStateError("model migration events require a destination schema") - create_statements: tuple[str, ...] = ( - ( - build_create_table_sql( - database=event.destination.database, - schema=event.destination.schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - transient=transient, - ), - ) - if create_table - else () - ) - statements: tuple[str, ...] = ( - *create_statements, - build_insert_sql(event=event, render_qualified_name=render_qualified_name), + existing_sql: str = build_existing_event_sql( + event=event, render_qualified_name=render_qualified_name ) + insert_sql: str = build_insert_sql(event=event, render_qualified_name=render_qualified_name) attempt: int for attempt in range(attempts): try: - statement: str - for statement in statements: - _ = execute(connection=connection, sql=statement) + if create_table_sql is not None: + _ = execute(connection=connection, sql=create_table_sql) + if not execute(connection=connection, sql=existing_sql).fetchall(): + _ = execute(connection=connection, sql=insert_sql) return except Exception as error: if attempt + 1 == attempts: diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index f33acf56f6..ad6e6a8eef 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -11,7 +11,6 @@ from sqlbuild.compiler.migrations.main.deterministic_event_id import ( deterministic_migration_event_id, ) -from sqlbuild.compiler.migrations.main.ensure_table import ensure_migration_table from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.main.write_event import write_migration_event from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation @@ -35,14 +34,11 @@ def promote_and_record( _promote(adapter=adapter, connection=connection, names=names) record_event(adapter=adapter, connection=connection, event=event) return - ensure_migration_table( + _ = adapter.execute( connection=connection, - execute=adapter.execute, - database=event.destination.database, - schema=event.destination.schema or "", - render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - transient=adapter.state_tables_transient, + sql=adapter.render_create_migration_state_table_sql( + database=event.destination.database, schema=event.destination.schema or "" + ), ) with adapter.transaction(connection): rebinds: tuple[str, ...] = ( @@ -79,9 +75,13 @@ def record_event( execute=adapter.execute, event=event, render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - transient=adapter.state_tables_transient, - create_table=create_table, + create_table_sql=( + adapter.render_create_migration_state_table_sql( + database=event.destination.database, schema=event.destination.schema or "" + ) + if create_table + else None + ), attempts=attempts, ) diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/_test_types.py b/tests/unit/src/sqlbuild/adapters/sqlserver/_test_types.py index b4eaeaff78..d4b2389d6f 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/_test_types.py @@ -142,3 +142,25 @@ class SqlServerPruneSqlTestCase: schema: str retain_versions: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SqlServerMigrationEventWriteTestCase: + description: str + existing_rows: tuple[tuple[object, ...], ...] + expected_statement_prefixes: tuple[str, ...] + + +@dataclass(frozen=True) +class SqlServerMigrationTableTestCase: + description: str + schema: str + expected_prefix: str + expected_suffix: str + + +@dataclass(frozen=True) +class SqlServerMigrationEventReadTestCase: + description: str + stored_created_at: object + expected_read_suffix: str diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/helpers.py b/tests/unit/src/sqlbuild/adapters/sqlserver/helpers.py index 7fab1b2779..d6fa9df15f 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/helpers.py @@ -7,3 +7,23 @@ def __init__(self) -> None: def execute(self, sql: str) -> None: self.executed_sql.append(sql) + + +class RowsCursor: + def __init__(self, rows: tuple[tuple[object, ...], ...]) -> None: + self.rows: tuple[tuple[object, ...], ...] = rows + + def fetchall(self) -> list[tuple[object, ...]]: + return list(self.rows) + + +class ScriptedSqlServerConnection: + """Record statements and answer every query with the same scripted rows.""" + + def __init__(self, rows: tuple[tuple[object, ...], ...] = ()) -> None: + self.rows: tuple[tuple[object, ...], ...] = rows + self.executed_sql: list[str] = [] + + def execute(self, sql: str) -> RowsCursor: + self.executed_sql.append(sql) + return RowsCursor(self.rows) diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py new file mode 100644 index 0000000000..302b428eac --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py @@ -0,0 +1,168 @@ +"""SQL Server rendering coverage for the append-only model migration state table.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.migrations.main._read_events import read_migration_events +from sqlbuild.compiler.migrations.main.write_event import write_migration_event +from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from tests.unit.src.sqlbuild.adapters.sqlserver._test_types import ( + SqlServerMigrationEventReadTestCase, + SqlServerMigrationEventWriteTestCase, + SqlServerMigrationTableTestCase, +) +from tests.unit.src.sqlbuild.adapters.sqlserver.helpers import ScriptedSqlServerConnection + +_CREATED_AT: datetime = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) +_EVENT: MigrationEvent = MigrationEvent( + event_id="event-1", + target_name="dev", + origin_model="stg_orders", + origin=MigrationRelation(database=None, schema="dbo", name="stg_orders"), + destination_model="stg_customer_orders", + destination=MigrationRelation(database=None, schema="dbo", name="stg_customer_orders"), + origin_version_hash="hash-1", + discovery=MigrationDiscovery.MANUAL, + decision=MigrationDecision.MIGRATE, + run_id="run-1", + created_at=_CREATED_AT, +) +_GUARDED_CREATE: str = ( + "IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'dbo' " + "AND table_name = '_sqlbuild_migrations') CREATE TABLE dbo._sqlbuild_migrations (" +) +_EXISTING: str = "SELECT event_id FROM dbo._sqlbuild_migrations WHERE event_id = 'event-1'" +_INSERT: str = "INSERT INTO dbo._sqlbuild_migrations (event_id, target_name" + + +@pytest.mark.parametrize( + "test_case", + [ + SqlServerMigrationTableTestCase( + description="state table creation is guarded by a catalog check", + schema="dbo", + expected_prefix=_GUARDED_CREATE, + expected_suffix="run_id NVARCHAR(MAX), created_at DATETIME2)", + ) + ], + ids=lambda case: case.description, +) +def test_given_sqlserver_when_rendering_migration_table_then_uses_guarded_create( + test_case: SqlServerMigrationTableTestCase, +) -> None: + """SQL Server has no CREATE TABLE IF NOT EXISTS, so creation is guarded by a catalog check.""" + + sql: str = SqlServerAdapter().render_create_migration_state_table_sql( + database=None, schema=test_case.schema + ) + + assert sql.startswith(test_case.expected_prefix) + assert "IF NOT EXISTS dbo" not in sql + assert sql.endswith(test_case.expected_suffix) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlServerMigrationEventWriteTestCase( + description="new event is checked then inserted with values", + existing_rows=(), + expected_statement_prefixes=(_GUARDED_CREATE, _EXISTING, _INSERT), + ), + SqlServerMigrationEventWriteTestCase( + description="already recorded event is not inserted again", + existing_rows=(("event-1",),), + expected_statement_prefixes=(_GUARDED_CREATE, _EXISTING), + ), + ], + ids=lambda case: case.description, +) +def test_given_sqlserver_when_writing_migration_event_then_statements_are_tsql( + test_case: SqlServerMigrationEventWriteTestCase, +) -> None: + """Event writes avoid SELECT-without-FROM guards and ANSI TIMESTAMP casts.""" + + adapter: SqlServerAdapter = SqlServerAdapter() + connection: ScriptedSqlServerConnection = ScriptedSqlServerConnection( + rows=test_case.existing_rows + ) + + write_migration_event( + connection=connection, + execute=adapter.execute, + event=_EVENT, + render_qualified_name=adapter.render_qualified_name, + create_table_sql=adapter.render_create_migration_state_table_sql( + database=None, schema="dbo" + ), + ) + + assert tuple( + sql[: len(prefix)] + for sql, prefix in zip( + connection.executed_sql, test_case.expected_statement_prefixes, strict=True + ) + ) == (test_case.expected_statement_prefixes) + assert all("AS TIMESTAMP" not in sql for sql in connection.executed_sql) + assert all("NOT EXISTS (SELECT 1 FROM dbo." not in sql for sql in connection.executed_sql) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlServerMigrationEventReadTestCase( + description="naive datetime2 values decode as utc", + stored_created_at=datetime(2026, 1, 2, 3, 4, 5), + expected_read_suffix="FROM dbo._sqlbuild_migrations ORDER BY created_at, event_id", + ) + ], + ids=lambda case: case.description, +) +def test_given_sqlserver_datetime2_rows_when_reading_events_then_decodes_utc_events( + test_case: SqlServerMigrationEventReadTestCase, +) -> None: + """Reads use plain ORDER BY and decode naive DATETIME2 values as UTC.""" + + adapter: SqlServerAdapter = SqlServerAdapter() + connection: ScriptedSqlServerConnection = ScriptedSqlServerConnection( + rows=( + ( + "event-1", + "dev", + "stg_orders", + None, + "dbo", + "stg_orders", + "stg_customer_orders", + None, + "dbo", + "stg_customer_orders", + "hash-1", + "manual", + "migrate", + "run-1", + test_case.stored_created_at, + ), + ) + ) + + events: tuple[MigrationEvent, ...] = read_migration_events( + connection=connection, + execute=adapter.execute, + database=None, + schema="dbo", + render_qualified_name=adapter.render_qualified_name, + ) + + assert events == (_EVENT,) + assert connection.executed_sql[0].endswith(test_case.expected_read_suffix) + assert "LIMIT" not in connection.executed_sql[0] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py index 5484ce2683..87d21f00ee 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -92,8 +92,9 @@ def test_given_repeated_event_write_when_reading_then_event_is_stored_once( execute=adapter.execute, event=event, render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - transient=False, + create_table_sql=adapter.render_create_migration_state_table_sql( + database=None, schema="main" + ), ) stored: tuple[MigrationEvent, ...] = read_migration_events( connection=connection, From 8911e0899820f18ac4778f55d973e9e71df48379 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:14:48 +0100 Subject: [PATCH 20/20] fix(migrations): fall back from clone to copy only on recognized clone refusals The BigQuery stage fallback previously copied after any clone failure. Adapters now supply a clone-refusal classifier; BigQuery accepts only HTTP 400 invalid/invalidQuery errors whose details describe a clone limitation, so transient, quota, and permission errors are re-raised. If the stage exists after a refusal it is never adopted. Plan text and JSON show the copy fallback, and build output reports the transfer used. --- src/sqlbuild/adapter/contract/models.py | 3 +- .../bigquery/_helpers/clone_refusal.py | 40 ++++++ .../bigquery/classes/bigquery_adapter.py | 2 + src/sqlbuild/adapters/bigquery/constants.py | 14 ++ src/sqlbuild/cli/output/_helpers/plan_json.py | 3 + .../planner/_helpers/migrations/planning.py | 3 +- .../main/pre_build/pre_build_work_display.py | 7 +- src/sqlbuild/compiler/planner/models.py | 1 + .../executor/migrations/_helpers/staging.py | 27 +++- .../executor/migrations/main/_apply.py | 9 +- .../main/model_migrations/_test_types.py | 3 + .../commands/main/model_migrations/helpers.py | 15 +++ .../model_migrations/test_staged_execution.py | 33 ++++- .../migrations/_helpers/_test_types.py | 18 +++ .../executor/migrations/_helpers/helpers.py | 51 ++++++++ .../migrations/_helpers/test_staging.py | 122 ++++++++++++++++++ .../sqlbuild/adapters/bigquery/_test_types.py | 7 + .../adapters/bigquery/test_clone_refusal.py | 100 ++++++++++++++ 18 files changed, 448 insertions(+), 10 deletions(-) create mode 100644 src/sqlbuild/adapters/bigquery/_helpers/clone_refusal.py create mode 100644 tests/integration/src/sqlbuild/executor/migrations/_helpers/test_staging.py create mode 100644 tests/unit/src/sqlbuild/adapters/bigquery/test_clone_refusal.py diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index b89dbc85eb..81426da31b 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -187,11 +187,12 @@ class RenderedRetentionChange: @dataclass(frozen=True) class MigrationStagePlan: - """Statements that create a fresh migration stage, plus a copy fallback if clone is refused.""" + """Fresh migration stage statements, plus a copy used only for recognized clone refusals.""" transfer: MigrationTransfer statements: tuple[str, ...] fallback_statements: tuple[str, ...] = () + is_clone_refusal: Callable[[BaseException], bool] | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/adapters/bigquery/_helpers/clone_refusal.py b/src/sqlbuild/adapters/bigquery/_helpers/clone_refusal.py new file mode 100644 index 0000000000..ee761f9b04 --- /dev/null +++ b/src/sqlbuild/adapters/bigquery/_helpers/clone_refusal.py @@ -0,0 +1,40 @@ +"""Recognize BigQuery refusals to clone a table where a physical copy can still succeed.""" + +from __future__ import annotations + +from sqlbuild.adapters.bigquery.constants import ( + CLONE_REFUSAL_ERROR_CODE, + CLONE_REFUSAL_PHRASES, + CLONE_REFUSAL_REASONS, + MAX_CLONE_REFUSAL_ERROR_CHAIN, +) + + +def is_bigquery_clone_refusal(error: BaseException) -> bool: + """Return whether a driver error in the cause chain is a clone-capability refusal.""" + + current: BaseException | None = error + depth: int = 0 + while current is not None and depth < MAX_CLONE_REFUSAL_ERROR_CHAIN: + if _is_refusal(current): + return True + current = current.__cause__ or current.__context__ + depth += 1 + return False + + +def _is_refusal(error: BaseException) -> bool: + details: object | None = getattr(error, "errors", None) + if getattr(error, "code", None) != CLONE_REFUSAL_ERROR_CODE or not isinstance(details, list): + return False + reasons: set[str] = { + str(detail.get("reason", "")) for detail in details if isinstance(detail, dict) + } + messages: str = " ".join( + str(detail.get("message", "")) for detail in details if isinstance(detail, dict) + ).lower() + return ( + bool(reasons) + and reasons <= CLONE_REFUSAL_REASONS + and any(phrase in messages for phrase in CLONE_REFUSAL_PHRASES) + ) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 93f7f9cd9e..73b6943b25 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -95,6 +95,7 @@ from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.bigquery._helpers.clone_refusal import is_bigquery_clone_refusal from sqlbuild.adapters.bigquery._helpers.statement_telemetry import affected_rows from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection from sqlbuild.adapters.bigquery.classes.bigquery_cursor import _BigQueryCursor @@ -1741,6 +1742,7 @@ def render_migration_stage( transfer=MigrationTransfer.CLONE, statements=(f"CREATE TABLE {quoted_stage} CLONE {quoted_origin}",), fallback_statements=(f"CREATE TABLE {quoted_stage} COPY {quoted_origin}",), + is_clone_refusal=is_bigquery_clone_refusal, ) def capture_dependent_view_rebinds( diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index 1454478697..8419dafa9f 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -17,3 +17,17 @@ SELECT_STATEMENT_TYPE: str = "SELECT" TABLE_NAME_WILDCARD: str = "*" TIMESTAMP_TYPE_TOKEN: str = "TIMESTAMP" + +CLONE_REFUSAL_ERROR_CODE: int = 400 +CLONE_REFUSAL_REASONS: frozenset[str] = frozenset({"invalid", "invalidQuery"}) +CLONE_REFUSAL_PHRASES: tuple[str, ...] = ( + "cannot clone", + "can't clone", + "clone is not supported", + "clones are not supported", + "cloning is not supported", + "not supported for clone", + "clones and snapshots", + "clone or snapshot", +) +MAX_CLONE_REFUSAL_ERROR_CHAIN: int = 8 diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 7720e21e1b..2da44895a5 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -487,6 +487,9 @@ def _serialize_model_migrations(plan: PlanOutput) -> list[dict[str, object]]: "target": entry.target_name, "origin_version_hash": entry.origin_version_hash, "transfer": entry.transfer.value if entry.transfer is not None else None, + "transfer_fallback": ( + entry.transfer_fallback.value if entry.transfer_fallback is not None else None + ), "storage_transition": entry.storage_transition, "promotion": entry.promotion.value if entry.promotion is not None else None, "completed_at": ( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 657e73f757..8020c59751 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -7,7 +7,7 @@ from datetime import UTC, datetime from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo -from sqlbuild.adapter.contract.types import BuiltinAdapter +from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.models import CompiledModel, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType @@ -533,6 +533,7 @@ def _with_execution( entry, stage_is_transient=stage_is_transient, transfer=stage.transfer, + transfer_fallback=MigrationTransfer.COPY if stage.fallback_statements else None, promotion=_promotion(runtime=runtime, decision=entry.decision), ) diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py index 3f35eaa104..bb4ec60431 100644 --- a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py +++ b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py @@ -97,7 +97,12 @@ def _migration_lines(entry: ModelMigrationPlanEntry) -> list[str]: rows.append(f" compatibility {entry.compatibility.value}") if entry.transfer is not None: details: tuple[str | None, ...] = ( - entry.transfer.label, + entry.transfer.label + + ( + "" + if entry.transfer_fallback is None + else f" ({entry.transfer_fallback.label} if refused)" + ), entry.storage_transition, None if entry.promotion is None else f"promote by {entry.promotion.label}", ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 6e2084e298..7121851f5a 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -982,6 +982,7 @@ class ModelMigrationPlanEntry: origin_is_transient: bool = False stage_is_transient: bool | None = None transfer: MigrationTransfer | None = None + transfer_fallback: MigrationTransfer | None = None promotion: MigrationPromotion | None = None completed_at: datetime | None = None compatibility_findings: tuple[str, ...] = () diff --git a/src/sqlbuild/executor/migrations/_helpers/staging.py b/src/sqlbuild/executor/migrations/_helpers/staging.py index 5e1693f536..b954e24ad0 100644 --- a/src/sqlbuild/executor/migrations/_helpers/staging.py +++ b/src/sqlbuild/executor/migrations/_helpers/staging.py @@ -31,8 +31,17 @@ def create_stage( try: _execute_all(adapter=adapter, connection=connection, statements=plan.statements) except Exception as error: - if not plan.fallback_statements: + if ( + not plan.fallback_statements + or plan.is_clone_refusal is None + or not plan.is_clone_refusal(error) + ): raise + if _stage_exists(adapter=adapter, connection=connection, entry=entry, names=names): + raise ExecutorInputError( + f"migration clone into {names.stage_qualified} was refused but the stage now " + "exists; refusing to adopt a stage of unknown completeness" + ) from error logging.getLogger("sqlbuild.migrations").warning( "migration clone into '%s' was refused; copying instead: %s", names.stage_name, @@ -105,6 +114,22 @@ def verify_stage( ) +def _stage_exists( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelMigrationPlanEntry, + names: MigrationArtifactNames, +) -> bool: + listed: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=entry.destination.database, + schemas=(entry.destination.schema,) if entry.destination.schema is not None else None, + names=(names.stage_name,), + ) + return any(relation.name.lower() == names.stage_name.lower() for relation in listed) + + def _execute_all(*, adapter: BaseAdapter, connection: Any, statements: tuple[str, ...]) -> None: statement: str for statement in statements: diff --git a/src/sqlbuild/executor/migrations/main/_apply.py b/src/sqlbuild/executor/migrations/main/_apply.py index 4887be3402..d341b483a5 100644 --- a/src/sqlbuild/executor/migrations/main/_apply.py +++ b/src/sqlbuild/executor/migrations/main/_apply.py @@ -72,7 +72,7 @@ def _apply_one( on_progress(f"Migrating {origin_label} -> {destination_label} ({entry.decision.value})...") started: float = time.monotonic() try: - _ = _stage_promote_and_record( + transfer: MigrationTransfer = _stage_promote_and_record( entry=entry, adapter=adapter, connection=connection, run_id=run_id ) except Exception as error: @@ -86,13 +86,15 @@ def _apply_one( ) from error if on_progress is not None: on_progress( - f"Migrated {origin_label} -> {destination_label}. ({time.monotonic() - started:.2f}s)" + f"Migrated {origin_label} -> {destination_label} by {transfer.label}" + f"{' (clone refused)' if transfer != entry.transfer else ''}. " + f"({time.monotonic() - started:.2f}s)" ) def _stage_promote_and_record( *, entry: ModelMigrationPlanEntry, adapter: BaseAdapter, connection: Any, run_id: str -) -> None: +) -> MigrationTransfer: adapter.ensure_schema( connection=connection, database=entry.destination.database, @@ -115,3 +117,4 @@ def _stage_promote_and_record( names=names, event=migration_event(entry=entry, run_id=run_id), ) + return transfer diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index c490b22af0..034efd68b6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -96,6 +96,9 @@ class StagedReplaceTestCase: @dataclass(frozen=True) class StagedPlanOutputTestCase: description: str + install_stage: Callable[[pytest.MonkeyPatch], None] + expected_transfer_fallback: str | None + expected_build_fragment: str expected_transfer: str expected_promotion: str expected_storage_transition: str | None diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index fec375de75..1a3364ef7b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -393,6 +393,21 @@ def fail_clone(monkeypatch: pytest.MonkeyPatch) -> None: ) +def refuse_clone_then_copy(monkeypatch: pytest.MonkeyPatch) -> None: + """Plan a clone with a copy fallback, as BigQuery does, and refuse the clone.""" + + monkeypatch.setattr( + DuckDbAdapter, + "render_migration_stage", + lambda self, *, origin, stage, **_: MigrationStagePlan( + transfer=MigrationTransfer.CLONE, + statements=("SELECT error('simulated clone refusal')",), + fallback_statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + is_clone_refusal=lambda error: "simulated clone refusal" in str(error), + ), + ) + + def fail_partial_stage(monkeypatch: pytest.MonkeyPatch) -> None: """Leave an empty, incomplete stage behind and then fail the copy.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py index a3b6d2e9ad..8c18d03f8e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_staged_execution.py @@ -31,6 +31,7 @@ order_ids, plan_json, prepare_forced_replace, + refuse_clone_then_copy, run_sqb, state_tables, ) @@ -121,30 +122,56 @@ def test_given_forced_replace_when_interrupted_then_retry_converges_without_losi [ StagedPlanOutputTestCase( description="duckdb plans a physical copy promoted in one transaction", + install_stage=no_failure, + expected_transfer_fallback=None, + expected_build_fragment="by physical copy. (", expected_transfer="copy", expected_promotion="transactional_rename", expected_storage_transition=None, expected_text_fragment="└── transfer physical copy, promote by transactional rename", - ) + ), + StagedPlanOutputTestCase( + description="refused clone plans its copy fallback and builds by copy", + install_stage=refuse_clone_then_copy, + expected_transfer_fallback="copy", + expected_build_fragment="by physical copy (clone refused).", + expected_transfer="clone", + expected_promotion="transactional_rename", + expected_storage_transition=None, + expected_text_fragment=( + "└── transfer zero-copy clone (physical copy if refused), " + "promote by transactional rename" + ), + ), ], ids=lambda case: case.description, ) def test_given_forced_replace_when_planning_then_reports_transfer_and_promotion( - test_case: StagedPlanOutputTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: StagedPlanOutputTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Plan text and JSON state how the migration is staged and promoted.""" + """Plan text and JSON state the planned transfer; build output reports the one used.""" prepare_forced_replace(project_dir=tmp_path, capsys=capsys) + test_case.install_stage(monkeypatch) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) text: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + built: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) migration: dict[str, Any] = plan["migrations"][0] assert migration["transfer"] == test_case.expected_transfer + assert migration["transfer_fallback"] == test_case.expected_transfer_fallback assert migration["promotion"] == test_case.expected_promotion assert migration["storage_transition"] == test_case.expected_storage_transition assert "statement" not in migration assert test_case.expected_text_fragment in text.output + assert test_case.expected_build_fragment in built.output + assert order_ids(project_dir=tmp_path, relation=f"main.{DESTINATION_MODEL}") == tuple( + range(1, 6) + ) @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py index c0eccdb248..52eaba518e 100644 --- a/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/_test_types.py @@ -14,3 +14,21 @@ class ArtifactNamingTestCase: expected_stage_name: str expected_displaced_name: str expected_destination_exists: bool + + +@dataclass(frozen=True) +class CloneFallbackTestCase: + description: str + clone_statements: tuple[str, ...] + refused_messages: tuple[str, ...] + expected_transfer: str + expected_stage_rows: int + + +@dataclass(frozen=True) +class CloneFailureTestCase: + description: str + clone_statements: tuple[str, ...] + refused_messages: tuple[str, ...] + expected_error_fragment: str + expected_stage_exists: bool diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py index bb46c82e98..88b2e1653e 100644 --- a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -2,6 +2,9 @@ from __future__ import annotations +from sqlbuild.adapter.contract.models import MigrationStagePlan +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.types import ( MigrationCompatibility, @@ -9,6 +12,7 @@ MigrationDiscovery, ) from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild.executor.migrations.models import MigrationArtifactNames def orders_migration_entry() -> ModelMigrationPlanEntry: @@ -31,3 +35,50 @@ def orders_migration_entry() -> ModelMigrationPlanEntry: ), target_name=None, ) + + +class CloneRefusingDuckDbAdapter(DuckDbAdapter): + """DuckDB adapter whose stage clone is scripted to fail with a copy fallback.""" + + def __init__( + self, *, clone_statements: tuple[str, ...], refused_messages: tuple[str, ...] + ) -> None: + super().__init__() + self.clone_statements: tuple[str, ...] = clone_statements + self.refused_messages: tuple[str, ...] = refused_messages + + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.CLONE, + statements=tuple( + statement.format(stage=stage, origin=origin) for statement in self.clone_statements + ), + fallback_statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + is_clone_refusal=self.is_clone_refusal, + ) + + def is_clone_refusal(self, error: BaseException) -> bool: + """Treat errors naming one of the scripted refusal messages as clone refusals.""" + + return any(message in str(error) for message in self.refused_messages) + + +def artifact_names_for(*, stage_name: str) -> MigrationArtifactNames: + """Return artifact names for a fresh stage beside main.orders_v2.""" + + return MigrationArtifactNames( + stage_name=stage_name, + stage_qualified=f"main.{stage_name}", + displaced_name="_sqb_archive__20260102t030405z__migration_previous__orders_v2", + displaced_qualified="main._sqb_archive__20260102t030405z__migration_previous__orders_v2", + destination_qualified="main.orders_v2", + destination_exists=False, + ) diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_staging.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_staging.py new file mode 100644 index 0000000000..d056da8208 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/test_staging.py @@ -0,0 +1,122 @@ +"""Integration coverage for the clone-refusal copy fallback when creating a migration stage.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.executor.migrations._helpers.staging import create_stage +from sqlbuild.executor.migrations.models import MigrationArtifactNames +from tests.integration.src.sqlbuild.executor.migrations._helpers._test_types import ( + CloneFailureTestCase, + CloneFallbackTestCase, +) +from tests.integration.src.sqlbuild.executor.migrations._helpers.helpers import ( + CloneRefusingDuckDbAdapter, + artifact_names_for, + orders_migration_entry, +) + +_STAGE: str = "_sqb_archive__20260102t030405z__migration_stage__orders_v2" + + +@pytest.mark.parametrize( + "test_case", + [ + CloneFallbackTestCase( + description="recognized clone refusal falls back to a physical copy", + clone_statements=("SELECT error('clone refused by warehouse')",), + refused_messages=("clone refused by warehouse",), + expected_transfer=MigrationTransfer.COPY.value, + expected_stage_rows=2, + ), + CloneFallbackTestCase( + description="successful clone reports the planned clone transfer", + clone_statements=("CREATE TABLE {stage} AS SELECT * FROM {origin}",), + refused_messages=("clone refused by warehouse",), + expected_transfer=MigrationTransfer.CLONE.value, + expected_stage_rows=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_clone_outcome_when_creating_stage_then_reports_actual_transfer( + test_case: CloneFallbackTestCase, +) -> None: + """The returned transfer is the one that actually populated the stage.""" + + adapter: CloneRefusingDuckDbAdapter = CloneRefusingDuckDbAdapter( + clone_statements=test_case.clone_statements, refused_messages=test_case.refused_messages + ) + connection: Any = adapter.connect({"database": ":memory:"}) + _ = adapter.execute( + connection=connection, sql="CREATE TABLE main.orders AS SELECT * FROM range(2) t(id)" + ) + names: MigrationArtifactNames = artifact_names_for(stage_name=_STAGE) + + transfer: MigrationTransfer = create_stage( + adapter=adapter, connection=connection, entry=orders_migration_entry(), names=names + ) + + assert transfer.value == test_case.expected_transfer + assert adapter.execute( + connection=connection, sql=f"SELECT COUNT(*) FROM {names.stage_qualified}" + ).fetchall() == [(test_case.expected_stage_rows,)] + + +@pytest.mark.parametrize( + "test_case", + [ + CloneFailureTestCase( + description="unrecognized failure is re-raised without attempting a copy", + clone_statements=("SELECT error('connection reset by peer')",), + refused_messages=("clone refused by warehouse",), + expected_error_fragment="connection reset by peer", + expected_stage_exists=False, + ), + CloneFailureTestCase( + description="refusal after the stage appeared fails instead of adopting it", + clone_statements=( + "CREATE TABLE {stage} AS SELECT * FROM {origin}", + "SELECT error('clone refused by warehouse')", + ), + refused_messages=("clone refused by warehouse",), + expected_error_fragment="refusing to adopt a stage of unknown completeness", + expected_stage_exists=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_unsafe_clone_failure_when_creating_stage_then_raises_without_copy( + test_case: CloneFailureTestCase, +) -> None: + """Only clean, recognized refusals fall back; anything else surfaces to the caller.""" + + adapter: CloneRefusingDuckDbAdapter = CloneRefusingDuckDbAdapter( + clone_statements=test_case.clone_statements, refused_messages=test_case.refused_messages + ) + connection: Any = adapter.connect({"database": ":memory:"}) + _ = adapter.execute( + connection=connection, sql="CREATE TABLE main.orders AS SELECT * FROM range(2) t(id)" + ) + names: MigrationArtifactNames = artifact_names_for(stage_name=_STAGE) + + with pytest.raises(Exception, match=test_case.expected_error_fragment): + _ = create_stage( + adapter=adapter, connection=connection, entry=orders_migration_entry(), names=names + ) + + stage_rows: list[tuple[object, ...]] = adapter.execute( + connection=connection, + sql=( + "SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = 'main' AND table_name = '{_STAGE}'" + ), + ).fetchall() + assert bool(stage_rows) is test_case.expected_stage_exists + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index 5ed09e990c..581f8a21da 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -210,3 +210,10 @@ class BigQueryRelationAgeMetadataTestCase: metadata_rows: tuple[tuple[object, ...], ...] expected_timestamps: tuple[tuple[str, str], ...] relation_names: tuple[str, ...] = ("old_orders", "old_orders_view") + + +@dataclass(frozen=True) +class BigQueryCloneRefusalTestCase: + description: str + driver_error: Exception + expected_refusal: bool diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_clone_refusal.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_clone_refusal.py new file mode 100644 index 0000000000..b6b784fd98 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_clone_refusal.py @@ -0,0 +1,100 @@ +"""BigQuery classification of stage clone failures that may fall back to a physical copy.""" + +from __future__ import annotations + +import pytest +from google.api_core.exceptions import BadRequest, Forbidden, InternalServerError, TooManyRequests + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import MigrationStagePlan +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.bigquery.classes.bigquery_connection import _BigQueryConnection +from tests.unit.src.sqlbuild.adapters.bigquery._test_types import BigQueryCloneRefusalTestCase +from tests.unit.src.sqlbuild.adapters.bigquery.helpers import FakeBigQueryFailingClient + + +@pytest.mark.parametrize( + "test_case", + [ + BigQueryCloneRefusalTestCase( + description="clone chain depth limit is a refusal", + driver_error=BadRequest( + "clone chain too deep", + errors=[ + { + "reason": "invalid", + "message": "Cannot clone table: exceeded the limit of 3 clones or " + "snapshots in a chain; use a copy operation instead.", + } + ], + ), + expected_refusal=True, + ), + BigQueryCloneRefusalTestCase( + description="unsupported clone source is a refusal", + driver_error=BadRequest( + "unsupported clone source", + errors=[{"reason": "invalidQuery", "message": "Clone is not supported for tables"}], + ), + expected_refusal=True, + ), + BigQueryCloneRefusalTestCase( + description="backend error is transient and not a refusal", + driver_error=InternalServerError( + "backend", + errors=[{"reason": "backendError", "message": "Cannot clone right now"}], + ), + expected_refusal=False, + ), + BigQueryCloneRefusalTestCase( + description="rate limit is transient and not a refusal", + driver_error=TooManyRequests( + "rate", + errors=[{"reason": "rateLimitExceeded", "message": "Cannot clone: rate limit"}], + ), + expected_refusal=False, + ), + BigQueryCloneRefusalTestCase( + description="permission denial is not a refusal", + driver_error=Forbidden( + "denied", + errors=[{"reason": "accessDenied", "message": "Access Denied: cannot clone"}], + ), + expected_refusal=False, + ), + BigQueryCloneRefusalTestCase( + description="bad request without clone wording is not a refusal", + driver_error=BadRequest( + "missing", + errors=[{"reason": "invalid", "message": "Dataset main was not found"}], + ), + expected_refusal=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_clone_failure_when_classifying_then_only_capability_refusals_fall_back( + test_case: BigQueryCloneRefusalTestCase, +) -> None: + """Refusals are 400 invalid errors with clone-limitation wording, seen through the adapter.""" + + adapter: BigQueryAdapter = BigQueryAdapter() + plan: MigrationStagePlan = adapter.render_migration_stage( + origin="analytics.orders", stage="analytics.orders_stage" + ) + connection: _BigQueryConnection = _BigQueryConnection( + client=FakeBigQueryFailingClient(query_error=test_case.driver_error), location="US" + ) + + with pytest.raises(AdapterUserError) as raised: + _ = adapter.execute(connection=connection, sql=plan.statements[0]) + + assert plan.is_clone_refusal is not None + assert plan.is_clone_refusal(raised.value) is test_case.expected_refusal + assert plan.fallback_statements == ( + "CREATE TABLE `analytics.orders_stage` COPY `analytics.orders`", + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])