From 833316a4932f2104de4dd7e5b333d6e973063893 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 07:42:34 +0100 Subject: [PATCH 1/9] feat(migrations): preserve column data on renames in incremental models Add column-level migrate_from for incremental, microbatch and snapshot models. Exact expression renames are detected automatically, near matches produce a plan hint, and declared renames run as in-place RENAME COLUMN before the delta with append-only, deterministic-ID events in _sqlbuild_column_migrations. Pure renames no longer trigger replay_on_change. Refs CHI-393 --- .../adapter/contract/classes/base_adapter.py | 44 ++ .../contract/classes/duckdb_backed_adapter.py | 50 ++ .../contract/classes/materialization.py | 18 + .../contract/classes/strict_adapter.py | 14 + .../contract/main/render_rename_column_sql.py | 11 + .../bigquery/classes/bigquery_adapter.py | 50 ++ .../databricks/classes/databricks_adapter.py | 78 +++ src/sqlbuild/adapters/databricks/constants.py | 2 + .../postgres/classes/postgres_adapter.py | 50 ++ .../snowflake/classes/snowflake_adapter.py | 51 ++ .../sqlserver/classes/sqlserver_adapter.py | 52 ++ .../_helpers/build_planning/plan_policies.py | 36 +- src/sqlbuild/cli/output/_helpers/plan_json.py | 31 ++ src/sqlbuild/cli/output/_helpers/plan_text.py | 22 +- .../_helpers/attachment/model_config.py | 16 +- .../_helpers/config/model_validation.py | 57 ++ src/sqlbuild/compiler/compile/constants.py | 1 + .../discovery/_helpers/sql/schema_columns.py | 23 +- .../discovery/main/_model_schema_columns.py | 2 + .../migrations/_helpers/column_sql.py | 132 +++++ .../compiler/migrations/_helpers/sql.py | 74 ++- .../compiler/migrations/_helpers/writes.py | 43 ++ src/sqlbuild/compiler/migrations/constants.py | 22 + .../migrations/main/_newest_column_event.py | 30 + .../migrations/main/_read_column_events.py | 43 ++ .../main/column_create_table_sql.py | 27 + .../main/deterministic_column_event_id.py | 30 + .../migrations/main/write_column_event.py | 41 ++ .../compiler/migrations/main/write_event.py | 40 +- src/sqlbuild/compiler/migrations/models.py | 37 +- src/sqlbuild/compiler/migrations/types.py | 58 ++ .../_helpers/migrations/column_renames.py | 89 +++ .../planner/_helpers/migrations/columns.py | 525 ++++++++++++++++++ .../planner/_helpers/migrations/display.py | 52 +- .../_helpers/migrations/fingerprint.py | 10 +- .../planner/_helpers/migrations/planning.py | 61 +- .../_helpers/migrations/projections.py | 130 +++++ .../_helpers/planning/output_assembly.py | 31 +- .../_helpers/planning/warehouse_state.py | 24 +- .../_helpers/sql_tests/fixture_formatting.py | 1 + .../planner/_helpers/warehouse/snapshot.py | 17 +- .../classes/migration_state_inspection.py | 53 +- src/sqlbuild/compiler/planner/models.py | 80 +++ src/sqlbuild/executor/build/main/_execute.py | 8 + src/sqlbuild/executor/janitor/constants.py | 6 +- .../executor/migrations/_helpers/columns.py | 142 +++++ .../migrations/main/_apply_columns.py | 79 +++ src/sqlbuild/spec/contracts/models.py | 1 + 48 files changed, 2390 insertions(+), 104 deletions(-) create mode 100644 src/sqlbuild/adapter/contract/main/render_rename_column_sql.py create mode 100644 src/sqlbuild/compiler/migrations/_helpers/column_sql.py create mode 100644 src/sqlbuild/compiler/migrations/_helpers/writes.py create mode 100644 src/sqlbuild/compiler/migrations/main/_newest_column_event.py create mode 100644 src/sqlbuild/compiler/migrations/main/_read_column_events.py create mode 100644 src/sqlbuild/compiler/migrations/main/column_create_table_sql.py create mode 100644 src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py create mode 100644 src/sqlbuild/compiler/migrations/main/write_column_event.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/columns.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/projections.py create mode 100644 src/sqlbuild/executor/migrations/_helpers/columns.py create mode 100644 src/sqlbuild/executor/migrations/main/_apply_columns.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 9c253c3119..87f5350bd5 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -775,6 +775,16 @@ def render_alter_column_types( for col in columns ) + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + """Render SQL that renames one table column in place, keeping its data.""" + + return ( + f"ALTER TABLE {destination} RENAME COLUMN {self.render_identifier(old_name)} " + f"TO {self.render_identifier(new_name)}", + ) + def render_merge( self, *, @@ -1350,6 +1360,23 @@ def drop_columns( ) -> None: raise AdapterUserError(message="drop_columns requires an engine-specific implementation") + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + raise AdapterUserError(message="rename_column requires an engine-specific implementation") + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + """Return why this table cannot rename a column in place, or None when it can.""" + + del connection, destination + return None + def alter_column_types( self, *, @@ -2133,6 +2160,23 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem transient=self.state_tables_transient, ) + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + """Render DDL that creates the column migration event table when it is missing.""" + + from sqlbuild.compiler.migrations.main.column_create_table_sql import ( + build_column_migration_state_create_table_sql, + ) + + return build_column_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 5638d35122..48c8a1d5f1 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -866,6 +866,20 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem render_framework_type=self.render_framework_type, ) + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + from sqlbuild.compiler.migrations.main.column_create_table_sql import ( + build_column_migration_state_create_table_sql, + ) + + return build_column_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1777,6 +1791,21 @@ def render_drop_columns( for col_name in column_names ) + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + from sqlbuild.adapter.contract.main.render_rename_column_sql import ( + render_alter_rename_column_sql, + ) + + return ( + render_alter_rename_column_sql( + destination=destination, + old_identifier=self.render_exact_identifier(old_name), + new_identifier=self.render_identifier(new_name), + ), + ) + def render_alter_column_types( self, *, destination: str, columns: tuple[ColumnInfo, ...] ) -> tuple[str, ...]: @@ -1818,6 +1847,27 @@ def drop_columns( for stmt in statements: self.execute(connection=connection, sql=stmt) + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename_column( + destination=destination, old_name=old_name, new_name=new_name + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + del connection, destination + return None + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/materialization.py b/src/sqlbuild/adapter/contract/classes/materialization.py index 7077efda58..bc499048cc 100644 --- a/src/sqlbuild/adapter/contract/classes/materialization.py +++ b/src/sqlbuild/adapter/contract/classes/materialization.py @@ -212,6 +212,24 @@ def drop_columns( """Drop columns from an existing table.""" ... + @abstractmethod + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + """Rename one column of an existing table in place, keeping its data.""" + ... + + @abstractmethod + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + """Return why this table cannot rename a column in place, or None when it can.""" + ... + @abstractmethod def alter_column_types( self, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 1017aaa377..1420b3bb21 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -427,6 +427,13 @@ def render_alter_column_types( """Render SQL statements that alter column types on a table.""" ... + @abstractmethod + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + """Render SQL that renames one table column in place, keeping its data.""" + ... + @abstractmethod def render_merge( self, @@ -880,6 +887,13 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem """Render DDL that creates the model migration event table when it is missing.""" ... + @abstractmethod + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + """Render DDL that creates the column migration event table when it is missing.""" + ... + @abstractmethod def render_prune_fingerprint_history_sql( self, diff --git a/src/sqlbuild/adapter/contract/main/render_rename_column_sql.py b/src/sqlbuild/adapter/contract/main/render_rename_column_sql.py new file mode 100644 index 0000000000..020a42c76e --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/render_rename_column_sql.py @@ -0,0 +1,11 @@ +"""Shared ALTER TABLE ... RENAME COLUMN rendering for adapters that support it.""" + +from __future__ import annotations + + +def render_alter_rename_column_sql( + *, destination: str, old_identifier: str, new_identifier: str +) -> str: + """Render one in-place column rename from already-quoted identifiers.""" + + return f"ALTER TABLE {destination} RENAME COLUMN {old_identifier} TO {new_identifier}" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 5a6e057a9c..10e0284ffc 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -337,6 +337,20 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem render_framework_type=self.render_framework_type, ) + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + from sqlbuild.compiler.migrations.main.column_create_table_sql import ( + build_column_migration_state_create_table_sql, + ) + + return build_column_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -742,6 +756,27 @@ def drop_columns( ) -> None: raise AdapterUserError(message="drop_columns requires an engine-specific implementation") + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename_column( + destination=destination, old_name=old_name, new_name=new_name + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + del connection, destination + return None + def alter_column_types( self, *, @@ -876,6 +911,21 @@ def render_drop_columns( for col_name in column_names ) + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + from sqlbuild.adapter.contract.main.render_rename_column_sql import ( + render_alter_rename_column_sql, + ) + + return ( + render_alter_rename_column_sql( + destination=destination, + old_identifier=self.render_exact_identifier(old_name), + new_identifier=self.render_identifier(new_name), + ), + ) + def render_alter_column_types( self, *, destination: str, columns: tuple[ColumnInfo, ...] ) -> tuple[str, ...]: diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 65af1de664..0ff80f36d6 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -97,6 +97,8 @@ from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.adapters.databricks.classes.databricks_connection import _DatabricksConnection from sqlbuild.adapters.databricks.constants import ( + DELTA_COLUMN_MAPPING_NAME_MODE, + DELTA_COLUMN_MAPPING_PROPERTY, DELTA_DEFAULT_DELETED_FILE_RETENTION_DAYS, DELTA_DEFAULT_LOG_RETENTION_DAYS, DELTA_RELATION_FORMAT, @@ -430,6 +432,20 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem render_framework_type=self.render_framework_type, ) + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + from sqlbuild.compiler.migrations.main.column_create_table_sql import ( + build_column_migration_state_create_table_sql, + ) + + return build_column_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -1810,6 +1826,21 @@ def render_drop_columns( for column_name in column_names ) + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + from sqlbuild.adapter.contract.main.render_rename_column_sql import ( + render_alter_rename_column_sql, + ) + + return ( + render_alter_rename_column_sql( + destination=destination, + old_identifier=self.render_exact_identifier(old_name), + new_identifier=self.render_identifier(new_name), + ), + ) + def render_alter_column_types( self, *, @@ -2230,6 +2261,53 @@ def drop_columns( for statement in statements: self.execute(connection=connection, sql=statement) + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + reason: str | None = self.column_rename_unavailable_reason( + connection=connection, destination=destination + ) + if reason is not None: + raise AdapterUserError(message=reason) + statements: tuple[str, ...] = self.render_rename_column( + destination=destination, old_name=old_name, new_name=new_name + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + rows: list[tuple[Any, ...]] = self.execute( + connection=connection, + sql=f"SHOW TBLPROPERTIES {destination} ('{DELTA_COLUMN_MAPPING_PROPERTY}')", + ).fetchall() + mode: str | None = next( + ( + str(row[1]).strip().lower() + for row in rows + if len(row) > 1 and str(row[0]) == DELTA_COLUMN_MAPPING_PROPERTY + ), + None, + ) + if mode == DELTA_COLUMN_MAPPING_NAME_MODE: + return None + return ( + f"Databricks renames a column in place only when the Delta table uses column " + f"mapping ('{DELTA_COLUMN_MAPPING_PROPERTY}' = '{DELTA_COLUMN_MAPPING_NAME_MODE}'), " + f"and {destination} has {'no column mapping' if mode is None else repr(mode)}. " + f"Enable it with ALTER TABLE {destination} SET TBLPROPERTIES " + f"('{DELTA_COLUMN_MAPPING_PROPERTY}' = '{DELTA_COLUMN_MAPPING_NAME_MODE}', " + "'delta.minReaderVersion' = '2', 'delta.minWriterVersion' = '5'), which upgrades " + "the table protocol, or full-refresh the model" + ) + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index 4328cee8d1..e957c550a8 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -5,3 +5,5 @@ DELTA_RELATION_FORMAT: str = "delta" DELTA_DEFAULT_LOG_RETENTION_DAYS: int = 30 DELTA_DEFAULT_DELETED_FILE_RETENTION_DAYS: int = 7 +DELTA_COLUMN_MAPPING_PROPERTY: str = "delta.columnMapping.mode" +DELTA_COLUMN_MAPPING_NAME_MODE: str = "name" diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 57b6d091b7..c0b5621393 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -653,6 +653,21 @@ def render_drop_columns( for col_name in column_names ) + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + from sqlbuild.adapter.contract.main.render_rename_column_sql import ( + render_alter_rename_column_sql, + ) + + return ( + render_alter_rename_column_sql( + destination=destination, + old_identifier=self.render_exact_identifier(old_name), + new_identifier=self.render_identifier(new_name.lower()), + ), + ) + def render_alter_column_types( self, *, destination: str, columns: tuple[ColumnInfo, ...] ) -> tuple[str, ...]: @@ -1529,6 +1544,20 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem render_framework_type=self.render_framework_type, ) + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + from sqlbuild.compiler.migrations.main.column_create_table_sql import ( + build_column_migration_state_create_table_sql, + ) + + return build_column_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_read_latest_source_freshness_sql( self, *, @@ -1972,6 +2001,27 @@ def drop_columns( for stmt in statements: self.execute(connection=connection, sql=stmt) + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename_column( + destination=destination, old_name=old_name, new_name=new_name + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + del connection, destination + return None + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 7afd631c09..f89fdb88ca 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -513,6 +513,21 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem transient=self.state_tables_transient, ) + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + from sqlbuild.compiler.migrations.main.column_create_table_sql import ( + build_column_migration_state_create_table_sql, + ) + + return build_column_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + transient=self.state_tables_transient, + ) + def render_prune_fingerprint_history_sql( self, *, @@ -871,6 +886,21 @@ def render_drop_columns( for col_name in column_names ) + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + from sqlbuild.adapter.contract.main.render_rename_column_sql import ( + render_alter_rename_column_sql, + ) + + return ( + render_alter_rename_column_sql( + destination=destination, + old_identifier=self.render_exact_identifier(old_name), + new_identifier=self.render_identifier(new_name), + ), + ) + def render_alter_column_types( self, *, destination: str, columns: tuple[ColumnInfo, ...] ) -> tuple[str, ...]: @@ -2474,6 +2504,27 @@ def drop_columns( for statement in statements: self.execute(connection=connection, sql=statement) + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename_column( + destination=destination, old_name=old_name, new_name=new_name + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + del connection, destination + return None + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 527ef2eb58..a3cff18841 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -635,6 +635,27 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem table_name=MIGRATION_TABLE_NAME, ) + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME + from sqlbuild.compiler.migrations.main.column_create_table_sql import ( + build_column_migration_state_create_table_sql, + ) + + create_sql: str = build_column_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ).replace("CREATE TABLE IF NOT EXISTS", "CREATE TABLE", 1) + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=COLUMN_MIGRATION_TABLE_NAME, + ) + def render_read_latest_source_freshness_sql( self, *, @@ -1408,6 +1429,27 @@ def drop_columns( ) -> None: raise AdapterUserError(message="drop_columns requires an engine-specific implementation") + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename_column( + destination=destination, old_name=old_name, new_name=new_name + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + del connection, destination + return None + def drop_view( self, *, @@ -2025,6 +2067,16 @@ def render_drop_columns( for col_name in column_names ) + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + column: str = ( + f"{self._sp_rename_relation_name(destination)}.{self.render_identifier(old_name)}" + ) + escaped_column: str = column.replace("'", "''") + escaped_name: str = new_name.replace("'", "''") + return (f"EXEC sp_rename N'{escaped_column}', N'{escaped_name}', 'COLUMN'",) + def render_loader_rows_select( self, *, diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py index 3fcabe3ac4..594b49cbf5 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py @@ -19,8 +19,19 @@ ) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import BuildCommandRequest, BuildInvocation -from sqlbuild.compiler.migrations.types import MigrationDecision -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput +from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision +from sqlbuild.compiler.planner.models import ( + ColumnMigrationPlanEntry, + ModelMigrationPlanEntry, + PlanOutput, +) + +_COLUMN_BLOCK_CODES: dict[ColumnMigrationDecision, str] = { + ColumnMigrationDecision.SOURCE_MISSING: "M109", + ColumnMigrationDecision.CONFLICT: "M110", + ColumnMigrationDecision.STILL_PRODUCED: "M111", + ColumnMigrationDecision.UNSUPPORTED: "M112", +} def enforce_build_plan_policies( @@ -29,6 +40,7 @@ def enforce_build_plan_policies( """Apply migration, execution-limit, and storage safety gates in their required order.""" _enforce_model_migration_policy(plan=plan) + _enforce_column_migration_policy(plan=plan) enforce_model_execution_limit( model_count=executable_model_count(plan=plan), target_name=invocation.effective_target_name, @@ -98,3 +110,23 @@ def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: code="M104", help="Run sqb plan to see the blocking schema differences.", ) + + +def _enforce_column_migration_policy(*, plan: PlanOutput) -> None: + """Refuse to build while any declared column rename cannot be applied safely.""" + + blocked: tuple[ColumnMigrationPlanEntry, ...] = tuple( + entry for entry in plan.column_migration_entries if entry.blocks_build + ) + if not blocked: + return + raise CliUserError( + "column migrations are blocked for " + + ", ".join( + f"'{entry.model_name}' ({entry.destination_column} migrate_from " + f"{entry.origin_column}: {entry.decision.label})" + for entry in blocked + ), + code=_COLUMN_BLOCK_CODES.get(blocked[0].decision, "M109"), + help="Run sqb plan to see why each column rename cannot be applied.", + ) diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 535d274922..112c1c5f94 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -98,6 +98,7 @@ def format_plan_json( "retention": retention, "table_type_conversions": table_type_conversions, "migrations": _serialize_model_migrations(plan), + "column_migrations": _serialize_column_migrations(plan), } if plan.metadata: result["metadata"] = plan.metadata @@ -192,6 +193,17 @@ def _serialize_model_entry(entry: ModelPlanEntry) -> dict[str, object]: if entry.destination.qualified_name is not None: model["qualified_name"] = entry.destination.qualified_name + if entry.column_rename_hints: + model["column_rename_hints"] = [ + { + "column": hint.added_column, + "candidates": list(hint.candidate_columns), + "identical": hint.identical, + "hint": hint.message, + } + for hint in entry.column_rename_hints + ] + return model @@ -465,6 +477,25 @@ def _serialize_provider_usage(usage: PlanProviderUsage) -> dict[str, object]: return payload +def _serialize_column_migrations(plan: PlanOutput) -> list[dict[str, object]]: + return [ + { + "kind": "column_migration", + "model": entry.model_name, + "relation": entry.destination.qualified_name or entry.destination.name, + "origin_column": entry.origin_column, + "destination_column": entry.destination_column, + "discovery": entry.discovery.value, + "decision": entry.decision.value, + "target": entry.target_name, + "completed_at": ( + entry.completed_at.isoformat() if entry.completed_at is not None else None + ), + } + for entry in plan.column_migration_entries + ] + + def _serialize_model_migrations(plan: PlanOutput) -> list[dict[str, object]]: return [ { diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index 9b27986b3a..924028c704 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -27,6 +27,7 @@ ) from sqlbuild.compiler.planner.models import ( CascadeResult, + ColumnRenameHint, CursorBounds, FunctionPlanEntry, ModelPlanEntry, @@ -1024,11 +1025,28 @@ def _append_policy_line(*, lines: list[str], entry: ModelPlanEntry) -> list[str] def _append_schema_diff(*, lines: list[str], entry: ModelPlanEntry) -> list[str]: """Append schema diff lines if findings exist.""" - if not entry.schema_findings: + if not entry.schema_findings and not entry.column_rename_hints: return lines style: CliStyle = CliStyle(use_color=True) lines.append(style.label(" schema diff:")) - lines.extend(_format_schema_findings(entry.schema_findings)) + finding_lines: list[str] = _format_schema_findings(entry.schema_findings) + hinted: set[str] = set() + finding: SchemaFinding + line: str + for finding, line in zip(entry.schema_findings, finding_lines, strict=True): + lines.append(line) + if finding.kind != SchemaChangeKind.COLUMN_ADDED: + continue + hint: ColumnRenameHint + for hint in entry.column_rename_hints: + if hint.added_column.lower() == finding.column_name.lower(): + lines.append(f" {style.warning(hint.message)}") + hinted.add(hint.added_column.lower()) + lines.extend( + f" {style.warning(f'{hint.added_column}: {hint.message}')}" + for hint in entry.column_rename_hints + if hint.added_column.lower() not in hinted + ) return lines diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index 4e090e3bad..343d3afd9c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -15,6 +15,9 @@ from sqlbuild.compiler.compile._helpers.config.dynamic_columns import ( parse_dynamic_column_families, ) +from sqlbuild.compiler.compile._helpers.config.model_validation import ( + validate_column_migration_config, +) from sqlbuild.compiler.compile._helpers.render.templating import ( contains_template_data, ) @@ -318,6 +321,9 @@ def build_model_header_schema_entry( model_name=model_name, file_path=file_path, ) + validate_column_migration_config( + config_values=model_header_values, model_name=model_name, columns=columns + ) type_enforcement: bool | None = ( True if any(column.type is not None for column in columns) or dynamic_columns else None ) @@ -366,6 +372,7 @@ def _parse_model_header_columns( label="model", error_class=CompileInputError, column_locations=column_locations, + allow_migrate_from=True, ) cached: CachedModelHeaderColumns | None = column_cache.get(raw_columns) if cached is None: @@ -375,6 +382,7 @@ def _parse_model_header_columns( label="model", error_class=CompileInputError, column_locations=column_locations, + allow_migrate_from=True, ) cached = CachedModelHeaderColumns( raw_columns=raw_columns, @@ -405,6 +413,7 @@ def _schema_column_at_location( nullable=column.nullable, description=column.description, meta=column.meta, + migrate_from=column.migrate_from, audits=tuple( SchemaAuditInstance( definition_name=audit.definition_name, @@ -458,6 +467,7 @@ def _merge_model_schema_columns( inherited_audits=named_column.audits, local_audits=local_column.audits, ), + migrate_from=local_column.migrate_from, ) return (*merged_named_columns, *additional_columns) @@ -500,11 +510,11 @@ def _validate_model_schema_audit_augmentation( f"{', '.join(overridden_fields)} for named-schema column '{local_column.name}' from " f"{named_origin}; only audit augmentation is supported" ) - if not local_column.audits: + if not local_column.audits and local_column.migrate_from is None: raise CompileInputError( f"model '{model_name}' in {file_path} redeclares named-schema column " - f"'{local_column.name}' from {named_origin} without audits; only audit augmentation " - "is supported" + f"'{local_column.name}' from {named_origin} without audits or migrate_from; only " + "audit augmentation and column migrations are supported" ) diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index 6ef14cc18f..bafc661f53 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -48,6 +48,7 @@ ) from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction +_QUALIFIED_NAME_SEPARATOR: str = "." _VALID_STRATEGIES: frozenset[str] = frozenset(s.value for s in IncrementalStrategy) _VALID_CURSOR_TYPES: frozenset[str] = frozenset(ct.value for ct in CursorType) _VALID_CURSOR_GRAINS: frozenset[str] = frozenset(cg.value for cg in CursorGrain) @@ -976,6 +977,62 @@ def validate_model_migration_config(*, config: CompileModelConfig, model_name: s raise CompileInputError(f"model '{model_name}': migrate_force must be true or false") +def validate_column_migration_config( + *, config_values: dict[str, object], model_name: str, columns: tuple[SchemaColumn, ...] +) -> None: + """Validate column-level migrate_from declarations as one rename set.""" + + declared: tuple[SchemaColumn, ...] = tuple( + column for column in columns if column.migrate_from is not None + ) + if not declared: + return + materialized: str | None = get_config_str(values=config_values, key="materialized") + if materialized not in {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT}: + raise CompileInputError( + f"model '{model_name}': column '{declared[0].name}' declares migrate_from, which is " + f"only valid for incremental and snapshot models; '{materialized}' models are " + "rebuilt with their new columns", + help="remove migrate_from from the column", + ) + sources: dict[str, str] = {} + column: SchemaColumn + for column in declared: + origin: str = (column.migrate_from or "").strip() + if not origin or _QUALIFIED_NAME_SEPARATOR in origin: + raise CompileInputError( + f"model '{model_name}': column '{column.name}' migrate_from must name one " + "column of the model's existing table", + help="write the old column name, for example revenue (migrate_from amount)", + ) + if origin.lower() == column.name.lower(): + raise CompileInputError( + f"model '{model_name}': column '{column.name}' migrate_from cannot name the " + "column itself" + ) + claimed_by: str | None = sources.get(origin.lower()) + if claimed_by is not None: + raise CompileInputError( + f"model '{model_name}': columns '{claimed_by}' and '{column.name}' both " + f"declare migrate_from {origin}; a column can be renamed to only one new name", + help="keep migrate_from on the column that should receive the existing data", + ) + sources[origin.lower()] = column.name + destinations: dict[str, str] = {column.name.lower(): column.name for column in declared} + for column in declared: + chained: str | None = destinations.get((column.migrate_from or "").strip().lower()) + if chained is not None: + raise CompileInputError( + f"model '{model_name}': column '{column.name}' migrates from " + f"'{column.migrate_from}', which itself declares migrate_from; chained or " + "swapped column renames are not supported", + help=( + "a chain or swap cannot be renamed in place safely; give the new column a " + "name no other renamed column uses, or rebuild the model with --full-refresh" + ), + ) + + def validate_snapshot_config( *, config: CompileModelConfig, diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 59fa1c641d..dba6ddb660 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -22,6 +22,7 @@ CURSOR_INPUTS_CONFIG_KEY: str = "cursor_inputs" MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" MIGRATE_FORCE_CONFIG_KEY: str = "migrate_force" +COLUMN_MIGRATE_FROM_KEY: str = "migrate_from" MAX_MICROBATCHES_CONFIG_KEY: str = "max_microbatches" MICROBATCH_LIMIT_MAX_BATCHES_KEY: str = "max_batches" MICROBATCH_LIMIT_ACTION_KEY: str = "action" diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/schema_columns.py b/src/sqlbuild/compiler/discovery/_helpers/sql/schema_columns.py index e42eb709bc..0fd07761bf 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/schema_columns.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/schema_columns.py @@ -9,13 +9,16 @@ from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.authored_values.main._optional_named_bool import optional_named_bool from sqlbuild.compiler.authored_values.main._optional_named_string import optional_named_string -from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME +from sqlbuild.compiler.compile.constants import COLUMN_MIGRATE_FROM_KEY, NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaColumn, SourceLocation type _SchemaColumnParseError = type[CompileInputError] | type[DeclarationParseError] +_SCHEMA_COLUMN_KEYS: frozenset[str] = frozenset({"type", "nullable", "description", "audits"}) +_MODEL_COLUMN_KEYS: frozenset[str] = _SCHEMA_COLUMN_KEYS | {COLUMN_MIGRATE_FROM_KEY} + def parse_schema_columns( *, @@ -25,6 +28,7 @@ def parse_schema_columns( error_class: _SchemaColumnParseError, column_locations: dict[str, SourceLocation] | None = None, require_columns: bool = False, + allow_migrate_from: bool = False, ) -> tuple[SchemaColumn, ...]: """Parse one MODEL or SCHEMA columns mapping into shared contract columns.""" @@ -58,12 +62,10 @@ def parse_schema_columns( f"{file_path} {label} column '{raw_column_name}' metadata must be a mapping" ) column_metadata: dict[str, object] = cast(dict[str, object], raw_column_metadata) - unknown_keys: set[str] = set(column_metadata) - { - "type", - "nullable", - "description", - "audits", - } + allowed_keys: frozenset[str] = ( + _MODEL_COLUMN_KEYS if allow_migrate_from else _SCHEMA_COLUMN_KEYS + ) + unknown_keys: set[str] = set(column_metadata) - allowed_keys if unknown_keys: raise error_class( f"{file_path} {label} column '{raw_column_name}' has unknown metadata keys: " @@ -122,6 +124,13 @@ def parse_schema_columns( ), audits=audits, location=column_location, + migrate_from=optional_named_string( + raw_value=column_metadata.get(COLUMN_MIGRATE_FROM_KEY), + file_path=file_path, + label=f"{label} column '{raw_column_name}'", + key=COLUMN_MIGRATE_FROM_KEY, + error_class=error_class, + ), ) ) return tuple(parsed_columns) diff --git a/src/sqlbuild/compiler/discovery/main/_model_schema_columns.py b/src/sqlbuild/compiler/discovery/main/_model_schema_columns.py index e856bd5ffa..3bc9263ddc 100644 --- a/src/sqlbuild/compiler/discovery/main/_model_schema_columns.py +++ b/src/sqlbuild/compiler/discovery/main/_model_schema_columns.py @@ -20,6 +20,7 @@ def parse_schema_columns( error_class: type[CompileInputError] | type[DeclarationParseError], column_locations: dict[str, SourceLocation] | None = None, require_columns: bool = False, + allow_migrate_from: bool = False, ) -> tuple[SchemaColumn, ...]: """Parse authored model columns through the discovery-owned implementation.""" @@ -30,4 +31,5 @@ def parse_schema_columns( error_class=error_class, column_locations=column_locations, require_columns=require_columns, + allow_migrate_from=allow_migrate_from, ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py new file mode 100644 index 0000000000..338205cc9b --- /dev/null +++ b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py @@ -0,0 +1,132 @@ +"""Portable SQL for append-only column migration events.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) +from sqlbuild.compiler.migrations._helpers.sql import ( + decode_event_timestamp, + optional_text, + render_event_exists_sql, + render_event_insert_sql, +) +from sqlbuild.compiler.migrations.constants import ( + COLUMN_MIGRATION_COLUMN_TYPES, + COLUMN_MIGRATION_COLUMNS, + COLUMN_MIGRATION_TABLE_NAME, +) +from sqlbuild.compiler.migrations.exceptions import MigrationStateError +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery + + +def qualified_column_migration_table( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: + table: str | None = render_qualified_name( + database=database, schema=schema, name=COLUMN_MIGRATION_TABLE_NAME + ) + if table is None: + raise MigrationStateError("column migration state requires a target schema") + return table + + +def build_column_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool, +) -> str: + return render_state_table_create_sql( + qualified_name=qualified_column_migration_table( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=COLUMN_MIGRATION_COLUMNS, + column_types=COLUMN_MIGRATION_COLUMN_TYPES, + required_columns=frozenset(), + render_framework_type=render_framework_type, + transient=transient, + ) + + +def build_column_existing_event_sql( + *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] +) -> str: + return render_event_exists_sql( + table=_event_table(event=event, render_qualified_name=render_qualified_name), + event_id=event.event_id, + ) + + +def build_column_insert_sql( + *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] +) -> str: + return render_event_insert_sql( + table=_event_table(event=event, render_qualified_name=render_qualified_name), + columns=COLUMN_MIGRATION_COLUMNS, + column_types=COLUMN_MIGRATION_COLUMN_TYPES, + values=( + event.event_id, + event.target_name, + event.model_name, + event.relation.database, + event.relation.schema, + event.relation.name, + event.origin_column, + event.destination_column, + event.discovery.value, + event.decision.value, + event.run_id, + event.created_at, + ), + ) + + +def build_column_read_sql( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: + table: str = qualified_column_migration_table( + database=database, schema=schema, render_qualified_name=render_qualified_name + ) + return ( + f"SELECT {', '.join(COLUMN_MIGRATION_COLUMNS)} FROM {table} ORDER BY created_at, event_id" + ) + + +def decode_column_event_row(row: tuple[Any, ...]) -> ColumnMigrationEvent: + values: dict[str, Any] = dict(zip(COLUMN_MIGRATION_COLUMNS, row, strict=False)) + return ColumnMigrationEvent( + event_id=str(values["event_id"]), + target_name=optional_text(values["target_name"]), + model_name=str(values["model_name"]), + relation=MigrationRelation( + database=optional_text(values["relation_database"]), + schema=optional_text(values["relation_schema"]), + name=str(values["relation_name"]), + ), + origin_column=str(values["origin_column"]), + destination_column=str(values["destination_column"]), + discovery=MigrationDiscovery(str(values["discovery"])), + decision=ColumnMigrationDecision(str(values["decision"])), + run_id=str(values["run_id"]), + created_at=decode_event_timestamp(values["created_at"]), + ) + + +def _event_table( + *, event: ColumnMigrationEvent, render_qualified_name: Callable[..., str | None] +) -> str: + if event.relation.schema is None: + raise MigrationStateError("column migration events require a relation schema") + return qualified_column_migration_table( + database=event.relation.database, + schema=event.relation.schema, + render_qualified_name=render_qualified_name, + ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py index 8e4a05257f..04477f0810 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/sql.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import UTC, datetime from typing import Any @@ -57,22 +57,44 @@ def build_existing_event_sql( *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] ) -> str: table: str = _event_table(event=event, render_qualified_name=render_qualified_name) - event_id_literal: str = render_state_sql_literal( - value=event.event_id, declared_type=StateSqlValueType.STRING - ) - return f"SELECT event_id FROM {table} WHERE event_id = {event_id_literal}" + return render_event_exists_sql(table=table, event_id=event.event_id) def build_insert_sql( *, event: MigrationEvent, render_qualified_name: Callable[..., str | None] ) -> str: table: str = _event_table(event=event, render_qualified_name=render_qualified_name) - values: tuple[object | None, ...] = _event_values(event) + return render_event_insert_sql( + table=table, + columns=MIGRATION_COLUMNS, + column_types=MIGRATION_COLUMN_TYPES, + values=_event_values(event), + ) + + +def render_event_exists_sql(*, table: str, event_id: str) -> str: + """Render the lookup that makes an append-only event write idempotent.""" + + event_id_literal: str = render_state_sql_literal( + value=event_id, declared_type=StateSqlValueType.STRING + ) + return f"SELECT event_id FROM {table} WHERE event_id = {event_id_literal}" + + +def render_event_insert_sql( + *, + table: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + values: tuple[object | None, ...], +) -> str: + """Render one append-only event row as INSERT ... VALUES with typed literals.""" + literals: str = ", ".join( - render_state_sql_literal(value=value, declared_type=MIGRATION_COLUMN_TYPES[column]) - for column, value in zip(MIGRATION_COLUMNS, values, strict=True) + render_state_sql_literal(value=value, declared_type=column_types[column]) + for column, value in zip(columns, values, strict=True) ) - return f"INSERT INTO {table} ({', '.join(MIGRATION_COLUMNS)}) VALUES ({literals})" + return f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({literals})" def _event_table(*, event: MigrationEvent, render_qualified_name: Callable[..., str | None]) -> str: @@ -96,27 +118,20 @@ def build_read_sql( def decode_event_row(row: tuple[Any, ...]) -> MigrationEvent: values: dict[str, Any] = dict(zip(MIGRATION_COLUMNS, row, strict=False)) - raw_created_at: Any = values["created_at"] - created_at: datetime = ( - raw_created_at - if isinstance(raw_created_at, datetime) - else datetime.fromisoformat(str(raw_created_at)) - ) - if created_at.tzinfo is None: - created_at = created_at.replace(tzinfo=UTC) + created_at: datetime = decode_event_timestamp(values["created_at"]) return MigrationEvent( event_id=str(values["event_id"]), - target_name=_optional_text(values["target_name"]), - origin_model=_optional_text(values["origin_model"]), + target_name=optional_text(values["target_name"]), + origin_model=optional_text(values["origin_model"]), origin=MigrationRelation( - database=_optional_text(values["origin_database"]), - schema=_optional_text(values["origin_schema"]), + database=optional_text(values["origin_database"]), + schema=optional_text(values["origin_schema"]), name=str(values["origin_name"]), ), destination_model=str(values["destination_model"]), destination=MigrationRelation( - database=_optional_text(values["destination_database"]), - schema=_optional_text(values["destination_schema"]), + database=optional_text(values["destination_database"]), + schema=optional_text(values["destination_schema"]), name=str(values["destination_name"]), ), origin_version_hash=str(values["origin_version_hash"] or ""), @@ -147,5 +162,16 @@ def _event_values(event: MigrationEvent) -> tuple[object | None, ...]: ) -def _optional_text(value: object | None) -> str | None: +def decode_event_timestamp(raw: object) -> datetime: + """Decode a stored event timestamp as an aware UTC datetime.""" + + created_at: datetime = raw if isinstance(raw, datetime) else datetime.fromisoformat(str(raw)) + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=UTC) + return created_at + + +def optional_text(value: object | None) -> str | None: + """Return a stored nullable text value as a string or None.""" + return None if value is None else str(value) diff --git a/src/sqlbuild/compiler/migrations/_helpers/writes.py b/src/sqlbuild/compiler/migrations/_helpers/writes.py new file mode 100644 index 0000000000..e01b7c8b72 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/_helpers/writes.py @@ -0,0 +1,43 @@ +"""Idempotent append of one migration event row with bounded retries.""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_RETRY_BASE_SECONDS + + +def append_event_row( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + existing_sql: str, + insert_sql: str, + create_table_sql: str | None, + attempts: int, + subject: str, +) -> None: + """Create the state table when given, then insert the row unless its event ID exists.""" + + attempt: int + for attempt in range(attempts): + try: + if create_table_sql is not None: + _ = execute(connection=connection, sql=create_table_sql) + if not execute(connection=connection, sql=existing_sql).fetchall(): + _ = execute(connection=connection, sql=insert_sql) + return + except Exception as error: + if attempt + 1 == attempts: + raise + logging.getLogger("sqlbuild.migrations").warning( + "migration event write attempt %s/%s failed for '%s'; retrying: %s", + attempt + 1, + attempts, + subject, + error, + ) + time.sleep(MIGRATION_WRITE_RETRY_BASE_SECONDS * (attempt + 1)) diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py index c08a9dcc3e..8a1713d3bd 100644 --- a/src/sqlbuild/compiler/migrations/constants.py +++ b/src/sqlbuild/compiler/migrations/constants.py @@ -30,3 +30,25 @@ **{column: StateSqlValueType.STRING for column in MIGRATION_COLUMNS}, "created_at": StateSqlValueType.TEXT_TIMESTAMP, } + +COLUMN_MIGRATION_TABLE_NAME: str = "_sqlbuild_column_migrations" + +COLUMN_MIGRATION_COLUMNS: tuple[str, ...] = ( + "event_id", + "target_name", + "model_name", + "relation_database", + "relation_schema", + "relation_name", + "origin_column", + "destination_column", + "discovery", + "decision", + "run_id", + "created_at", +) + +COLUMN_MIGRATION_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in COLUMN_MIGRATION_COLUMNS}, + "created_at": StateSqlValueType.TEXT_TIMESTAMP, +} diff --git a/src/sqlbuild/compiler/migrations/main/_newest_column_event.py b/src/sqlbuild/compiler/migrations/main/_newest_column_event.py new file mode 100644 index 0000000000..6158873f46 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/_newest_column_event.py @@ -0,0 +1,30 @@ +"""Project the newest column migration event that mentions a column pair.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation + + +def newest_column_migration_event_mentioning( + *, + events: Iterable[ColumnMigrationEvent], + relation: MigrationRelation, + origin_column: str, + destination_column: str, +) -> ColumnMigrationEvent | None: + """Return the latest event on the relation that renamed either column.""" + + columns: frozenset[str] = frozenset({origin_column.lower(), destination_column.lower()}) + newest: ColumnMigrationEvent | None = None + event: ColumnMigrationEvent + for event in events: + if not event.mentions(relation=relation, columns=columns): + continue + if newest is None or (event.created_at, event.event_id) > ( + newest.created_at, + newest.event_id, + ): + newest = event + return newest diff --git a/src/sqlbuild/compiler/migrations/main/_read_column_events.py b/src/sqlbuild/compiler/migrations/main/_read_column_events.py new file mode 100644 index 0000000000..a8b0b045f8 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/_read_column_events.py @@ -0,0 +1,43 @@ +"""Read append-only column migration events from one schema.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.compiler.migrations._helpers.column_sql import ( + build_column_read_sql, + decode_column_event_row, +) +from sqlbuild.compiler.migrations.exceptions import MigrationStateError +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent + + +def read_column_migration_events( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], +) -> tuple[ColumnMigrationEvent, ...]: + """Read every column migration event this version understands from one schema.""" + + sql: str = build_column_read_sql( + database=database, schema=schema, render_qualified_name=render_qualified_name + ) + try: + rows: list[tuple[Any, ...]] = execute(connection=connection, sql=sql).fetchall() + except Exception as error: + raise MigrationStateError( + f"Unable to read column migration events from schema '{schema}': {error}" + ) from error + events: list[ColumnMigrationEvent] = [] + row: tuple[Any, ...] + for row in rows: + try: + events.append(decode_column_event_row(tuple(row))) + except ValueError: + continue + return tuple(events) diff --git a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py new file mode 100644 index 0000000000..e790b3320a --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py @@ -0,0 +1,27 @@ +"""Column migration state table SQL entrypoint.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.compiler.migrations._helpers.column_sql import build_column_create_table_sql + + +def build_column_migration_state_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + transient: bool = False, +) -> str: + """Build DDL that creates the column migration event table when it is missing.""" + + return build_column_create_table_sql( + database=database, + schema=schema, + render_qualified_name=render_qualified_name, + render_framework_type=render_framework_type, + transient=transient, + ) diff --git a/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py b/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py new file mode 100644 index 0000000000..9f7882154b --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/deterministic_column_event_id.py @@ -0,0 +1,30 @@ +"""Build stable identifiers for immutable column migration events.""" + +from __future__ import annotations + +import hashlib +import json + +from sqlbuild.compiler.migrations.models import MigrationRelation + + +def deterministic_column_migration_event_id( + *, + run_id: str, + target_name: str | None, + relation: MigrationRelation, + origin_column: str, + destination_column: str, +) -> str: + """Return the content-addressed ID for one column rename performed by one run.""" + + identity: tuple[object, ...] = ( + "column", + run_id, + target_name, + relation.identity, + origin_column.lower(), + destination_column.lower(), + ) + encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/compiler/migrations/main/write_column_event.py b/src/sqlbuild/compiler/migrations/main/write_column_event.py new file mode 100644 index 0000000000..2847fd7cf7 --- /dev/null +++ b/src/sqlbuild/compiler/migrations/main/write_column_event.py @@ -0,0 +1,41 @@ +"""Append one column migration event idempotently.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.types import AdapterExecute +from sqlbuild.compiler.migrations._helpers.column_sql import ( + build_column_existing_event_sql, + build_column_insert_sql, +) +from sqlbuild.compiler.migrations._helpers.writes import append_event_row +from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent + + +def write_column_migration_event( + *, + connection: Any, + execute: AdapterExecute[Any, Any], + event: ColumnMigrationEvent, + render_qualified_name: Callable[..., str | None], + create_table_sql: str | None, + attempts: int = MIGRATION_WRITE_ATTEMPTS, +) -> None: + """Create the state table from adapter DDL when given, then insert the event if absent.""" + + _ = append_event_row( + connection=connection, + execute=execute, + existing_sql=build_column_existing_event_sql( + event=event, render_qualified_name=render_qualified_name + ), + insert_sql=build_column_insert_sql( + event=event, render_qualified_name=render_qualified_name + ), + create_table_sql=create_table_sql, + attempts=attempts, + subject=f"{event.model_name}.{event.destination_column}", + ) diff --git a/src/sqlbuild/compiler/migrations/main/write_event.py b/src/sqlbuild/compiler/migrations/main/write_event.py index fd0c04edb3..a34abb2ce8 100644 --- a/src/sqlbuild/compiler/migrations/main/write_event.py +++ b/src/sqlbuild/compiler/migrations/main/write_event.py @@ -2,17 +2,13 @@ from __future__ import annotations -import logging -import time from collections.abc import Callable from typing import Any from sqlbuild.adapter.contract.types import AdapterExecute from sqlbuild.compiler.migrations._helpers.sql import build_existing_event_sql, build_insert_sql -from sqlbuild.compiler.migrations.constants import ( - MIGRATION_WRITE_ATTEMPTS, - MIGRATION_WRITE_RETRY_BASE_SECONDS, -) +from sqlbuild.compiler.migrations._helpers.writes import append_event_row +from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.compiler.migrations.models import MigrationEvent @@ -27,26 +23,14 @@ def write_migration_event( ) -> None: """Create the state table from adapter DDL when given, then insert the event if absent.""" - existing_sql: str = build_existing_event_sql( - event=event, render_qualified_name=render_qualified_name + _ = append_event_row( + connection=connection, + execute=execute, + existing_sql=build_existing_event_sql( + event=event, render_qualified_name=render_qualified_name + ), + insert_sql=build_insert_sql(event=event, render_qualified_name=render_qualified_name), + create_table_sql=create_table_sql, + attempts=attempts, + subject=event.destination_model, ) - insert_sql: str = build_insert_sql(event=event, render_qualified_name=render_qualified_name) - attempt: int - for attempt in range(attempts): - try: - if create_table_sql is not None: - _ = execute(connection=connection, sql=create_table_sql) - if not execute(connection=connection, sql=existing_sql).fetchall(): - _ = execute(connection=connection, sql=insert_sql) - return - except Exception as error: - if attempt + 1 == attempts: - raise - logging.getLogger("sqlbuild.migrations").warning( - "model migration event write attempt %s/%s failed for '%s'; retrying: %s", - attempt + 1, - attempts, - event.destination_model, - error, - ) - time.sleep(MIGRATION_WRITE_RETRY_BASE_SECONDS * (attempt + 1)) diff --git a/src/sqlbuild/compiler/migrations/models.py b/src/sqlbuild/compiler/migrations/models.py index cc83d5a4e3..124fe17454 100644 --- a/src/sqlbuild/compiler/migrations/models.py +++ b/src/sqlbuild/compiler/migrations/models.py @@ -5,7 +5,11 @@ from dataclasses import dataclass from datetime import datetime -from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from sqlbuild.compiler.migrations.types import ( + ColumnMigrationDecision, + MigrationDecision, + MigrationDiscovery, +) @dataclass(frozen=True) @@ -56,3 +60,34 @@ def mentions(self, relation: MigrationRelation) -> bool: """Return whether this event names the relation as origin or destination.""" return self.origin.matches(relation) or self.destination.matches(relation) + + +@dataclass(frozen=True) +class ColumnMigrationEvent: + """One immutable fact that a column of a model relation was renamed in place.""" + + event_id: str + target_name: str | None + model_name: str + relation: MigrationRelation + origin_column: str + destination_column: str + discovery: MigrationDiscovery + decision: ColumnMigrationDecision + run_id: str + created_at: datetime + + def mentions(self, *, relation: MigrationRelation, columns: frozenset[str]) -> bool: + """Return whether this event renamed one of the columns of the relation.""" + + return self.relation.matches(relation) and bool( + {self.origin_column.lower(), self.destination_column.lower()} & columns + ) + + def renamed(self, *, origin_column: str, destination_column: str) -> bool: + """Return whether this event renamed exactly this origin column to this destination.""" + + return ( + self.origin_column.lower() == origin_column.lower() + and self.destination_column.lower() == destination_column.lower() + ) diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py index 6fc0791987..7e8fa7fbbb 100644 --- a/src/sqlbuild/compiler/migrations/types.py +++ b/src/sqlbuild/compiler/migrations/types.py @@ -82,3 +82,61 @@ class MigrationCompatibility(StrEnum): COMPATIBLE = "compatible" INCOMPATIBLE = "incompatible" NOT_CHECKED = "not_checked" + + +class ColumnMigrationDecision(StrEnum): + """Per-run decision for one declared or detected column rename.""" + + RENAME = "rename" + RECORD = "record" + DONE = "done" + CONFLICT = "conflict" + SOURCE_MISSING = "source_missing" + STILL_PRODUCED = "still_produced" + UNSUPPORTED = "unsupported" + + @property + def renames(self) -> bool: + """Return whether this decision renames the warehouse column.""" + + return self == ColumnMigrationDecision.RENAME + + @property + def records_event(self) -> bool: + """Return whether this decision appends a column migration event.""" + + return self in _RECORDING_COLUMN_DECISIONS + + @property + def blocks_build(self) -> bool: + """Return whether this decision stops a build before any execution.""" + + return self in _BLOCKING_COLUMN_DECISIONS + + @property + def label(self) -> str: + """Return the human-readable plan label for this decision.""" + + return _COLUMN_DECISION_LABELS[self] + + +_RECORDING_COLUMN_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( + {ColumnMigrationDecision.RENAME, ColumnMigrationDecision.RECORD} +) +_BLOCKING_COLUMN_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( + { + ColumnMigrationDecision.CONFLICT, + ColumnMigrationDecision.SOURCE_MISSING, + ColumnMigrationDecision.STILL_PRODUCED, + ColumnMigrationDecision.UNSUPPORTED, + } +) +_COLUMN_DECISION_LABELS: dict[ColumnMigrationDecision, str] = { + ColumnMigrationDecision.RENAME: "rename in place", + ColumnMigrationDecision.RECORD: "already renamed, record", + ColumnMigrationDecision.DONE: "done", + ColumnMigrationDecision.CONFLICT: "conflict", + ColumnMigrationDecision.SOURCE_MISSING: "source missing", + ColumnMigrationDecision.STILL_PRODUCED: "source still produced", + ColumnMigrationDecision.UNSUPPORTED: "unsupported", +} diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py b/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py new file mode 100644 index 0000000000..c1c92efa11 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py @@ -0,0 +1,89 @@ +"""Recognize renamed output columns by comparing formatting-insensitive expressions.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner.models import ColumnRenameHint, QueryProjection, QueryShape + + +def identical_renames( + *, previous: QueryShape, current: QueryShape, excluded: frozenset[str] +) -> tuple[tuple[str, str], ...]: + """Return (old, new) pairs where exactly one removed and one added expression are equal.""" + + removed: tuple[QueryProjection, ...] + added: tuple[QueryProjection, ...] + removed, added = _removed_and_added(previous=previous, current=current, excluded=excluded) + pairs: list[tuple[str, str]] = [] + column: QueryProjection + for column in added: + matches: tuple[QueryProjection, ...] = tuple( + origin for origin in removed if origin.expression == column.expression + ) + if len(matches) != 1: + continue + claims: int = sum(1 for other in added if other.expression == matches[0].expression) + if claims == 1: + pairs.append((matches[0].name, column.name)) + return tuple(pairs) + + +def rename_hints( + *, + model_name: str, + previous: QueryShape, + current: QueryShape, + excluded: frozenset[str], + live_columns: frozenset[str], +) -> tuple[ColumnRenameHint, ...]: + """Point added columns that resemble removed live columns at an explicit migrate_from.""" + + removed: tuple[QueryProjection, ...] + added: tuple[QueryProjection, ...] + removed, added = _removed_and_added(previous=previous, current=current, excluded=excluded) + live_removed: tuple[QueryProjection, ...] = tuple( + origin for origin in removed if origin.key in live_columns + ) + hints: list[ColumnRenameHint] = [] + column: QueryProjection + for column in added: + if column.key in live_columns: + continue + identical: tuple[str, ...] = tuple( + origin.name for origin in live_removed if origin.expression == column.expression + ) + similar: tuple[str, ...] = tuple( + origin.name + for origin in live_removed + if origin.references and origin.references == column.references + ) + if identical or similar: + hints.append( + ColumnRenameHint( + model_name=model_name, + added_column=column.name, + candidate_columns=identical or similar, + identical=bool(identical), + ) + ) + return tuple(hints) + + +def _removed_and_added( + *, previous: QueryShape, current: QueryShape, excluded: frozenset[str] +) -> tuple[tuple[QueryProjection, ...], tuple[QueryProjection, ...]]: + """Return previous-only and current-only output columns by case-insensitive name.""" + + previous_names: frozenset[str] = frozenset(item.key for item in previous.projections) + current_names: frozenset[str] = frozenset(item.key for item in current.projections) + return ( + tuple( + item + for item in previous.projections + if item.key not in current_names and item.key not in excluded + ), + tuple( + item + for item in current.projections + if item.key not in previous_names and item.key not in excluded + ), + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py new file mode 100644 index 0000000000..d2ec77f466 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -0,0 +1,525 @@ +"""Per-run decisions for in-place column renames of incremental and snapshot models.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, replace + +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.migrations.main._newest_column_event import ( + newest_column_migration_event_mentioning, +) +from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent +from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery +from sqlbuild.compiler.planner._helpers.migrations.column_renames import ( + identical_renames, + rename_hints, +) +from sqlbuild.compiler.planner._helpers.migrations.planning import ( + planning_database, + redirected_cursor_snapshots, +) +from sqlbuild.compiler.planner._helpers.migrations.projections import ( + parse_query_shape, + renames_explain_change, +) +from sqlbuild.compiler.planner.classes.migration_state_inspection import ( + MigrationStateInspection, +) +from sqlbuild.compiler.planner.models import ( + ColumnMigrationPlanEntry, + ColumnMigrationPlanning, + ColumnRenameHint, + DeferralInputs, + PlannerOverrides, + PlannerRuntime, + PlannerScope, + PlanWarning, + QueryShape, + WarehouseFingerprints, + WarehouseSnapshot, +) +from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity +from sqlbuild.spec.contracts.main.get_config_str import get_config_str + +_HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( + {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} +) + + +@dataclass(frozen=True) +class _ModelColumns: + """Inputs gathered for one model that may rename columns this run.""" + + model: CompiledModel + warehouse: dict[str, ColumnInfo] + declared: dict[str, str] + fingerprint: Fingerprint | None + previous: QueryShape | None + current: QueryShape | None + + +@dataclass(frozen=True) +class _Pair: + """One old-to-new column rename and how it was requested.""" + + origin: str + destination: str + discovery: MigrationDiscovery + + +def plan_column_migrations( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + full_refresh_model_names: frozenset[str], + physical_relations: Mapping[str, str], + overrides: PlannerOverrides, + deferral: DeferralInputs, +) -> ColumnMigrationPlanning: + """Decide declared and detected column renames and project the renamed snapshot.""" + + dialect: str | None = runtime.adapter.sql_analysis_dialect() + candidates: tuple[_ModelColumns, ...] = tuple( + candidate + for key in scope.execution_order + if ( + candidate := _candidate( + key=key, + scope=scope, + snapshot=snapshot, + full_refresh_model_names=full_refresh_model_names, + dialect=dialect, + ) + ) + is not None + ) + if not candidates: + return ColumnMigrationPlanning(snapshot=snapshot) + state: MigrationStateInspection = MigrationStateInspection( + adapter=runtime.adapter, + connection=runtime.connection, + database=planning_database(runtime=runtime), + ) + state.inspect_column_events( + schemas={ + candidate.model.destination.schema + for candidate in candidates + if candidate.model.destination.schema is not None + } + ) + entries: list[ColumnMigrationPlanEntry] = [] + hints: list[ColumnRenameHint] = [] + warnings: list[PlanWarning] = [] + candidate: _ModelColumns + for candidate in candidates: + model_entries: tuple[ColumnMigrationPlanEntry, ...] + model_warnings: tuple[PlanWarning, ...] + model_entries, model_warnings = _checked_entries( + runtime=runtime, + candidate=candidate, + entries=_decide(runtime=runtime, candidate=candidate, events=state.column_events), + physical_relation=physical_relations.get(candidate.model.name), + ) + entries.extend(model_entries) + warnings.extend(model_warnings) + hints.extend(_hints(candidate=candidate, entries=model_entries)) + warnings.extend(warning for entry in entries if (warning := _entry_warning(entry))) + overlaid: WarehouseSnapshot = _overlay_snapshot( + snapshot=snapshot, candidates=candidates, entries=tuple(entries) + ) + return ColumnMigrationPlanning( + snapshot=_with_renamed_cursors( + runtime=runtime, + scope=scope, + snapshot=overlaid, + entries=tuple(entries), + physical_relations=physical_relations, + overrides=overrides, + deferral=deferral, + ), + entries=tuple(entries), + hints=tuple(hints), + warnings=tuple(warnings), + ) + + +def _candidate( + *, + key: CompiledObjectKey, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + full_refresh_model_names: frozenset[str], + dialect: str | None, +) -> _ModelColumns | None: + if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: + return None + model: CompiledModel | None = scope.models_by_name.get(key.name) + if ( + model is None + or get_config_str(values=model.config.values, key="materialized") + not in _HISTORY_MATERIALIZATIONS + or model.name in full_refresh_model_names + or model.name not in snapshot.existing_relations + ): + return None + warehouse: tuple[ColumnInfo, ...] = snapshot.existing_columns.get(model.name, ()) + declared: dict[str, str] = { + column.name: column.migrate_from.strip() + for column in (model.schema_entry.columns if model.schema_entry is not None else ()) + if column.migrate_from is not None + } + fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model.name) + changed: bool = fingerprint is not None and fingerprint.definition_hash != compute_query_hash( + model.query_sql + ) + if not warehouse or (not declared and not changed): + return None + return _ModelColumns( + model=model, + warehouse={column.name.lower(): column for column in warehouse}, + declared=declared, + fingerprint=fingerprint, + previous=( + parse_query_shape(query_sql=fingerprint.definition, dialect=dialect) + if changed and fingerprint is not None + else None + ), + current=parse_query_shape(query_sql=model.query_sql, dialect=dialect), + ) + + +def _decide( + *, + runtime: PlannerRuntime, + candidate: _ModelColumns, + events: tuple[ColumnMigrationEvent, ...], +) -> tuple[ColumnMigrationPlanEntry, ...]: + entries: list[ColumnMigrationPlanEntry] = [] + pair: _Pair + for pair in (*_declared_pairs(candidate), *_detected_pairs(candidate)): + entry: ColumnMigrationPlanEntry | None = _decide_pair( + runtime=runtime, candidate=candidate, pair=pair, events=events + ) + if entry is not None: + entries.append(entry) + return tuple(entries) + + +def _declared_pairs(candidate: _ModelColumns) -> tuple[_Pair, ...]: + return tuple( + _Pair(origin=origin, destination=destination, discovery=MigrationDiscovery.MANUAL) + for destination, origin in candidate.declared.items() + ) + + +def _detected_pairs(candidate: _ModelColumns) -> tuple[_Pair, ...]: + if candidate.previous is None or candidate.current is None: + return () + return tuple( + _Pair(origin=origin, destination=destination, discovery=MigrationDiscovery.AUTOMATIC) + for origin, destination in identical_renames( + previous=candidate.previous, + current=candidate.current, + excluded=_declared_names(candidate), + ) + ) + + +def _declared_names(candidate: _ModelColumns) -> frozenset[str]: + return frozenset( + {name.lower() for name in candidate.declared} + | {origin.lower() for origin in candidate.declared.values()} + ) + + +def _decide_pair( + *, + runtime: PlannerRuntime, + candidate: _ModelColumns, + pair: _Pair, + events: tuple[ColumnMigrationEvent, ...], +) -> ColumnMigrationPlanEntry | None: + """Decide one rename from live columns first, then from recorded events.""" + + origin: ColumnInfo | None = candidate.warehouse.get(pair.origin.lower()) + has_destination: bool = pair.destination.lower() in candidate.warehouse + newest: ColumnMigrationEvent | None = newest_column_migration_event_mentioning( + events=events, + relation=migration_relation_for_location(candidate.model.destination), + origin_column=pair.origin, + destination_column=pair.destination, + ) + recorded: bool = newest is not None and newest.renamed( + origin_column=pair.origin, destination_column=pair.destination + ) + decision: ColumnMigrationDecision + if recorded and (has_destination or origin is None): + decision = ColumnMigrationDecision.DONE + elif pair.origin.lower() in _output_names(candidate): + decision = ColumnMigrationDecision.STILL_PRODUCED + elif origin is not None and not has_destination: + decision = ColumnMigrationDecision.RENAME + elif origin is None and has_destination: + decision = ColumnMigrationDecision.RECORD + elif origin is not None: + decision = ColumnMigrationDecision.CONFLICT + else: + decision = ColumnMigrationDecision.SOURCE_MISSING + if pair.discovery == MigrationDiscovery.AUTOMATIC and decision.blocks_build: + return None + return ColumnMigrationPlanEntry( + model_name=candidate.model.name, + destination=candidate.model.destination, + origin_column=origin.name if origin is not None else pair.origin, + destination_column=pair.destination, + discovery=pair.discovery, + decision=decision, + target_name=( + newest.target_name + if decision == ColumnMigrationDecision.DONE and newest is not None + else runtime.project.effective_target_name + ), + completed_at=( + newest.created_at + if decision == ColumnMigrationDecision.DONE and newest is not None + else None + ), + ) + + +def _output_names(candidate: _ModelColumns) -> frozenset[str]: + if candidate.current is not None: + return frozenset(projection.key for projection in candidate.current.projections) + return frozenset(column.name.lower() for column in candidate.model.inferred_columns or ()) + + +def _checked_entries( + *, + runtime: PlannerRuntime, + candidate: _ModelColumns, + entries: tuple[ColumnMigrationPlanEntry, ...], + physical_relation: str | None, +) -> tuple[tuple[ColumnMigrationPlanEntry, ...], tuple[PlanWarning, ...]]: + """Ask the adapter once per model whether pending renames can run in place.""" + + if not any(entry.decision.renames for entry in entries): + return entries, () + relation: str = ( + physical_relation + or candidate.model.destination.qualified_name + or candidate.model.destination.name + ) + reason: str | None = runtime.adapter.column_rename_unavailable_reason( + connection=runtime.connection, destination=relation + ) + if reason is None: + return entries, () + checked: list[ColumnMigrationPlanEntry] = [] + warnings: list[PlanWarning] = [] + entry: ColumnMigrationPlanEntry + for entry in entries: + if not entry.decision.renames: + checked.append(entry) + elif entry.discovery == MigrationDiscovery.MANUAL: + checked.append( + replace(entry, decision=ColumnMigrationDecision.UNSUPPORTED, message=reason) + ) + else: + warnings.append( + PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.WARNING, + message=( + f"column {entry.destination_column} looks renamed from " + f"{entry.origin_column}, but it cannot be renamed in place: {reason}; " + "the change follows on_schema_change instead" + ), + code="M112", + ) + ) + return tuple(checked), tuple(warnings) + + +def _hints( + *, candidate: _ModelColumns, entries: tuple[ColumnMigrationPlanEntry, ...] +) -> tuple[ColumnRenameHint, ...]: + if candidate.previous is None or candidate.current is None: + return () + claimed: frozenset[str] = frozenset( + {entry.origin_column.lower() for entry in entries} + | {entry.destination_column.lower() for entry in entries} + ) + return rename_hints( + model_name=candidate.model.name, + previous=candidate.previous, + current=candidate.current, + excluded=_declared_names(candidate) | claimed, + live_columns=frozenset(candidate.warehouse), + ) + + +def _overlay_snapshot( + *, + snapshot: WarehouseSnapshot, + candidates: tuple[_ModelColumns, ...], + entries: tuple[ColumnMigrationPlanEntry, ...], +) -> WarehouseSnapshot: + """Plan each model as if its columns were already renamed and its query unchanged.""" + + if not entries: + return snapshot + blocked: frozenset[str] = frozenset(entry.model_name for entry in entries if entry.blocks_build) + columns: dict[str, tuple[ColumnInfo, ...]] = dict(snapshot.existing_columns) + fingerprints: dict[str, Fingerprint] = dict(snapshot.fingerprints.models) + candidate: _ModelColumns + for candidate in candidates: + renames: dict[str, str] = { + entry.origin_column: entry.destination_column + for entry in entries + if entry.model_name == candidate.model.name + } + if not renames or candidate.model.name in blocked: + continue + pending: dict[str, str] = { + entry.origin_column.lower(): entry.destination_column + for entry in entries + if entry.model_name == candidate.model.name and entry.decision.renames + } + columns[candidate.model.name] = tuple( + replace(column, name=pending.get(column.name.lower(), column.name)) + for column in snapshot.existing_columns.get(candidate.model.name, ()) + ) + if ( + candidate.fingerprint is not None + and candidate.previous is not None + and candidate.current is not None + and renames_explain_change( + previous=candidate.previous, current=candidate.current, renames=renames + ) + ): + fingerprints[candidate.model.name] = replace( + candidate.fingerprint, + definition=candidate.model.query_sql, + definition_hash=compute_query_hash(candidate.model.query_sql), + ) + return replace( + snapshot, + existing_columns=columns, + fingerprints=WarehouseFingerprints( + models=fingerprints, + functions=snapshot.fingerprints.functions, + seeds=snapshot.fingerprints.seeds, + python_nodes=snapshot.fingerprints.python_nodes, + ), + ) + + +def _with_renamed_cursors( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + entries: tuple[ColumnMigrationPlanEntry, ...], + physical_relations: Mapping[str, str], + overrides: PlannerOverrides, + deferral: DeferralInputs, +) -> WarehouseSnapshot: + """Read the target watermark from the old column when the cursor column is renamed.""" + + old_cursor_columns: dict[str, str] = {} + entry: ColumnMigrationPlanEntry + for entry in entries: + model: CompiledModel | None = scope.models_by_name.get(entry.model_name) + cursor: str | None = ( + get_config_str(values=model.config.values, key="cursor") if model else None + ) + if ( + entry.decision.renames + and cursor is not None + and cursor.lower() == entry.destination_column.lower() + ): + old_cursor_columns[entry.model_name] = entry.origin_column + if not old_cursor_columns: + return snapshot + return replace( + snapshot, + cursor_snapshots={ + **snapshot.cursor_snapshots, + **redirected_cursor_snapshots( + runtime=runtime, + scope=scope, + existing_relations=snapshot.existing_relations, + target_relations={ + name: physical_relations.get(name) + or scope.models_by_name[name].destination.qualified_name + or name + for name in old_cursor_columns + }, + overrides=overrides, + deferral=deferral, + origin_cursor_columns=old_cursor_columns, + ), + }, + ) + + +def _entry_warning(entry: ColumnMigrationPlanEntry) -> PlanWarning | None: + relation: str = entry.destination.qualified_name or entry.destination.name + declaration: str = f"column '{entry.destination_column}' migrate_from {entry.origin_column}" + if entry.decision == ColumnMigrationDecision.DONE: + if entry.discovery != MigrationDiscovery.MANUAL: + return None + completed: str = entry.completed_at.isoformat() if entry.completed_at else "unknown time" + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.WARNING, + message=( + f"column rename {entry.origin_column} -> {entry.destination_column} on " + f"{relation} completed at {completed} on target " + f"'{entry.target_name or 'default'}'; " + f"migrate_from can be removed from column '{entry.destination_column}' of " + f"'{entry.model_name}'" + ), + code="M108", + ) + messages: dict[ColumnMigrationDecision, tuple[str, str]] = { + ColumnMigrationDecision.SOURCE_MISSING: ( + "M109", + f"model '{entry.model_name}': {declaration}: column {entry.origin_column} does not " + f"exist in {relation} and no recorded rename into {entry.destination_column} was " + "found; if the rename already happened elsewhere or is no longer needed, remove " + "migrate_from from the column", + ), + ColumnMigrationDecision.CONFLICT: ( + "M110", + f"model '{entry.model_name}': {declaration}: {relation} already has both " + f"{entry.origin_column} and {entry.destination_column}, so the column cannot be " + "renamed in place; drop one of them, or remove migrate_from and let " + "on_schema_change handle the columns", + ), + ColumnMigrationDecision.STILL_PRODUCED: ( + "M111", + f"model '{entry.model_name}': {declaration}: the model still produces " + f"{entry.origin_column}; remove it from the query, or remove migrate_from", + ), + ColumnMigrationDecision.UNSUPPORTED: ( + "M112", + f"model '{entry.model_name}': {declaration}: {entry.message}", + ), + } + coded: tuple[str, str] | None = messages.get(entry.decision) + if coded is None: + return None + return PlanWarning( + model_name=entry.model_name, + severity=WarningSeverity.ERROR, + message=coded[1], + code=coded[0], + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py index 7183e87708..fa5e1d07fb 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py @@ -5,11 +5,16 @@ from collections.abc import Callable, Sequence from sqlbuild.compiler.migrations.types import ( + ColumnMigrationDecision, MigrationCompatibility, MigrationDecision, MigrationDiscovery, ) -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput +from sqlbuild.compiler.planner.models import ( + ColumnMigrationPlanEntry, + ModelMigrationPlanEntry, + PlanOutput, +) from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.main.append_overflow_line import append_overflow_line from sqlbuild.presentation.main.count_header import count_header_style @@ -51,9 +56,54 @@ def format_model_migrations( indent=" ", options=display_options, ) + return _format_column_migrations( + lines=result, plan=plan, style=style, header_style=header_style + ) + + +def _format_column_migrations( + *, + lines: list[str], + plan: PlanOutput, + style: CliStyle, + header_style: Callable[[str], str], +) -> list[str]: + """Format in-place column renames grouped under the model whose table they change.""" + + entries: tuple[ColumnMigrationPlanEntry, ...] = plan.column_migration_entries + if not entries: + return lines + result: list[str] = list(lines) + result.append(header_style(f"Column migrations ({len(entries)})")) + model_names: tuple[str, ...] = tuple(dict.fromkeys(entry.model_name for entry in entries)) + model_name: str + for model_name in model_names: + result.append(f" {style.object_name(model_name)} {style.muted('migrate columns')}") + entry: ColumnMigrationPlanEntry + for entry in entries: + if entry.model_name == model_name: + result.append(f" {_column_migration_text(entry=entry, style=style)}") return result +def _column_migration_text(*, entry: ColumnMigrationPlanEntry, style: CliStyle) -> str: + rename: str = f"{entry.origin_column} -> {entry.destination_column}" + decision: str + if entry.decision.blocks_build: + decision = style.error_strong(entry.decision.label) + elif entry.decision == ColumnMigrationDecision.DONE: + decision = style.muted(entry.decision.label) + else: + decision = style.accent_strong(entry.decision.label) + details: list[str] = [] + if entry.discovery != MigrationDiscovery.MANUAL: + details.append(entry.discovery.value) + if entry.decision == ColumnMigrationDecision.DONE and entry.completed_at is not None: + details.append(f"completed {entry.completed_at.isoformat()}") + suffix: str = f" {style.muted(f'({", ".join(details)})')}" if details else "" + return f"{rename} {decision}{suffix}" + + def _migration_lines(*, entry: ModelMigrationPlanEntry, style: CliStyle) -> list[str]: origin: str = entry.origin.qualified_name or entry.origin.name destination: str = entry.destination.qualified_name or entry.destination.name diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py index 423a009235..f47906b769 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/fingerprint.py @@ -110,7 +110,7 @@ def _normalized_query_sql( ).to_dict() except polyglot.PolyglotError: return None - stripped: Any = _strip_formatting(parsed) + stripped: Any = strip_ast_formatting(parsed) if not _has_unambiguous_local_scopes(stripped): return None definitions: list[tuple[str, bool]] = _local_names(stripped) @@ -130,15 +130,17 @@ def _ref_placeholder(*, identity: str) -> str: return f"{MIGRATION_REF_PLACEHOLDER_PREFIX}{digest}" -def _strip_formatting(node: Any) -> Any: +def strip_ast_formatting(node: Any) -> Any: + """Drop source spans and comments from a parsed SQL tree.""" + if isinstance(node, dict): return { - key: [] if key.endswith(_COMMENTS_SUFFIX) else _strip_formatting(value) + key: [] if key.endswith(_COMMENTS_SUFFIX) else strip_ast_formatting(value) for key, value in node.items() if key != _SPAN_KEY } if isinstance(node, list): - return [_strip_formatting(value) for value in node] + return [strip_ast_formatting(value) for value in node] return node diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 8020c59751..5f23334a2b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -648,28 +648,13 @@ def _overlay_snapshot( redirected[entry.model_name] = entry.origin.qualified_name cursor_snapshots: dict[str, ModelCursorSnapshot] = { **snapshot.cursor_snapshots, - **gather_redirected_cursor_snapshots( - project=runtime.project, - adapter=runtime.adapter, - connection=runtime.connection, + **redirected_cursor_snapshots( + runtime=runtime, + scope=scope, existing_relations=relations, target_relations=redirected, - cursor_scope=CursorSnapshotScope( - model_keys=scope.selected_keys, - runtime_producer_keys=scope.selected_keys, - invocation_time=runtime.invocation_time, - start_cursor_config=( - runtime.project_config.cursors.start - if runtime.project_config is not None - else None - ), - cursor_overrides=overrides.cursor_overrides, - ), - full_refresh_model_names=effectively_full_refreshed_model_names( - project=runtime.project, cli_full_refresh=overrides.full_refresh - ), - deferred_locations=deferral.deferred_locations, - on_progress=runtime.on_progress, + overrides=overrides, + deferral=deferral, ), } return replace( @@ -686,6 +671,42 @@ def _overlay_snapshot( ) +def redirected_cursor_snapshots( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + existing_relations: dict[str, RelationInfo], + target_relations: dict[str, str], + overrides: PlannerOverrides, + deferral: DeferralInputs, + origin_cursor_columns: dict[str, str] | None = None, +) -> dict[str, ModelCursorSnapshot]: + """Re-read target cursors from where history lives before this run's migrations.""" + + return gather_redirected_cursor_snapshots( + project=runtime.project, + adapter=runtime.adapter, + connection=runtime.connection, + existing_relations=existing_relations, + target_relations=target_relations, + cursor_scope=CursorSnapshotScope( + model_keys=scope.selected_keys, + runtime_producer_keys=scope.selected_keys, + invocation_time=runtime.invocation_time, + start_cursor_config=( + runtime.project_config.cursors.start if runtime.project_config is not None else None + ), + cursor_overrides=overrides.cursor_overrides, + ), + full_refresh_model_names=effectively_full_refreshed_model_names( + project=runtime.project, cli_full_refresh=overrides.full_refresh + ), + deferred_locations=deferral.deferred_locations, + on_progress=runtime.on_progress, + origin_cursor_columns=origin_cursor_columns, + ) + + def _has_cursor(*, runtime: PlannerRuntime, model_name: str) -> bool: model: CompiledModel | None = next( (model for model in runtime.project.models if model.name == model_name), None diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py b/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py new file mode 100644 index 0000000000..65aec9f2b7 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py @@ -0,0 +1,130 @@ +"""Formatting-insensitive top-level projections used to recognize column renames.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from typing import Any + +from sqlbuild.compiler.planner._helpers.migrations.fingerprint import strip_ast_formatting +from sqlbuild.compiler.planner.models import QueryProjection, QueryShape +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql + +_GENERIC_DIALECT: str = "generic" +_SELECT_KEY: str = "select" +_EXPRESSIONS_KEY: str = "expressions" +_ALIAS_KEY: str = "alias" +_THIS_KEY: str = "this" +_COLUMN_KEY: str = "column" +_NAME_KEY: str = "name" +_QUOTED_KEY: str = "quoted" + + +def parse_query_shape(*, query_sql: str, dialect: str | None) -> QueryShape | None: + """Return the named top-level projections of one plain SELECT, or None when unknowable.""" + + polyglot: Any = import_polyglot_sql() + try: + parsed: Any = polyglot.parse_one(query_sql, dialect=dialect or _GENERIC_DIALECT).to_dict() + except polyglot.PolyglotError: + return None + root: Any = strip_ast_formatting(parsed) + if not isinstance(root, dict) or set(root) != {_SELECT_KEY}: + return None + select: Any = root[_SELECT_KEY] + if not isinstance(select, dict) or not isinstance(select.get(_EXPRESSIONS_KEY), list): + return None + projections: list[QueryProjection] = [] + expression: Any + for expression in select[_EXPRESSIONS_KEY]: + projection: QueryProjection | None = _projection(expression) + if projection is None: + return None + projections.append(projection) + if len({projection.key for projection in projections}) != len(projections): + return None + body: dict[str, Any] = {key: value for key, value in select.items() if key != _EXPRESSIONS_KEY} + return QueryShape(projections=tuple(projections), body=_encode(_canonical(body))) + + +def renames_explain_change( + *, previous: QueryShape, current: QueryShape, renames: Mapping[str, str] +) -> bool: + """Return whether renaming previous output columns alone turns previous into current.""" + + if previous.body != current.body or len(previous.projections) != len(current.projections): + return False + lowered: dict[str, str] = {old.lower(): new.lower() for old, new in renames.items()} + return all( + lowered.get(before.key, before.key) == after.key and before.expression == after.expression + for before, after in zip(previous.projections, current.projections, strict=True) + ) + + +def _projection(expression: Any) -> QueryProjection | None: + if not isinstance(expression, dict) or len(expression) != 1: + return None + if _ALIAS_KEY in expression: + alias: Any = expression[_ALIAS_KEY] + name: str | None = ( + _identifier_name(alias.get(_ALIAS_KEY)) if isinstance(alias, dict) else None + ) + body: Any = alias.get(_THIS_KEY) if isinstance(alias, dict) else None + if name is None or body is None: + return None + return QueryProjection( + name=name, expression=_encode(_canonical(body)), references=_references(body) + ) + if _COLUMN_KEY in expression: + column: Any = expression[_COLUMN_KEY] + name = _identifier_name(column.get(_NAME_KEY)) if isinstance(column, dict) else None + if name is None: + return None + return QueryProjection( + name=name, + expression=_encode(_canonical(expression)), + references=_references(expression), + ) + return None + + +def _identifier_name(identifier: Any) -> str | None: + if not isinstance(identifier, dict) or not isinstance(identifier.get(_NAME_KEY), str): + return None + return str(identifier[_NAME_KEY]) + + +def _references(node: Any) -> frozenset[str]: + found: set[str] = set() + pending: list[Any] = [node] + while pending: + current: Any = pending.pop() + if isinstance(current, list): + pending.extend(current) + continue + if not isinstance(current, dict): + continue + column: Any = current.get(_COLUMN_KEY) + if isinstance(column, dict): + name: str | None = _identifier_name(column.get(_NAME_KEY)) + if name is not None: + found.add(name.lower()) + pending.extend(current.values()) + return frozenset(found) + + +def _canonical(node: Any) -> Any: + """Lower-case unquoted identifiers so letter case alone never looks like a change.""" + + if isinstance(node, list): + return [_canonical(value) for value in node] + if not isinstance(node, dict): + return node + canonical: dict[str, Any] = {key: _canonical(value) for key, value in node.items()} + if isinstance(canonical.get(_NAME_KEY), str) and canonical.get(_QUOTED_KEY) is False: + canonical[_NAME_KEY] = str(canonical[_NAME_KEY]).lower() + return canonical + + +def _encode(node: Any) -> str: + return json.dumps(node, sort_keys=True, separators=(",", ":"), default=str) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py index 271fac940e..c8e7b4de8c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py @@ -12,6 +12,8 @@ build_stale_out_of_selection_warnings, ) from sqlbuild.compiler.planner.models import ( + ColumnRenameHint, + ModelPlanEntry, PlannedSqlTests, PlannerChangeReconciliation, PlannerChangeResults, @@ -83,12 +85,16 @@ def with_storage_policies( return replace( plan_output, - model_entries=with_migration_fingerprints( - entries=plan_output.model_entries, - models_by_name={model.name: model for model in runtime.project.models}, - dialect=runtime.adapter.sql_analysis_dialect(), + model_entries=with_column_rename_hints( + entries=with_migration_fingerprints( + entries=plan_output.model_entries, + models_by_name={model.name: model for model in runtime.project.models}, + dialect=runtime.adapter.sql_analysis_dialect(), + ), + hints=warehouse.column_rename_hints, ), migration_entries=warehouse.migration_entries, + column_migration_entries=warehouse.column_migration_entries, warnings=(*warehouse.migration_warnings, *plan_output.warnings), table_type_entries=plan_table_types( runtime=runtime, warehouse=warehouse, scope=scopes.selected_scope @@ -99,6 +105,23 @@ def with_storage_policies( ) +def with_column_rename_hints( + *, entries: tuple[ModelPlanEntry, ...], hints: tuple[ColumnRenameHint, ...] +) -> tuple[ModelPlanEntry, ...]: + """Attach near-match column rename hints to the model entries they describe.""" + + by_model: dict[str, list[ColumnRenameHint]] = {} + hint: ColumnRenameHint + for hint in hints: + by_model.setdefault(hint.model_name, []).append(hint) + return tuple( + replace(entry, column_rename_hints=tuple(by_model[entry.name])) + if entry.name in by_model + else entry + for entry in entries + ) + + def with_plan_warnings( *, runtime: PlannerRuntime, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index 0e809de25b..2d7924275b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -4,6 +4,7 @@ import time +from sqlbuild.compiler.planner._helpers.migrations.columns import plan_column_migrations from sqlbuild.compiler.planner._helpers.migrations.planning import plan_model_migrations from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( @@ -11,6 +12,7 @@ ) from sqlbuild.compiler.planner._helpers.warehouse.snapshot import gather_warehouse_snapshot from sqlbuild.compiler.planner.models import ( + ColumnMigrationPlanning, CursorSnapshotScope, DeferralInputs, ModelMigrationPlanning, @@ -64,6 +66,22 @@ def gather_planner_warehouse_state( overrides=overrides, deferral=deferral, ) + columns: ColumnMigrationPlanning = plan_column_migrations( + runtime=runtime, + scope=scopes.selected_scope, + snapshot=migrations.snapshot, + full_refresh_model_names=effectively_full_refreshed_model_names( + project=runtime.project, + cli_full_refresh=overrides.full_refresh, + ), + physical_relations={ + entry.model_name: entry.origin.qualified_name or entry.origin.name + for entry in migrations.entries + if entry.decision.moves_data + }, + overrides=overrides, + deferral=deferral, + ) inspection_relations: PlannerRelationsContext = build_planner_relations_context( project=runtime.project, adapter=runtime.adapter, @@ -79,8 +97,10 @@ def gather_planner_warehouse_state( ) runtime.on_progress("Generating plan...") return PlannerWarehouseState( - snapshot=migrations.snapshot, + snapshot=columns.snapshot, inspection_relations=inspection_relations, migration_entries=migrations.entries, - migration_warnings=migrations.warnings, + migration_warnings=(*migrations.warnings, *columns.warnings), + column_migration_entries=columns.entries, + column_rename_hints=columns.hints, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py index 3f6b5e3da4..f2f591c8e4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py @@ -178,6 +178,7 @@ def _fixture_relations( label="model", error_class=CompileInputError, column_locations=model_file.header_column_locations, + allow_migrate_from=True, ) if not columns: continue diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index 69da86940f..02c7559085 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -134,6 +134,13 @@ class _CursorModelInfo: has_start_override: bool = False cursor_watermark_mode: CursorWatermarkMode = CursorWatermarkMode.ALL microbatch_strategy: str | None = None + origin_cursor_column: str | None = None + + @property + def physical_cursor_column(self) -> str: + """Return the target column holding the cursor now, before any pending rename.""" + + return self.origin_cursor_column or self.cursor_column @dataclass(frozen=True) @@ -148,6 +155,7 @@ class _CursorGatherInputs: start_cursor_config: StartCursorsConfig | None cursor_overrides: CursorOverrides | None target_relation_overrides: dict[str, str] | None = None + origin_cursor_column_overrides: dict[str, str] | None = None def build_warehouse_snapshot( @@ -299,8 +307,9 @@ def gather_redirected_cursor_snapshots( full_refresh_model_names: frozenset[str], deferred_locations: dict[str, CompiledRelationLocation] | None = None, on_progress: Callable[[str], None] | None = None, + origin_cursor_columns: dict[str, str] | None = None, ) -> dict[str, ModelCursorSnapshot]: - """Gather cursor snapshots for models whose target history lives in another relation.""" + """Gather cursor snapshots for models whose history lives in another relation or column.""" if not target_relations: return {} @@ -324,6 +333,7 @@ def gather_redirected_cursor_snapshots( start_cursor_config=cursor_scope.start_cursor_config, cursor_overrides=cursor_scope.cursor_overrides, target_relation_overrides=target_relations, + origin_cursor_column_overrides=origin_cursor_columns, ), ) model_map: dict[str, CompiledModel] = {model.name: model for model in project.models} @@ -840,6 +850,7 @@ def _collect_cursor_models( model_name=model.name, target_tag=target_tag, target_relation=target_relation, + origin_cursor_column=(inputs.origin_cursor_column_overrides or {}).get(model.name), cursor_column=cursor_column, upstreams=tuple(upstreams), cursor_type=cursor_type, @@ -880,7 +891,7 @@ def _build_cursor_queries(cursor_models: list[_CursorModelInfo]) -> list[_Physic if info.target_tag is not None and info.target_relation is not None: target_key: tuple[str, str, str] = ( info.target_relation, - info.cursor_column, + info.physical_cursor_column, info.cursor_type or CursorType.TIMESTAMP, ) target_tags: tuple[list[str], list[str]] = grouped_tags.setdefault(target_key, ([], [])) @@ -1086,7 +1097,7 @@ def _gather_eligible_target_maxes( horizon_is_date: bool = isinstance(horizon, DateValue) eligible_sql: str = adapter.render_max_cursor_at_or_before( relation=info.target_relation, - cursor_column=info.cursor_column, + cursor_column=info.physical_cursor_column, maximum_allowed=render(value=horizon), cursor_type=info.cursor_type, is_date=horizon_is_date, diff --git a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py index dea9293152..bd4161c5d7 100644 --- a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py +++ b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py @@ -10,9 +10,17 @@ from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL from sqlbuild.compiler.fingerprints.main.read import read_latest_fingerprints from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME +from sqlbuild.compiler.migrations.constants import ( + COLUMN_MIGRATION_TABLE_NAME, + MIGRATION_TABLE_NAME, +) +from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events from sqlbuild.compiler.migrations.main._read_events import read_migration_events -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.models import ( + ColumnMigrationEvent, + MigrationEvent, + MigrationRelation, +) class MigrationStateInspection: @@ -27,6 +35,8 @@ def __init__(self, *, adapter: BaseAdapter, connection: Any, database: str | Non self._events: list[MigrationEvent] = [] self._relations: dict[tuple[str, str], RelationInfo] = {} self._columns: dict[tuple[str, str], tuple[ColumnInfo, ...]] = {} + self._column_event_schemas: set[str] = set() + self._column_events: list[ColumnMigrationEvent] = [] @property def database(self) -> str | None: @@ -40,6 +50,45 @@ def events(self) -> tuple[MigrationEvent, ...]: return tuple(self._events) + @property + def column_events(self) -> tuple[ColumnMigrationEvent, ...]: + """Return every column migration event read so far.""" + + return tuple(self._column_events) + + def inspect_column_events(self, *, schemas: set[str]) -> None: + """Read the column migration state table once for every schema not inspected yet.""" + + pending: tuple[str, ...] = tuple( + sorted(schema for schema in schemas if schema.lower() not in self._column_event_schemas) + ) + if not pending: + return + self._column_event_schemas.update(schema.lower() for schema in pending) + listed: tuple[RelationInfo, ...] = self._adapter.list_relations( + connection=self._connection, + database=self._database, + schemas=pending, + names=(COLUMN_MIGRATION_TABLE_NAME,), + ) + present: set[str] = { + (relation.schema or "").lower() + for relation in listed + if relation.name.lower() == COLUMN_MIGRATION_TABLE_NAME + } + schema: str + for schema in pending: + if schema.lower() in present: + self._column_events.extend( + read_column_migration_events( + connection=self._connection, + execute=self._adapter.execute, + database=self._database, + schema=schema, + render_qualified_name=self._adapter.render_qualified_name, + ) + ) + def inspect_schemas(self, *, schemas: set[str]) -> None: """Read state tables once for every schema not inspected yet.""" diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index c51eed111a..508b7eced2 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -32,6 +32,7 @@ from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.types import ( + ColumnMigrationDecision, MigrationCompatibility, MigrationDecision, MigrationDiscovery, @@ -937,6 +938,71 @@ def blocks_build(self) -> bool: ) +@dataclass(frozen=True) +class ColumnMigrationPlanEntry: + """One declared or detected in-place column rename and its per-run decision.""" + + model_name: str + destination: CompiledRelationLocation + origin_column: str + destination_column: str + discovery: MigrationDiscovery + decision: ColumnMigrationDecision + target_name: str | None + completed_at: datetime | None = None + message: str | None = None + + @property + def blocks_build(self) -> bool: + """Return whether this column migration must stop a build before any execution.""" + + return self.decision.blocks_build + + +@dataclass(frozen=True) +class ColumnRenameHint: + """A removed column that an added column may have been renamed from.""" + + model_name: str + added_column: str + candidate_columns: tuple[str, ...] + identical: bool = False + + @property + def message(self) -> str: + """Return the one-line hint shown beside the added column.""" + + relation: str = "same expression as" if self.identical else "similar to" + origin: str = self.candidate_columns[0] if len(self.candidate_columns) == 1 else "" + return ( + f"{relation} {', '.join(self.candidate_columns)}; if this is a rename, add " + f"{self.added_column} (migrate_from {origin})" + ) + + +@dataclass(frozen=True) +class QueryProjection: + """One named top-level output column of a query and its canonical expression.""" + + name: str + expression: str + references: frozenset[str] + + @property + def key(self) -> str: + """Return the case-insensitive output name.""" + + return self.name.lower() + + +@dataclass(frozen=True) +class QueryShape: + """Canonical top-level projections and the rest of one query, ignoring formatting.""" + + projections: tuple[QueryProjection, ...] + body: str + + @dataclass(frozen=True) class PlanOutputExtras: """Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.""" @@ -1037,6 +1103,7 @@ class ModelPlanEntry: custom_materialization_name: str | None = None custom_config: dict[str, object] = field(default_factory=dict) custom_placeholders: dict[str, str] = field(default_factory=dict) + column_rename_hints: tuple[ColumnRenameHint, ...] = field(default_factory=tuple) @dataclass(frozen=True) @@ -1345,6 +1412,7 @@ class PlanOutput: retention_entries: tuple[RetentionPlanEntry, ...] = field(default_factory=tuple) table_type_entries: tuple[TableTypePlanEntry, ...] = field(default_factory=tuple) migration_entries: tuple[ModelMigrationPlanEntry, ...] = field(default_factory=tuple) + column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] = field(default_factory=tuple) upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = field( default_factory=dict ) @@ -1441,6 +1509,8 @@ class PlannerWarehouseState: inspection_relations: PlannerRelationsContext migration_entries: tuple[ModelMigrationPlanEntry, ...] = () migration_warnings: tuple[PlanWarning, ...] = () + column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] = () + column_rename_hints: tuple[ColumnRenameHint, ...] = () @dataclass(frozen=True) @@ -1491,6 +1561,16 @@ class ModelMigrationPlanning: warnings: tuple[PlanWarning, ...] = () +@dataclass(frozen=True) +class ColumnMigrationPlanning: + """Column rename decisions plus the warehouse snapshot as it will look after renames.""" + + snapshot: WarehouseSnapshot + entries: tuple[ColumnMigrationPlanEntry, ...] = () + hints: tuple[ColumnRenameHint, ...] = () + warnings: tuple[PlanWarning, ...] = () + + @dataclass(frozen=True) class PlannerIdentityContext: """Expected version identities for inspection and stale-warning scopes.""" diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 1d49045db1..310d6e4a92 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -34,6 +34,7 @@ from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.migrations.main._apply import apply_model_migrations +from sqlbuild.executor.migrations.main._apply_columns import apply_column_migrations from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult @@ -80,6 +81,13 @@ def execute_build_plan( run_id=runtime.run_id, on_progress=resolved_callbacks.on_migration_progress, ) + _ = apply_column_migrations( + plan=plan, + adapter=adapter, + connection=scheduler_connection, + run_id=runtime.run_id, + on_progress=resolved_callbacks.on_migration_progress, + ) _ = apply_retention_phase( plan=plan, adapter=adapter, diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 506aa4c3d8..d56339d28f 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -3,7 +3,10 @@ from __future__ import annotations from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME +from sqlbuild.compiler.migrations.constants import ( + COLUMN_MIGRATION_TABLE_NAME, + MIGRATION_TABLE_NAME, +) from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.audit_results.constants import AUDIT_RESULTS_TABLE_NAME from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME @@ -18,6 +21,7 @@ AUDIT_RESULTS_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MIGRATION_TABLE_NAME, + COLUMN_MIGRATION_TABLE_NAME, ) DIRECT_DEFAULT_RETENTION_DAYS: int = 14 diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py new file mode 100644 index 0000000000..4d5c82781e --- /dev/null +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -0,0 +1,142 @@ +"""Rename one model's columns in place and append their column migration events.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS +from sqlbuild.compiler.migrations.main.deterministic_column_event_id import ( + deterministic_column_migration_event_id, +) +from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location +from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation +from sqlbuild.compiler.migrations.types import ColumnMigrationDecision +from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry + + +def apply_model_column_renames( + *, + adapter: BaseAdapter, + connection: Any, + entries: tuple[ColumnMigrationPlanEntry, ...], + run_id: str, +) -> None: + """Rename and record every pending column of one model, re-checking live columns first.""" + + first: ColumnMigrationPlanEntry = entries[0] + adapter.ensure_schema( + connection=connection, + database=first.destination.database, + schema=first.destination.schema, + statement_recorder=StatementRecorder(), + ) + create_table_sql: str = adapter.render_create_column_migration_state_table_sql( + database=first.destination.database, schema=first.destination.schema or "" + ) + live: frozenset[str] = frozenset( + column.name.lower() + for column in _live_columns(adapter=adapter, connection=connection, entry=first) + ) + if not adapter.supports_transactional_ddl(): + entry: ColumnMigrationPlanEntry + for entry in entries: + _rename_and_record( + adapter=adapter, + connection=connection, + entry=entry, + live=live, + run_id=run_id, + create_table_sql=create_table_sql, + ) + return + _ = adapter.execute(connection=connection, sql=create_table_sql) + with adapter.transaction(connection): + for entry in entries: + _rename_and_record( + adapter=adapter, + connection=connection, + entry=entry, + live=live, + run_id=run_id, + create_table_sql=None, + attempts=1, + ) + + +def _rename_and_record( + *, + adapter: BaseAdapter, + connection: Any, + entry: ColumnMigrationPlanEntry, + live: frozenset[str], + run_id: str, + create_table_sql: str | None, + attempts: int = MIGRATION_WRITE_ATTEMPTS, +) -> None: + already_renamed: bool = ( + entry.origin_column.lower() not in live and entry.destination_column.lower() in live + ) + decision: ColumnMigrationDecision = entry.decision + if entry.decision.renames and not already_renamed: + adapter.rename_column( + connection=connection, + destination=entry.destination.qualified_name or entry.destination.name, + old_name=entry.origin_column, + new_name=entry.destination_column, + statement_recorder=StatementRecorder(), + ) + elif entry.decision.renames: + decision = ColumnMigrationDecision.RECORD + event: ColumnMigrationEvent = column_migration_event( + entry=entry, decision=decision, run_id=run_id + ) + write_column_migration_event( + connection=connection, + execute=adapter.execute, + event=event, + render_qualified_name=adapter.render_qualified_name, + create_table_sql=create_table_sql, + attempts=attempts, + ) + + +def _live_columns( + *, adapter: BaseAdapter, connection: Any, entry: ColumnMigrationPlanEntry +) -> tuple[ColumnInfo, ...]: + return adapter.get_columns( + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + name=entry.destination.name, + ) + + +def column_migration_event( + *, entry: ColumnMigrationPlanEntry, decision: ColumnMigrationDecision, run_id: str +) -> ColumnMigrationEvent: + """Build the append-only event recording one completed column rename.""" + + relation: MigrationRelation = migration_relation_for_location(entry.destination) + return ColumnMigrationEvent( + event_id=deterministic_column_migration_event_id( + run_id=run_id, + target_name=entry.target_name, + relation=relation, + origin_column=entry.origin_column, + destination_column=entry.destination_column, + ), + target_name=entry.target_name, + model_name=entry.model_name, + relation=relation, + origin_column=entry.origin_column, + destination_column=entry.destination_column, + discovery=entry.discovery, + decision=decision, + run_id=run_id, + created_at=datetime.now(tz=UTC), + ) diff --git a/src/sqlbuild/executor/migrations/main/_apply_columns.py b/src/sqlbuild/executor/migrations/main/_apply_columns.py new file mode 100644 index 0000000000..4229d79fc9 --- /dev/null +++ b/src/sqlbuild/executor/migrations/main/_apply_columns.py @@ -0,0 +1,79 @@ +"""Execute planned in-place column renames before any build node runs.""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry, PlanOutput +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.migrations._helpers.columns import apply_model_column_renames + + +def apply_column_migrations( + *, + plan: PlanOutput, + adapter: BaseAdapter, + connection: Any, + run_id: str, + on_progress: Callable[[str], None] | None = None, +) -> None: + """Rename each model's pending columns and record them, one model at a time.""" + + blocked: tuple[ColumnMigrationPlanEntry, ...] = tuple( + entry for entry in plan.column_migration_entries if entry.blocks_build + ) + if blocked: + raise ExecutorInputError( + "column migrations are blocked for " + + ", ".join(f"'{entry.model_name}' ({entry.decision.value})" for entry in blocked), + code="M110", + ) + pending: tuple[ColumnMigrationPlanEntry, ...] = tuple( + entry for entry in plan.column_migration_entries if entry.decision.records_event + ) + model_name: str + for model_name in dict.fromkeys(entry.model_name for entry in pending): + _ = _apply_model( + entries=tuple(entry for entry in pending if entry.model_name == model_name), + adapter=adapter, + connection=connection, + run_id=run_id, + on_progress=on_progress, + ) + + +def _apply_model( + *, + entries: tuple[ColumnMigrationPlanEntry, ...], + adapter: BaseAdapter, + connection: Any, + run_id: str, + on_progress: Callable[[str], None] | None, +) -> None: + relation: str = entries[0].destination.qualified_name or entries[0].destination.name + renames: str = ", ".join( + f"{entry.origin_column} -> {entry.destination_column}" for entry in entries + ) + if on_progress is not None: + on_progress(f"Renaming columns of {relation} ({renames})...") + started: float = time.monotonic() + try: + _ = apply_model_column_renames( + adapter=adapter, connection=connection, entries=entries, run_id=run_id + ) + except Exception as error: + raise ExecutorInputError( + f"column migration {renames} on {relation} failed: {error}", + code="M113", + help=( + "Re-run the build; column renames are re-evaluated from the table's current " + "columns and recorded events, so completed renames are not repeated." + ), + ) from error + if on_progress is not None: + on_progress( + f"Renamed columns of {relation} ({renames}). ({time.monotonic() - started:.2f}s)" + ) diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index 812bab2ff7..044f123326 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -474,6 +474,7 @@ class SchemaColumn: meta: dict[str, object] = field(default_factory=dict) audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) location: SourceLocation | None = None + migrate_from: str | None = None @dataclass(frozen=True) From 406f1ae416d4e45073896ae4c7ceef5529494df9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 07:42:35 +0100 Subject: [PATCH 2/9] test(migrations): cover column rename detection, planning and execution Refs CHI-393 --- .../main/model_migrations/_test_types.py | 12 + .../commands/main/model_migrations/helpers.py | 122 +++++ .../test_column_migration_lifecycle.py | 127 ++++++ .../cli/commands/main/postgres/_test_types.py | 18 + .../cli/commands/main/postgres/helpers.py | 68 +++ .../main/postgres/test_column_migrations.py | 208 +++++++++ .../main/column_migrations/__init__.py | 1 + .../main/column_migrations/_test_types.py | 104 +++++ .../main/column_migrations/helpers.py | 318 +++++++++++++ .../test_column_rename_safety.py | 207 +++++++++ .../column_migrations/test_column_renames.py | 424 ++++++++++++++++++ .../unit/src/sqlbuild/adapters/_test_types.py | 24 + tests/unit/src/sqlbuild/adapters/helpers.py | 31 ++ .../sqlbuild/adapters/test_rename_column.py | 253 +++++++++++ .../sqlbuild/cli/output/main/plan/helpers.py | 3 + .../cli/output/main/plan/test_plan_text.py | 34 ++ .../compiler/migrations/main/_test_types.py | 17 + .../compiler/migrations/main/helpers.py | 39 +- .../main/test_column_migration_events.py | 128 ++++++ .../_helpers/migrations/_test_types.py | 34 ++ .../planner/_helpers/migrations/helpers.py | 10 + .../migrations/test_column_renames.py | 219 +++++++++ .../planner/main/pre_build/_test_types.py | 21 +- .../planner/main/pre_build/helpers.py | 30 +- .../pre_build/test_pre_build_work_display.py | 138 +++++- 25 files changed, 2585 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_column_migration_lifecycle.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_column_migrations.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/column_migrations/__init__.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py create mode 100644 tests/unit/src/sqlbuild/adapters/helpers.py create mode 100644 tests/unit/src/sqlbuild/adapters/test_rename_column.py create mode 100644 tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index 9a8644d8fb..5d2b72d9f3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -12,3 +12,15 @@ class MigrationLifecycleE2ETestCase: expected_final_ids: tuple[int, ...] expected_previous_archive_ids: tuple[tuple[int, ...], ...] expected_events: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class ColumnMigrationLifecycleE2ETestCase: + description: str + expected_automatic_plan: str + expected_retry_decisions: tuple[str, ...] + expected_revenue: tuple[tuple[int, int], ...] + expected_declared_notice: str + expected_near_match_hint: str + expected_final_columns: tuple[str, ...] + expected_events: tuple[tuple[str, str, str, str], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 070c2dfd3b..7005eb1b3c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -125,3 +125,125 @@ def migration_events(*, project_dir: Path) -> tuple[tuple[str, str, str], ...]: ), ) ) + + +_REPLAY_LINES: dict[bool, str] = {True: " replay_on_change full,\n", False: ""} + + +def fct_orders_sql(*, columns: str, extra_config: str = "", replay: bool = True) -> str: + """Return an incremental orders fact model projecting the given value columns.""" + + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + f"{_REPLAY_LINES[replay]}" + f"{extra_config}" + ");\n\n" + f'SELECT order_id, order_date, {columns} FROM __source("raw_orders")\n' + ) + + +def write_fct_orders_project(*, tmp_path: Path, model_sql: str) -> Path: + """Write a project containing exactly the incremental orders fact model.""" + + return prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": _SOURCES_YML, + "models/fct_orders.sql": model_sql, + }, + ) + + +def load_raw_order_amounts( + *, + project_dir: Path, + last_day: int, + changed_day: int = 0, + failing_day: int = 0, +) -> None: + """Replace raw orders with a view whose amounts can change or fail for one day.""" + + amount: str = ( + f"CASE WHEN i = {failing_day} THEN CAST('broken' AS BIGINT) " + f"WHEN i = {changed_day} THEN 999 ELSE 100 + i END" + ) + execute_duckdb(db_path=project_dir / DATABASE_FILE, sql="DROP VIEW IF EXISTS main.raw_orders") + execute_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "CREATE VIEW main.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) AS order_date, " + f"{amount} AS amount FROM range(1, {last_day + 1}) AS t(i)" + ), + ) + + +def plan_payload(*, project_dir: Path) -> dict[str, Any]: + """Run sqb plan --json in a subprocess and parse stdout.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--json"), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + return json.loads(result.stdout) + + +def plan_output(*, project_dir: Path) -> str: + """Run sqb plan in a subprocess and return its text output.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan"), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + return result.stdout + + +def fct_order_values(*, project_dir: Path, column: str) -> tuple[tuple[int, int], ...]: + """Return (order_id, value) for every built order.""" + + return tuple( + (int(row[0]), int(row[1])) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=f"SELECT order_id, {column} FROM main.fct_orders ORDER BY 1", + ) + ) + + +def fct_order_columns(*, project_dir: Path) -> tuple[str, ...]: + """Return the physical columns of the orders fact table in order.""" + + return tuple( + str(row[0]) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT column_name FROM information_schema.columns WHERE table_schema = 'main' " + "AND table_name = 'fct_orders' ORDER BY ordinal_position" + ), + ) + ) + + +def column_migration_events(*, project_dir: Path) -> tuple[tuple[str, str, str, str], ...]: + """Return (origin, destination, discovery, decision) for every recorded column rename.""" + + return tuple( + (str(row[0]), str(row[1]), str(row[2]), str(row[3])) + for row in query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT origin_column, destination_column, discovery, decision " + "FROM main._sqlbuild_column_migrations ORDER BY created_at, event_id" + ), + ) + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_column_migration_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_column_migration_lifecycle.py new file mode 100644 index 0000000000..a7ef765472 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_column_migration_lifecycle.py @@ -0,0 +1,127 @@ +"""CLI e2e coverage for renaming incremental model columns in place across builds.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( + ColumnMigrationLifecycleE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + build, + column_migration_events, + fct_order_columns, + fct_order_values, + fct_orders_sql, + load_raw_order_amounts, + plan_output, + plan_payload, + write_fct_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnMigrationLifecycleE2ETestCase( + description="automatic, retried, declared, and near-match renames", + expected_automatic_plan=( + "Column migrations (1)\n└── fct_orders migrate columns\n" + " └── amount -> revenue rename in place (automatic)\n" + ), + expected_retry_decisions=("done",), + expected_revenue=((1, 101), (2, 102), (3, 103), (4, 104), (5, 105)), + expected_declared_notice=( + "migrate_from can be removed from column 'net_revenue' of 'fct_orders'" + ), + expected_near_match_hint=( + "similar to net_revenue; if this is a rename, add gross (migrate_from net_revenue)" + ), + expected_final_columns=("order_id", "order_date", "net_revenue", "gross"), + expected_events=( + ("amount", "revenue", "automatic", "rename"), + ("revenue", "net_revenue", "manual", "rename"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_columns_when_building_through_the_cli_then_history_follows_each_rename( + tmp_path: Path, test_case: ColumnMigrationLifecycleE2ETestCase +) -> None: + """The real CLI renames in place, survives a failed delta, and hints near matches.""" + + project_dir: Path = write_fct_orders_project( + tmp_path=tmp_path, model_sql=fct_orders_sql(columns="amount") + ) + load_raw_order_amounts(project_dir=project_dir, last_day=3) + initial: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + + _ = write_fct_orders_project( + tmp_path=tmp_path, model_sql=fct_orders_sql(columns="amount AS revenue") + ) + load_raw_order_amounts(project_dir=project_dir, last_day=5, changed_day=1, failing_day=5) + automatic_plan: str = plan_output(project_dir=project_dir) + failed: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + columns_after_failure: tuple[str, ...] = fct_order_columns(project_dir=project_dir) + + load_raw_order_amounts(project_dir=project_dir, last_day=5, changed_day=1) + retry_plan: dict[str, Any] = plan_payload(project_dir=project_dir) + retried: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + revenue: tuple[tuple[int, int], ...] = fct_order_values( + project_dir=project_dir, column="revenue" + ) + + _ = write_fct_orders_project( + tmp_path=tmp_path, + model_sql=fct_orders_sql( + columns="amount AS net_revenue", + extra_config=" columns (net_revenue (migrate_from revenue)),\n", + ), + ) + declared: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + recorded: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + + _ = write_fct_orders_project( + tmp_path=tmp_path, + model_sql=fct_orders_sql( + columns="amount AS net_revenue, ROUND(amount * 1.5) AS gross", replay=False + ), + ) + near_match_plan: str = plan_output(project_dir=project_dir) + _ = write_fct_orders_project( + tmp_path=tmp_path, + model_sql=fct_orders_sql(columns="ROUND(amount * 1.5) AS gross", replay=False), + ) + near_match_hint_plan: str = plan_output(project_dir=project_dir) + near_match: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + after_near_match_plan: str = plan_output(project_dir=project_dir) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert test_case.expected_automatic_plan in automatic_plan + assert failed.returncode != 0 + assert columns_after_failure == ("order_id", "order_date", "revenue") + assert tuple(entry["decision"] for entry in retry_plan["column_migrations"]) == ( + test_case.expected_retry_decisions + ) + assert retried.returncode == 0, retried.stdout + retried.stderr + assert "Renaming columns" not in retried.stdout + retried.stderr + assert revenue == test_case.expected_revenue + assert declared.returncode == 0, declared.stdout + declared.stderr + assert recorded.returncode == 0, recorded.stdout + recorded.stderr + assert recorded.stdout.count(test_case.expected_declared_notice) == 1 + assert "Completed successfully" in recorded.stdout.rstrip().splitlines()[-1] + assert "Column migrations" not in near_match_plan + assert test_case.expected_near_match_hint in near_match_hint_plan + assert near_match.returncode == 0, near_match.stdout + near_match.stderr + assert test_case.expected_near_match_hint not in after_near_match_plan + assert fct_order_columns(project_dir=project_dir) == test_case.expected_final_columns + assert column_migration_events(project_dir=project_dir) == test_case.expected_events + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index f76a553705..1de6d26f50 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -153,3 +153,21 @@ class PostgresMigrationRollbackE2ETestCase: expected_destination_ids_after_failure: tuple[tuple[object, ...], ...] expected_previous_archives_after_failure: int expected_final_destination_ids: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class PostgresColumnMigrationE2ETestCase: + description: str + expected_columns: tuple[tuple[object, ...], ...] + expected_amounts: tuple[tuple[object, ...], ...] + expected_view_amounts: tuple[tuple[object, ...], ...] + expected_events: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class PostgresColumnMigrationRollbackE2ETestCase: + description: str + expected_failure_fragment: str + expected_columns_after_failure: tuple[tuple[object, ...], ...] + expected_final_columns: tuple[tuple[object, ...], ...] + expected_events: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py index a30f2848ac..f12dac554c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py @@ -612,3 +612,71 @@ def view_catalog( ), config=config, ) + + +FACT_ORDERS_MODEL: str = "fct_orders" +_COLUMN_MIGRATION_TEXT_COLUMNS: tuple[str, ...] = ( + "event_id", + "target_name", + "model_name", + "relation_database", + "relation_schema", + "relation_name", + "origin_column", + "destination_column", + "discovery", + "decision", + "run_id", +) + + +def fact_orders_sql(*, amount: str = "amount_cents", columns: str = "") -> str: + """Return an incremental orders fact model with an optional column migrations block.""" + + declarations: str = {"": ""}.get(columns, f" columns ({columns}),\n") + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key order_id,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + " replay_on_change full,\n" + f"{declarations}" + ");\n\n" + f'SELECT order_id, order_date, {amount} FROM __source("raw_orders")\n' + ) + + +def fact_order_columns( + *, schema_name: str, config: dict[str, object] +) -> tuple[tuple[object, ...], ...]: + """Return the physical column names of the orders fact table in order.""" + + return fetch_postgres_rows( + sql=( + "SELECT column_name FROM information_schema.columns " + f"WHERE table_schema = '{schema_name}' AND table_name = '{FACT_ORDERS_MODEL}' " + "ORDER BY ordinal_position" + ), + config=config, + ) + + +def create_unwritable_column_migration_table( + *, schema_name: str, config: dict[str, object] +) -> None: + """Occupy the column migration state name with a read-only view so the insert fails.""" + + columns: str = ", ".join( + f"CAST(NULL AS TEXT) AS {column}" for column in _COLUMN_MIGRATION_TEXT_COLUMNS + ) + execute_postgres_sql( + sql=( + f"CREATE VIEW {schema_name}._sqlbuild_column_migrations AS SELECT {columns}, " + "CAST(NULL AS TIMESTAMP) AS created_at WHERE false" + ), + config=config, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_column_migrations.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_column_migrations.py new file mode 100644 index 0000000000..7f6baf76c7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_column_migrations.py @@ -0,0 +1,208 @@ +"""PostgreSQL e2e coverage for in-place column renames of incremental models.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.postgres._test_types import ( + PostgresColumnMigrationE2ETestCase, + PostgresColumnMigrationRollbackE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.postgres.helpers import ( + FACT_ORDERS_MODEL, + build_unique_schema_name, + cleanup_postgres_schema, + create_unwritable_column_migration_table, + ensure_postgres_schema_ready, + execute_postgres_sql, + fact_order_columns, + fact_orders_sql, + fetch_postgres_rows, + load_raw_orders, + write_migration_project, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +_AMOUNTS: tuple[tuple[object, ...], ...] = ((1, 101), (2, 102), (3, 103), (4, 104), (5, 105)) + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresColumnMigrationE2ETestCase( + description="automatic rename keeps history and dependent views keep reading", + expected_columns=(("order_id",), ("order_date",), ("amount",)), + expected_amounts=_AMOUNTS, + expected_view_amounts=_AMOUNTS, + expected_events=(("amount_cents", "amount", "automatic", "rename"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_column_when_building_on_postgres_then_history_keeps_its_values( + tmp_path: Path, + test_case: PostgresColumnMigrationE2ETestCase, + postgres_e2e_config: dict[str, object], +) -> None: + """ALTER TABLE ... RENAME COLUMN keeps rows, and PostgreSQL re-points dependent views.""" + + schema_name: str = build_unique_schema_name(prefix="sqlbuild_column_migration") + raw_schema_name: str = f"{schema_name}_raw" + ensure_postgres_schema_ready(schema_name=schema_name, config=postgres_e2e_config) + load_raw_orders(raw_schema_name=raw_schema_name, config=postgres_e2e_config) + project_dir: Path = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={FACT_ORDERS_MODEL: fact_orders_sql()}, + ) + try: + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + execute_postgres_sql( + sql=( + f"CREATE VIEW {raw_schema_name}.order_amounts AS SELECT order_id, amount_cents " + f"FROM {schema_name}.{FACT_ORDERS_MODEL}" + ), + config=postgres_e2e_config, + ) + project_dir = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={FACT_ORDERS_MODEL: fact_orders_sql(amount="amount_cents AS amount")}, + ) + renamed: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert renamed.returncode == 0, renamed.stdout + renamed.stderr + assert "amount_cents -> amount" in renamed.stdout + renamed.stderr + assert ( + fact_order_columns(schema_name=schema_name, config=postgres_e2e_config) + == test_case.expected_columns + ) + assert ( + fetch_postgres_rows( + sql=f"SELECT order_id, amount FROM {schema_name}.{FACT_ORDERS_MODEL} ORDER BY 1", + config=postgres_e2e_config, + ) + == test_case.expected_amounts + ) + assert ( + fetch_postgres_rows( + sql=f"SELECT order_id, amount_cents FROM {raw_schema_name}.order_amounts ORDER BY 1", + config=postgres_e2e_config, + ) + == test_case.expected_view_amounts + ) + assert ( + fetch_postgres_rows( + sql=( + "SELECT origin_column, destination_column, discovery, decision " + f"FROM {schema_name}._sqlbuild_column_migrations" + ), + config=postgres_e2e_config, + ) + == test_case.expected_events + ) + finally: + cleanup_postgres_schema(schema_name=raw_schema_name, config=postgres_e2e_config) + cleanup_postgres_schema(schema_name=schema_name, config=postgres_e2e_config) + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresColumnMigrationRollbackE2ETestCase( + description="failed event insert rolls the column rename back", + expected_failure_fragment="M113", + expected_columns_after_failure=(("order_id",), ("order_date",), ("amount_cents",)), + expected_final_columns=(("order_id",), ("order_date",), ("amount",)), + expected_events=(("amount_cents", "amount", "manual", "rename"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_unwritable_event_table_when_renaming_column_then_rename_rolls_back( + tmp_path: Path, + test_case: PostgresColumnMigrationRollbackE2ETestCase, + postgres_e2e_config: dict[str, object], +) -> None: + """The column rename and its event commit or roll back together on PostgreSQL.""" + + schema_name: str = build_unique_schema_name(prefix="sqlbuild_column_rollback") + raw_schema_name: str = f"{schema_name}_raw" + ensure_postgres_schema_ready(schema_name=schema_name, config=postgres_e2e_config) + load_raw_orders(raw_schema_name=raw_schema_name, config=postgres_e2e_config) + project_dir: Path = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={FACT_ORDERS_MODEL: fact_orders_sql()}, + ) + try: + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + create_unwritable_column_migration_table( + schema_name=schema_name, config=postgres_e2e_config + ) + project_dir = write_migration_project( + tmp_path=tmp_path, + schema_name=schema_name, + raw_schema_name=raw_schema_name, + config=postgres_e2e_config, + models={ + FACT_ORDERS_MODEL: fact_orders_sql( + amount="amount_cents AS amount", columns="amount (migrate_from amount_cents)" + ) + }, + ) + failed: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + columns_after_failure: tuple[tuple[object, ...], ...] = fact_order_columns( + schema_name=schema_name, config=postgres_e2e_config + ) + execute_postgres_sql( + sql=f"DROP VIEW {schema_name}._sqlbuild_column_migrations", config=postgres_e2e_config + ) + retried: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert initial.returncode == 0, initial.stdout + initial.stderr + assert failed.returncode != 0 + assert test_case.expected_failure_fragment in failed.stdout + failed.stderr + assert columns_after_failure == test_case.expected_columns_after_failure + assert retried.returncode == 0, retried.stdout + retried.stderr + assert ( + fact_order_columns(schema_name=schema_name, config=postgres_e2e_config) + == test_case.expected_final_columns + ) + assert ( + fetch_postgres_rows( + sql=( + "SELECT origin_column, destination_column, discovery, decision " + f"FROM {schema_name}._sqlbuild_column_migrations" + ), + config=postgres_e2e_config, + ) + == test_case.expected_events + ) + finally: + cleanup_postgres_schema(schema_name=raw_schema_name, config=postgres_e2e_config) + cleanup_postgres_schema(schema_name=schema_name, config=postgres_e2e_config) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/__init__.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/__init__.py new file mode 100644 index 0000000000..9acf514e89 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/__init__.py @@ -0,0 +1 @@ +"""Integration tests for direct-mode column migrations.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py new file mode 100644 index 0000000000..a5d2f34c17 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py @@ -0,0 +1,104 @@ +"""Test case types for column migration integration tests.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import pytest + + +@dataclass(frozen=True) +class ColumnRenameOutcomeTestCase: + description: str + renamed_sql: str + expected_planned: tuple[tuple[str, str, str, str], ...] + expected_plan_fragment: str + expected_columns: tuple[str, ...] + expected_values: tuple[tuple[int, Any], ...] + expected_events: tuple[tuple[str, str, str, str], ...] + value_column: str = "revenue" + + +@dataclass(frozen=True) +class CompletedColumnRenameTestCase: + description: str + renamed_sql: str + expected_planned: tuple[tuple[str, str, str, str], ...] + expected_notice: str + expected_final_line: str + + +@dataclass(frozen=True) +class NearMatchPolicyTestCase: + description: str + renamed_sql: str + expected_hint: str + expected_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class RenameWithPolicyTestCase: + description: str + renamed_sql: str + expected_columns: tuple[str, ...] + expected_values: tuple[tuple[int, Any], ...] + added_column: str = "revenue" + expected_added_values: tuple[tuple[int, Any], ...] = ( + (1, 101), + (2, 102), + (3, 103), + (4, 104), + (5, 105), + ) + + +@dataclass(frozen=True) +class ColumnRenameRecoveryTestCase: + description: str + renamed_sql: str + install_failure: Callable[[pytest.MonkeyPatch], None] + expected_retry_planned: tuple[tuple[str, str, str, str], ...] + expected_events: tuple[tuple[str, str, str, str], ...] + expected_values: tuple[tuple[int, Any], ...] + + +@dataclass(frozen=True) +class BlockedColumnRenameTestCase: + description: str + renamed_sql: str + expected_decision: str + expected_error: str + prepare_sql: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class ColumnMigrationCompileErrorTestCase: + description: str + model_sql: str + expected_fragment: str + + +@dataclass(frozen=True) +class SnapshotColumnRenameTestCase: + description: str + renamed_sql: str + expected_planned: tuple[tuple[str, str, str, str], ...] + expected_values: tuple[tuple[int, Any], ...] + + +@dataclass(frozen=True) +class CursorColumnRenameTestCase: + description: str + renamed_sql: str + expected_planned: tuple[tuple[str, str, str, str], ...] + expected_order_ids: tuple[int, ...] + + +@dataclass(frozen=True) +class FirstRunDeclarationTestCase: + description: str + model_sql: str + expected_planned: tuple[tuple[str, str, str, str], ...] + expected_columns: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py new file mode 100644 index 0000000000..bb386b58c1 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -0,0 +1,318 @@ +"""Project builders and CLI runners for column migration integration tests.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest +from _pytest.capture import CaptureResult + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.executor.build._helpers import scheduler as scheduler_module +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + execute, + query, + write_project, +) + +MODEL_NAME: str = "fct_orders" +REPLAY_FULL: str = " replay_on_change full,\n" +RELATION: str = f"main.{MODEL_NAME}" +_INCREMENTAL_HEADER: str = ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy {strategy},\n" + " unique_key order_id,\n" + " cursor {cursor},\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + "{extra_config}" + ");\n\n" +) +_SNAPSHOT_HEADER: str = ( + "MODEL (\n" + " materialized snapshot,\n" + " unique_key [order_id],\n" + " snapshot_strategy timestamp,\n" + " updated_at order_date,\n" + "{extra_config}" + ");\n\n" +) + + +@dataclass(frozen=True) +class CliRun: + """Captured result of one in-process sqb invocation.""" + + exit_code: int + stdout: str + stderr: str + + @property + def output(self) -> str: + """Return stdout followed by stderr.""" + + return self.stdout + self.stderr + + +def orders_sql( + *, + columns: str = "amount", + extra_config: str = "", + strategy: str = "delete_insert", + cursor: str = "order_date", + order_date: str = "order_date", +) -> str: + """Return an incremental orders model projecting the given value columns.""" + + return ( + _INCREMENTAL_HEADER.format(strategy=strategy, cursor=cursor, extra_config=extra_config) + + f'SELECT order_id, {order_date}, {columns} FROM __source("raw_orders")\n' + ) + + +MICROBATCH_CONFIG: str = ( + " incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n" + " cursor_watermark_mode all,\n" + " batch_size 1d,\n" + " cursor_inputs (raw_orders (column order_date, roles [filter, watermark])),\n" +) + + +def state_table_names(*, project_dir: Path) -> tuple[str, ...]: + """Return the SQLBuild state tables in the main schema.""" + + return tuple( + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' " + "AND starts_with(table_name, '_sqlbuild_') ORDER BY 1" + ), + ) + ) + + +def snapshot_sql(*, columns: str = "amount", extra_config: str = "") -> str: + """Return a timestamp snapshot of raw orders projecting the given value columns.""" + + return ( + _SNAPSHOT_HEADER.format(extra_config=extra_config) + + f'SELECT order_id, order_date, {columns} FROM __source("raw_orders")\n' + ) + + +def build_initial( + *, project_dir: Path, capsys: pytest.CaptureFixture[str], sql: str | None = None +) -> None: + """Build the orders model over three days, then load five days of raw orders.""" + + write_model(project_dir=project_dir, sql=sql or orders_sql(extra_config=REPLAY_FULL)) + load_orders(project_dir=project_dir, last_day=3) + _ = build_ok(project_dir=project_dir, capsys=capsys) + load_orders(project_dir=project_dir, last_day=5) + + +def last_visible_line(output: str) -> str: + """Return the final non-blank line of command output.""" + + return output.rstrip().splitlines()[-1] + + +def migrate_columns(declarations: str) -> str: + """Return a MODEL header columns block declaring column migrations.""" + + return f" columns ({declarations}),\n" + + +def write_model(*, project_dir: Path, sql: str) -> None: + """Replace the project with exactly the orders model.""" + + write_project(project_dir=project_dir, models={MODEL_NAME: sql}) + + +def load_orders(*, project_dir: Path, last_day: int, failing_day: int | None = None) -> None: + """Replace raw orders with one order per January day, optionally failing on one day.""" + + amount: str = {None: "100 + i"}.get( + failing_day, f"CASE WHEN i = {failing_day} THEN CAST('broken' AS BIGINT) ELSE 100 + i END" + ) + execute(project_dir=project_dir, sql="DROP VIEW IF EXISTS main.raw_orders") + execute(project_dir=project_dir, sql="DROP TABLE IF EXISTS main.raw_orders") + execute( + project_dir=project_dir, + sql=( + "CREATE VIEW main.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) AS order_date, " + f"{amount} AS amount, i AS tax " + f"FROM range(1, {last_day + 1}) AS t(i)" + ), + ) + + +def run_sqb( + *, project_dir: Path, args: tuple[str, ...], capsys: pytest.CaptureFixture[str] +) -> CliRun: + """Run sqb in-process and capture stdout and stderr separately.""" + + _ = capsys.readouterr() + try: + exit_code: int = main(["--project-dir", str(project_dir), "--no-color", *args]) + except Exception as error: + captured_error: CaptureResult[str] = capsys.readouterr() + return CliRun(exit_code=1, stdout=captured_error.out, stderr=str(error)) + captured: CaptureResult[str] = capsys.readouterr() + return CliRun(exit_code=exit_code, stdout=captured.out, stderr=captured.err) + + +def build(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: + """Run sqb build.""" + + return run_sqb(project_dir=project_dir, args=("build",), capsys=capsys) + + +def build_ok(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> CliRun: + """Run sqb build and require success.""" + + result: CliRun = build(project_dir=project_dir, capsys=capsys) + assert result.exit_code == 0, result.output + return result + + +def plan_text(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> str: + """Run sqb plan and return its text output.""" + + result: CliRun = run_sqb(project_dir=project_dir, args=("plan",), capsys=capsys) + assert result.exit_code == 0, result.output + return result.stdout + + +def plan_json(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> dict[str, Any]: + """Run sqb plan --json and parse stdout.""" + + result: CliRun = run_sqb(project_dir=project_dir, args=("plan", "--json"), capsys=capsys) + assert result.exit_code == 0, result.output + return json.loads(result.stdout) + + +def planned_column_migrations(plan: dict[str, Any]) -> tuple[tuple[str, str, str, str], ...]: + """Return (origin, destination, discovery, decision) for every planned column migration.""" + + return tuple( + ( + str(entry["origin_column"]), + str(entry["destination_column"]), + str(entry["discovery"]), + str(entry["decision"]), + ) + for entry in plan["column_migrations"] + ) + + +def model_plan(plan: dict[str, Any]) -> dict[str, Any]: + """Return the serialized plan entry of the orders model.""" + + return {model["name"]: model for model in plan["models"]}[MODEL_NAME] + + +def warning_codes(plan: dict[str, Any]) -> tuple[str, ...]: + """Return every plan warning code in order.""" + + return tuple(str(warning.get("code")) for warning in plan["warnings"]) + + +def column_names(*, project_dir: Path, relation: str = RELATION) -> tuple[str, ...]: + """Return the physical column names of one relation in order.""" + + schema: str + name: str + schema, name = relation.split(".") + return tuple( + str(row[0]) + for row in query( + project_dir=project_dir, + sql=( + "SELECT column_name FROM information_schema.columns " + f"WHERE table_schema = '{schema}' AND table_name = '{name}' " + "ORDER BY ordinal_position" + ), + ) + ) + + +def column_values( + *, project_dir: Path, column: str, relation: str = RELATION +) -> tuple[tuple[int, Any], ...]: + """Return (order_id, value) for every row of one relation in order.""" + + return tuple( + (int(row[0]), row[1]) + for row in query( + project_dir=project_dir, + sql=f"SELECT order_id, {column} FROM {relation} ORDER BY order_id, {column}", + ) + ) + + +def column_events(*, project_dir: Path) -> tuple[tuple[str, str, str, str], ...]: + """Return (origin, destination, discovery, decision) for every recorded column rename.""" + + return tuple( + (str(row[0]), str(row[1]), str(row[2]), str(row[3])) + for row in query( + project_dir=project_dir, + sql=( + "SELECT origin_column, destination_column, discovery, decision " + "FROM main._sqlbuild_column_migrations ORDER BY created_at, event_id" + ), + ) + ) + + +def change_historical_amount(*, project_dir: Path) -> None: + """Change day one's raw amount so any replay would be visible in built history.""" + + execute( + project_dir=project_dir, + sql=( + "CREATE OR REPLACE VIEW main.raw_orders AS SELECT order_id, order_date, " + "CASE WHEN order_id = 1 THEN 999 ELSE amount END AS amount, tax " + "FROM (SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) AS order_date, " + "100 + i AS amount, i AS tax FROM range(1, 6) AS t(i))" + ), + ) + + +def _raise_interruption(*_: object, **__: object) -> None: + raise RuntimeError("simulated interruption") + + +def fail_model_build(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the orders model's incremental build raise after column migrations have run.""" + + incremental: Callable[..., Any] = scheduler_module.execute_incremental_entry + failing: dict[str, Callable[..., Any]] = {MODEL_NAME: _raise_interruption} + monkeypatch.setattr( + scheduler_module, + "execute_incremental_entry", + lambda **kwargs: failing.get(kwargs["context"].entry.name, incremental)(**kwargs), + ) + + +def fail_non_transactional_record(monkeypatch: pytest.MonkeyPatch) -> None: + """Rename without a transaction, then crash before the event is recorded.""" + + monkeypatch.setattr(DuckDbAdapter, "supports_transactional_ddl", lambda self: False) + monkeypatch.setattr( + "sqlbuild.executor.migrations._helpers.columns.write_column_migration_event", + _raise_interruption, + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py new file mode 100644 index 0000000000..bdf8cafd20 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py @@ -0,0 +1,207 @@ +"""Integration coverage for column rename retries, blocked renames, and header errors.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.column_migrations._test_types import ( + BlockedColumnRenameTestCase, + ColumnMigrationCompileErrorTestCase, + ColumnRenameRecoveryTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.column_migrations.helpers import ( + REPLAY_FULL, + CliRun, + build, + build_initial, + build_ok, + column_events, + column_names, + column_values, + fail_model_build, + fail_non_transactional_record, + migrate_columns, + model_plan, + orders_sql, + plan_json, + planned_column_migrations, + run_sqb, + write_model, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import execute + +_HISTORY: tuple[tuple[int, Any], ...] = ((1, 101), (2, 102), (3, 103), (4, 104), (5, 105)) + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnRenameRecoveryTestCase( + description="build failure after a recorded rename resumes without replay", + renamed_sql=orders_sql(columns="amount AS revenue", extra_config=REPLAY_FULL), + install_failure=fail_model_build, + expected_retry_planned=(("amount", "revenue", "automatic", "done"),), + expected_events=(("amount", "revenue", "automatic", "rename"),), + expected_values=_HISTORY, + ), + ColumnRenameRecoveryTestCase( + description="non-transactional crash before the event records the renamed column", + renamed_sql=orders_sql( + columns="amount AS revenue", + extra_config=REPLAY_FULL + migrate_columns("revenue (migrate_from amount)"), + ), + install_failure=fail_non_transactional_record, + expected_retry_planned=(("amount", "revenue", "manual", "record"),), + expected_events=(("amount", "revenue", "manual", "record"),), + expected_values=_HISTORY, + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupted_rename_when_retrying_then_column_is_renamed_once_and_delta_lands( + test_case: ColumnRenameRecoveryTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + + with monkeypatch.context() as patched: + test_case.install_failure(patched) + interrupted: CliRun = build(project_dir=tmp_path, capsys=capsys) + columns_after_interruption: tuple[str, ...] = column_names(project_dir=tmp_path) + retry_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert interrupted.exit_code != 0 + assert columns_after_interruption == ("order_id", "order_date", "revenue") + assert planned_column_migrations(retry_plan) == test_case.expected_retry_planned + assert model_plan(retry_plan)["backfill"]["action"] == "forward" + assert column_events(project_dir=tmp_path) == test_case.expected_events + assert column_values(project_dir=tmp_path, column="revenue") == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + [ + BlockedColumnRenameTestCase( + description="both columns already exist", + renamed_sql=orders_sql( + columns="amount AS revenue", + extra_config=migrate_columns("revenue (migrate_from amount)"), + ), + prepare_sql=("ALTER TABLE main.fct_orders ADD COLUMN revenue BIGINT",), + expected_decision="conflict", + expected_error="error[M110]", + ), + BlockedColumnRenameTestCase( + description="source column never existed", + renamed_sql=orders_sql( + columns="amount AS revenue", + extra_config=migrate_columns("revenue (migrate_from gross_amount)"), + ), + expected_decision="source_missing", + expected_error="error[M109]", + ), + BlockedColumnRenameTestCase( + description="source column is still produced", + renamed_sql=orders_sql( + columns="amount, amount AS revenue", + extra_config=migrate_columns("revenue (migrate_from amount)"), + ), + expected_decision="still_produced", + expected_error="error[M111]", + ), + ], + ids=lambda case: case.description, +) +def test_given_unsafe_declared_rename_when_building_then_build_stops_before_changes( + test_case: BlockedColumnRenameTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys) + statement: str + for statement in test_case.prepare_sql: + execute(project_dir=tmp_path, sql=statement) + columns_before: tuple[str, ...] = column_names(project_dir=tmp_path) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build(project_dir=tmp_path, capsys=capsys) + + assert [entry["decision"] for entry in plan["column_migrations"]] == [ + test_case.expected_decision + ] + assert result.exit_code != 0 + assert result.output.count(test_case.expected_error) == 1, result.output + assert column_names(project_dir=tmp_path) == columns_before + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnMigrationCompileErrorTestCase( + description="table materialization", + model_sql=( + "MODEL (materialized table, columns (revenue (migrate_from amount)));\n" + 'SELECT amount AS revenue FROM __source("raw_orders")\n' + ), + expected_fragment="only valid for incremental and snapshot models", + ), + ColumnMigrationCompileErrorTestCase( + description="column migrates from itself", + model_sql=orders_sql( + columns="amount AS revenue", + extra_config=migrate_columns("revenue (migrate_from revenue)"), + ), + expected_fragment="migrate_from cannot name the column itself", + ), + ColumnMigrationCompileErrorTestCase( + description="two columns claim one source", + model_sql=orders_sql( + columns="amount AS revenue, amount AS gross", + extra_config=migrate_columns( + "revenue (migrate_from amount), gross (migrate_from amount)" + ), + ), + expected_fragment="both declare migrate_from amount", + ), + ColumnMigrationCompileErrorTestCase( + description="chained renames", + model_sql=orders_sql( + columns="amount AS revenue, tax AS amount", + extra_config=migrate_columns( + "revenue (migrate_from amount), amount (migrate_from tax)" + ), + ), + expected_fragment="chained or swapped column renames are not supported", + ), + ColumnMigrationCompileErrorTestCase( + description="qualified source column", + model_sql=orders_sql( + columns="amount AS revenue", + extra_config=migrate_columns('revenue (migrate_from "fct_orders.amount")'), + ), + expected_fragment="migrate_from must name one column", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_column_migration_header_when_compiling_then_fails( + test_case: ColumnMigrationCompileErrorTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_model(project_dir=tmp_path, sql=test_case.model_sql) + + result: CliRun = run_sqb(project_dir=tmp_path, args=("compile",), capsys=capsys) + + assert result.exit_code != 0 + assert test_case.expected_fragment in result.output, result.output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py new file mode 100644 index 0000000000..e5288af099 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py @@ -0,0 +1,424 @@ +"""Integration coverage for in-place column renames through the real CLI on DuckDB.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.column_migrations._test_types import ( + ColumnRenameOutcomeTestCase, + CompletedColumnRenameTestCase, + CursorColumnRenameTestCase, + FirstRunDeclarationTestCase, + NearMatchPolicyTestCase, + RenameWithPolicyTestCase, + SnapshotColumnRenameTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.column_migrations.helpers import ( + MICROBATCH_CONFIG, + RELATION, + REPLAY_FULL, + CliRun, + build_initial, + build_ok, + change_historical_amount, + column_events, + column_names, + column_values, + last_visible_line, + load_orders, + migrate_columns, + model_plan, + orders_sql, + plan_json, + plan_text, + planned_column_migrations, + snapshot_sql, + state_table_names, + warning_codes, + write_model, +) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + execute, + query, +) + +_HISTORY: tuple[tuple[int, Any], ...] = ((1, 101), (2, 102), (3, 103), (4, 104), (5, 105)) + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnRenameOutcomeTestCase( + description="declared rename keeps history without replay", + renamed_sql=orders_sql( + columns="amount AS revenue", + extra_config=REPLAY_FULL + migrate_columns("revenue (migrate_from amount)"), + ), + expected_planned=(("amount", "revenue", "manual", "rename"),), + expected_plan_fragment=( + "Column migrations (1)\n└── fct_orders migrate columns\n" + " └── amount -> revenue rename in place\n" + ), + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + expected_events=(("amount", "revenue", "manual", "rename"),), + ), + ColumnRenameOutcomeTestCase( + description="pure rename is detected automatically", + renamed_sql=orders_sql(columns="amount AS revenue", extra_config=REPLAY_FULL), + expected_planned=(("amount", "revenue", "automatic", "rename"),), + expected_plan_fragment=( + "Column migrations (1)\n└── fct_orders migrate columns\n" + " └── amount -> revenue rename in place (automatic)\n" + ), + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + expected_events=(("amount", "revenue", "automatic", "rename"),), + ), + ColumnRenameOutcomeTestCase( + description="declared near match renames and follows replay for the new expression", + renamed_sql=orders_sql( + columns="amount * 10 AS revenue", + extra_config=migrate_columns("revenue (migrate_from amount)"), + ), + expected_planned=(("amount", "revenue", "manual", "rename"),), + expected_plan_fragment=" └── amount -> revenue rename in place\n", + expected_columns=("order_id", "order_date", "revenue"), + expected_values=((1, 101), (2, 102), (3, 1030), (4, 1040), (5, 1050)), + expected_events=(("amount", "revenue", "manual", "rename"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_column_when_building_then_history_moves_to_the_new_name( + test_case: ColumnRenameOutcomeTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + change_historical_amount(project_dir=tmp_path) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + text: str = plan_text(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == test_case.expected_planned + assert test_case.expected_plan_fragment in text + assert "Renamed columns of main.fct_orders (amount -> revenue)" in result.output + assert column_names(project_dir=tmp_path) == test_case.expected_columns + assert ( + column_values(project_dir=tmp_path, column=test_case.value_column) + == test_case.expected_values + ) + assert column_events(project_dir=tmp_path) == test_case.expected_events + + +@pytest.mark.parametrize( + "test_case", + [ + RenameWithPolicyTestCase( + description="pure rename with a full replay policy is not replayed", + renamed_sql=orders_sql(columns="amount AS revenue", extra_config=REPLAY_FULL), + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + ), + ], + ids=lambda case: case.description, +) +def test_given_pure_rename_with_replay_policy_when_planning_then_model_continues_forward( + test_case: RenameWithPolicyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert model_plan(plan)["reason"] == "normal_incremental" + assert model_plan(plan)["backfill"]["action"] == "forward" + assert column_names(project_dir=tmp_path) == test_case.expected_columns + assert column_values(project_dir=tmp_path, column="revenue") == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + [ + RenameWithPolicyTestCase( + description="append_new_columns renames first, then adds the new column", + renamed_sql=orders_sql(columns="amount AS revenue, tax * 2 AS double_tax"), + expected_columns=("order_id", "order_date", "revenue", "double_tax"), + expected_values=_HISTORY, + added_column="double_tax", + expected_added_values=((1, None), (2, None), (3, 6), (4, 8), (5, 10)), + ), + RenameWithPolicyTestCase( + description="fail accepts a pure rename because no difference remains", + renamed_sql=orders_sql( + columns="amount AS revenue", extra_config=" on_schema_change fail,\n" + ), + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + ), + RenameWithPolicyTestCase( + description="sync_all_columns renames first, then drops the removed column", + renamed_sql=orders_sql( + columns="amount AS revenue", + extra_config=" on_schema_change sync_all_columns,\n", + ), + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + ), + ], + ids=lambda case: case.description, +) +def test_given_rename_and_schema_policy_when_building_then_rename_runs_before_the_policy( + test_case: RenameWithPolicyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys, sql=orders_sql()) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert column_names(project_dir=tmp_path) == test_case.expected_columns + assert column_values(project_dir=tmp_path, column="revenue") == test_case.expected_values + assert ( + column_values(project_dir=tmp_path, column=test_case.added_column) + == test_case.expected_added_values + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompletedColumnRenameTestCase( + description="recorded declared rename reports that migrate_from can be removed", + renamed_sql=orders_sql( + columns="amount AS revenue", + extra_config=migrate_columns("revenue (migrate_from amount)"), + ), + expected_planned=(("amount", "revenue", "manual", "done"),), + expected_notice=("migrate_from can be removed from column 'revenue' of 'fct_orders'"), + expected_final_line="Completed successfully", + ), + ], + ids=lambda case: case.description, +) +def test_given_recorded_rename_when_rebuilding_then_notice_precedes_the_success_line( + test_case: CompletedColumnRenameTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == test_case.expected_planned + assert "M108" in warning_codes(plan) + assert result.stdout.count(test_case.expected_notice) == 1 + assert test_case.expected_final_line in last_visible_line(result.stdout) + assert "Renaming columns" not in result.output + assert len(column_events(project_dir=tmp_path)) == 1 + + +@pytest.mark.parametrize( + "test_case", + [ + NearMatchPolicyTestCase( + description="append_new_columns keeps the old column and adds the new one", + renamed_sql=orders_sql(columns="ROUND(amount * 1.5, 2) AS revenue"), + expected_hint=( + "similar to amount; if this is a rename, add revenue (migrate_from amount)" + ), + expected_columns=("order_id", "order_date", "amount", "revenue"), + ), + NearMatchPolicyTestCase( + description="sync_all_columns drops the old column and adds the new one", + renamed_sql=orders_sql( + columns="ROUND(amount * 1.5, 2) AS revenue", + extra_config=" on_schema_change sync_all_columns,\n", + ), + expected_hint=( + "similar to amount; if this is a rename, add revenue (migrate_from amount)" + ), + expected_columns=("order_id", "order_date", "revenue"), + ), + ], + ids=lambda case: case.description, +) +def test_given_near_match_when_building_then_hint_shows_once_and_policy_applies( + test_case: NearMatchPolicyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + + before: str = plan_text(project_dir=tmp_path, capsys=capsys) + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + after: str = plan_text(project_dir=tmp_path, capsys=capsys) + + assert test_case.expected_hint in before + assert "+ revenue" in before + assert planned_column_migrations(plan) == () + assert model_plan(plan)["column_rename_hints"][0]["hint"] == test_case.expected_hint + assert test_case.expected_hint not in after + assert column_names(project_dir=tmp_path) == test_case.expected_columns + assert not query( + project_dir=tmp_path, + sql="SELECT 1 FROM information_schema.tables WHERE table_name = " + "'_sqlbuild_column_migrations'", + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotColumnRenameTestCase( + description="snapshot history moves to the renamed column", + renamed_sql=snapshot_sql(columns="amount AS revenue"), + expected_planned=(("amount", "revenue", "automatic", "rename"),), + expected_values=((1, 101), (2, 102), (2, 1102), (3, 103)), + ), + ], + ids=lambda case: case.description, +) +def test_given_snapshot_column_rename_when_building_then_versions_keep_their_values( + test_case: SnapshotColumnRenameTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys, sql=snapshot_sql()) + load_orders(project_dir=tmp_path, last_day=3) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + execute( + project_dir=tmp_path, + sql=( + "CREATE OR REPLACE VIEW main.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) " + "+ CASE WHEN i = 2 THEN INTERVAL 1 DAY ELSE INTERVAL 0 DAY END AS order_date, " + "CASE WHEN i = 2 THEN 1102 ELSE 100 + i END AS amount, i AS tax " + "FROM range(1, 4) AS t(i)" + ), + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == test_case.expected_planned + assert "amount" not in column_names(project_dir=tmp_path) + assert column_values(project_dir=tmp_path, column="revenue") == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + [ + RenameWithPolicyTestCase( + description="sequential microbatch rename keeps history and stays stateless", + renamed_sql=orders_sql(columns="amount AS revenue", extra_config=MICROBATCH_CONFIG), + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + ), + ], + ids=lambda case: case.description, +) +def test_given_microbatch_rename_when_building_then_batches_write_the_renamed_column( + test_case: RenameWithPolicyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial( + project_dir=tmp_path, capsys=capsys, sql=orders_sql(extra_config=MICROBATCH_CONFIG) + ) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == (("amount", "revenue", "automatic", "rename"),) + assert column_names(project_dir=tmp_path) == test_case.expected_columns + assert column_values(project_dir=tmp_path, column="revenue") == test_case.expected_values + assert "_sqlbuild_microbatches" not in state_table_names(project_dir=tmp_path) + + +@pytest.mark.parametrize( + "test_case", + [ + CursorColumnRenameTestCase( + description="renamed cursor column keeps the append watermark", + renamed_sql=orders_sql( + strategy="append", + cursor="ordered_at", + order_date="order_date AS ordered_at", + extra_config=( + " cursor_inputs (raw_orders order_date),\n" + + migrate_columns("ordered_at (migrate_from order_date)") + ), + ), + expected_planned=(("order_date", "ordered_at", "manual", "rename"),), + expected_order_ids=(2, 3, 4, 5), + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_cursor_column_when_appending_then_no_history_is_reprocessed( + test_case: CursorColumnRenameTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial( + project_dir=tmp_path, + capsys=capsys, + sql=orders_sql( + strategy="append", + extra_config=" append_cursor_inclusive false,\n", + ), + ) + write_model( + project_dir=tmp_path, + sql=test_case.renamed_sql.replace( + " cursor_inputs", " append_cursor_inclusive false,\n cursor_inputs" + ), + ) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == test_case.expected_planned + assert model_plan(plan)["cursor_bounds"]["start"].startswith("2026-01-03") + assert ( + tuple( + int(row[0]) + for row in query( + project_dir=tmp_path, sql=f"SELECT order_id FROM {RELATION} ORDER BY 1" + ) + ) + == test_case.expected_order_ids + ) + + +@pytest.mark.parametrize( + "test_case", + [ + FirstRunDeclarationTestCase( + description="declaration on a first build has nothing to migrate", + model_sql=orders_sql( + columns="amount AS revenue", + extra_config=migrate_columns("revenue (migrate_from amount)"), + ), + expected_planned=(), + expected_columns=("order_id", "order_date", "revenue"), + ), + ], + ids=lambda case: case.description, +) +def test_given_declaration_on_first_run_when_building_then_model_is_created_normally( + test_case: FirstRunDeclarationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_model(project_dir=tmp_path, sql=test_case.model_sql) + load_orders(project_dir=tmp_path, last_day=3) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == test_case.expected_planned + assert column_names(project_dir=tmp_path) == test_case.expected_columns + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 7e89d6ee63..8c847ee7b7 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -72,3 +72,27 @@ class AdapterTransactionalDdlTestCase: description: str adapter: BaseAdapter expected_transactional: bool + + +@dataclass(frozen=True) +class AdapterRenameColumnSqlTestCase: + description: str + adapter: StrictAdapter + destination: str + old_name: str + new_name: str + expected_statements: tuple[str, ...] + + +@dataclass(frozen=True) +class AdapterColumnMigrationStateTableTestCase: + description: str + adapter: StrictAdapter + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DatabricksColumnMappingTestCase: + description: str + property_rows: list[tuple[str, str]] + expected_available: bool diff --git a/tests/unit/src/sqlbuild/adapters/helpers.py b/tests/unit/src/sqlbuild/adapters/helpers.py new file mode 100644 index 0000000000..7c5a3613fa --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/helpers.py @@ -0,0 +1,31 @@ +"""Shared fakes for adapter unit tests.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + + +class FetchedRows: + """A cursor result that returns fixed rows.""" + + def __init__(self, rows: list[tuple[str, str]]) -> None: + self._rows: list[tuple[str, str]] = rows + + def fetchall(self) -> list[tuple[str, str]]: + """Return the fixed rows.""" + + return self._rows + + +def recording_execute( + *, rows: list[tuple[str, str]], executed: list[str] +) -> Callable[..., FetchedRows]: + """Return an adapter execute stub that records SQL and returns fixed rows.""" + + def execute(*, connection: Any, sql: str) -> FetchedRows: + del connection + executed.append(sql) + return FetchedRows(rows) + + return execute diff --git a/tests/unit/src/sqlbuild/adapters/test_rename_column.py b/tests/unit/src/sqlbuild/adapters/test_rename_column.py new file mode 100644 index 0000000000..78067bfac1 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/test_rename_column.py @@ -0,0 +1,253 @@ +"""Unit coverage for per-adapter in-place column rename SQL and column migration state.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from tests.unit.src.sqlbuild.adapters._test_types import ( + AdapterColumnMigrationStateTableTestCase, + AdapterRenameColumnSqlTestCase, + DatabricksColumnMappingTestCase, +) +from tests.unit.src.sqlbuild.adapters.helpers import recording_execute + +_DATABRICKS_TABLE: str = "`main`.`analytics`.`fct_orders`" + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterRenameColumnSqlTestCase( + description="duckdb renames in place", + adapter=DuckDbAdapter(), + destination="main.fct_orders", + old_name="amount", + new_name="revenue", + expected_statements=( + 'ALTER TABLE main.fct_orders RENAME COLUMN "amount" TO "revenue"', + ), + ), + AdapterRenameColumnSqlTestCase( + description="motherduck renames in place", + adapter=MotherDuckAdapter(), + destination="orders.main.fct_orders", + old_name="amount", + new_name="revenue", + expected_statements=( + 'ALTER TABLE orders.main.fct_orders RENAME COLUMN "amount" TO "revenue"', + ), + ), + AdapterRenameColumnSqlTestCase( + description="postgres folds the unquoted new name to lower case", + adapter=PostgresAdapter(), + destination='"analytics"."fct_orders"', + old_name="amount", + new_name="Revenue", + expected_statements=( + 'ALTER TABLE "analytics"."fct_orders" RENAME COLUMN "amount" TO "revenue"', + ), + ), + AdapterRenameColumnSqlTestCase( + description="snowflake keeps the physical old name and upper-cases the new name", + adapter=SnowflakeAdapter(), + destination="ANALYTICS.PUBLIC.FCT_ORDERS", + old_name="AMOUNT", + new_name="revenue", + expected_statements=( + 'ALTER TABLE ANALYTICS.PUBLIC.FCT_ORDERS RENAME COLUMN "AMOUNT" TO "REVENUE"', + ), + ), + AdapterRenameColumnSqlTestCase( + description="bigquery renames with backtick identifiers", + adapter=BigQueryAdapter(), + destination="`orders-project`.`analytics`.`fct_orders`", + old_name="amount", + new_name="revenue", + expected_statements=( + "ALTER TABLE `orders-project`.`analytics`.`fct_orders` " + "RENAME COLUMN `amount` TO `revenue`", + ), + ), + AdapterRenameColumnSqlTestCase( + description="databricks renames with backtick identifiers", + adapter=DatabricksAdapter(), + destination="`main`.`analytics`.`fct_orders`", + old_name="amount", + new_name="revenue", + expected_statements=( + "ALTER TABLE `main`.`analytics`.`fct_orders` RENAME COLUMN `amount` TO `revenue`", + ), + ), + AdapterRenameColumnSqlTestCase( + description="sql server renames through sp_rename", + adapter=SqlServerAdapter(), + destination="[orders].[analytics].[fct_orders]", + old_name="amount", + new_name="revenue", + expected_statements=( + "EXEC sp_rename N'[analytics].[fct_orders].[amount]', N'revenue', 'COLUMN'", + ), + ), + AdapterRenameColumnSqlTestCase( + description="sql server escapes quotes in both names", + adapter=SqlServerAdapter(), + destination="[analytics].[fct_orders]", + old_name="o'amount", + new_name="o'revenue", + expected_statements=( + "EXEC sp_rename N'[analytics].[fct_orders].[o''amount]', N'o''revenue', 'COLUMN'", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_when_rendering_column_rename_then_uses_the_engine_statement( + test_case: AdapterRenameColumnSqlTestCase, +) -> None: + statements: tuple[str, ...] = test_case.adapter.render_rename_column( + destination=test_case.destination, + old_name=test_case.old_name, + new_name=test_case.new_name, + ) + + assert statements == test_case.expected_statements + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterColumnMigrationStateTableTestCase( + description="duckdb creates the column migration table when missing", + adapter=DuckDbAdapter(), + expected_fragments=( + "CREATE TABLE IF NOT EXISTS main._sqlbuild_column_migrations", + "origin_column", + "destination_column", + ), + ), + AdapterColumnMigrationStateTableTestCase( + description="snowflake state tables follow the transient state setting", + adapter=SnowflakeAdapter(), + expected_fragments=( + "CREATE TRANSIENT TABLE IF NOT EXISTS main._sqlbuild_column_migrations", + "origin_column", + ), + ), + AdapterColumnMigrationStateTableTestCase( + description="sql server guards the create with an existence check", + adapter=SqlServerAdapter(), + expected_fragments=( + "IF NOT EXISTS (SELECT 1 FROM information_schema.tables", + "table_name = '_sqlbuild_column_migrations'", + "CREATE TABLE main._sqlbuild_column_migrations (", + "created_at DATETIME2", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_when_rendering_column_migration_state_table_then_ddl_is_adapter_native( + test_case: AdapterColumnMigrationStateTableTestCase, +) -> None: + sql: str = test_case.adapter.render_create_column_migration_state_table_sql( + database=None, schema="main" + ) + + assert all(fragment in sql for fragment in test_case.expected_fragments), sql + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksColumnMappingTestCase( + description="name column mapping allows renames", + property_rows=[("delta.columnMapping.mode", "name")], + expected_available=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_databricks_column_mapping_when_renaming_then_renames_in_place( + test_case: DatabricksColumnMappingTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter: DatabricksAdapter = DatabricksAdapter() + executed: list[str] = [] + monkeypatch.setattr( + adapter, "execute", recording_execute(rows=test_case.property_rows, executed=executed) + ) + + reason: str | None = adapter.column_rename_unavailable_reason( + connection=None, destination=_DATABRICKS_TABLE + ) + adapter.rename_column( + connection=None, + destination=_DATABRICKS_TABLE, + old_name="amount", + new_name="revenue", + statement_recorder=StatementRecorder(), + ) + + assert (reason is None) is test_case.expected_available + assert executed[-1] == f"ALTER TABLE {_DATABRICKS_TABLE} RENAME COLUMN `amount` TO `revenue`" + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksColumnMappingTestCase( + description="missing column mapping refuses renames", + property_rows=[ + ( + "delta.columnMapping.mode", + "Table main.analytics.fct_orders does not have property: " + "delta.columnMapping.mode", + ) + ], + expected_available=False, + ), + DatabricksColumnMappingTestCase( + description="id column mapping refuses renames", + property_rows=[("delta.columnMapping.mode", "id")], + expected_available=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_databricks_without_name_mapping_when_renaming_then_fails_before_altering( + test_case: DatabricksColumnMappingTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter: DatabricksAdapter = DatabricksAdapter() + executed: list[str] = [] + monkeypatch.setattr( + adapter, "execute", recording_execute(rows=test_case.property_rows, executed=executed) + ) + + reason: str | None = adapter.column_rename_unavailable_reason( + connection=None, destination=_DATABRICKS_TABLE + ) + with pytest.raises(AdapterUserError, match="delta.columnMapping.mode"): + adapter.rename_column( + connection=None, + destination=_DATABRICKS_TABLE, + old_name="amount", + new_name="revenue", + statement_recorder=StatementRecorder(), + ) + + assert (reason is None) is test_case.expected_available + assert reason is not None + assert "SET TBLPROPERTIES ('delta.columnMapping.mode' = 'name'" in reason + assert all("RENAME COLUMN" not in sql for sql in executed) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index 739ad650e8..e12ddccf95 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -13,6 +13,7 @@ from sqlbuild.compiler.planner.models import ( BackfillResult, CascadeResult, + ColumnRenameHint, CursorBounds, CursorInputRelation, FunctionPlanEntry, @@ -74,6 +75,7 @@ def build_model_entry( microbatch_limit_count: int | None = None, microbatch_limit_action: MicrobatchLimitAction | None = None, microbatch_limit_warning: str | None = None, + column_rename_hints: tuple[ColumnRenameHint, ...] = (), ) -> ModelPlanEntry: """Build a minimal ModelPlanEntry for formatter tests.""" @@ -119,6 +121,7 @@ def build_model_entry( microbatch_limit_count=microbatch_limit_count, microbatch_limit_action=microbatch_limit_action, microbatch_limit_warning=microbatch_limit_warning, + column_rename_hints=column_rename_hints, ) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 0cc878cffa..c49569f7e1 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -12,6 +12,7 @@ from sqlbuild.compiler.planner.models import ( CascadeCause, CascadeResult, + ColumnRenameHint, CursorBounds, CursorInputRelation, ) @@ -428,6 +429,39 @@ "added", ), ), + FormatPlanTestCase( + description="near-match rename hint follows the added column", + plan_output=build_plan_output( + model_entries=( + build_model_entry( + name="fct_orders", + action=PlanAction.INCREMENTAL_APPEND, + reason=PlanReason.QUERY_CHANGED, + materialization_type=MaterializationType.INCREMENTAL, + backfill_action=BackfillAction.FORWARD_ONLY, + schema_findings=( + build_schema_finding( + kind=SchemaChangeKind.COLUMN_ADDED, column_name="revenue" + ), + build_schema_finding( + kind=SchemaChangeKind.COLUMN_REMOVED, column_name="amount" + ), + ), + column_rename_hints=( + ColumnRenameHint( + model_name="fct_orders", + added_column="revenue", + candidate_columns=("amount",), + ), + ), + ), + ), + ), + expected_fragments=( + "+ revenue (added)\n similar to amount; if this is a rename, add " + "revenue (migrate_from amount)\n - amount (removed)", + ), + ), FormatPlanTestCase( description="config changed shows config diff without query diff", plan_output=build_plan_output( diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index 8621359dd7..c320687fcc 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -18,3 +18,20 @@ class RepeatedEventWriteTestCase: description: str write_count: int expected_stored_count: int + + +@dataclass(frozen=True) +class ColumnEventStorageTestCase: + description: str + write_count: int + foreign_decisions: tuple[str, ...] + expected_stored_count: int + + +@dataclass(frozen=True) +class NewestColumnEventTestCase: + description: str + renames: tuple[tuple[str, str], ...] + origin_column: str + destination_column: str + expected_newest_index: int | None diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py index c39ce64147..68708910a6 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -4,11 +4,22 @@ from datetime import UTC, datetime, timedelta +from sqlbuild.compiler.migrations.main.deterministic_column_event_id import ( + deterministic_column_migration_event_id, +) from sqlbuild.compiler.migrations.main.deterministic_event_id import ( deterministic_migration_event_id, ) -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDecision, MigrationDiscovery +from sqlbuild.compiler.migrations.models import ( + ColumnMigrationEvent, + MigrationEvent, + MigrationRelation, +) +from sqlbuild.compiler.migrations.types import ( + ColumnMigrationDecision, + MigrationDecision, + MigrationDiscovery, +) _STARTED_AT: datetime = datetime(2026, 1, 1, tzinfo=UTC) @@ -40,3 +51,27 @@ def migration_event(*, origin: str, destination: str, day: int) -> MigrationEven run_id=f"run-{day}", created_at=_STARTED_AT + timedelta(days=day), ) + + +def column_migration_event(*, origin: str, destination: str, day: int) -> ColumnMigrationEvent: + """Return a manual column rename on main.fct_orders recorded on the given day.""" + + relation: MigrationRelation = main_relation("fct_orders") + return ColumnMigrationEvent( + event_id=deterministic_column_migration_event_id( + run_id=f"run-{day}", + target_name="dev", + relation=relation, + origin_column=origin, + destination_column=destination, + ), + target_name="dev", + model_name="fct_orders", + relation=relation, + origin_column=origin, + destination_column=destination, + discovery=MigrationDiscovery.MANUAL, + decision=ColumnMigrationDecision.RENAME, + run_id=f"run-{day}", + created_at=_STARTED_AT + timedelta(days=day), + ) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py new file mode 100644 index 0000000000..cd49f9f219 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py @@ -0,0 +1,128 @@ +"""Unit coverage for append-only column migration event storage and projection.""" + +from __future__ import annotations + +import duckdb +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.migrations.main._newest_column_event import ( + newest_column_migration_event_mentioning, +) +from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events +from sqlbuild.compiler.migrations.main.write_column_event import write_column_migration_event +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent +from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( + ColumnEventStorageTestCase, + NewestColumnEventTestCase, +) +from tests.unit.src.sqlbuild.compiler.migrations.main.helpers import ( + column_migration_event, + main_relation, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnEventStorageTestCase( + description="writing the same event twice stores it once", + write_count=2, + foreign_decisions=(), + expected_stored_count=1, + ), + ColumnEventStorageTestCase( + description="rows with a decision from a newer version are skipped", + write_count=1, + foreign_decisions=("renamed_by_a_future_version",), + expected_stored_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_column_events_when_writing_and_reading_then_rows_are_idempotent( + test_case: ColumnEventStorageTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + event: ColumnMigrationEvent = column_migration_event( + origin="amount", destination="revenue", day=1 + ) + + for _ in range(test_case.write_count): + write_column_migration_event( + connection=connection, + execute=adapter.execute, + event=event, + render_qualified_name=adapter.render_qualified_name, + create_table_sql=adapter.render_create_column_migration_state_table_sql( + database=None, schema="main" + ), + ) + for foreign_decision in test_case.foreign_decisions: + _ = connection.execute( + "INSERT INTO main._sqlbuild_column_migrations (event_id, model_name, relation_name, " + "origin_column, destination_column, discovery, decision, run_id, created_at) VALUES " + f"('future', 'fct_orders', 'fct_orders', 'tax', 'levy', 'manual', " + f"'{foreign_decision}', 'run-9', TIMESTAMP '2026-01-09')" + ) + stored: tuple[ColumnMigrationEvent, ...] = read_column_migration_events( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + ) + + assert len(stored) == test_case.expected_stored_count + assert stored == (event,) + + +@pytest.mark.parametrize( + "test_case", + [ + NewestColumnEventTestCase( + description="latest rename of either column wins", + renames=(("amount", "revenue"), ("revenue", "amount")), + origin_column="amount", + destination_column="revenue", + expected_newest_index=1, + ), + NewestColumnEventTestCase( + description="column names are case insensitive", + renames=(("AMOUNT", "Revenue"),), + origin_column="amount", + destination_column="revenue", + expected_newest_index=0, + ), + NewestColumnEventTestCase( + description="unrelated columns have no event", + renames=(("tax", "levy"),), + origin_column="amount", + destination_column="revenue", + expected_newest_index=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_column_event_history_when_projecting_newest_mention_then_returns_latest_event( + test_case: NewestColumnEventTestCase, +) -> None: + events: tuple[ColumnMigrationEvent, ...] = tuple( + column_migration_event(origin=origin, destination=destination, day=index + 1) + for index, (origin, destination) in enumerate(test_case.renames) + ) + events_by_index: dict[int | None, ColumnMigrationEvent] = dict(enumerate(events)) + + newest: ColumnMigrationEvent | None = newest_column_migration_event_mentioning( + events=reversed(events), + relation=main_relation("fct_orders"), + origin_column=test_case.origin_column, + destination_column=test_case.destination_column, + ) + + assert newest == events_by_index.get(test_case.expected_newest_index) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py index 5d3d916a71..ca5ba6bd9f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py @@ -23,3 +23,37 @@ class IneligibleFingerprintTestCase: description: str query_sql: str expected_fingerprint: str | None + + +@dataclass(frozen=True) +class IdenticalRenameTestCase: + description: str + previous_sql: str + current_sql: str + expected_renames: tuple[tuple[str, str], ...] + excluded: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class RenameHintTestCase: + description: str + previous_sql: str + current_sql: str + live_columns: frozenset[str] + expected_hints: tuple[str, ...] + + +@dataclass(frozen=True) +class RenameExplainsChangeTestCase: + description: str + previous_sql: str + current_sql: str + renames: dict[str, str] + expected_explained: bool + + +@dataclass(frozen=True) +class UnreadableQueryShapeTestCase: + description: str + query_sql: str + expected_shape: None = None diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index d5a94a4f24..6ea4fd3c2b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -5,6 +5,8 @@ import json from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint +from sqlbuild.compiler.planner._helpers.migrations.projections import parse_query_shape +from sqlbuild.compiler.planner.models import QueryShape BASE_CONFIG: dict[str, object] = { "materialized": "incremental", @@ -26,3 +28,11 @@ def model_fingerprint( ref_identities=ref_identities, dialect="duckdb", ) + + +def query_shape(query_sql: str) -> QueryShape: + """Return the DuckDB query shape of one readable model query.""" + + shape: QueryShape | None = parse_query_shape(query_sql=query_sql, dialect="duckdb") + assert shape is not None, query_sql + return shape diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py new file mode 100644 index 0000000000..c260f6f047 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py @@ -0,0 +1,219 @@ +"""Unit coverage for recognizing renamed output columns by expression.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.planner._helpers.migrations.column_renames import ( + identical_renames, + rename_hints, +) +from sqlbuild.compiler.planner._helpers.migrations.projections import ( + parse_query_shape, + renames_explain_change, +) +from sqlbuild.compiler.planner.models import ColumnRenameHint +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( + IdenticalRenameTestCase, + RenameExplainsChangeTestCase, + RenameHintTestCase, + UnreadableQueryShapeTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import query_shape + +_ORDERS: str = 'FROM __source("raw_orders")' + + +@pytest.mark.parametrize( + "test_case", + [ + IdenticalRenameTestCase( + description="aliasing a column is a pure rename", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS}", + expected_renames=(("amount", "revenue"),), + ), + IdenticalRenameTestCase( + description="formatting, comments, and keyword case are ignored", + previous_sql=f"SELECT order_id, ROUND(amount, 2) AS revenue {_ORDERS}", + current_sql=( + "select order_id,\n round( AMOUNT ,2 ) as net_revenue -- renamed\n" + 'from __source("raw_orders")' + ), + expected_renames=(("revenue", "net_revenue"),), + ), + IdenticalRenameTestCase( + description="two columns renamed crosswise pair by expression, not position", + previous_sql=f"SELECT order_id, amount AS gross, tax AS levy {_ORDERS}", + current_sql=f"SELECT order_id, tax AS tax_amount, amount AS gross_amount {_ORDERS}", + expected_renames=(("levy", "tax_amount"), ("gross", "gross_amount")), + ), + IdenticalRenameTestCase( + description="swapped names keep their names, so nothing is renamed", + previous_sql=f"SELECT order_id, amount AS gross, tax AS levy {_ORDERS}", + current_sql=f"SELECT order_id, tax AS gross, amount AS levy {_ORDERS}", + expected_renames=(), + ), + IdenticalRenameTestCase( + description="a near match is not a rename", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, ROUND(amount, 2) AS revenue {_ORDERS}", + expected_renames=(), + ), + IdenticalRenameTestCase( + description="two removed columns with the same expression are ambiguous", + previous_sql=f"SELECT order_id, amount AS gross, amount AS total {_ORDERS}", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS}", + expected_renames=(), + ), + IdenticalRenameTestCase( + description="two added columns claiming one removed column are ambiguous", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, amount AS gross, amount AS revenue {_ORDERS}", + expected_renames=(), + ), + IdenticalRenameTestCase( + description="declared columns are left to their declaration", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS}", + excluded=frozenset({"revenue"}), + expected_renames=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_previous_and_current_queries_when_detecting_renames_then_returns_unique_matches( + test_case: IdenticalRenameTestCase, +) -> None: + renames: tuple[tuple[str, str], ...] = identical_renames( + previous=query_shape(test_case.previous_sql), + current=query_shape(test_case.current_sql), + excluded=test_case.excluded, + ) + + assert renames == test_case.expected_renames + + +@pytest.mark.parametrize( + "test_case", + [ + RenameHintTestCase( + description="near match points at migrate_from", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, ROUND(amount, 2) AS revenue {_ORDERS}", + live_columns=frozenset({"order_id", "amount"}), + expected_hints=( + "similar to amount; if this is a rename, add revenue (migrate_from amount)", + ), + ), + RenameHintTestCase( + description="ambiguous identical matches name every candidate", + previous_sql=f"SELECT order_id, amount AS gross, amount AS total {_ORDERS}", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS}", + live_columns=frozenset({"order_id", "gross", "total"}), + expected_hints=( + "same expression as gross, total; if this is a rename, add revenue " + "(migrate_from )", + ), + ), + RenameHintTestCase( + description="hint disappears once the added column exists", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, ROUND(amount, 2) AS revenue {_ORDERS}", + live_columns=frozenset({"order_id", "amount", "revenue"}), + expected_hints=(), + ), + RenameHintTestCase( + description="unrelated new column gets no hint", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, discount * 2 AS promo {_ORDERS}", + live_columns=frozenset({"order_id", "amount"}), + expected_hints=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_changed_columns_when_building_hints_then_suggests_explicit_migrate_from( + test_case: RenameHintTestCase, +) -> None: + hints: tuple[ColumnRenameHint, ...] = rename_hints( + model_name="fct_orders", + previous=query_shape(test_case.previous_sql), + current=query_shape(test_case.current_sql), + excluded=frozenset(), + live_columns=test_case.live_columns, + ) + + assert tuple(hint.message for hint in hints) == test_case.expected_hints + + +@pytest.mark.parametrize( + "test_case", + [ + RenameExplainsChangeTestCase( + description="pure rename explains the whole change", + previous_sql=f"SELECT order_id, amount {_ORDERS} WHERE amount > 0", + current_sql=f"select order_id, amount as revenue {_ORDERS} where amount > 0", + renames={"amount": "revenue"}, + expected_explained=True, + ), + RenameExplainsChangeTestCase( + description="an expression change is more than a rename", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, ROUND(amount, 2) AS revenue {_ORDERS}", + renames={"amount": "revenue"}, + expected_explained=False, + ), + RenameExplainsChangeTestCase( + description="a filter change is more than a rename", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS} WHERE amount > 0", + renames={"amount": "revenue"}, + expected_explained=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_renames_when_comparing_queries_then_reports_whether_they_explain_the_change( + test_case: RenameExplainsChangeTestCase, +) -> None: + explained: bool = renames_explain_change( + previous=query_shape(test_case.previous_sql), + current=query_shape(test_case.current_sql), + renames=test_case.renames, + ) + + assert explained is test_case.expected_explained + + +@pytest.mark.parametrize( + "test_case", + [ + UnreadableQueryShapeTestCase( + description="star projection", query_sql=f"SELECT * {_ORDERS}" + ), + UnreadableQueryShapeTestCase( + description="unnamed expression", query_sql=f"SELECT amount + 1 {_ORDERS}" + ), + UnreadableQueryShapeTestCase( + description="set operation", + query_sql=f"SELECT amount {_ORDERS} UNION ALL SELECT amount {_ORDERS}", + ), + UnreadableQueryShapeTestCase( + description="duplicate output names", + query_sql=f"SELECT amount, tax AS amount {_ORDERS}", + ), + ], + ids=lambda case: case.description, +) +def test_given_query_without_readable_output_names_when_parsing_then_returns_none( + test_case: UnreadableQueryShapeTestCase, +) -> None: + assert ( + parse_query_shape(query_sql=test_case.query_sql, dialect="duckdb") + is test_case.expected_shape + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py index 58de063742..5604afa7d6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py @@ -1,6 +1,11 @@ from dataclasses import dataclass -from sqlbuild.compiler.migrations.types import MigrationCompatibility, MigrationDecision +from sqlbuild.compiler.migrations.types import ( + ColumnMigrationDecision, + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, +) @dataclass(frozen=True) @@ -17,3 +22,17 @@ class MigrationPlainTextTestCase: description: str decision: MigrationDecision expected_text: str + + +@dataclass(frozen=True) +class ColumnMigrationPlainTextTestCase: + description: str + entries: tuple[tuple[str, str, str, ColumnMigrationDecision, MigrationDiscovery], ...] + expected_text: str + + +@dataclass(frozen=True) +class ColumnMigrationStyleTestCase: + description: str + decision: ColumnMigrationDecision + expected_fragment: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py index 50b408132d..7091b8cc87 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py @@ -5,13 +5,18 @@ from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.types import ( + ColumnMigrationDecision, MigrationCompatibility, MigrationDecision, MigrationDiscovery, MigrationPromotion, ) from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import format_pre_build_work -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, PlanOutput +from sqlbuild.compiler.planner.models import ( + ColumnMigrationPlanEntry, + ModelMigrationPlanEntry, + PlanOutput, +) from sqlbuild.presentation.models import DisplayOptions @@ -53,3 +58,26 @@ def styled_migration_text(entry: ModelMigrationPlanEntry) -> str: lines=[], plan=PlanOutput(migration_entries=(entry,)), display_options=DisplayOptions() ) ) + + +def column_migration_entry( + *, + model_name: str, + origin_column: str, + destination_column: str, + decision: ColumnMigrationDecision, + discovery: MigrationDiscovery, +) -> ColumnMigrationPlanEntry: + """Return one column rename on a prod model relation.""" + + return ColumnMigrationPlanEntry( + model_name=model_name, + destination=CompiledRelationLocation( + database=None, schema="prod", name=model_name, qualified_name=f"prod.{model_name}" + ), + origin_column=origin_column, + destination_column=destination_column, + discovery=discovery, + decision=decision, + target_name=None, + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index d1a7356104..1e94d810b9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -5,13 +5,23 @@ import pytest from sqlbuild.cli.output.main.plan import format_plan -from sqlbuild.compiler.migrations.types import MigrationCompatibility, MigrationDecision +from sqlbuild.compiler.migrations.types import ( + ColumnMigrationDecision, + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, +) +from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import format_pre_build_work from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.presentation.models import DisplayOptions from tests.unit.src.sqlbuild.compiler.planner.main.pre_build._test_types import ( + ColumnMigrationPlainTextTestCase, + ColumnMigrationStyleTestCase, MigrationDecisionStyleTestCase, MigrationPlainTextTestCase, ) from tests.unit.src.sqlbuild.compiler.planner.main.pre_build.helpers import ( + column_migration_entry, migration_entry, styled_migration_text, ) @@ -146,5 +156,131 @@ def test_given_colour_off_when_formatting_plan_then_migration_text_is_plain( assert "\033[" not in rendered +@pytest.mark.parametrize( + "test_case", + ( + ColumnMigrationPlainTextTestCase( + description="one pending rename renders as the column migration tree", + entries=( + ( + "fact_orders", + "amount", + "revenue", + ColumnMigrationDecision.RENAME, + MigrationDiscovery.MANUAL, + ), + ), + expected_text=( + "Column migrations (1)\n" + "└── fact_orders migrate columns\n" + " └── amount -> revenue rename in place" + ), + ), + ColumnMigrationPlainTextTestCase( + description="renames group under each model and mark automatic discovery", + entries=( + ( + "fact_orders", + "amount", + "revenue", + ColumnMigrationDecision.RENAME, + MigrationDiscovery.AUTOMATIC, + ), + ( + "fact_orders", + "tax", + "tax_amount", + ColumnMigrationDecision.RECORD, + MigrationDiscovery.MANUAL, + ), + ( + "order_history", + "status", + "order_status", + ColumnMigrationDecision.RENAME, + MigrationDiscovery.MANUAL, + ), + ), + expected_text=( + "Column migrations (3)\n" + "├── fact_orders migrate columns\n" + " ├── amount -> revenue rename in place (automatic)\n" + " └── tax -> tax_amount already renamed, record\n" + "└── order_history migrate columns\n" + " └── status -> order_status rename in place" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_column_migrations_when_formatting_plan_then_renders_a_model_tree( + test_case: ColumnMigrationPlainTextTestCase, +) -> None: + rendered: str = format_plan( + plan=PlanOutput( + column_migration_entries=tuple( + column_migration_entry( + model_name=model_name, + origin_column=origin, + destination_column=destination, + decision=decision, + discovery=discovery, + ) + for model_name, origin, destination, decision, discovery in test_case.entries + ) + ), + use_color=False, + include_header=False, + ) + + assert rendered.strip("\n") == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + ( + ColumnMigrationStyleTestCase( + description="pending rename is bold blue", + decision=ColumnMigrationDecision.RENAME, + expected_fragment=f"{_BOLD_BLUE}rename in place{_RESET}", + ), + ColumnMigrationStyleTestCase( + description="done is dim", + decision=ColumnMigrationDecision.DONE, + expected_fragment=f"{_DIM}done{_RESET}", + ), + ColumnMigrationStyleTestCase( + description="conflict is bold red", + decision=ColumnMigrationDecision.CONFLICT, + expected_fragment=f"{_BOLD_RED}conflict{_RESET}", + ), + ), + ids=lambda case: case.description, +) +def test_given_column_migration_decision_when_formatting_then_applies_colour_roles( + test_case: ColumnMigrationStyleTestCase, +) -> None: + rendered: str = "\n".join( + format_pre_build_work( + lines=[], + plan=PlanOutput( + column_migration_entries=( + column_migration_entry( + model_name="fact_orders", + origin_column="amount", + destination_column="revenue", + decision=test_case.decision, + discovery=MigrationDiscovery.MANUAL, + ), + ) + ), + display_options=DisplayOptions(), + ) + ) + + assert f"\033[1mColumn migrations{_RESET} {_DIM}(1){_RESET}" in rendered + assert test_case.expected_fragment in rendered, rendered + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From f4ade5dd918ced940fea00452131338cd5a32d7a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 07:42:35 +0100 Subject: [PATCH 3/9] docs(migrations): document column migrations and regenerate skill references Refs CHI-393 --- .../sqlbuild/references/docs/CONTENTS.md | 1 + .../sqlbuild/references/docs/cli/plan.md | 17 +++ .../references/docs/concepts/incremental.md | 2 + .../docs/concepts/models/column-migrations.md | 135 ++++++++++++++++++ .../docs/concepts/models/configuration.md | 2 +- .../docs/concepts/models/migrations.md | 2 +- .../docs/concepts/models/schemas.md | 4 +- .../references/docs/concepts/snapshots.md | 3 + website/src/content/docs/docs/cli/plan.mdx | 10 ++ .../docs/docs/concepts/incremental.mdx | 2 + .../concepts/models/column-migrations.mdx | 123 ++++++++++++++++ .../docs/concepts/models/configuration.mdx | 2 +- .../docs/docs/concepts/models/migrations.mdx | 2 +- .../docs/docs/concepts/models/schemas.mdx | 4 +- .../content/docs/docs/concepts/snapshots.mdx | 3 + website/src/sidebar.json | 1 + 16 files changed, 307 insertions(+), 6 deletions(-) create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md create mode 100644 website/src/content/docs/docs/concepts/models/column-migrations.mdx diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index b9111d6836..40d21e2602 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -39,6 +39,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Type Enforcement](concepts/models/type-enforcement.md) (`concepts/models/type-enforcement`) - Understand declared model types, static checks, and runtime casting. - [Contracts](concepts/models/contracts.md) (`concepts/models/contracts`) - Validate required or exact model output schemas. - [Model migrations](concepts/models/migrations.md) (`concepts/models/migrations`) - Keep the history of an incremental or snapshot model when you rename it. +- [Column migrations](concepts/models/column-migrations.md) (`concepts/models/column-migrations`) - Keep a column's history when you rename it in an incremental or snapshot model. - [Hooks](concepts/models/hooks.md) (`concepts/models/hooks`) - Run SQL or Python lifecycle hooks around model materialization. - [SQL Hooks](concepts/models/hooks/sql.md) (`concepts/models/hooks/sql`) - Define, parameterize, compile, and invoke reusable or inline SQL lifecycle hooks. - [Python Hooks](concepts/models/hooks/python.md) (`concepts/models/hooks/python`) - Define Python lifecycle hooks with runtime context, providers, SQL access, and skips. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 7ee4cf7549..50518bdaab 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -6,6 +6,13 @@ Online: https://sqlbuild.com/docs/cli/plan/ +## Contents + +- Usage +- Flags +- Output +- Missing upstream dependencies + Shows the execution plan without making any changes. Useful for inspecting change detection, backfill policies, and selector scope before building. ## Usage @@ -76,6 +83,16 @@ Seeds (1) When query or schema changes are detected, the plan shows the affected models with backfill actions and cascade information. +Work that runs before models build is listed first. [Model migrations](../concepts/models/migrations.md) appear under `Migrations`, and in-place [column renames](../concepts/models/column-migrations.md) under `Column migrations`: + +``` +Column migrations (1) +└── fact_orders migrate columns + └── amount -> revenue rename in place +``` + +When a removed column and an added column look related but are not an exact rename, the schema diff shows a hint such as `similar to amount; if this is a rename, add revenue (migrate_from amount)` until the change is built. + ## Missing upstream dependencies Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index 4af85a13c2..04eebb38e0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -393,3 +393,5 @@ Controls how schema differences are handled at execution time when the increment | `sync_all_columns` | Add, drop, and alter columns to match the delta | | `ignore` | Log and continue without schema changes | | `fail` | Reject the build with an error | + +Columns are compared by name, so a renamed column looks like one column removed and one added. [Column migrations](models/column-migrations.md) rename the warehouse column in place before `on_schema_change` runs, so its history keeps its values; the policy then applies only to the differences that remain. Pure renames such as `amount` becoming `amount AS revenue` are detected automatically, and `sqb plan` suggests `migrate_from` for near matches. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md new file mode 100644 index 0000000000..61c534a2f1 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md @@ -0,0 +1,135 @@ + + +# Column migrations + +> Keep a column's history when you rename it in an incremental or snapshot model. + +Online: https://sqlbuild.com/docs/concepts/models/column-migrations/ + +## Contents + +- Declaring a rename +- Automatic detection +- Decisions +- How a rename runs +- State +- Adapter support + +`on_schema_change` compares columns by name, so renaming a column looks like one column removed and another added. With the default `append_new_columns`, the new column starts empty and older rows keep their values under the old name; with `sync_all_columns`, the old column and its data are dropped. A column migration renames the column in the warehouse instead, so every row keeps its value under the new name, with no leftover column and no replay. + +Column migrations apply to incremental models, including microbatch models, and snapshot models. Tables and views are rebuilt on every change and don't need them. + +## Declaring a rename + +Add `migrate_from` to the renamed column in the model header, naming the old column: + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + unique_key order_id, + cursor order_date, + columns ( + revenue (migrate_from amount), + ), +); + +SELECT order_id, order_date, amount AS revenue +FROM __source("raw_orders") +``` + +On the next build, SQLBuild renames `amount` to `revenue` in the existing table, records the rename, and then builds the model incrementally, so the new delta lands in `revenue`. + +A column entry may contain only `migrate_from`, or combine it with `type`, `nullable`, `description`, and `audits`. When the model binds a reusable schema with `model_schema`, an inherited column may add `migrate_from` without redeclaring its metadata. + +After the rename is recorded, `migrate_from` has no further effect, and SQLBuild reports (`M108`) that it can be removed. Keep it until every target that needs the rename has built. On a target where the table doesn't exist yet, the declaration does nothing: the table is created with the new column name. + +A declaration is rejected when compiling if: + +- the model is not incremental or snapshot; +- `migrate_from` names the column itself, or a qualified name instead of one column; +- two columns declare the same `migrate_from`; +- a declared column migrates from another column that itself declares `migrate_from`. Chains and swaps can't be renamed in place safely; rename through a name no other column uses, or rebuild with `--full-refresh`. + +## Automatic detection + +A pure rename is detected without a declaration: exactly one column that disappeared from the model has exactly the same expression as exactly one new column, for example `amount` becoming `amount AS revenue`, or `ROUND(amount, 2) AS revenue` becoming `ROUND(amount, 2) AS net_revenue`. Expressions are compared ignoring formatting, comments, and keyword and unquoted identifier case. Detection compares the query the table was last built from with the current query, so it applies to the build that makes the change. + +Detection never guesses: + +- Columns are matched by expression, not by position. Two columns renamed at once each pair with the column whose expression they kept; two columns that swap names keep their names and are not renamed. +- If more than one removed column has the same expression, or more than one new column does, nothing is migrated automatically. +- A near match, where the expressions reference the same columns but differ, for example `amount` becoming `ROUND(amount, 2) AS revenue`, is not a rename. It follows the model's `on_schema_change` policy, and `sqb plan` shows a hint next to the added column until the change has been built: + +```text + ├── schema diff: + + revenue (added) + similar to amount; if this is a rename, add revenue (migrate_from amount) + - amount (removed) +``` + +Add the suggested `migrate_from` to keep the history. A declared rename with a changed expression is still renamed in place, but it is a query change, so `replay_on_change` decides how much history is rebuilt with the new expression. + +## Decisions + +`sqb plan` and `sqb build` list column migrations before the models they change: + +```text +Column migrations (1) +└── fact_orders migrate columns + └── amount -> revenue rename in place +``` + +Automatically detected renames are marked `(automatic)`. The decision is based on the table's current columns first, then on recorded renames: + +| Decision | Meaning | +|----------|---------| +| `rename in place` | The old column exists and the new one doesn't. Rename it. | +| `already renamed, record` | The new column exists and the old one doesn't, but no rename is recorded, which is what an interrupted rename leaves behind. Record it without renaming again. | +| `done` | The rename is recorded and the new column exists. Nothing to do. | +| `conflict` | Both columns exist. The build stops (`M110`); drop one of them, or remove `migrate_from` and let `on_schema_change` handle the columns. | +| `source missing` | Neither column exists and no rename is recorded. The build stops (`M109`); remove `migrate_from` if the rename happened elsewhere or is no longer needed. | +| `source still produced` | The query still selects the old column. The build stops (`M111`). | +| `unsupported` | The warehouse can't rename this table's columns in place (see [Adapter support](#adapter-support)). The build stops (`M112`). | + +Automatically detected renames never stop a build: when they can't be applied, the columns follow `on_schema_change` as before. + +## How a rename runs + +Renames run after [model migrations](migrations.md) and before any model builds, one model at a time: + +1. The table's current columns are read again. A column that is already renamed is not renamed twice. +2. The column is renamed with `ALTER TABLE ... RENAME COLUMN` or the adapter's equivalent. +3. The rename is recorded. + +On adapters with transactional DDL (PostgreSQL, SQL Server, DuckDB, MotherDuck), each model's renames and their records commit together. On other adapters, a build interrupted between the rename and the record is recovered by the `already renamed, record` decision on the next build. A build that fails after the rename, for example while applying the delta, resumes on the next build without renaming again. + +After the rename, the remaining differences follow `on_schema_change`: `append_new_columns` adds any other new columns, `sync_all_columns` also drops removed ones and changes types, and `fail` only fails when a difference other than the rename remains. + +A pure rename is not a query change. It doesn't trigger `replay_on_change` for the model, and it doesn't cascade a replay to downstream models. Downstream models that you edit to use the new name have query changes of their own and follow their own `replay_on_change`. If the renamed column is the model's cursor, the next delta starts from the watermark stored under the old name. + +## State + +Each rename is recorded in an append-only `_sqlbuild_column_migrations` table in the model's schema, with the model, relation, old and new column, discovery (`manual` or `automatic`), and decision (`rename` or `record`). Janitor never archives it. + +## Adapter support + +| Adapter | Statement | +|---------|-----------| +| Snowflake, PostgreSQL, DuckDB, MotherDuck, BigQuery | `ALTER TABLE ... RENAME COLUMN ... TO ...` | +| Databricks | `ALTER TABLE ... RENAME COLUMN ... TO ...`, which requires Delta column mapping | +| SQL Server | `EXEC sp_rename '..', '', 'COLUMN'` | + +Databricks renames a Delta table column only when the table uses name-based column mapping (`delta.columnMapping.mode = 'name'`). SQLBuild checks this during planning and doesn't change the table protocol for you. A declared rename on a table without column mapping stops the build (`M112`) with the statement that enables it: + +```sql +ALTER TABLE
SET TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', + 'delta.minReaderVersion' = '2', + 'delta.minWriterVersion' = '5' +); +``` + +Enabling column mapping upgrades the table's Delta protocol, which older readers may not support. An automatically detected rename on such a table is skipped with a warning (`M112`) and follows `on_schema_change`. + +On PostgreSQL, views that select the renamed column keep working and keep their own output column names. On SQL Server, `sp_rename` warns that renaming can break scripts and stored procedures that reference the old name. On PostgreSQL, the new name is created in lower case, as an unquoted identifier would be. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 58a486a7a4..5c57343513 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -23,7 +23,7 @@ Online: https://sqlbuild.com/docs/concepts/models/configuration/ | `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | | `tags` | Tags used by selectors | | `description` | Human-readable model description | -| `columns` | Model-local column declarations or inherited-column audit augmentation | +| `columns` | Model-local column declarations, inherited-column audit augmentation, or column renames with `migrate_from` (incremental and snapshot models; see [Column migrations](column-migrations.md)) | | `model_schema` | Reusable column schema name | | `audits` | Model-level audit instances | | `enums` | Model-local enum declarations; names must begin with `_` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index 2192cff3cf..65baada20e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -8,7 +8,7 @@ Online: https://sqlbuild.com/docs/concepts/models/migrations/ Renaming an incremental or snapshot model normally means rebuilding it from scratch under the new name, which loses any history the source data can no longer reproduce. A model migration moves the existing relation's data to the new name instead, then continues building incrementally. -Migrations apply to incremental and snapshot models. +Migrations apply to incremental and snapshot models. To keep a column's history when you rename the column, see [Column migrations](column-migrations.md). ## Declaring a migration diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/schemas.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/schemas.md index a4731e04e9..962fc40bd8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/schemas.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/schemas.md @@ -39,6 +39,8 @@ MODEL ( See [Audits](../audits.md) for built-in audits, custom audits, arguments, severity, and incremental run scope. +In incremental and snapshot models, an inline column can also declare `migrate_from ` to rename the existing warehouse column in place and keep its history. See [Column migrations](column-migrations.md). Reusable `SCHEMA()` declarations don't accept `migrate_from`; a bound model adds it to an inherited column in its own header. + ## Reusable schemas When multiple models implement the same relation shape, declare it once in a schema role. SQLBuild discovers schema files recursively in each role. @@ -147,6 +149,6 @@ Reusable schemas intentionally support a narrow ownership model: - One optional parent, with transitive inheritance. - Additive child and model-local output columns. -- Audit-only model augmentation of inherited columns. +- Audit-only model augmentation of inherited columns, plus `migrate_from` column renames. - No general column overrides, multiple inheritance, composition, mixins, parameters, or generated projections. - No physical output ordinal enforcement. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md index 0fbaf5415a..0d318ce936 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md @@ -200,6 +200,9 @@ Reappearing keys create a new active version. | `valid_to_column` | No | Override generated column name. Default `valid_to`. | | `initial_valid_from` | No | First-version start time: `updated_at`, `observed_at`, or `execution_time`. See defaults below. | | `snapshot_full_refresh` | No | Model-level full-refresh safety: `deny`, `require_confirmation`, or `allow`. | +| `columns ( (migrate_from ))` | No | Rename a snapshot column in place so every version keeps its value. See [Column migrations](models/column-migrations.md). | + +Renaming a column in the snapshot query, such as `amount` becoming `amount AS revenue`, renames the existing snapshot column in place, so earlier versions keep their values under the new name. Renames are detected automatically when the expression is unchanged; declare `migrate_from` for anything else. See [Column migrations](models/column-migrations.md). ### Initial valid_from defaults diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index ebc55d1b62..5749f4f169 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -75,6 +75,16 @@ Seeds (1) When query or schema changes are detected, the plan shows the affected models with backfill actions and cascade information. +Work that runs before models build is listed first. [Model migrations](/docs/concepts/models/migrations) appear under `Migrations`, and in-place [column renames](/docs/concepts/models/column-migrations) under `Column migrations`: + +``` +Column migrations (1) +└── fact_orders migrate columns + └── amount -> revenue rename in place +``` + +When a removed column and an added column look related but are not an exact rename, the schema diff shows a hint such as `similar to amount; if this is a rename, add revenue (migrate_from amount)` until the change is built. + ## Missing upstream dependencies Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index f72936a98a..a953c7e067 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -382,3 +382,5 @@ Controls how schema differences are handled at execution time when the increment | `sync_all_columns` | Add, drop, and alter columns to match the delta | | `ignore` | Log and continue without schema changes | | `fail` | Reject the build with an error | + +Columns are compared by name, so a renamed column looks like one column removed and one added. [Column migrations](/docs/concepts/models/column-migrations) rename the warehouse column in place before `on_schema_change` runs, so its history keeps its values; the policy then applies only to the differences that remain. Pure renames such as `amount` becoming `amount AS revenue` are detected automatically, and `sqb plan` suggests `migrate_from` for near matches. diff --git a/website/src/content/docs/docs/concepts/models/column-migrations.mdx b/website/src/content/docs/docs/concepts/models/column-migrations.mdx new file mode 100644 index 0000000000..19df24da72 --- /dev/null +++ b/website/src/content/docs/docs/concepts/models/column-migrations.mdx @@ -0,0 +1,123 @@ +--- +title: "Column migrations" +description: "Keep a column's history when you rename it in an incremental or snapshot model." +--- + +`on_schema_change` compares columns by name, so renaming a column looks like one column removed and another added. With the default `append_new_columns`, the new column starts empty and older rows keep their values under the old name; with `sync_all_columns`, the old column and its data are dropped. A column migration renames the column in the warehouse instead, so every row keeps its value under the new name, with no leftover column and no replay. + +Column migrations apply to incremental models, including microbatch models, and snapshot models. Tables and views are rebuilt on every change and don't need them. + +## Declaring a rename + +Add `migrate_from` to the renamed column in the model header, naming the old column: + +```sql +MODEL ( + materialized incremental, + incremental_strategy delete_insert, + unique_key order_id, + cursor order_date, + columns ( + revenue (migrate_from amount), + ), +); + +SELECT order_id, order_date, amount AS revenue +FROM __source("raw_orders") +``` + +On the next build, SQLBuild renames `amount` to `revenue` in the existing table, records the rename, and then builds the model incrementally, so the new delta lands in `revenue`. + +A column entry may contain only `migrate_from`, or combine it with `type`, `nullable`, `description`, and `audits`. When the model binds a reusable schema with `model_schema`, an inherited column may add `migrate_from` without redeclaring its metadata. + +After the rename is recorded, `migrate_from` has no further effect, and SQLBuild reports (`M108`) that it can be removed. Keep it until every target that needs the rename has built. On a target where the table doesn't exist yet, the declaration does nothing: the table is created with the new column name. + +A declaration is rejected when compiling if: + +- the model is not incremental or snapshot; +- `migrate_from` names the column itself, or a qualified name instead of one column; +- two columns declare the same `migrate_from`; +- a declared column migrates from another column that itself declares `migrate_from`. Chains and swaps can't be renamed in place safely; rename through a name no other column uses, or rebuild with `--full-refresh`. + +## Automatic detection + +A pure rename is detected without a declaration: exactly one column that disappeared from the model has exactly the same expression as exactly one new column, for example `amount` becoming `amount AS revenue`, or `ROUND(amount, 2) AS revenue` becoming `ROUND(amount, 2) AS net_revenue`. Expressions are compared ignoring formatting, comments, and keyword and unquoted identifier case. Detection compares the query the table was last built from with the current query, so it applies to the build that makes the change. + +Detection never guesses: + +- Columns are matched by expression, not by position. Two columns renamed at once each pair with the column whose expression they kept; two columns that swap names keep their names and are not renamed. +- If more than one removed column has the same expression, or more than one new column does, nothing is migrated automatically. +- A near match, where the expressions reference the same columns but differ, for example `amount` becoming `ROUND(amount, 2) AS revenue`, is not a rename. It follows the model's `on_schema_change` policy, and `sqb plan` shows a hint next to the added column until the change has been built: + +```text + ├── schema diff: + + revenue (added) + similar to amount; if this is a rename, add revenue (migrate_from amount) + - amount (removed) +``` + +Add the suggested `migrate_from` to keep the history. A declared rename with a changed expression is still renamed in place, but it is a query change, so `replay_on_change` decides how much history is rebuilt with the new expression. + +## Decisions + +`sqb plan` and `sqb build` list column migrations before the models they change: + +```text +Column migrations (1) +└── fact_orders migrate columns + └── amount -> revenue rename in place +``` + +Automatically detected renames are marked `(automatic)`. The decision is based on the table's current columns first, then on recorded renames: + +| Decision | Meaning | +|----------|---------| +| `rename in place` | The old column exists and the new one doesn't. Rename it. | +| `already renamed, record` | The new column exists and the old one doesn't, but no rename is recorded, which is what an interrupted rename leaves behind. Record it without renaming again. | +| `done` | The rename is recorded and the new column exists. Nothing to do. | +| `conflict` | Both columns exist. The build stops (`M110`); drop one of them, or remove `migrate_from` and let `on_schema_change` handle the columns. | +| `source missing` | Neither column exists and no rename is recorded. The build stops (`M109`); remove `migrate_from` if the rename happened elsewhere or is no longer needed. | +| `source still produced` | The query still selects the old column. The build stops (`M111`). | +| `unsupported` | The warehouse can't rename this table's columns in place (see [Adapter support](#adapter-support)). The build stops (`M112`). | + +Automatically detected renames never stop a build: when they can't be applied, the columns follow `on_schema_change` as before. + +## How a rename runs + +Renames run after [model migrations](/docs/concepts/models/migrations) and before any model builds, one model at a time: + +1. The table's current columns are read again. A column that is already renamed is not renamed twice. +2. The column is renamed with `ALTER TABLE ... RENAME COLUMN` or the adapter's equivalent. +3. The rename is recorded. + +On adapters with transactional DDL (PostgreSQL, SQL Server, DuckDB, MotherDuck), each model's renames and their records commit together. On other adapters, a build interrupted between the rename and the record is recovered by the `already renamed, record` decision on the next build. A build that fails after the rename, for example while applying the delta, resumes on the next build without renaming again. + +After the rename, the remaining differences follow `on_schema_change`: `append_new_columns` adds any other new columns, `sync_all_columns` also drops removed ones and changes types, and `fail` only fails when a difference other than the rename remains. + +A pure rename is not a query change. It doesn't trigger `replay_on_change` for the model, and it doesn't cascade a replay to downstream models. Downstream models that you edit to use the new name have query changes of their own and follow their own `replay_on_change`. If the renamed column is the model's cursor, the next delta starts from the watermark stored under the old name. + +## State + +Each rename is recorded in an append-only `_sqlbuild_column_migrations` table in the model's schema, with the model, relation, old and new column, discovery (`manual` or `automatic`), and decision (`rename` or `record`). Janitor never archives it. + +## Adapter support + +| Adapter | Statement | +|---------|-----------| +| Snowflake, PostgreSQL, DuckDB, MotherDuck, BigQuery | `ALTER TABLE ... RENAME COLUMN ... TO ...` | +| Databricks | `ALTER TABLE ... RENAME COLUMN ... TO ...`, which requires Delta column mapping | +| SQL Server | `EXEC sp_rename '.
.', '', 'COLUMN'` | + +Databricks renames a Delta table column only when the table uses name-based column mapping (`delta.columnMapping.mode = 'name'`). SQLBuild checks this during planning and doesn't change the table protocol for you. A declared rename on a table without column mapping stops the build (`M112`) with the statement that enables it: + +```sql +ALTER TABLE
SET TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', + 'delta.minReaderVersion' = '2', + 'delta.minWriterVersion' = '5' +); +``` + +Enabling column mapping upgrades the table's Delta protocol, which older readers may not support. An automatically detected rename on such a table is skipped with a warning (`M112`) and follows `on_schema_change`. + +On PostgreSQL, views that select the renamed column keep working and keep their own output column names. On SQL Server, `sp_rename` warns that renaming can break scripts and stored procedures that reference the old name. On PostgreSQL, the new name is created in lower case, as an unquoted identifier would be. diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 717f9c53f5..7ad3f1895f 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -10,7 +10,7 @@ description: "MODEL() header fields and SQL-analysis controls." | `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | | `tags` | Tags used by selectors | | `description` | Human-readable model description | -| `columns` | Model-local column declarations or inherited-column audit augmentation | +| `columns` | Model-local column declarations, inherited-column audit augmentation, or column renames with `migrate_from` (incremental and snapshot models; see [Column migrations](/docs/concepts/models/column-migrations)) | | `model_schema` | Reusable column schema name | | `audits` | Model-level audit instances | | `enums` | Model-local enum declarations; names must begin with `_` | diff --git a/website/src/content/docs/docs/concepts/models/migrations.mdx b/website/src/content/docs/docs/concepts/models/migrations.mdx index cee105fb62..5d955663a5 100644 --- a/website/src/content/docs/docs/concepts/models/migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/migrations.mdx @@ -5,7 +5,7 @@ description: "Keep the history of an incremental or snapshot model when you rena Renaming an incremental or snapshot model normally means rebuilding it from scratch under the new name, which loses any history the source data can no longer reproduce. A model migration moves the existing relation's data to the new name instead, then continues building incrementally. -Migrations apply to incremental and snapshot models. +Migrations apply to incremental and snapshot models. To keep a column's history when you rename the column, see [Column migrations](/docs/concepts/models/column-migrations). ## Declaring a migration diff --git a/website/src/content/docs/docs/concepts/models/schemas.mdx b/website/src/content/docs/docs/concepts/models/schemas.mdx index f0ce3b7f79..e7b05a8d41 100644 --- a/website/src/content/docs/docs/concepts/models/schemas.mdx +++ b/website/src/content/docs/docs/concepts/models/schemas.mdx @@ -26,6 +26,8 @@ MODEL ( See [Audits](/docs/concepts/audits) for built-in audits, custom audits, arguments, severity, and incremental run scope. +In incremental and snapshot models, an inline column can also declare `migrate_from ` to rename the existing warehouse column in place and keep its history. See [Column migrations](/docs/concepts/models/column-migrations). Reusable `SCHEMA()` declarations don't accept `migrate_from`; a bound model adds it to an inherited column in its own header. + ## Reusable schemas When multiple models implement the same relation shape, declare it once in a schema role. SQLBuild discovers schema files recursively in each role. @@ -134,6 +136,6 @@ Reusable schemas intentionally support a narrow ownership model: - One optional parent, with transitive inheritance. - Additive child and model-local output columns. -- Audit-only model augmentation of inherited columns. +- Audit-only model augmentation of inherited columns, plus `migrate_from` column renames. - No general column overrides, multiple inheritance, composition, mixins, parameters, or generated projections. - No physical output ordinal enforcement. diff --git a/website/src/content/docs/docs/concepts/snapshots.mdx b/website/src/content/docs/docs/concepts/snapshots.mdx index e051dfd163..119e0cf7c8 100644 --- a/website/src/content/docs/docs/concepts/snapshots.mdx +++ b/website/src/content/docs/docs/concepts/snapshots.mdx @@ -184,6 +184,9 @@ Reappearing keys create a new active version. | `valid_to_column` | No | Override generated column name. Default `valid_to`. | | `initial_valid_from` | No | First-version start time: `updated_at`, `observed_at`, or `execution_time`. See defaults below. | | `snapshot_full_refresh` | No | Model-level full-refresh safety: `deny`, `require_confirmation`, or `allow`. | +| `columns ( (migrate_from ))` | No | Rename a snapshot column in place so every version keeps its value. See [Column migrations](/docs/concepts/models/column-migrations). | + +Renaming a column in the snapshot query, such as `amount` becoming `amount AS revenue`, renames the existing snapshot column in place, so earlier versions keep their values under the new name. Renames are detected automatically when the expression is unchanged; declare `migrate_from` for anything else. See [Column migrations](/docs/concepts/models/column-migrations). ### Initial valid_from defaults diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 433c45edab..487d278a8b 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -49,6 +49,7 @@ "docs/concepts/models/type-enforcement", "docs/concepts/models/contracts", "docs/concepts/models/migrations", + "docs/concepts/models/column-migrations", { "label": "Hooks", "collapsed": true, From acdfede4391f354c74b716e9f997066e273b07a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:09:54 +0100 Subject: [PATCH 4/9] fix(planner): skip target cursor reads when the cursor column is absent A planned rename of the cursor column made the first cursor-bounds read query the new column name, which does not exist yet, and log a misleading 'Failed cursor bounds' line before the old column was read. Skip the target read when inspected columns show the cursor column is missing; the outcome is unchanged and no failure is reported. Refs CHI-393 --- .../planner/_helpers/warehouse/snapshot.py | 35 ++++++++++++++++- .../main/column_migrations/_test_types.py | 9 +++++ .../main/column_migrations/helpers.py | 38 +++++++++++++++++++ .../column_migrations/test_column_renames.py | 11 +++++- 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index 02c7559085..311b16e4de 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -156,6 +156,7 @@ class _CursorGatherInputs: cursor_overrides: CursorOverrides | None target_relation_overrides: dict[str, str] | None = None origin_cursor_column_overrides: dict[str, str] | None = None + existing_columns: dict[str, tuple[ColumnInfo, ...]] | None = None def build_warehouse_snapshot( @@ -283,6 +284,7 @@ def gather_warehouse_snapshot( cursor_scope.start_cursor_config if cursor_scope is not None else None ), cursor_overrides=(cursor_scope.cursor_overrides if cursor_scope is not None else None), + existing_columns=columns, ), ) @@ -769,7 +771,15 @@ def _collect_cursor_models( and target_relation_info is not None and target_relation_info.name == model.name ): - target_tag = f"{model.name}__target__max" + target_tag = ( + f"{model.name}__target__max" + if _target_has_cursor_column( + model_name=model.name, + cursor_column=cursor_column, + existing_columns=inputs.existing_columns, + ) + else None + ) target_relation = model.destination.qualified_name cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") @@ -882,6 +892,29 @@ def _collect_cursor_models( return cursor_models +def _target_has_cursor_column( + *, + model_name: str, + cursor_column: str, + existing_columns: dict[str, tuple[ColumnInfo, ...]] | None, +) -> bool: + """Return whether the target may hold the cursor column, skipping reads known to fail.""" + + columns: tuple[ColumnInfo, ...] | None = (existing_columns or {}).get(model_name) + if not columns: + return True + wanted: str = cursor_column.strip('"`[]').lower() + if any(column.name.strip('"`[]').lower() == wanted for column in columns): + return True + log_debug_event( + logger=_DEBUG_LOGGER, + message="target relation lacks the cursor column; skipping its cursor bounds read", + sqlbuild_model=model_name, + sqlbuild_cursor_column=cursor_column, + ) + return False + + def _build_cursor_queries(cursor_models: list[_CursorModelInfo]) -> list[_PhysicalCursorQuery]: """Group logical cursor requests by physical relation and column.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py index a5d2f34c17..9aaccc14dd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py @@ -94,6 +94,7 @@ class CursorColumnRenameTestCase: renamed_sql: str expected_planned: tuple[tuple[str, str, str, str], ...] expected_order_ids: tuple[int, ...] + expected_failure_lines: tuple[str, ...] @dataclass(frozen=True) @@ -102,3 +103,11 @@ class FirstRunDeclarationTestCase: model_sql: str expected_planned: tuple[tuple[str, str, str, str], ...] expected_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class MaintenanceNoticeOrderTestCase: + description: str + command: tuple[str, ...] + expected_notices: tuple[str, ...] + expected_final_line: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index bb386b58c1..3670f28c79 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -121,12 +121,50 @@ def build_initial( load_orders(project_dir=project_dir, last_day=5) +def build_with_completed_migrations( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Apply one model rename and one column rename, keeping both migrate_from declarations.""" + + write_project( + project_dir=project_dir, + models={MODEL_NAME: orders_sql(), "old_orders": orders_sql()}, + ) + load_orders(project_dir=project_dir, last_day=3) + _ = build_ok(project_dir=project_dir, capsys=capsys) + write_project( + project_dir=project_dir, + models={ + MODEL_NAME: orders_sql( + columns="amount AS revenue", + extra_config=migrate_columns("revenue (migrate_from amount)"), + ), + "new_orders": orders_sql(extra_config=' migrate_from "old_orders",\n'), + }, + ) + _ = build_ok(project_dir=project_dir, capsys=capsys) + + def last_visible_line(output: str) -> str: """Return the final non-blank line of command output.""" return output.rstrip().splitlines()[-1] +def failure_lines(output: str) -> tuple[str, ...]: + """Return every output line that reports a failure.""" + + return tuple(filter(lambda line: "Failed" in line, output.splitlines())) + + +def run_log_text(*, project_dir: Path) -> str: + """Return the concatenated text of every persisted run log.""" + + return "\n".join( + path.read_text(encoding="utf-8") for path in sorted((project_dir / "logs").rglob("*.log")) + ) + + def migrate_columns(declarations: str) -> str: """Return a MODEL header columns block declaring column migrations.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py index e5288af099..3f72420518 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py @@ -27,6 +27,7 @@ column_events, column_names, column_values, + failure_lines, last_visible_line, load_orders, migrate_columns, @@ -35,6 +36,8 @@ plan_json, plan_text, planned_column_migrations, + run_log_text, + run_sqb, snapshot_sql, state_table_names, warning_codes, @@ -354,6 +357,7 @@ def test_given_microbatch_rename_when_building_then_batches_write_the_renamed_co ), expected_planned=(("order_date", "ordered_at", "manual", "rename"),), expected_order_ids=(2, 3, 4, 5), + expected_failure_lines=(), ), ], ids=lambda case: case.description, @@ -377,10 +381,15 @@ def test_given_renamed_cursor_column_when_appending_then_no_history_is_reprocess ) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) - _ = build_ok(project_dir=tmp_path, capsys=capsys) + planned: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + built: CliRun = build_ok(project_dir=tmp_path, capsys=capsys) assert planned_column_migrations(plan) == test_case.expected_planned assert model_plan(plan)["cursor_bounds"]["start"].startswith("2026-01-03") + assert ( + failure_lines(planned.output + built.output + run_log_text(project_dir=tmp_path)) + == test_case.expected_failure_lines + ) assert ( tuple( int(row[0]) From 43229967fab34898c16d99e8f285082719e2f04b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:09:54 +0100 Subject: [PATCH 5/9] fix(cli): end text plan output with a completion line Maintenance notices such as M101 and M108 were the last visible output of a successful sqb plan. Text plan output now ends with a 'Plan complete' line after the Warnings section; JSON is unchanged. Refs CHI-393 --- .../sqlbuild/references/docs/cli/plan.md | 6 +++ .../sqlbuild/references/docs/quickstart.md | 4 ++ .../cli/commands/_helpers/plan/outputs.py | 2 +- src/sqlbuild/cli/output/_helpers/plan_text.py | 32 +++++++++++ src/sqlbuild/cli/output/main/plan.py | 13 ++++- .../test_maintenance_notice_order.py | 54 +++++++++++++++++++ website/src/content/docs/docs/cli/plan.mdx | 6 +++ website/src/content/docs/docs/quickstart.mdx | 4 ++ 8 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_maintenance_notice_order.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 50518bdaab..06a440fc17 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -62,6 +62,8 @@ First run (12) Seeds (1) waffle_types + +✓ Plan complete 13 selected ``` Steady state: @@ -79,10 +81,14 @@ Normal (12) Seeds (1) waffle_types + +✓ Plan complete 13 selected ``` When query or schema changes are detected, the plan shows the affected models with backfill actions and cascade information. +Warnings and maintenance notices, such as `migrate_from can be removed`, are listed in a `Warnings` section before the final `Plan complete` line. `sqb plan --json` output is unchanged. + Work that runs before models build is listed first. [Model migrations](../concepts/models/migrations.md) appear under `Migrations`, and in-place [column renames](../concepts/models/column-migrations.md) under `Column migrations`: ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md index 7453c17f8a..1dcebfc5d5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md @@ -68,6 +68,8 @@ First run (12) Seeds (1) waffle_types + +✓ Plan complete 13 selected ``` Source data is defined as inline expressions in `sources/raw.yml`, so SQLBuild resolves everything at compile time - no seeding step needed. @@ -159,6 +161,8 @@ Normal (12) Seeds (1) waffle_types + +✓ Plan complete 13 selected ``` All models now show `Normal` instead of `First run`. diff --git a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py b/src/sqlbuild/cli/commands/_helpers/plan/outputs.py index 8044b5a2cc..95fda411d1 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/outputs.py @@ -39,6 +39,6 @@ def write_plan_command_output( display_options=display_options, python_plan_entries=pipeline_result.python_plan_entries, include_direct_freshness_diagnostics=(False), + include_completion=True, ) - + "\n" ) diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index 924028c704..d54e3c6433 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -50,12 +50,14 @@ from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.main.aligned_name_value import format_aligned_name_value from sqlbuild.presentation.main.append_overflow_line import append_overflow_line +from sqlbuild.presentation.main.completion_line import format_completion_line from sqlbuild.presentation.main.count_header import count_header_style from sqlbuild.presentation.main.resolve_name_column_width import resolve_name_column_width from sqlbuild.presentation.main.surface_header import format_surface_header from sqlbuild.presentation.main.tree_connector import tree_connector from sqlbuild.presentation.main.visible_entries import visible_entries from sqlbuild.presentation.models import DisplayOptions +from sqlbuild.presentation.types import CompletionState from sqlbuild.runtime.contracts.types import ExecutionResourceKind _DIFF_HEADER_MARKER: str = "# " @@ -407,6 +409,36 @@ def _format_full_refresh( return lines +def format_plan_completion( + *, + plan: PlanOutput, + full_refresh: bool, + use_color: bool, + python_plan_entries: tuple[PythonPlanEntry, ...], +) -> str: + """Render the terminal completion line that closes text plan output.""" + + has_errors: bool = any(warning.severity == WarningSeverity.ERROR for warning in plan.warnings) + active_model_entries: tuple[ModelPlanEntry, ...] = tuple( + entry for entry in plan.model_entries if entry.action != PlanAction.SKIP + ) + all_active_models_full_refresh: bool = bool(active_model_entries) and all( + entry.reason == PlanReason.FULL_REFRESH for entry in active_model_entries + ) + return format_completion_line( + style=CliStyle(use_color=use_color), + state=CompletionState.WARN if has_errors else CompletionState.OK, + label="Plan complete with errors" if has_errors else "Plan complete", + summary=_plan_ready_header( + selected_count=_selected_count(plan), + source_load_entries=plan.source_load_entries, + python_plan_entries=python_plan_entries, + full_refresh=all_active_models_full_refresh + or (full_refresh and not active_model_entries), + ), + ) + + def _selected_count(plan: PlanOutput) -> int: """Count selected executable resources shown in plan output.""" diff --git a/src/sqlbuild/cli/output/main/plan.py b/src/sqlbuild/cli/output/main/plan.py index 48eedf9c31..340baee71a 100644 --- a/src/sqlbuild/cli/output/main/plan.py +++ b/src/sqlbuild/cli/output/main/plan.py @@ -5,6 +5,7 @@ from collections.abc import Callable from sqlbuild.cli.output._helpers.plan_text import format_plan as _format_plan +from sqlbuild.cli.output._helpers.plan_text import format_plan_completion from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.presentation.models import DisplayOptions @@ -20,10 +21,11 @@ def format_plan( section_header_style: Callable[[str], str] | None = None, python_plan_entries: tuple[PythonPlanEntry, ...] = (), include_direct_freshness_diagnostics: bool = True, + include_completion: bool = False, ) -> str: """Format plan output grouped by reason with inline detail.""" - return _format_plan( + plan_text: str = _format_plan( plan=plan, full_refresh=full_refresh, use_color=use_color, @@ -33,3 +35,12 @@ def format_plan( python_plan_entries=python_plan_entries, include_direct_freshness_diagnostics=include_direct_freshness_diagnostics, ) + if not include_completion: + return plan_text + completion: str = format_plan_completion( + plan=plan, + full_refresh=full_refresh, + use_color=use_color, + python_plan_entries=python_plan_entries, + ) + return f"{plan_text}\n\n{completion}" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_maintenance_notice_order.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_maintenance_notice_order.py new file mode 100644 index 0000000000..12354efc1a --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_maintenance_notice_order.py @@ -0,0 +1,54 @@ +"""Integration coverage that migration maintenance notices precede the terminal success line.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.column_migrations._test_types import ( + MaintenanceNoticeOrderTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.column_migrations.helpers import ( + CliRun, + build_with_completed_migrations, + last_visible_line, + run_sqb, +) + +_MODEL_NOTICE: str = "migrate_from can be removed from 'new_orders'" +_COLUMN_NOTICE: str = "migrate_from can be removed from column 'revenue' of 'fct_orders'" + + +@pytest.mark.parametrize( + "test_case", + [ + MaintenanceNoticeOrderTestCase( + description="plan reports M101 and M108 before its completion line", + command=("plan",), + expected_notices=(_MODEL_NOTICE, _COLUMN_NOTICE), + expected_final_line="✓ Plan complete 2 selected", + ), + MaintenanceNoticeOrderTestCase( + description="build reports M101 and M108 before its completion line", + command=("build",), + expected_notices=(_MODEL_NOTICE, _COLUMN_NOTICE), + expected_final_line="✓ Completed successfully", + ), + ], + ids=lambda case: case.description, +) +def test_given_completed_migrations_when_running_command_then_success_line_is_last( + test_case: MaintenanceNoticeOrderTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_with_completed_migrations(project_dir=tmp_path, capsys=capsys) + + result: CliRun = run_sqb(project_dir=tmp_path, args=test_case.command, capsys=capsys) + + assert result.exit_code == 0, result.output + assert tuple(result.stdout.count(notice) for notice in test_case.expected_notices) == (1, 1) + assert last_visible_line(result.stdout).startswith(test_case.expected_final_line) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 5749f4f169..2049f0df59 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -54,6 +54,8 @@ First run (12) Seeds (1) waffle_types + +✓ Plan complete 13 selected ``` Steady state: @@ -71,10 +73,14 @@ Normal (12) Seeds (1) waffle_types + +✓ Plan complete 13 selected ``` When query or schema changes are detected, the plan shows the affected models with backfill actions and cascade information. +Warnings and maintenance notices, such as `migrate_from can be removed`, are listed in a `Warnings` section before the final `Plan complete` line. `sqb plan --json` output is unchanged. + Work that runs before models build is listed first. [Model migrations](/docs/concepts/models/migrations) appear under `Migrations`, and in-place [column renames](/docs/concepts/models/column-migrations) under `Column migrations`: ``` diff --git a/website/src/content/docs/docs/quickstart.mdx b/website/src/content/docs/docs/quickstart.mdx index 530954868c..303aae4010 100644 --- a/website/src/content/docs/docs/quickstart.mdx +++ b/website/src/content/docs/docs/quickstart.mdx @@ -54,6 +54,8 @@ First run (12) Seeds (1) waffle_types + +✓ Plan complete 13 selected ``` Source data is defined as inline expressions in `sources/raw.yml`, so SQLBuild resolves everything at compile time - no seeding step needed. @@ -145,6 +147,8 @@ Normal (12) Seeds (1) waffle_types + +✓ Plan complete 13 selected ``` All models now show `Normal` instead of `First run`. From a8ba44695a2a3d3ac6c8c65a2d179590ee236d72 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:28:30 +0100 Subject: [PATCH 6/9] fix(planner): rename columns automatically only when the rename is the whole change Automatic detection matched projection text alone, so an expression rebound by a changed CTE, join, or filter was renamed in place and bypassed on_schema_change. Replay suppression compared projections by position and treated alias references as query changes, so a pure rename with ORDER BY on the new alias, or a reordered projection list, replayed and overwrote history. Prove renames against the canonical AST: everything except the output names must match, projections are matched by name through the rename map, ORDER BY and QUALIFY references to a renamed alias are restored to the old name, and reorders are refused under positional grouping or ordering. Unproven changes get no automatic rename and only a 'similar to' hint; declared renames keep the model's replay decision unchanged. Refs CHI-393 --- .../_helpers/migrations/column_renames.py | 22 +- .../planner/_helpers/migrations/columns.py | 4 + .../_helpers/migrations/projections.py | 97 ++++++++- src/sqlbuild/compiler/planner/models.py | 2 + .../main/column_migrations/_test_types.py | 29 +++ .../main/column_migrations/helpers.py | 12 ++ .../test_column_rename_equivalence.py | 196 ++++++++++++++++++ .../column_migrations/test_column_renames.py | 5 +- .../_helpers/migrations/_test_types.py | 1 + .../migrations/test_column_renames.py | 87 ++++++++ 10 files changed, 442 insertions(+), 13 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py b/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py index c1c92efa11..fe58300798 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py @@ -1,14 +1,21 @@ -"""Recognize renamed output columns by comparing formatting-insensitive expressions.""" +"""Recognize renamed output columns only when the rest of the query is provably unchanged.""" from __future__ import annotations +from collections.abc import Mapping + +from sqlbuild.compiler.planner._helpers.migrations.projections import renames_explain_change from sqlbuild.compiler.planner.models import ColumnRenameHint, QueryProjection, QueryShape def identical_renames( - *, previous: QueryShape, current: QueryShape, excluded: frozenset[str] + *, + previous: QueryShape, + current: QueryShape, + excluded: frozenset[str], + declared: Mapping[str, str], ) -> tuple[tuple[str, str], ...]: - """Return (old, new) pairs where exactly one removed and one added expression are equal.""" + """Return same-expression (old, new) pairs only when renaming is the whole change.""" removed: tuple[QueryProjection, ...] added: tuple[QueryProjection, ...] @@ -24,6 +31,10 @@ def identical_renames( claims: int = sum(1 for other in added if other.expression == matches[0].expression) if claims == 1: pairs.append((matches[0].name, column.name)) + if not pairs or not renames_explain_change( + previous=previous, current=current, renames={**declared, **dict(pairs)} + ): + return () return tuple(pairs) @@ -43,13 +54,16 @@ def rename_hints( live_removed: tuple[QueryProjection, ...] = tuple( origin for origin in removed if origin.key in live_columns ) + same_binding: bool = previous.body == current.body hints: list[ColumnRenameHint] = [] column: QueryProjection for column in added: if column.key in live_columns: continue identical: tuple[str, ...] = tuple( - origin.name for origin in live_removed if origin.expression == column.expression + origin.name + for origin in live_removed + if same_binding and origin.expression == column.expression ) similar: tuple[str, ...] = tuple( origin.name diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index d2ec77f466..a3988d7f8c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -222,12 +222,16 @@ def _declared_pairs(candidate: _ModelColumns) -> tuple[_Pair, ...]: def _detected_pairs(candidate: _ModelColumns) -> tuple[_Pair, ...]: if candidate.previous is None or candidate.current is None: return () + declared: dict[str, str] = { + origin: destination for destination, origin in candidate.declared.items() + } return tuple( _Pair(origin=origin, destination=destination, discovery=MigrationDiscovery.AUTOMATIC) for origin, destination in identical_renames( previous=candidate.previous, current=candidate.current, excluded=_declared_names(candidate), + declared=declared, ) ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py b/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py index 65aec9f2b7..5ecb9e867e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py @@ -1,4 +1,4 @@ -"""Formatting-insensitive top-level projections used to recognize column renames.""" +"""Formatting-insensitive query shapes used to prove that a change is only a column rename.""" from __future__ import annotations @@ -18,6 +18,14 @@ _COLUMN_KEY: str = "column" _NAME_KEY: str = "name" _QUOTED_KEY: str = "quoted" +_TABLE_KEY: str = "table" +_LITERAL_KEY: str = "literal" +_LITERAL_TYPE_KEY: str = "literal_type" +_NUMBER_LITERAL: str = "number" +_GROUP_BY_KEY: str = "group_by" +_ORDER_BY_KEY: str = "order_by" +_DISTINCT_ON_KEY: str = "distinct_on" +_ALIAS_SCOPED_KEYS: tuple[str, ...] = (_ORDER_BY_KEY, "qualify") def parse_query_shape(*, query_sql: str, dialect: str | None) -> QueryShape | None: @@ -43,22 +51,95 @@ def parse_query_shape(*, query_sql: str, dialect: str | None) -> QueryShape | No projections.append(projection) if len({projection.key for projection in projections}) != len(projections): return None - body: dict[str, Any] = {key: value for key, value in select.items() if key != _EXPRESSIONS_KEY} - return QueryShape(projections=tuple(projections), body=_encode(_canonical(body))) + canonical: dict[str, Any] = _canonical(select) + body: dict[str, Any] = { + key: value + for key, value in canonical.items() + if key != _EXPRESSIONS_KEY and key not in _ALIAS_SCOPED_KEYS + } + return QueryShape( + projections=tuple(projections), + body=_encode(body), + alias_clauses=_encode({key: canonical.get(key) for key in _ALIAS_SCOPED_KEYS}), + positional=_has_positional_reference(canonical), + ) def renames_explain_change( *, previous: QueryShape, current: QueryShape, renames: Mapping[str, str] ) -> bool: - """Return whether renaming previous output columns alone turns previous into current.""" + """Return whether renaming output columns, matched by name, is the whole query change.""" - if previous.body != current.body or len(previous.projections) != len(current.projections): + if not renames or previous.body != current.body: return False lowered: dict[str, str] = {old.lower(): new.lower() for old, new in renames.items()} - return all( - lowered.get(before.key, before.key) == after.key and before.expression == after.expression - for before, after in zip(previous.projections, current.projections, strict=True) + renamed_previous: dict[str, str] = { + lowered.get(item.key, item.key): item.expression for item in previous.projections + } + current_projections: dict[str, str] = { + item.key: item.expression for item in current.projections + } + if ( + len(renamed_previous) != len(previous.projections) + or renamed_previous != current_projections + ): + return False + reordered: bool = tuple(lowered.get(item.key, item.key) for item in previous.projections) != ( + tuple(item.key for item in current.projections) + ) + if reordered and (previous.positional or current.positional): + return False + restored: Any = _substitute_alias_references( + node=json.loads(current.alias_clauses), + names={new: old for old, new in lowered.items()}, ) + return _encode(restored) == previous.alias_clauses + + +def _substitute_alias_references(*, node: Any, names: Mapping[str, str]) -> Any: + """Rename unqualified references to output aliases, leaving nested queries untouched.""" + + if isinstance(node, list): + return [_substitute_alias_references(node=value, names=names) for value in node] + if not isinstance(node, dict) or _SELECT_KEY in node: + return node + column: Any = node.get(_COLUMN_KEY) + if isinstance(column, dict) and column.get(_TABLE_KEY) is None: + identifier: Any = column.get(_NAME_KEY) + name: str | None = _identifier_name(identifier) + if name is not None and identifier.get(_QUOTED_KEY) is False and name in names: + return { + **node, + _COLUMN_KEY: {**column, _NAME_KEY: {**identifier, _NAME_KEY: names[name]}}, + } + return { + key: _substitute_alias_references(node=value, names=names) for key, value in node.items() + } + + +def _has_positional_reference(select: Mapping[str, Any]) -> bool: + """Return whether grouping, ordering, or DISTINCT ON refer to output columns by position.""" + + items: list[Any] = [*_clause_items(select.get(_GROUP_BY_KEY))] + items.extend( + item.get(_THIS_KEY) if isinstance(item, dict) else item + for item in _clause_items(select.get(_ORDER_BY_KEY)) + ) + items.extend(_clause_items(select.get(_DISTINCT_ON_KEY))) + return any( + isinstance(item, dict) + and isinstance(item.get(_LITERAL_KEY), dict) + and item[_LITERAL_KEY].get(_LITERAL_TYPE_KEY) == _NUMBER_LITERAL + for item in items + ) + + +def _clause_items(clause: Any) -> list[Any]: + if isinstance(clause, list): + return clause + if isinstance(clause, dict) and isinstance(clause.get(_EXPRESSIONS_KEY), list): + return list(clause[_EXPRESSIONS_KEY]) + return [] def _projection(expression: Any) -> QueryProjection | None: diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 508b7eced2..dcf9e6b5e7 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -1001,6 +1001,8 @@ class QueryShape: projections: tuple[QueryProjection, ...] body: str + alias_clauses: str = "{}" + positional: bool = False @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py index 9aaccc14dd..6754e12a6f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py @@ -111,3 +111,32 @@ class MaintenanceNoticeOrderTestCase: command: tuple[str, ...] expected_notices: tuple[str, ...] expected_final_line: str + + +@dataclass(frozen=True) +class RebindingRenameTestCase: + description: str + rebound_sql: str + expected_build_succeeds: bool + expected_columns: tuple[str, ...] + expected_hint: str + + +@dataclass(frozen=True) +class EquivalentRenameTestCase: + description: str + initial_sql: str + renamed_sql: str + expected_planned: tuple[tuple[str, str, str, str], ...] + expected_backfill: str + expected_columns: tuple[str, ...] + expected_values: tuple[tuple[int, Any], ...] + + +@dataclass(frozen=True) +class UnprovenRenameReplayTestCase: + description: str + undeclared_sql: str + declared_sql: str + expected_backfill: dict[str, Any] + expected_values: tuple[tuple[int, Any], ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index 3670f28c79..6444d6c31d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -77,6 +77,18 @@ def orders_sql( ) +def incremental_sql(*, body: str, extra_config: str = "") -> str: + """Return a delete_insert orders model with an arbitrary query body.""" + + return ( + _INCREMENTAL_HEADER.format( + strategy="delete_insert", cursor="order_date", extra_config=extra_config + ) + + body + + "\n" + ) + + MICROBATCH_CONFIG: str = ( " incremental_mode microbatch,\n" " microbatch_strategy watermark,\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py new file mode 100644 index 0000000000..887114ac9c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py @@ -0,0 +1,196 @@ +"""Integration coverage that only provably pure renames move history or skip replay.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.column_migrations._test_types import ( + EquivalentRenameTestCase, + RebindingRenameTestCase, + UnprovenRenameReplayTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.column_migrations.helpers import ( + REPLAY_FULL, + CliRun, + build, + build_initial, + build_ok, + change_historical_amount, + column_names, + column_values, + incremental_sql, + migrate_columns, + model_plan, + orders_sql, + plan_json, + planned_column_migrations, + state_table_names, + write_model, +) + +_ORDERS: str = 'FROM __source("raw_orders")' +_REBOUND_BODY: str = ( + f"WITH changed AS (SELECT order_id, order_date, tax AS amount {_ORDERS}) " + "SELECT order_id, order_date, amount AS revenue FROM changed" +) +_FILTERED_BODY: str = f"SELECT order_id, order_date, amount AS revenue {_ORDERS} WHERE amount > 0" +_REPLAY_BOUNDED: str = " replay_on_change bounded-2d,\n" +_HISTORY: tuple[tuple[int, Any], ...] = ((1, 101), (2, 102), (3, 103), (4, 104), (5, 105)) + + +@pytest.mark.parametrize( + "test_case", + [ + RebindingRenameTestCase( + description="on_schema_change fail rejects a rebound column instead of renaming it", + rebound_sql=incremental_sql( + body=_REBOUND_BODY, extra_config=" on_schema_change fail,\n" + ), + expected_build_succeeds=False, + expected_columns=("order_id", "order_date", "amount"), + expected_hint=( + "similar to amount; if this is a rename, add revenue (migrate_from amount)" + ), + ), + RebindingRenameTestCase( + description="append_new_columns keeps amount history and adds the rebound column", + rebound_sql=incremental_sql(body=_REBOUND_BODY), + expected_build_succeeds=True, + expected_columns=("order_id", "order_date", "amount", "revenue"), + expected_hint=( + "similar to amount; if this is a rename, add revenue (migrate_from amount)" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_expression_rebound_by_a_changed_cte_when_building_then_nothing_is_renamed( + test_case: RebindingRenameTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys, sql=orders_sql()) + write_model(project_dir=tmp_path, sql=test_case.rebound_sql) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + result: CliRun = build(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == () + assert model_plan(plan)["column_rename_hints"][0]["hint"] == test_case.expected_hint + assert (result.exit_code == 0) is test_case.expected_build_succeeds, result.output + assert column_names(project_dir=tmp_path) == test_case.expected_columns + assert column_values(project_dir=tmp_path, column="amount")[:2] == _HISTORY[:2] + assert "_sqlbuild_column_migrations" not in state_table_names(project_dir=tmp_path) + + +@pytest.mark.parametrize( + "test_case", + [ + EquivalentRenameTestCase( + description="automatic rename with an ORDER BY on the new alias is not replayed", + initial_sql=incremental_sql( + body=f"SELECT order_id, order_date, amount {_ORDERS} ORDER BY amount", + extra_config=REPLAY_FULL, + ), + renamed_sql=incremental_sql( + body=f"SELECT order_id, order_date, amount AS revenue {_ORDERS} ORDER BY revenue", + extra_config=REPLAY_FULL, + ), + expected_planned=(("amount", "revenue", "automatic", "rename"),), + expected_backfill="forward", + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + ), + EquivalentRenameTestCase( + description="automatic rename that also reorders projections is not replayed", + initial_sql=orders_sql(extra_config=REPLAY_FULL), + renamed_sql=incremental_sql( + body=f"SELECT amount AS revenue, order_id, order_date {_ORDERS}", + extra_config=REPLAY_FULL, + ), + expected_planned=(("amount", "revenue", "automatic", "rename"),), + expected_backfill="forward", + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + ), + EquivalentRenameTestCase( + description="declared rename with an ORDER BY on the new alias is not replayed", + initial_sql=incremental_sql( + body=f"SELECT order_id, order_date, amount {_ORDERS} ORDER BY amount", + extra_config=REPLAY_FULL, + ), + renamed_sql=incremental_sql( + body=f"SELECT order_id, order_date, amount AS revenue {_ORDERS} ORDER BY revenue", + extra_config=REPLAY_FULL + migrate_columns("revenue (migrate_from amount)"), + ), + expected_planned=(("amount", "revenue", "manual", "rename"),), + expected_backfill="forward", + expected_columns=("order_id", "order_date", "revenue"), + expected_values=_HISTORY, + ), + ], + ids=lambda case: case.description, +) +def test_given_rename_equivalent_modulo_aliases_when_building_then_history_is_kept( + test_case: EquivalentRenameTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys, sql=test_case.initial_sql) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + change_historical_amount(project_dir=tmp_path) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == test_case.expected_planned + assert model_plan(plan)["backfill"]["action"] == test_case.expected_backfill + assert column_names(project_dir=tmp_path) == test_case.expected_columns + assert column_values(project_dir=tmp_path, column="revenue") == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + [ + UnprovenRenameReplayTestCase( + description="a full replay policy rebuilds exactly as it would without the rename", + undeclared_sql=incremental_sql(body=_FILTERED_BODY, extra_config=REPLAY_FULL), + declared_sql=incremental_sql( + body=_FILTERED_BODY, + extra_config=REPLAY_FULL + migrate_columns("revenue (migrate_from amount)"), + ), + expected_backfill={"action": "full", "duration": None}, + expected_values=((1, 999), *_HISTORY[1:]), + ), + UnprovenRenameReplayTestCase( + description="a bounded replay policy keeps renamed history outside its window", + undeclared_sql=incremental_sql(body=_FILTERED_BODY, extra_config=_REPLAY_BOUNDED), + declared_sql=incremental_sql( + body=_FILTERED_BODY, + extra_config=_REPLAY_BOUNDED + migrate_columns("revenue (migrate_from amount)"), + ), + expected_backfill={"action": "bounded", "duration": "2d"}, + expected_values=_HISTORY, + ), + ], + ids=lambda case: case.description, +) +def test_given_rename_with_other_query_changes_when_building_then_replay_matches_no_rename( + test_case: UnprovenRenameReplayTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_initial(project_dir=tmp_path, capsys=capsys, sql=orders_sql()) + change_historical_amount(project_dir=tmp_path) + write_model(project_dir=tmp_path, sql=test_case.undeclared_sql) + undeclared: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + write_model(project_dir=tmp_path, sql=test_case.declared_sql) + declared: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(undeclared) == () + assert planned_column_migrations(declared) == (("amount", "revenue", "manual", "rename"),) + assert model_plan(undeclared)["backfill"] == test_case.expected_backfill + assert model_plan(declared)["backfill"] == test_case.expected_backfill + assert column_values(project_dir=tmp_path, column="revenue") == test_case.expected_values + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py index 3f72420518..0d11039be6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_renames.py @@ -150,7 +150,10 @@ def test_given_pure_rename_with_replay_policy_when_planning_then_model_continues [ RenameWithPolicyTestCase( description="append_new_columns renames first, then adds the new column", - renamed_sql=orders_sql(columns="amount AS revenue, tax * 2 AS double_tax"), + renamed_sql=orders_sql( + columns="amount AS revenue, tax * 2 AS double_tax", + extra_config=migrate_columns("revenue (migrate_from amount)"), + ), expected_columns=("order_id", "order_date", "revenue", "double_tax"), expected_values=_HISTORY, added_column="double_tax", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py index ca5ba6bd9f..c05dcb3020 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py @@ -32,6 +32,7 @@ class IdenticalRenameTestCase: current_sql: str expected_renames: tuple[tuple[str, str], ...] excluded: frozenset[str] = frozenset() + declared: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py index c260f6f047..556a5cf63c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py @@ -79,6 +79,44 @@ excluded=frozenset({"revenue"}), expected_renames=(), ), + IdenticalRenameTestCase( + description="a changed CTE rebinding the same expression is not a rename", + previous_sql=f"SELECT order_id, order_date, amount {_ORDERS}", + current_sql=( + f"WITH changed AS (SELECT order_id, order_date, tax AS amount {_ORDERS}) " + "SELECT order_id, order_date, amount AS revenue FROM changed" + ), + expected_renames=(), + ), + IdenticalRenameTestCase( + description="a new join alongside the rename is not a rename", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=( + f"SELECT order_id, amount AS revenue {_ORDERS} " + 'JOIN __source("raw_refunds") USING (order_id)' + ), + expected_renames=(), + ), + IdenticalRenameTestCase( + description="another projection change alongside the rename is not automatic", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=f"SELECT order_id, amount AS revenue, tax {_ORDERS}", + expected_renames=(), + ), + IdenticalRenameTestCase( + description="an ORDER BY that follows the renamed alias is still a rename", + previous_sql=f"SELECT order_id, amount {_ORDERS} ORDER BY amount", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS} ORDER BY revenue", + expected_renames=(("amount", "revenue"),), + ), + IdenticalRenameTestCase( + description="a declared rename is part of the proof for a detected one", + previous_sql=f"SELECT order_id, amount, tax {_ORDERS}", + current_sql=f"SELECT order_id, amount AS revenue, tax AS levy {_ORDERS}", + excluded=frozenset({"tax", "levy"}), + declared={"tax": "levy"}, + expected_renames=(("amount", "revenue"),), + ), ], ids=lambda case: case.description, ) @@ -89,6 +127,7 @@ def test_given_previous_and_current_queries_when_detecting_renames_then_returns_ previous=query_shape(test_case.previous_sql), current=query_shape(test_case.current_sql), excluded=test_case.excluded, + declared=test_case.declared, ) assert renames == test_case.expected_renames @@ -130,6 +169,18 @@ def test_given_previous_and_current_queries_when_detecting_renames_then_returns_ live_columns=frozenset({"order_id", "amount"}), expected_hints=(), ), + RenameHintTestCase( + description="a rebound expression is only similar, never the same", + previous_sql=f"SELECT order_id, amount {_ORDERS}", + current_sql=( + f"WITH changed AS (SELECT order_id, tax AS amount {_ORDERS}) " + "SELECT order_id, amount AS revenue FROM changed" + ), + live_columns=frozenset({"order_id", "amount"}), + expected_hints=( + "similar to amount; if this is a rename, add revenue (migrate_from amount)", + ), + ), ], ids=lambda case: case.description, ) @@ -171,6 +222,42 @@ def test_given_changed_columns_when_building_hints_then_suggests_explicit_migrat renames={"amount": "revenue"}, expected_explained=False, ), + RenameExplainsChangeTestCase( + description="ORDER BY and QUALIFY references to the renamed alias are equivalent", + previous_sql=( + f"SELECT order_id, amount {_ORDERS} " + "QUALIFY row_number() OVER (PARTITION BY order_id ORDER BY amount) = 1 " + "ORDER BY amount" + ), + current_sql=( + f"SELECT order_id, amount AS revenue {_ORDERS} " + "QUALIFY row_number() OVER (PARTITION BY order_id ORDER BY revenue) = 1 " + "ORDER BY revenue" + ), + renames={"amount": "revenue"}, + expected_explained=True, + ), + RenameExplainsChangeTestCase( + description="reordering projections while renaming matches by name", + previous_sql=f"SELECT order_id, order_date, amount {_ORDERS}", + current_sql=f"SELECT amount AS revenue, order_id, order_date {_ORDERS}", + renames={"amount": "revenue"}, + expected_explained=True, + ), + RenameExplainsChangeTestCase( + description="reordering under positional grouping changes the result", + previous_sql=f"SELECT order_id, sum(amount) AS amount {_ORDERS} GROUP BY 1", + current_sql=f"SELECT sum(amount) AS revenue, order_id {_ORDERS} GROUP BY 1", + renames={"amount": "revenue"}, + expected_explained=False, + ), + RenameExplainsChangeTestCase( + description="a WHERE on a renamed alias is not treated as an alias reference", + previous_sql=f"SELECT order_id, amount {_ORDERS} WHERE amount > 0", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS} WHERE revenue > 0", + renames={"amount": "revenue"}, + expected_explained=False, + ), ], ids=lambda case: case.description, ) From 2e3219136bb20f0d73f30f7682b6c5a8547fbcaa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:28:30 +0100 Subject: [PATCH 7/9] fix(executor): re-read live columns before each in-place column rename Live columns were read once per model, so an overlapping build that renamed a later column between two renames made the second ALTER fail. Re-read them immediately before each rename and record a column that is already renamed. Refs CHI-393 --- fensu.toml | 6 ++ .../executor/migrations/_helpers/columns.py | 13 ++-- .../sqlbuild/executor/migrations/__init__.py | 0 .../executor/migrations/_helpers/__init__.py | 0 .../migrations/_helpers/_test_types.py | 13 ++++ .../executor/migrations/_helpers/helpers.py | 67 +++++++++++++++++ .../migrations/_helpers/test_columns.py | 73 +++++++++++++++++++ 7 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 tests/unit/src/sqlbuild/executor/migrations/__init__.py create mode 100644 tests/unit/src/sqlbuild/executor/migrations/_helpers/__init__.py create mode 100644 tests/unit/src/sqlbuild/executor/migrations/_helpers/_test_types.py create mode 100644 tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py create mode 100644 tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py diff --git a/fensu.toml b/fensu.toml index 6c84a5a076..1fda03ea27 100644 --- a/fensu.toml +++ b/fensu.toml @@ -93,6 +93,12 @@ rule = "FFR601" path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" reason = "The SQL Server adapter intentionally keeps overrideable backend behavior on one class." +[[targets.python.rule_exceptions]] +rule = "XSB051" +path = "src/sqlbuild/executor/migrations/_helpers/columns.py" +symbols = ["apply_model_column_renames"] +reason = "Each in-place column rename must re-read live columns immediately before its ALTER so overlapping builds reconcile instead of failing." + [[targets.python.rule_exceptions]] rule = "FFS120" path = "src/sqlbuild/integrations/dbt/_helpers/profile/render.py" diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index 4d5c82781e..ab0bb19620 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -26,7 +26,7 @@ def apply_model_column_renames( entries: tuple[ColumnMigrationPlanEntry, ...], run_id: str, ) -> None: - """Rename and record every pending column of one model, re-checking live columns first.""" + """Rename and record every pending column of one model, re-checking live columns per rename.""" first: ColumnMigrationPlanEntry = entries[0] adapter.ensure_schema( @@ -38,10 +38,6 @@ def apply_model_column_renames( create_table_sql: str = adapter.render_create_column_migration_state_table_sql( database=first.destination.database, schema=first.destination.schema or "" ) - live: frozenset[str] = frozenset( - column.name.lower() - for column in _live_columns(adapter=adapter, connection=connection, entry=first) - ) if not adapter.supports_transactional_ddl(): entry: ColumnMigrationPlanEntry for entry in entries: @@ -49,7 +45,6 @@ def apply_model_column_renames( adapter=adapter, connection=connection, entry=entry, - live=live, run_id=run_id, create_table_sql=create_table_sql, ) @@ -61,7 +56,6 @@ def apply_model_column_renames( adapter=adapter, connection=connection, entry=entry, - live=live, run_id=run_id, create_table_sql=None, attempts=1, @@ -73,11 +67,14 @@ def _rename_and_record( adapter: BaseAdapter, connection: Any, entry: ColumnMigrationPlanEntry, - live: frozenset[str], run_id: str, create_table_sql: str | None, attempts: int = MIGRATION_WRITE_ATTEMPTS, ) -> None: + live: frozenset[str] = frozenset( + column.name.lower() + for column in _live_columns(adapter=adapter, connection=connection, entry=entry) + ) already_renamed: bool = ( entry.origin_column.lower() not in live and entry.destination_column.lower() in live ) diff --git a/tests/unit/src/sqlbuild/executor/migrations/__init__.py b/tests/unit/src/sqlbuild/executor/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/__init__.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/_test_types.py new file mode 100644 index 0000000000..43a93c0c51 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/_test_types.py @@ -0,0 +1,13 @@ +"""Test case types for executor column rename helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class OverlappingColumnRenameTestCase: + description: str + transactional: bool + expected_columns: tuple[str, ...] + expected_decisions: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py new file mode 100644 index 0000000000..a9c27990b7 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -0,0 +1,67 @@ +"""Fixtures for executor column rename helpers.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import duckdb + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery +from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry + +RELATION: str = "main.fct_orders" + + +def orders_connection() -> duckdb.DuckDBPyConnection: + """Return an in-memory warehouse holding one orders table with amount and tax.""" + + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + _ = connection.execute( + f"CREATE TABLE {RELATION} AS SELECT 1 AS order_id, 101 AS amount, 7 AS tax" + ) + return connection + + +def pending_rename(*, origin_column: str, destination_column: str) -> ColumnMigrationPlanEntry: + """Return one planned in-place rename on the orders table.""" + + return ColumnMigrationPlanEntry( + model_name="fct_orders", + destination=CompiledRelationLocation( + database=None, schema="main", name="fct_orders", qualified_name=RELATION + ), + origin_column=origin_column, + destination_column=destination_column, + discovery=MigrationDiscovery.MANUAL, + decision=ColumnMigrationDecision.RENAME, + target_name="dev", + ) + + +def rename_then_overlap( + *, adapter: DuckDbAdapter, connection: duckdb.DuckDBPyConnection +) -> Callable[..., None]: + """Return a rename that lets an overlapping build rename tax to levy right after it.""" + + original: Callable[..., None] = adapter.rename_column + + def rename(**kwargs: Any) -> None: + original(**kwargs) + _ = connection.execute(f"ALTER TABLE {RELATION} RENAME COLUMN tax TO levy") + + return rename + + +def live_column_names(connection: duckdb.DuckDBPyConnection) -> tuple[str, ...]: + """Return the orders table's physical column names in order.""" + + return tuple( + str(row[0]) + for row in connection.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'fct_orders' ORDER BY ordinal_position" + ).fetchall() + ) diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py new file mode 100644 index 0000000000..f1bdb26f8f --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py @@ -0,0 +1,73 @@ +"""Unit coverage for re-checking live columns before each in-place column rename.""" + +from __future__ import annotations + +import duckdb +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.migrations.main._read_column_events import read_column_migration_events +from sqlbuild.compiler.migrations.models import ColumnMigrationEvent +from sqlbuild.executor.migrations._helpers.columns import apply_model_column_renames +from tests.unit.src.sqlbuild.executor.migrations._helpers._test_types import ( + OverlappingColumnRenameTestCase, +) +from tests.unit.src.sqlbuild.executor.migrations._helpers.helpers import ( + live_column_names, + orders_connection, + pending_rename, + rename_then_overlap, +) + + +@pytest.mark.parametrize( + "test_case", + [ + OverlappingColumnRenameTestCase( + description="transactional warehouse records the column another build renamed", + transactional=True, + expected_columns=("order_id", "revenue", "levy"), + expected_decisions=("rename", "record"), + ), + OverlappingColumnRenameTestCase( + description="non-transactional warehouse records the column another build renamed", + transactional=False, + expected_columns=("order_id", "revenue", "levy"), + expected_decisions=("rename", "record"), + ), + ], + ids=lambda case: case.description, +) +def test_given_overlapping_build_between_renames_when_applying_then_reconciles_from_live_columns( + test_case: OverlappingColumnRenameTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: duckdb.DuckDBPyConnection = orders_connection() + monkeypatch.setattr(adapter, "supports_transactional_ddl", lambda: test_case.transactional) + monkeypatch.setattr( + adapter, "rename_column", rename_then_overlap(adapter=adapter, connection=connection) + ) + + apply_model_column_renames( + adapter=adapter, + connection=connection, + entries=( + pending_rename(origin_column="amount", destination_column="revenue"), + pending_rename(origin_column="tax", destination_column="levy"), + ), + run_id="run-1", + ) + events: tuple[ColumnMigrationEvent, ...] = read_column_migration_events( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + ) + + assert live_column_names(connection) == test_case.expected_columns + assert tuple(str(event.decision) for event in events) == test_case.expected_decisions + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 3fcb43893ee7b8be40b842b3e5738e12d97bd3fc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:28:30 +0100 Subject: [PATCH 8/9] docs(migrations): document when column renames are detected automatically Refs CHI-393 --- .../references/docs/concepts/models/column-migrations.md | 8 +++++--- .../docs/docs/concepts/models/column-migrations.mdx | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md index 61c534a2f1..f0e11be428 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md @@ -55,11 +55,13 @@ A declaration is rejected when compiling if: A pure rename is detected without a declaration: exactly one column that disappeared from the model has exactly the same expression as exactly one new column, for example `amount` becoming `amount AS revenue`, or `ROUND(amount, 2) AS revenue` becoming `ROUND(amount, 2) AS net_revenue`. Expressions are compared ignoring formatting, comments, and keyword and unquoted identifier case. Detection compares the query the table was last built from with the current query, so it applies to the build that makes the change. +A rename is only automatic when it is the whole change. Apart from the renamed output names, the query must be the same: the same CTEs, `FROM`, joins, filters, grouping, and other columns. References to the new name in `ORDER BY` or `QUALIFY` count as references to the old name, and the output columns may be reordered unless the query groups, orders, or uses `DISTINCT ON` by position. If anything else changed, an unchanged expression such as `amount` may now read different data, for example from a CTE that redefines `amount`, so nothing is renamed automatically and the change follows `on_schema_change`. Declare `migrate_from` to rename the column anyway. + Detection never guesses: - Columns are matched by expression, not by position. Two columns renamed at once each pair with the column whose expression they kept; two columns that swap names keep their names and are not renamed. - If more than one removed column has the same expression, or more than one new column does, nothing is migrated automatically. -- A near match, where the expressions reference the same columns but differ, for example `amount` becoming `ROUND(amount, 2) AS revenue`, is not a rename. It follows the model's `on_schema_change` policy, and `sqb plan` shows a hint next to the added column until the change has been built: +- A near match, where the expressions reference the same columns but differ or the rest of the query changed, for example `amount` becoming `ROUND(amount, 2) AS revenue`, is not a rename. It follows the model's `on_schema_change` policy, and `sqb plan` shows a hint next to the added column until the change has been built: ```text ├── schema diff: @@ -68,7 +70,7 @@ Detection never guesses: - amount (removed) ``` -Add the suggested `migrate_from` to keep the history. A declared rename with a changed expression is still renamed in place, but it is a query change, so `replay_on_change` decides how much history is rebuilt with the new expression. +Add the suggested `migrate_from` to keep the history. A declared rename with a changed expression or other query changes is still renamed in place, but it is a query change, so `replay_on_change` decides how much history is rebuilt, exactly as it would without the rename. ## Decisions @@ -106,7 +108,7 @@ On adapters with transactional DDL (PostgreSQL, SQL Server, DuckDB, MotherDuck), After the rename, the remaining differences follow `on_schema_change`: `append_new_columns` adds any other new columns, `sync_all_columns` also drops removed ones and changes types, and `fail` only fails when a difference other than the rename remains. -A pure rename is not a query change. It doesn't trigger `replay_on_change` for the model, and it doesn't cascade a replay to downstream models. Downstream models that you edit to use the new name have query changes of their own and follow their own `replay_on_change`. If the renamed column is the model's cursor, the next delta starts from the watermark stored under the old name. +A pure rename is not a query change. This applies to declared renames too, as long as the rename is the whole change by the rule above. It doesn't trigger `replay_on_change` for the model, and it doesn't cascade a replay to downstream models. Downstream models that you edit to use the new name have query changes of their own and follow their own `replay_on_change`. If the renamed column is the model's cursor, the next delta starts from the watermark stored under the old name. ## State diff --git a/website/src/content/docs/docs/concepts/models/column-migrations.mdx b/website/src/content/docs/docs/concepts/models/column-migrations.mdx index 19df24da72..52cc26c2ee 100644 --- a/website/src/content/docs/docs/concepts/models/column-migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/column-migrations.mdx @@ -43,11 +43,13 @@ A declaration is rejected when compiling if: A pure rename is detected without a declaration: exactly one column that disappeared from the model has exactly the same expression as exactly one new column, for example `amount` becoming `amount AS revenue`, or `ROUND(amount, 2) AS revenue` becoming `ROUND(amount, 2) AS net_revenue`. Expressions are compared ignoring formatting, comments, and keyword and unquoted identifier case. Detection compares the query the table was last built from with the current query, so it applies to the build that makes the change. +A rename is only automatic when it is the whole change. Apart from the renamed output names, the query must be the same: the same CTEs, `FROM`, joins, filters, grouping, and other columns. References to the new name in `ORDER BY` or `QUALIFY` count as references to the old name, and the output columns may be reordered unless the query groups, orders, or uses `DISTINCT ON` by position. If anything else changed, an unchanged expression such as `amount` may now read different data, for example from a CTE that redefines `amount`, so nothing is renamed automatically and the change follows `on_schema_change`. Declare `migrate_from` to rename the column anyway. + Detection never guesses: - Columns are matched by expression, not by position. Two columns renamed at once each pair with the column whose expression they kept; two columns that swap names keep their names and are not renamed. - If more than one removed column has the same expression, or more than one new column does, nothing is migrated automatically. -- A near match, where the expressions reference the same columns but differ, for example `amount` becoming `ROUND(amount, 2) AS revenue`, is not a rename. It follows the model's `on_schema_change` policy, and `sqb plan` shows a hint next to the added column until the change has been built: +- A near match, where the expressions reference the same columns but differ or the rest of the query changed, for example `amount` becoming `ROUND(amount, 2) AS revenue`, is not a rename. It follows the model's `on_schema_change` policy, and `sqb plan` shows a hint next to the added column until the change has been built: ```text ├── schema diff: @@ -56,7 +58,7 @@ Detection never guesses: - amount (removed) ``` -Add the suggested `migrate_from` to keep the history. A declared rename with a changed expression is still renamed in place, but it is a query change, so `replay_on_change` decides how much history is rebuilt with the new expression. +Add the suggested `migrate_from` to keep the history. A declared rename with a changed expression or other query changes is still renamed in place, but it is a query change, so `replay_on_change` decides how much history is rebuilt, exactly as it would without the rename. ## Decisions @@ -94,7 +96,7 @@ On adapters with transactional DDL (PostgreSQL, SQL Server, DuckDB, MotherDuck), After the rename, the remaining differences follow `on_schema_change`: `append_new_columns` adds any other new columns, `sync_all_columns` also drops removed ones and changes types, and `fail` only fails when a difference other than the rename remains. -A pure rename is not a query change. It doesn't trigger `replay_on_change` for the model, and it doesn't cascade a replay to downstream models. Downstream models that you edit to use the new name have query changes of their own and follow their own `replay_on_change`. If the renamed column is the model's cursor, the next delta starts from the watermark stored under the old name. +A pure rename is not a query change. This applies to declared renames too, as long as the rename is the whole change by the rule above. It doesn't trigger `replay_on_change` for the model, and it doesn't cascade a replay to downstream models. Downstream models that you edit to use the new name have query changes of their own and follow their own `replay_on_change`. If the renamed column is the model's cursor, the next delta starts from the watermark stored under the old name. ## State From b9231fc041c7cba97adaf0da7484973a08681b38 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:46:40 +0100 Subject: [PATCH 9/9] fix(planner): prove alias bindings before treating renamed references as equivalent ORDER BY and QUALIFY references to a renamed output were rewritten to the old name without proving they bind to the alias. When the new name was also an input column, DuckDB bound QUALIFY to the input, so the queries differed but the rename was applied automatically and replay was suppressed. Substitute only when the binding is proven: every relation the SELECT reads must have known columns (inspected physical columns of sources and upstream models, or named CTE and subquery outputs), the new name must not be an input column, and the old name must be absent from the inputs or passed through unchanged. Otherwise the change is not a rename, so nothing is renamed automatically and declared renames follow the normal replay policy. Refs CHI-393 --- .../docs/concepts/models/column-migrations.md | 2 +- .../_helpers/migrations/column_renames.py | 7 +- .../planner/_helpers/migrations/columns.py | 35 ++- .../_helpers/migrations/projections.py | 238 +++++++++++++++++- .../_helpers/planning/warehouse_state.py | 19 +- .../planner/classes/known_input_columns.py | 31 +++ src/sqlbuild/compiler/planner/constants.py | 2 + src/sqlbuild/compiler/planner/models.py | 4 + src/sqlbuild/compiler/planner/types.py | 2 + .../main/column_migrations/_test_types.py | 10 + .../main/column_migrations/helpers.py | 15 ++ .../test_column_rename_equivalence.py | 65 +++++ .../_helpers/migrations/_test_types.py | 11 +- .../planner/_helpers/migrations/helpers.py | 10 + .../migrations/test_column_renames.py | 83 +++++- .../concepts/models/column-migrations.mdx | 2 +- 16 files changed, 514 insertions(+), 22 deletions(-) create mode 100644 src/sqlbuild/compiler/planner/classes/known_input_columns.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md index f0e11be428..6ad05a0950 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md @@ -55,7 +55,7 @@ A declaration is rejected when compiling if: A pure rename is detected without a declaration: exactly one column that disappeared from the model has exactly the same expression as exactly one new column, for example `amount` becoming `amount AS revenue`, or `ROUND(amount, 2) AS revenue` becoming `ROUND(amount, 2) AS net_revenue`. Expressions are compared ignoring formatting, comments, and keyword and unquoted identifier case. Detection compares the query the table was last built from with the current query, so it applies to the build that makes the change. -A rename is only automatic when it is the whole change. Apart from the renamed output names, the query must be the same: the same CTEs, `FROM`, joins, filters, grouping, and other columns. References to the new name in `ORDER BY` or `QUALIFY` count as references to the old name, and the output columns may be reordered unless the query groups, orders, or uses `DISTINCT ON` by position. If anything else changed, an unchanged expression such as `amount` may now read different data, for example from a CTE that redefines `amount`, so nothing is renamed automatically and the change follows `on_schema_change`. Declare `migrate_from` to rename the column anyway. +A rename is only automatic when it is the whole change. Apart from the renamed output names, the query must be the same: the same CTEs, `FROM`, joins, filters, grouping, and other columns. References to the new name in `ORDER BY` or `QUALIFY` count as references to the old name only when SQLBuild can prove they bind to the output column: the columns of every relation the query reads must be known, from the inspected warehouse tables of its sources and upstream models or from CTEs and subqueries with named columns, the new name must not be an input column, and the old name must either not be an input column or be passed through unchanged. Otherwise the change is not treated as a rename, because engines differ on whether such a name refers to the output alias or the input column. The output columns may be reordered unless the query groups, orders, or uses `DISTINCT ON` by position. If anything else changed, an unchanged expression such as `amount` may now read different data, for example from a CTE that redefines `amount`, so nothing is renamed automatically and the change follows `on_schema_change`. Declare `migrate_from` to rename the column anyway. Detection never guesses: diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py b/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py index fe58300798..6b38fba99f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/column_renames.py @@ -6,6 +6,7 @@ from sqlbuild.compiler.planner._helpers.migrations.projections import renames_explain_change from sqlbuild.compiler.planner.models import ColumnRenameHint, QueryProjection, QueryShape +from sqlbuild.compiler.planner.types import InputColumns def identical_renames( @@ -14,6 +15,7 @@ def identical_renames( current: QueryShape, excluded: frozenset[str], declared: Mapping[str, str], + input_columns: InputColumns, ) -> tuple[tuple[str, str], ...]: """Return same-expression (old, new) pairs only when renaming is the whole change.""" @@ -32,7 +34,10 @@ def identical_renames( if claims == 1: pairs.append((matches[0].name, column.name)) if not pairs or not renames_explain_change( - previous=previous, current=current, renames={**declared, **dict(pairs)} + previous=previous, + current=current, + renames={**declared, **dict(pairs)}, + input_columns=input_columns, ): return () return tuple(pairs) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index a3988d7f8c..2db0141960 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -28,6 +28,7 @@ parse_query_shape, renames_explain_change, ) +from sqlbuild.compiler.planner.classes.known_input_columns import KnownInputColumns from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) @@ -44,7 +45,7 @@ WarehouseFingerprints, WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity +from sqlbuild.compiler.planner.types import InputColumns, MaterializationType, WarningSeverity from sqlbuild.spec.contracts.main.get_config_str import get_config_str _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( @@ -82,6 +83,7 @@ def plan_column_migrations( physical_relations: Mapping[str, str], overrides: PlannerOverrides, deferral: DeferralInputs, + source_columns: Mapping[str, tuple[ColumnInfo, ...]], ) -> ColumnMigrationPlanning: """Decide declared and detected column renames and project the renamed snapshot.""" @@ -114,6 +116,10 @@ def plan_column_migrations( if candidate.model.destination.schema is not None } ) + input_columns: InputColumns = KnownInputColumns( + model_columns=snapshot.existing_columns, + source_columns=source_columns, + ) entries: list[ColumnMigrationPlanEntry] = [] hints: list[ColumnRenameHint] = [] warnings: list[PlanWarning] = [] @@ -124,7 +130,12 @@ def plan_column_migrations( model_entries, model_warnings = _checked_entries( runtime=runtime, candidate=candidate, - entries=_decide(runtime=runtime, candidate=candidate, events=state.column_events), + entries=_decide( + runtime=runtime, + candidate=candidate, + events=state.column_events, + input_columns=input_columns, + ), physical_relation=physical_relations.get(candidate.model.name), ) entries.extend(model_entries) @@ -132,7 +143,10 @@ def plan_column_migrations( hints.extend(_hints(candidate=candidate, entries=model_entries)) warnings.extend(warning for entry in entries if (warning := _entry_warning(entry))) overlaid: WarehouseSnapshot = _overlay_snapshot( - snapshot=snapshot, candidates=candidates, entries=tuple(entries) + snapshot=snapshot, + candidates=candidates, + entries=tuple(entries), + input_columns=input_columns, ) return ColumnMigrationPlanning( snapshot=_with_renamed_cursors( @@ -200,10 +214,14 @@ def _decide( runtime: PlannerRuntime, candidate: _ModelColumns, events: tuple[ColumnMigrationEvent, ...], + input_columns: InputColumns, ) -> tuple[ColumnMigrationPlanEntry, ...]: entries: list[ColumnMigrationPlanEntry] = [] pair: _Pair - for pair in (*_declared_pairs(candidate), *_detected_pairs(candidate)): + for pair in ( + *_declared_pairs(candidate), + *_detected_pairs(candidate=candidate, input_columns=input_columns), + ): entry: ColumnMigrationPlanEntry | None = _decide_pair( runtime=runtime, candidate=candidate, pair=pair, events=events ) @@ -219,7 +237,7 @@ def _declared_pairs(candidate: _ModelColumns) -> tuple[_Pair, ...]: ) -def _detected_pairs(candidate: _ModelColumns) -> tuple[_Pair, ...]: +def _detected_pairs(*, candidate: _ModelColumns, input_columns: InputColumns) -> tuple[_Pair, ...]: if candidate.previous is None or candidate.current is None: return () declared: dict[str, str] = { @@ -232,6 +250,7 @@ def _detected_pairs(candidate: _ModelColumns) -> tuple[_Pair, ...]: current=candidate.current, excluded=_declared_names(candidate), declared=declared, + input_columns=input_columns, ) ) @@ -374,6 +393,7 @@ def _overlay_snapshot( snapshot: WarehouseSnapshot, candidates: tuple[_ModelColumns, ...], entries: tuple[ColumnMigrationPlanEntry, ...], + input_columns: InputColumns, ) -> WarehouseSnapshot: """Plan each model as if its columns were already renamed and its query unchanged.""" @@ -405,7 +425,10 @@ def _overlay_snapshot( and candidate.previous is not None and candidate.current is not None and renames_explain_change( - previous=candidate.previous, current=candidate.current, renames=renames + previous=candidate.previous, + current=candidate.current, + renames=renames, + input_columns=input_columns, ) ): fingerprints[candidate.model.name] = replace( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py b/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py index 5ecb9e867e..ac50991eb0 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/projections.py @@ -7,7 +7,9 @@ from typing import Any from sqlbuild.compiler.planner._helpers.migrations.fingerprint import strip_ast_formatting +from sqlbuild.compiler.planner.constants import REF_INPUT_FUNCTION, SOURCE_INPUT_FUNCTION from sqlbuild.compiler.planner.models import QueryProjection, QueryShape +from sqlbuild.compiler.planner.types import InputColumns from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _GENERIC_DIALECT: str = "generic" @@ -26,6 +28,21 @@ _ORDER_BY_KEY: str = "order_by" _DISTINCT_ON_KEY: str = "distinct_on" _ALIAS_SCOPED_KEYS: tuple[str, ...] = (_ORDER_BY_KEY, "qualify") +_FROM_KEY: str = "from" +_JOINS_KEY: str = "joins" +_WITH_KEY: str = "with" +_ON_KEY: str = "on" +_LATERAL_VIEWS_KEY: str = "lateral_views" +_CTES_KEY: str = "ctes" +_COLUMNS_KEY: str = "columns" +_COLUMN_ALIASES_KEY: str = "column_aliases" +_FUNCTION_KEY: str = "function" +_ARGS_KEY: str = "args" +_SUBQUERY_KEY: str = "subquery" +_RELATION_FUNCTIONS: frozenset[str] = frozenset({SOURCE_INPUT_FUNCTION, REF_INPUT_FUNCTION}) +_UNSCANNED_CLAUSE_KEYS: frozenset[str] = frozenset( + {_EXPRESSIONS_KEY, _WITH_KEY, _FROM_KEY, _JOINS_KEY} +) def parse_query_shape(*, query_sql: str, dialect: str | None) -> QueryShape | None: @@ -57,16 +74,27 @@ def parse_query_shape(*, query_sql: str, dialect: str | None) -> QueryShape | No for key, value in canonical.items() if key != _EXPRESSIONS_KEY and key not in _ALIAS_SCOPED_KEYS } + inputs: tuple[tuple[tuple[str, str], ...], frozenset[str]] | None = _select_inputs(canonical) return QueryShape( projections=tuple(projections), body=_encode(body), alias_clauses=_encode({key: canonical.get(key) for key in _ALIAS_SCOPED_KEYS}), positional=_has_positional_reference(canonical), + input_relations=inputs[0] if inputs is not None else None, + local_input_columns=inputs[1] if inputs is not None else frozenset(), + clause_references=_unqualified_references( + [value for key, value in canonical.items() if key not in _UNSCANNED_CLAUSE_KEYS] + + [join.get(_ON_KEY) for join in _dicts(canonical.get(_JOINS_KEY))] + ), ) def renames_explain_change( - *, previous: QueryShape, current: QueryShape, renames: Mapping[str, str] + *, + previous: QueryShape, + current: QueryShape, + renames: Mapping[str, str], + input_columns: InputColumns, ) -> bool: """Return whether renaming output columns, matched by name, is the whole query change.""" @@ -89,6 +117,17 @@ def renames_explain_change( ) if reordered and (previous.positional or current.positional): return False + if not all( + _binding_proven( + previous=previous, + current=current, + origin=origin, + destination=destination, + input_columns=input_columns, + ) + for origin, destination in lowered.items() + ): + return False restored: Any = _substitute_alias_references( node=json.loads(current.alias_clauses), names={new: old for old, new in lowered.items()}, @@ -96,6 +135,187 @@ def renames_explain_change( return _encode(restored) == previous.alias_clauses +def _binding_proven( + *, + previous: QueryShape, + current: QueryShape, + origin: str, + destination: str, + input_columns: InputColumns, +) -> bool: + """Return whether references to a renamed column provably bind the same in both queries.""" + + referenced: frozenset[str] = ( + previous.clause_references + | current.clause_references + | _other_projection_references(shape=previous, excluded=origin) + | _other_projection_references(shape=current, excluded=destination) + ) + if origin not in referenced and destination not in referenced: + return True + inputs: frozenset[str] | None = _known_input_columns(shape=current, input_columns=input_columns) + if inputs is None or destination in inputs: + return False + if origin in inputs: + return any(item.key == origin and item.passthrough for item in previous.projections) + return origin not in ( + current.clause_references | _other_projection_references(shape=current, excluded="") + ) + + +def _other_projection_references(*, shape: QueryShape, excluded: str) -> frozenset[str]: + return frozenset().union( + *(item.references for item in shape.projections if item.key != excluded) + ) + + +def _known_input_columns( + *, shape: QueryShape, input_columns: InputColumns +) -> frozenset[str] | None: + if shape.input_relations is None: + return None + known: set[str] = set(shape.local_input_columns) + kind: str + name: str + for kind, name in shape.input_relations: + columns: frozenset[str] | None = input_columns(kind, name) + if columns is None: + return None + known.update(columns) + return frozenset(known) + + +def _select_inputs( + select: Mapping[str, Any], +) -> tuple[tuple[tuple[str, str], ...], frozenset[str]] | None: + """Return a SELECT's referenced relations and derived-table columns, or None when unknown.""" + + if _dicts(select.get(_LATERAL_VIEWS_KEY)): + return None + ctes: dict[str, frozenset[str] | None] = _cte_columns(select.get(_WITH_KEY)) + items: list[Any] = list(_clause_items(select.get(_FROM_KEY))) + items.extend(join.get(_THIS_KEY) for join in _dicts(select.get(_JOINS_KEY))) + relations: list[tuple[str, str]] = [] + local: set[str] = set() + item: Any + for item in items: + resolved: tuple[tuple[str, str] | None, frozenset[str] | None] = _input_item( + item=item, ctes=ctes + ) + if resolved[0] is not None: + relations.append(resolved[0]) + elif resolved[1] is not None: + local.update(resolved[1]) + else: + return None + return tuple(relations), frozenset(local) + + +def _input_item( + *, item: Any, ctes: Mapping[str, frozenset[str] | None] +) -> tuple[tuple[str, str] | None, frozenset[str] | None]: + """Resolve one FROM or JOIN item to a relation name or to its known columns.""" + + if not isinstance(item, dict) or len(item) != 1: + return None, None + if _ALIAS_KEY in item: + alias: Any = item[_ALIAS_KEY] + if not isinstance(alias, dict): + return None, None + renamed: tuple[str, ...] = _identifier_names(alias.get(_COLUMN_ALIASES_KEY)) + inner: tuple[tuple[str, str] | None, frozenset[str] | None] = _input_item( + item=alias.get(_THIS_KEY), ctes=ctes + ) + if renamed and (inner[0] is not None or inner[1] is not None): + return None, frozenset(renamed) + return inner + function: Any = item.get(_FUNCTION_KEY) + if isinstance(function, dict): + args: Any = function.get(_ARGS_KEY) + kind: str = str(function.get(_NAME_KEY, "")).lower() + if ( + kind not in _RELATION_FUNCTIONS + or not isinstance(args, list) + or len(args) != 1 + or not isinstance(args[0], dict) + or not isinstance(args[0].get(_COLUMN_KEY), dict) + ): + return None, None + relation: str | None = _identifier_name(args[0][_COLUMN_KEY].get(_NAME_KEY)) + return (None if relation is None else (kind, relation)), None + table: Any = item.get(_TABLE_KEY) + if isinstance(table, dict): + name: str | None = _identifier_name(table.get(_NAME_KEY)) + if name is None or table.get("schema") is not None or table.get("catalog") is not None: + return None, None + return None, ctes.get(name) + subquery: Any = item.get(_SUBQUERY_KEY) + if isinstance(subquery, dict): + return None, _output_names(subquery.get(_THIS_KEY)) + return None, None + + +def _cte_columns(with_clause: Any) -> dict[str, frozenset[str] | None]: + if not isinstance(with_clause, dict): + return {} + columns: dict[str, frozenset[str] | None] = {} + cte: dict[str, Any] + for cte in _dicts(with_clause.get(_CTES_KEY)): + name: str | None = _identifier_name(cte.get(_ALIAS_KEY)) + if name is None: + continue + declared: tuple[str, ...] = _identifier_names(cte.get(_COLUMNS_KEY)) + columns[name] = frozenset(declared) if declared else _output_names(cte.get(_THIS_KEY)) + return columns + + +def _output_names(query: Any) -> frozenset[str] | None: + """Return the lower-cased output names of a plain SELECT, or None when any is unknown.""" + + if not isinstance(query, dict) or set(query) != {_SELECT_KEY}: + return None + expressions: Any = query[_SELECT_KEY].get(_EXPRESSIONS_KEY) + if not isinstance(expressions, list): + return None + projections: list[QueryProjection | None] = [_projection(value) for value in expressions] + if any(projection is None for projection in projections): + return None + return frozenset(projection.key for projection in projections if projection is not None) + + +def _identifier_names(identifiers: Any) -> tuple[str, ...]: + return tuple( + name.lower() + for identifier in (identifiers if isinstance(identifiers, list) else []) + if (name := _identifier_name(identifier)) is not None + ) + + +def _dicts(value: Any) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def _unqualified_references(node: Any) -> frozenset[str]: + """Return lower-cased names of unqualified column references anywhere under node.""" + + found: set[str] = set() + pending: list[Any] = [node] + while pending: + current: Any = pending.pop() + if isinstance(current, list): + pending.extend(current) + continue + if not isinstance(current, dict): + continue + column: Any = current.get(_COLUMN_KEY) + if isinstance(column, dict) and column.get(_TABLE_KEY) is None: + name: str | None = _identifier_name(column.get(_NAME_KEY)) + if name is not None: + found.add(name.lower()) + pending.extend(current.values()) + return frozenset(found) + + def _substitute_alias_references(*, node: Any, names: Mapping[str, str]) -> Any: """Rename unqualified references to output aliases, leaving nested queries untouched.""" @@ -154,7 +374,10 @@ def _projection(expression: Any) -> QueryProjection | None: if name is None or body is None: return None return QueryProjection( - name=name, expression=_encode(_canonical(body)), references=_references(body) + name=name, + expression=_encode(_canonical(body)), + references=_references(body), + passthrough=_is_bare_column(node=body, name=name), ) if _COLUMN_KEY in expression: column: Any = expression[_COLUMN_KEY] @@ -165,10 +388,21 @@ def _projection(expression: Any) -> QueryProjection | None: name=name, expression=_encode(_canonical(expression)), references=_references(expression), + passthrough=_is_bare_column(node=expression, name=name), ) return None +def _is_bare_column(*, node: Any, name: str) -> bool: + """Return whether node is an unqualified reference to a column with the given name.""" + + column: Any = node.get(_COLUMN_KEY) if isinstance(node, dict) else None + if not isinstance(column, dict) or column.get(_TABLE_KEY) is not None: + return False + referenced: str | None = _identifier_name(column.get(_NAME_KEY)) + return referenced is not None and referenced.lower() == name.lower() + + def _identifier_name(identifier: Any) -> str | None: if not isinstance(identifier, dict) or not isinstance(identifier.get(_NAME_KEY), str): return None diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py index 2d7924275b..7ea4b50db9 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/warehouse_state.py @@ -66,6 +66,15 @@ def gather_planner_warehouse_state( overrides=overrides, deferral=deferral, ) + inspection_relations: PlannerRelationsContext = build_planner_relations_context( + project=runtime.project, + adapter=runtime.adapter, + connection=runtime.connection, + scope=scopes.inspection_scope, + deferral=deferral, + project_config=runtime.project_config, + local_config=runtime.local_config, + ) columns: ColumnMigrationPlanning = plan_column_migrations( runtime=runtime, scope=scopes.selected_scope, @@ -81,15 +90,7 @@ def gather_planner_warehouse_state( }, overrides=overrides, deferral=deferral, - ) - inspection_relations: PlannerRelationsContext = build_planner_relations_context( - project=runtime.project, - adapter=runtime.adapter, - connection=runtime.connection, - scope=scopes.inspection_scope, - deferral=deferral, - project_config=runtime.project_config, - local_config=runtime.local_config, + source_columns=inspection_relations.source_warehouse_columns, ) if runtime.on_progress is not None: runtime.on_progress( diff --git a/src/sqlbuild/compiler/planner/classes/known_input_columns.py b/src/sqlbuild/compiler/planner/classes/known_input_columns.py new file mode 100644 index 0000000000..e8d30db508 --- /dev/null +++ b/src/sqlbuild/compiler/planner/classes/known_input_columns.py @@ -0,0 +1,31 @@ +"""Physical columns of the relations a model reads, as inspected by the planner.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.compiler.planner.constants import REF_INPUT_FUNCTION, SOURCE_INPUT_FUNCTION + + +class KnownInputColumns: + """Look up the lower-cased physical columns a query binds against, or None when unknown.""" + + def __init__( + self, + *, + model_columns: Mapping[str, tuple[ColumnInfo, ...]], + source_columns: Mapping[str, tuple[ColumnInfo, ...]], + ) -> None: + self._inspected: dict[str, Mapping[str, tuple[ColumnInfo, ...]]] = { + REF_INPUT_FUNCTION: model_columns, + SOURCE_INPUT_FUNCTION: source_columns, + } + + def __call__(self, kind: str, name: str) -> frozenset[str] | None: + """Return the inspected columns of one __source or __ref relation, or None.""" + + columns: tuple[ColumnInfo, ...] = self._inspected.get(kind, {}).get(name, ()) + if not columns: + return None + return frozenset(column.name.lower() for column in columns) diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 73829c5887..5e5bdc928b 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -80,3 +80,5 @@ ) SNAPSHOT_DEFAULT_VALID_FROM_COLUMN: str = "valid_from" SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str = "valid_to" +SOURCE_INPUT_FUNCTION: str = "__source" +REF_INPUT_FUNCTION: str = "__ref" diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index dcf9e6b5e7..bb7c757098 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -987,6 +987,7 @@ class QueryProjection: name: str expression: str references: frozenset[str] + passthrough: bool = False @property def key(self) -> str: @@ -1003,6 +1004,9 @@ class QueryShape: body: str alias_clauses: str = "{}" positional: bool = False + input_relations: tuple[tuple[str, str], ...] | None = None + local_input_columns: frozenset[str] = frozenset() + clause_references: frozenset[str] = frozenset() @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index b41eae84bb..201122727e 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -2,12 +2,14 @@ from __future__ import annotations +from collections.abc import Callable from enum import StrEnum from typing import TYPE_CHECKING, Protocol from sqlbuild.compiler.compile.types import CompiledResourceType type FixtureKey = tuple[CompiledResourceType, str] +type InputColumns = Callable[[str, str], frozenset[str] | None] type FixtureGroups = tuple[tuple[CompiledResourceType, dict[str, str]], ...] if TYPE_CHECKING: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py index 6754e12a6f..8115baca26 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/_test_types.py @@ -140,3 +140,13 @@ class UnprovenRenameReplayTestCase: declared_sql: str expected_backfill: dict[str, Any] expected_values: tuple[tuple[int, Any], ...] + + +@dataclass(frozen=True) +class CollidingAliasRenameTestCase: + description: str + initial_sql: str + renamed_sql: str + expected_planned: tuple[tuple[str, str, str, str], ...] + expected_backfill: dict[str, Any] + expected_values: tuple[tuple[int, Any], ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index 6444d6c31d..bb4537f6df 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -327,6 +327,21 @@ def column_events(*, project_dir: Path) -> tuple[tuple[str, str, str, str], ...] ) +def load_descending_amounts(*, project_dir: Path, last_day: int) -> None: + """Replace raw orders so amounts fall while tax rises, making the two orderings differ.""" + + execute(project_dir=project_dir, sql="DROP VIEW IF EXISTS main.raw_orders") + execute(project_dir=project_dir, sql="DROP TABLE IF EXISTS main.raw_orders") + execute( + project_dir=project_dir, + sql=( + "CREATE VIEW main.raw_orders AS SELECT i AS order_id, " + "TIMESTAMP '2026-01-01' + to_days(CAST(i - 1 AS INTEGER)) AS order_date, " + f"100 - i AS amount, i AS tax FROM range(1, {last_day + 1}) AS t(i)" + ), + ) + + def change_historical_amount(*, project_dir: Path) -> None: """Change day one's raw amount so any replay would be visible in built history.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py index 887114ac9c..3abaad406a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py @@ -8,6 +8,7 @@ import pytest from tests.integration.src.sqlbuild.cli.commands.main.column_migrations._test_types import ( + CollidingAliasRenameTestCase, EquivalentRenameTestCase, RebindingRenameTestCase, UnprovenRenameReplayTestCase, @@ -22,6 +23,7 @@ column_names, column_values, incremental_sql, + load_descending_amounts, migrate_columns, model_plan, orders_sql, @@ -36,6 +38,13 @@ f"WITH changed AS (SELECT order_id, order_date, tax AS amount {_ORDERS}) " "SELECT order_id, order_date, amount AS revenue FROM changed" ) +_QUALIFY_BEFORE: str = ( + f"SELECT order_id, order_date, amount {_ORDERS} QUALIFY row_number() OVER (ORDER BY amount) = 1" +) +_QUALIFY_AFTER: str = ( + f"SELECT order_id, order_date, amount AS tax {_ORDERS} " + "QUALIFY row_number() OVER (ORDER BY tax) = 1" +) _FILTERED_BODY: str = f"SELECT order_id, order_date, amount AS revenue {_ORDERS} WHERE amount > 0" _REPLAY_BOUNDED: str = " replay_on_change bounded-2d,\n" _HISTORY: tuple[tuple[int, Any], ...] = ((1, 101), (2, 102), (3, 103), (4, 104), (5, 105)) @@ -192,5 +201,61 @@ def test_given_rename_with_other_query_changes_when_building_then_replay_matches assert column_values(project_dir=tmp_path, column="revenue") == test_case.expected_values +@pytest.mark.parametrize( + "test_case", + [ + CollidingAliasRenameTestCase( + description="QUALIFY on a new name that is an input column replays as configured", + initial_sql=incremental_sql(body=_QUALIFY_BEFORE, extra_config=REPLAY_FULL), + renamed_sql=incremental_sql(body=_QUALIFY_AFTER, extra_config=REPLAY_FULL), + expected_planned=(), + expected_backfill={"action": "full", "duration": None}, + expected_values=((1, 99),), + ), + CollidingAliasRenameTestCase( + description="a declared rename with a colliding QUALIFY name still replays fully", + initial_sql=incremental_sql(body=_QUALIFY_BEFORE, extra_config=REPLAY_FULL), + renamed_sql=incremental_sql( + body=_QUALIFY_AFTER, + extra_config=REPLAY_FULL + migrate_columns("tax (migrate_from amount)"), + ), + expected_planned=(("amount", "tax", "manual", "rename"),), + expected_backfill={"action": "full", "duration": None}, + expected_values=((1, 99),), + ), + CollidingAliasRenameTestCase( + description="ORDER BY on a new name that is an input column replays as configured", + initial_sql=incremental_sql( + body=f"SELECT order_id, order_date, amount {_ORDERS} ORDER BY amount LIMIT 1", + extra_config=REPLAY_FULL, + ), + renamed_sql=incremental_sql( + body=f"SELECT order_id, order_date, amount AS tax {_ORDERS} ORDER BY tax LIMIT 1", + extra_config=REPLAY_FULL, + ), + expected_planned=(), + expected_backfill={"action": "full", "duration": None}, + expected_values=((5, 95),), + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_alias_colliding_with_an_input_column_when_building_then_it_replays( + test_case: CollidingAliasRenameTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_model(project_dir=tmp_path, sql=test_case.initial_sql) + load_descending_amounts(project_dir=tmp_path, last_day=3) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + load_descending_amounts(project_dir=tmp_path, last_day=5) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + + plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert planned_column_migrations(plan) == test_case.expected_planned + assert model_plan(plan)["backfill"] == test_case.expected_backfill + assert column_values(project_dir=tmp_path, column="tax") == test_case.expected_values + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py index c05dcb3020..345a72273b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/_test_types.py @@ -4,7 +4,10 @@ from dataclasses import dataclass, field -from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import BASE_CONFIG +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import ( + BASE_CONFIG, + ORDERS_INPUTS, +) @dataclass(frozen=True) @@ -33,6 +36,9 @@ class IdenticalRenameTestCase: expected_renames: tuple[tuple[str, str], ...] excluded: frozenset[str] = frozenset() declared: dict[str, str] = field(default_factory=dict) + input_columns: dict[tuple[str, str], frozenset[str]] = field( + default_factory=lambda: dict(ORDERS_INPUTS) + ) @dataclass(frozen=True) @@ -51,6 +57,9 @@ class RenameExplainsChangeTestCase: current_sql: str renames: dict[str, str] expected_explained: bool + input_columns: dict[tuple[str, str], frozenset[str]] = field( + default_factory=lambda: dict(ORDERS_INPUTS) + ) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index 6ea4fd3c2b..d75a770cd9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.planner._helpers.migrations.fingerprint import build_migration_fingerprint from sqlbuild.compiler.planner._helpers.migrations.projections import parse_query_shape from sqlbuild.compiler.planner.models import QueryShape +from sqlbuild.compiler.planner.types import InputColumns BASE_CONFIG: dict[str, object] = { "materialized": "incremental", @@ -14,6 +15,9 @@ "unique_key": "order_id", "cursor": "order_date", } +ORDERS_INPUTS: dict[tuple[str, str], frozenset[str]] = { + ("__source", "raw_orders"): frozenset({"order_id", "order_date", "amount", "tax"}) +} ORDERS_SQL: str = 'SELECT o.order_id, o.amount_cents FROM __ref("stg_orders") AS o' @@ -36,3 +40,9 @@ def query_shape(query_sql: str) -> QueryShape: shape: QueryShape | None = parse_query_shape(query_sql=query_sql, dialect="duckdb") assert shape is not None, query_sql return shape + + +def input_lookup(relations: dict[tuple[str, str], frozenset[str]]) -> InputColumns: + """Return an input-column lookup over (reference kind, relation name) pairs.""" + + return lambda kind, name: relations.get((kind, name)) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py index 556a5cf63c..8d90f66d5b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py @@ -19,7 +19,10 @@ RenameHintTestCase, UnreadableQueryShapeTestCase, ) -from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import query_shape +from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations.helpers import ( + input_lookup, + query_shape, +) _ORDERS: str = 'FROM __source("raw_orders")' @@ -128,6 +131,7 @@ def test_given_previous_and_current_queries_when_detecting_renames_then_returns_ current=query_shape(test_case.current_sql), excluded=test_case.excluded, declared=test_case.declared, + input_columns=input_lookup(test_case.input_columns), ) assert renames == test_case.expected_renames @@ -251,6 +255,82 @@ def test_given_changed_columns_when_building_hints_then_suggests_explicit_migrat renames={"amount": "revenue"}, expected_explained=False, ), + RenameExplainsChangeTestCase( + description="QUALIFY on a new name that is also an input column binds the input", + previous_sql=( + f"SELECT order_id, order_date, amount {_ORDERS} " + "QUALIFY row_number() OVER (ORDER BY amount) = 1" + ), + current_sql=( + f"SELECT order_id, order_date, amount AS tax {_ORDERS} " + "QUALIFY row_number() OVER (ORDER BY tax) = 1" + ), + renames={"amount": "tax"}, + expected_explained=False, + ), + RenameExplainsChangeTestCase( + description="ORDER BY on a new name that is also an input column binds the input", + previous_sql=f"SELECT order_id, amount {_ORDERS} ORDER BY amount LIMIT 1", + current_sql=f"SELECT order_id, amount AS tax {_ORDERS} ORDER BY tax LIMIT 1", + renames={"amount": "tax"}, + expected_explained=False, + ), + RenameExplainsChangeTestCase( + description="an old name that is a different input column fails closed", + previous_sql=f"SELECT order_id, amount * 2 AS tax {_ORDERS} ORDER BY tax", + current_sql=f"SELECT order_id, amount * 2 AS doubled {_ORDERS} ORDER BY doubled", + renames={"tax": "doubled"}, + expected_explained=False, + ), + RenameExplainsChangeTestCase( + description="unknown source columns fail closed for alias references", + previous_sql=f"SELECT order_id, amount {_ORDERS} ORDER BY amount", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS} ORDER BY revenue", + renames={"amount": "revenue"}, + input_columns={}, + expected_explained=False, + ), + RenameExplainsChangeTestCase( + description="an unresolvable FROM relation fails closed for alias references", + previous_sql="SELECT order_id, amount FROM main.raw_orders ORDER BY amount", + current_sql="SELECT order_id, amount AS revenue FROM main.raw_orders ORDER BY revenue", + renames={"amount": "revenue"}, + expected_explained=False, + ), + RenameExplainsChangeTestCase( + description="a star CTE has unknown columns and fails closed", + previous_sql=( + f"WITH o AS (SELECT * {_ORDERS}) SELECT order_id, amount FROM o ORDER BY amount" + ), + current_sql=( + f"WITH o AS (SELECT * {_ORDERS}) " + "SELECT order_id, amount AS revenue FROM o ORDER BY revenue" + ), + renames={"amount": "revenue"}, + expected_explained=False, + ), + RenameExplainsChangeTestCase( + description="a CTE with named outputs proves the alias binding on its own", + previous_sql=( + f"WITH o AS (SELECT order_id, amount {_ORDERS}) " + "SELECT order_id, amount FROM o ORDER BY amount" + ), + current_sql=( + f"WITH o AS (SELECT order_id, amount {_ORDERS}) " + "SELECT order_id, amount AS revenue FROM o ORDER BY revenue" + ), + renames={"amount": "revenue"}, + input_columns={}, + expected_explained=True, + ), + RenameExplainsChangeTestCase( + description="a rename nothing else references needs no input columns", + previous_sql=f"SELECT order_id, amount {_ORDERS} ORDER BY order_id", + current_sql=f"SELECT order_id, amount AS revenue {_ORDERS} ORDER BY order_id", + renames={"amount": "revenue"}, + input_columns={}, + expected_explained=True, + ), RenameExplainsChangeTestCase( description="a WHERE on a renamed alias is not treated as an alias reference", previous_sql=f"SELECT order_id, amount {_ORDERS} WHERE amount > 0", @@ -268,6 +348,7 @@ def test_given_renames_when_comparing_queries_then_reports_whether_they_explain_ previous=query_shape(test_case.previous_sql), current=query_shape(test_case.current_sql), renames=test_case.renames, + input_columns=input_lookup(test_case.input_columns), ) assert explained is test_case.expected_explained diff --git a/website/src/content/docs/docs/concepts/models/column-migrations.mdx b/website/src/content/docs/docs/concepts/models/column-migrations.mdx index 52cc26c2ee..921e8f8d4d 100644 --- a/website/src/content/docs/docs/concepts/models/column-migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/column-migrations.mdx @@ -43,7 +43,7 @@ A declaration is rejected when compiling if: A pure rename is detected without a declaration: exactly one column that disappeared from the model has exactly the same expression as exactly one new column, for example `amount` becoming `amount AS revenue`, or `ROUND(amount, 2) AS revenue` becoming `ROUND(amount, 2) AS net_revenue`. Expressions are compared ignoring formatting, comments, and keyword and unquoted identifier case. Detection compares the query the table was last built from with the current query, so it applies to the build that makes the change. -A rename is only automatic when it is the whole change. Apart from the renamed output names, the query must be the same: the same CTEs, `FROM`, joins, filters, grouping, and other columns. References to the new name in `ORDER BY` or `QUALIFY` count as references to the old name, and the output columns may be reordered unless the query groups, orders, or uses `DISTINCT ON` by position. If anything else changed, an unchanged expression such as `amount` may now read different data, for example from a CTE that redefines `amount`, so nothing is renamed automatically and the change follows `on_schema_change`. Declare `migrate_from` to rename the column anyway. +A rename is only automatic when it is the whole change. Apart from the renamed output names, the query must be the same: the same CTEs, `FROM`, joins, filters, grouping, and other columns. References to the new name in `ORDER BY` or `QUALIFY` count as references to the old name only when SQLBuild can prove they bind to the output column: the columns of every relation the query reads must be known, from the inspected warehouse tables of its sources and upstream models or from CTEs and subqueries with named columns, the new name must not be an input column, and the old name must either not be an input column or be passed through unchanged. Otherwise the change is not treated as a rename, because engines differ on whether such a name refers to the output alias or the input column. The output columns may be reordered unless the query groups, orders, or uses `DISTINCT ON` by position. If anything else changed, an unchanged expression such as `amount` may now read different data, for example from a CTE that redefines `amount`, so nothing is renamed automatically and the change follows `on_schema_change`. Declare `migrate_from` to rename the column anyway. Detection never guesses: