From 097c65ea450beb3cbfbac98876210b1f88707d9b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:56:54 +0100 Subject: [PATCH 1/2] fix(snowflake): create state tables permanent with maximum time travel --- .../docs/concepts/adapters/snowflake.md | 13 + .../references/docs/concepts/planning.md | 4 + .../adapter/contract/classes/base_adapter.py | 22 +- .../contract/classes/duckdb_backed_adapter.py | 13 +- .../contract/classes/strict_adapter.py | 5 + .../adapter/state_sql/_helpers/table_ddl.py | 4 +- .../main/render_state_table_create_sql.py | 2 - .../bigquery/classes/bigquery_adapter.py | 13 +- .../databricks/classes/databricks_adapter.py | 13 +- .../postgres/classes/postgres_adapter.py | 13 +- .../_helpers/state_table_retention.py | 24 + .../snowflake/classes/snowflake_adapter.py | 146 ++++-- .../state_table_retention_fallbacks.py | 59 +++ src/sqlbuild/adapters/snowflake/constants.py | 4 + .../sqlserver/classes/sqlserver_adapter.py | 12 + .../_helpers/freshness/source_freshness.py | 2 +- .../compiler/fingerprints/_helpers/sql.py | 4 +- .../fingerprints/main/create_table_sql.py | 2 - .../migrations/_helpers/column_sql.py | 2 - .../migrations/_helpers/old_name_sql.py | 2 - .../compiler/migrations/_helpers/sql.py | 2 - .../main/column_create_table_sql.py | 2 - .../migrations/main/create_table_sql.py | 2 - .../main/old_name_view_create_table_sql.py | 2 - .../compiler/source_freshness/_helpers/sql.py | 31 +- ...reate_table_sql.py => create_table_sql.py} | 2 - .../compiler/source_freshness/main/write.py | 2 - .../executor/audit_results/_helpers/sql.py | 2 - .../audit_results/main/create_table_sql.py | 2 - .../executor/janitor_events/_helpers/sql.py | 2 - .../janitor_events/main/create_table_sql.py | 2 - .../executor/node_results/_helpers/sql.py | 31 +- .../node_results/main/create_table_sql.py | 2 - .../classes/statement_lifecycle.py | 8 + .../build/snowflake_metadata/_test_types.py | 10 + .../build/snowflake_metadata/helpers.py | 40 ++ .../test_state_table_retention.py | 60 +++ .../unit/src/sqlbuild/adapters/_test_types.py | 8 + .../adapters/snowflake/_test_types.py | 59 +++ .../sqlbuild/adapters/snowflake/helpers.py | 148 ++++++ .../snowflake/test_state_table_ddl.py | 442 ++++++++++++++++++ .../sqlbuild/adapters/test_rename_column.py | 5 +- .../sqlbuild/adapters/test_state_table_ddl.py | 57 +++ .../sqlbuild/cli/commands/_helpers/helpers.py | 13 +- .../main/shared/_helpers/_test_types.py | 1 - .../main/shared/_helpers/test_sql.py | 11 - .../main/shared/_helpers/_test_types.py | 1 - .../main/shared/_helpers/test_sql.py | 11 - .../_helpers/test_old_name_views.py | 4 +- .../docs/docs/concepts/adapters/snowflake.mdx | 12 + .../content/docs/docs/concepts/planning.mdx | 4 + 51 files changed, 1189 insertions(+), 148 deletions(-) create mode 100644 src/sqlbuild/adapters/snowflake/_helpers/state_table_retention.py create mode 100644 src/sqlbuild/adapters/snowflake/classes/state_table_retention_fallbacks.py rename src/sqlbuild/compiler/source_freshness/main/{_create_table_sql.py => create_table_sql.py} (93%) create mode 100644 tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_state_table_retention.py create mode 100644 tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py create mode 100644 tests/unit/src/sqlbuild/adapters/test_state_table_ddl.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index ebc4156195..a48a472d01 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/snowflake/ - Session initialization - Planning metadata - Build metadata +- State tables - Shared connections across targets - Cost estimates @@ -75,6 +76,18 @@ During a build, checks for whether one relation exists use `SHOW TERSE TABLES` a need a running warehouse. Each build reuses these answers until it changes that relation, for example by replacing a delta table or adding a column, and then reads it again. +## State tables + +SQLBuild records its state in `_sqlbuild_*` tables, such as `_sqlbuild_fingerprints`, +`_sqlbuild_source_freshness`, and `_sqlbuild_microbatches`. On Snowflake, SQLBuild creates every +state table as a permanent table with `DATA_RETENTION_TIME_IN_DAYS = 90`, the maximum time travel, +so the table also has Fail-safe. On editions that allow only 1 day of time travel, Snowflake +rejects 90 and SQLBuild creates the table with 1 day instead. + +The retention is set only in the statement that creates the table. SQLBuild never alters an +existing state table, so a state table that already exists keeps its current table type and +retention. Models are not affected: their table type follows the model `table_type` setting. + ## Shared connections across targets Use multiple targets to reuse one Snowflake connection while selecting different authoritative diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md index 1c69d187a6..e44318b6ba 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md @@ -133,6 +133,10 @@ SQLBuild records append-only planning evidence in the warehouse: The planner reads the latest applicable facts and appends new facts after successful work. +On Snowflake, SQLBuild creates its state tables as permanent tables with the maximum time travel: +90 days, or 1 day on editions that allow only 1. The retention is set when SQLBuild creates the +table. See [Snowflake state tables](adapters/snowflake.md#state-tables). + ## Related topics - [Replay decisions](planning/replay-decisions.md) diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index b9ea630cf2..d43aef5d3d 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -106,7 +106,6 @@ class BaseAdapter(RetentionAdapterMixin, StrictAdapter): sql_analysis_dialect_name: ClassVar[str | None] = None sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax() max_identifier_length: ClassVar[int] = 63 - state_tables_transient: ClassVar[bool] = False relation_grants_supported: ClassVar[bool] = True views_read_with_reader_access: ClassVar[bool] = False allows_implicit_managed_write_schema: ClassVar[bool] = False @@ -2092,7 +2091,6 @@ def render_create_fingerprint_table_sql( schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: @@ -2161,6 +2159,20 @@ def render_create_fingerprint_index_sqls( del database, schema return () + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the source freshness table when it is missing.""" + + from sqlbuild.compiler.source_freshness.main.create_table_sql import ( + build_create_table_sql as build_source_freshness_create_table_sql, + ) + + return build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_source_freshness_index_sqls( self, *, @@ -2205,7 +2217,6 @@ def render_create_node_result_table_sql( schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_node_result_index_sqls( @@ -2230,7 +2241,6 @@ def render_create_audit_result_table_sql(self, *, database: str | None, schema: schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_audit_result_index_sqls( @@ -2252,7 +2262,6 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: @@ -2267,7 +2276,6 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_old_name_view_state_table_sql( @@ -2284,7 +2292,6 @@ def render_create_old_name_view_state_table_sql( schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_column_migration_state_table_sql( @@ -2301,7 +2308,6 @@ def render_create_column_migration_state_table_sql( schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_prune_fingerprint_history_sql( diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index c7a5c1ad76..7db0414c0a 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -804,6 +804,18 @@ def render_read_latest_source_freshness_sql( render_qualified_name=self.render_qualified_name, ) + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.source_freshness.main.create_table_sql import ( + build_create_table_sql as build_source_freshness_create_table_sql, + ) + + return build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_source_freshness_index_sqls( self, *, @@ -907,7 +919,6 @@ def render_create_old_name_view_state_table_sql( schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_column_migration_state_table_sql( diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 71f3bcf5ae..04d5a6519b 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -910,6 +910,11 @@ def render_create_fingerprint_index_sqls( """Render optional fingerprint table index DDL statements.""" ... + @abstractmethod + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the source freshness table when it is missing.""" + ... + @abstractmethod def render_create_source_freshness_index_sqls( self, diff --git a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py index bbf7c9b285..5312637c8c 100644 --- a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py +++ b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py @@ -15,7 +15,6 @@ def render_state_table_create_sql_impl( column_types: Mapping[str, StateSqlValueType], required_columns: frozenset[str], render_framework_type: Callable[[FrameworkType], str], - transient: bool, ) -> str: """Render create-if-missing DDL for string, integer, and text-timestamp state columns.""" @@ -28,5 +27,4 @@ def render_state_table_create_sql_impl( for column in columns: required: str = " NOT NULL" if column in required_columns else "" definitions.append(f"{column} {rendered_types[column_types[column]]}{required}") - table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" - return f"CREATE {table_kind} IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" + return f"CREATE TABLE IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" diff --git a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py index 7d0a48d69a..e0fe88aac6 100644 --- a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py +++ b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py @@ -14,7 +14,6 @@ def render_state_table_create_sql( column_types: Mapping[str, StateSqlValueType], required_columns: frozenset[str], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Render create-if-missing DDL for one append-only state table.""" @@ -24,5 +23,4 @@ def render_state_table_create_sql( column_types=column_types, required_columns=required_columns, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index b36671c154..4a131f271a 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -269,6 +269,18 @@ def render_read_latest_source_freshness_sql( render_qualified_name=self.render_qualified_name, ) + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.source_freshness.main.create_table_sql import ( + build_create_table_sql as build_source_freshness_create_table_sql, + ) + + return build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_source_freshness_index_sqls( self, *, @@ -372,7 +384,6 @@ def render_create_old_name_view_state_table_sql( schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_column_migration_state_table_sql( diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index abbffa3210..d8aceadc17 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -364,6 +364,18 @@ def render_read_latest_source_freshness_sql( render_qualified_name=self.render_qualified_name, ) + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.source_freshness.main.create_table_sql import ( + build_create_table_sql as build_source_freshness_create_table_sql, + ) + + return build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_source_freshness_index_sqls( self, *, @@ -467,7 +479,6 @@ def render_create_old_name_view_state_table_sql( schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_column_migration_state_table_sql( diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 0d3c7ca00c..93c15906bc 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1576,6 +1576,18 @@ def render_prune_fingerprint_history_sql( ")" ) + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.source_freshness.main.create_table_sql import ( + build_create_table_sql as build_source_freshness_create_table_sql, + ) + + return build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_source_freshness_index_sqls( self, *, @@ -1723,7 +1735,6 @@ def render_create_old_name_view_state_table_sql( schema=schema, render_qualified_name=self.render_qualified_name, render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_column_migration_state_table_sql( diff --git a/src/sqlbuild/adapters/snowflake/_helpers/state_table_retention.py b/src/sqlbuild/adapters/snowflake/_helpers/state_table_retention.py new file mode 100644 index 0000000000..2d13ecad3c --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/state_table_retention.py @@ -0,0 +1,24 @@ +"""Time travel retention for SQLBuild state tables created on Snowflake.""" + +from __future__ import annotations + +from sqlbuild.adapters.snowflake.constants import ( + INVALID_PARAMETER_VALUE_ERRNO, + STATE_TABLE_RETENTION_PARAMETER, +) + +_INVALID_RETENTION_MARKER: str = f"for parameter '{STATE_TABLE_RETENTION_PARAMETER}'" + + +def render_state_table_retention_sql(*, create_sql: str, retention_days: int) -> str: + """Append the time travel retention clause to one state-table CREATE statement.""" + + return f"{create_sql} {STATE_TABLE_RETENTION_PARAMETER} = {retention_days}" + + +def is_invalid_retention_error(error: Exception) -> bool: + """Return whether Snowflake rejected the value as ``001008 ... for parameter ''``.""" + + return getattr( + error, "errno", None + ) == INVALID_PARAMETER_VALUE_ERRNO and _INVALID_RETENTION_MARKER in str(error) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 1afd84cb3a..f56a8bf6a7 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -126,7 +126,13 @@ listed_relation_from_show_view, show_like_pattern, ) +from sqlbuild.adapters.snowflake._helpers.state_table_retention import ( + is_invalid_retention_error, +) from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection +from sqlbuild.adapters.snowflake.classes.state_table_retention_fallbacks import ( + StateTableRetentionFallbacks, +) from sqlbuild.adapters.snowflake.constants import ( BASE_TABLE_METADATA_TYPE, CURRENT_DATABASE_ATTRIBUTE, @@ -149,12 +155,14 @@ from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from sqlbuild.diagnostics.main.log_sql import log_sql +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS from sqlbuild.spec.contracts.models import SeedCsvSettings from sqlbuild.spec.contracts.types import TableType from sqlbuild.sql_values.models import SqlValue _EXACT_COLUMN_INSPECTION_LIMIT: int = 32 +_STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" _METADATA_INSPECTION_CONCURRENCY: int = 8 _SHOW_RESULT_LIMIT: int = 10_000 _SHOW_COLUMN_KIND: str = "COLUMN" @@ -179,7 +187,6 @@ class SnowflakeAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): exact_column_inspection_limit: ClassVar[int] = _EXACT_COLUMN_INSPECTION_LIMIT max_identifier_length: ClassVar[int] = 255 - state_tables_transient: ClassVar[bool] = True supports_session_warehouse: ClassVar[bool] = True connection_routing_keys: ClassVar[frozenset[str]] = frozenset( {"source", "profile", "target", "project_dir", "profiles_dir"} @@ -535,6 +542,22 @@ def render_read_latest_source_freshness_sql( render_qualified_name=self.render_qualified_name, ) + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.source_freshness.main.create_table_sql import ( + build_create_table_sql as build_source_freshness_create_table_sql, + ) + + return self._with_state_table_retention( + create_sql=build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ), + database=database, + schema=schema, + ) + def render_create_source_freshness_index_sqls( self, *, @@ -544,6 +567,34 @@ def render_create_source_freshness_index_sqls( del database, schema return () + def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: + return self._with_state_table_retention( + create_sql=super().render_create_microbatch_state_table_sql( + database=database, schema=schema + ), + database=database, + schema=schema, + ) + + def _with_state_table_retention( + self, *, create_sql: str, database: str | None, schema: str + ) -> str: + """Render state-table DDL with the maximum time travel its schema accepts.""" + + return self._state_table_retention_fallbacks().render( + create_sql=create_sql, database=database, schema=schema + ) + + def _state_table_retention_fallbacks(self) -> StateTableRetentionFallbacks: + fallbacks: StateTableRetentionFallbacks | None = self.__dict__.get( + _STATE_TABLE_RETENTION_ATTRIBUTE + ) + if fallbacks is not None: + return fallbacks + return self.__dict__.setdefault( + _STATE_TABLE_RETENTION_ATTRIBUTE, StateTableRetentionFallbacks() + ) + def render_insert_source_freshness_records_sql( self, *, @@ -568,12 +619,15 @@ def render_create_node_result_table_sql( build_node_results_create_table_sql, ) - return build_node_results_create_table_sql( + return self._with_state_table_retention( + create_sql=build_node_results_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ), database=database, schema=schema, - render_qualified_name=self.render_qualified_name, - render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_node_result_index_sqls( @@ -590,12 +644,15 @@ def render_create_audit_result_table_sql(self, *, database: str | None, schema: build_audit_results_create_table_sql, ) - return build_audit_results_create_table_sql( + return self._with_state_table_retention( + create_sql=build_audit_results_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ), database=database, schema=schema, - render_qualified_name=self.render_qualified_name, - render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_audit_result_index_sqls( @@ -609,12 +666,15 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: build_janitor_events_create_table_sql, ) - return build_janitor_events_create_table_sql( + return self._with_state_table_retention( + create_sql=build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ), database=database, schema=schema, - render_qualified_name=self.render_qualified_name, - render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: @@ -622,12 +682,15 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem build_migration_state_create_table_sql, ) - return build_migration_state_create_table_sql( + return self._with_state_table_retention( + create_sql=build_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ), database=database, schema=schema, - render_qualified_name=self.render_qualified_name, - render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_old_name_view_state_table_sql( @@ -637,12 +700,15 @@ def render_create_old_name_view_state_table_sql( build_old_name_view_state_create_table_sql, ) - return build_old_name_view_state_create_table_sql( + return self._with_state_table_retention( + create_sql=build_old_name_view_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ), database=database, schema=schema, - render_qualified_name=self.render_qualified_name, - render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_create_column_migration_state_table_sql( @@ -652,12 +718,15 @@ def render_create_column_migration_state_table_sql( build_column_migration_state_create_table_sql, ) - return build_column_migration_state_create_table_sql( + return self._with_state_table_retention( + create_sql=build_column_migration_state_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ), database=database, schema=schema, - render_qualified_name=self.render_qualified_name, - render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_prune_fingerprint_history_sql( @@ -1724,6 +1793,22 @@ def connect(self, config: dict[str, Any]) -> _SnowflakeConnection: def _execute(self, *, connection: _SnowflakeConnection, sql: str) -> Any: """Execute a SQL statement against a Snowflake connection.""" + fallbacks: StateTableRetentionFallbacks = self._state_table_retention_fallbacks() + fallback_sql: str | None = fallbacks.fallback_for(sql) + if fallback_sql is None: + return self._execute_statement(connection=connection, sql=sql) + if fallbacks.is_fallen_back(sql): + return self._execute_statement(connection=connection, sql=fallback_sql) + try: + return self._execute_statement(connection=connection, sql=sql) + except Exception as error: + if not is_invalid_retention_error(error): + raise + fallbacks.remember_fallback(sql) + StatementLifecycle.discard_failed_attempt() + return self._execute_statement(connection=connection, sql=fallback_sql) + + def _execute_statement(self, *, connection: _SnowflakeConnection, sql: str) -> Any: log_sql(logger=logging.getLogger("sqlbuild.adapter.snowflake"), sql=sql) return connection.execute(sql) @@ -2621,12 +2706,15 @@ def render_create_fingerprint_table_sql( build_create_table_sql, ) - return build_create_table_sql( + return self._with_state_table_retention( + create_sql=build_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ), database=database, schema=schema, - render_qualified_name=self.render_qualified_name, - render_framework_type=self.render_framework_type, - transient=self.state_tables_transient, ) def render_table_function_call(self, *, target: str, call_suffix_sql: str) -> str: diff --git a/src/sqlbuild/adapters/snowflake/classes/state_table_retention_fallbacks.py b/src/sqlbuild/adapters/snowflake/classes/state_table_retention_fallbacks.py new file mode 100644 index 0000000000..a38dec234a --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/classes/state_table_retention_fallbacks.py @@ -0,0 +1,59 @@ +"""Per-schema memory of Snowflake state-table retention fallbacks.""" + +from __future__ import annotations + +import threading + +from sqlbuild.adapters.snowflake._helpers.state_table_retention import ( + render_state_table_retention_sql, +) +from sqlbuild.adapters.snowflake.constants import ( + STATE_TABLE_FALLBACK_RETENTION_DAYS, + STATE_TABLE_RETENTION_DAYS, +) + + +class StateTableRetentionFallbacks: + """Track rendered state-table DDL and the schemas that rejected maximum retention.""" + + def __init__(self) -> None: + self._lock: threading.Lock = threading.Lock() + self._fallbacks: dict[str, tuple[tuple[str | None, str], str]] = {} + self._fallback_schemas: set[tuple[str | None, str]] = set() + + def render(self, *, create_sql: str, database: str | None, schema: str) -> str: + """Return the state-table DDL with the retention this schema currently accepts.""" + + key: tuple[str | None, str] = (database, schema) + preferred_sql: str = render_state_table_retention_sql( + create_sql=create_sql, retention_days=STATE_TABLE_RETENTION_DAYS + ) + fallback_sql: str = render_state_table_retention_sql( + create_sql=create_sql, retention_days=STATE_TABLE_FALLBACK_RETENTION_DAYS + ) + with self._lock: + self._fallbacks[preferred_sql] = (key, fallback_sql) + uses_fallback: bool = key in self._fallback_schemas + return fallback_sql if uses_fallback else preferred_sql + + def fallback_for(self, sql: str) -> str | None: + """Return the 1-day DDL for a rendered maximum-retention state-table statement.""" + + with self._lock: + entry: tuple[tuple[str | None, str], str] | None = self._fallbacks.get(sql) + return None if entry is None else entry[1] + + def is_fallen_back(self, sql: str) -> bool: + """Return whether the schema of a rendered state-table statement already fell back.""" + + with self._lock: + entry: tuple[tuple[str | None, str], str] | None = self._fallbacks.get(sql) + return entry is not None and entry[0] in self._fallback_schemas + + def remember_fallback(self, sql: str) -> None: + """Remember that the schema of a rendered state-table statement rejected 90 days.""" + + with self._lock: + entry: tuple[tuple[str | None, str], str] | None = self._fallbacks.get(sql) + if entry is not None: + self._fallback_schemas.add(entry[0]) diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 69692ed103..26a745d85c 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -24,3 +24,7 @@ TRUE_METADATA_VALUE: str = "YES" SHOW_LIKE_ESCAPE_CHARACTER: str = "\\" CURRENT_DATABASE_ATTRIBUTE: str = "_sqlbuild_current_database" +STATE_TABLE_RETENTION_DAYS: int = 90 +STATE_TABLE_FALLBACK_RETENTION_DAYS: int = 1 +STATE_TABLE_RETENTION_PARAMETER: str = "DATA_RETENTION_TIME_IN_DAYS" +INVALID_PARAMETER_VALUE_ERRNO: int = 1008 diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index abd8a9d163..341356ef4a 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -445,6 +445,18 @@ def render_prune_fingerprint_history_sql( f"DELETE FROM __sqlbuild_ranked WHERE __sqlbuild_history_rank > {retain_versions}" ) + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.compiler.source_freshness.main.create_table_sql import ( + build_create_table_sql as build_source_freshness_create_table_sql, + ) + + return build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_source_freshness_index_sqls( self, *, diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py index 3b46dc7a9e..90fd1430ad 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py @@ -145,7 +145,7 @@ def _append_records_by_location( render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, + render_create_table_sql=adapter.render_create_source_freshness_table_sql, render_create_index_sqls=adapter.render_create_source_freshness_index_sqls, ), - transient=adapter.state_tables_transient, ) diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/sql.py b/src/sqlbuild/compiler/fingerprints/_helpers/sql.py index 967c68c9cd..2869b5325a 100644 --- a/src/sqlbuild/compiler/fingerprints/_helpers/sql.py +++ b/src/sqlbuild/compiler/fingerprints/_helpers/sql.py @@ -49,7 +49,6 @@ def build_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Build a CREATE TABLE IF NOT EXISTS statement for the fingerprint table.""" @@ -60,9 +59,8 @@ def build_create_table_sql( ) string_type: str = render_framework_type(FrameworkType.STRING) timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) - table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" return ( - f"CREATE {table_kind} IF NOT EXISTS {qualified_name} (" + f"CREATE TABLE IF NOT EXISTS {qualified_name} (" f"{COLUMN_NODE_TYPE} {string_type} NOT NULL, " f"{COLUMN_NODE_NAME} {string_type} NOT NULL, " f"{COLUMN_TARGET_DATABASE} {string_type}, " diff --git a/src/sqlbuild/compiler/fingerprints/main/create_table_sql.py b/src/sqlbuild/compiler/fingerprints/main/create_table_sql.py index c1aeed0c7b..565aedbe2b 100644 --- a/src/sqlbuild/compiler/fingerprints/main/create_table_sql.py +++ b/src/sqlbuild/compiler/fingerprints/main/create_table_sql.py @@ -16,7 +16,6 @@ def build_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Build fingerprint-table DDL through the public fingerprint entrypoint.""" @@ -25,5 +24,4 @@ def build_create_table_sql( schema=schema, render_qualified_name=render_qualified_name, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py index 338205cc9b..f70055039d 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/column_sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/column_sql.py @@ -42,7 +42,6 @@ def build_column_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool, ) -> str: return render_state_table_create_sql( qualified_name=qualified_column_migration_table( @@ -52,7 +51,6 @@ def build_column_create_table_sql( column_types=COLUMN_MIGRATION_COLUMN_TYPES, required_columns=frozenset(), render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py b/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py index ea5a3a3996..f14967b3a1 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/old_name_sql.py @@ -43,7 +43,6 @@ def build_old_name_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool, ) -> str: return render_state_table_create_sql( qualified_name=qualified_old_name_view_table( @@ -53,7 +52,6 @@ def build_old_name_create_table_sql( column_types=OLD_NAME_VIEW_COLUMN_TYPES, required_columns=frozenset(), render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/compiler/migrations/_helpers/sql.py b/src/sqlbuild/compiler/migrations/_helpers/sql.py index 04477f0810..9773f4a003 100644 --- a/src/sqlbuild/compiler/migrations/_helpers/sql.py +++ b/src/sqlbuild/compiler/migrations/_helpers/sql.py @@ -39,7 +39,6 @@ def build_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool, ) -> str: return render_state_table_create_sql( qualified_name=qualified_migration_table( @@ -49,7 +48,6 @@ def build_create_table_sql( column_types=MIGRATION_COLUMN_TYPES, required_columns=frozenset(), render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py index e790b3320a..37a22d0e4d 100644 --- a/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py +++ b/src/sqlbuild/compiler/migrations/main/column_create_table_sql.py @@ -14,7 +14,6 @@ def build_column_migration_state_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Build DDL that creates the column migration event table when it is missing.""" @@ -23,5 +22,4 @@ def build_column_migration_state_create_table_sql( schema=schema, render_qualified_name=render_qualified_name, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/compiler/migrations/main/create_table_sql.py b/src/sqlbuild/compiler/migrations/main/create_table_sql.py index c2464b188c..75f9c3eea5 100644 --- a/src/sqlbuild/compiler/migrations/main/create_table_sql.py +++ b/src/sqlbuild/compiler/migrations/main/create_table_sql.py @@ -14,7 +14,6 @@ def build_migration_state_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Build DDL that creates the model migration event table when it is missing.""" @@ -23,5 +22,4 @@ def build_migration_state_create_table_sql( schema=schema, render_qualified_name=render_qualified_name, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py b/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py index 4539d6bf19..c489c71eee 100644 --- a/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py +++ b/src/sqlbuild/compiler/migrations/main/old_name_view_create_table_sql.py @@ -14,7 +14,6 @@ def build_old_name_view_state_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Build DDL that creates the old-name view fact table when it is missing.""" @@ -23,5 +22,4 @@ def build_old_name_view_state_create_table_sql( schema=schema, render_qualified_name=render_qualified_name, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/sql.py b/src/sqlbuild/compiler/source_freshness/_helpers/sql.py index b078640079..e907ae02ca 100644 --- a/src/sqlbuild/compiler/source_freshness/_helpers/sql.py +++ b/src/sqlbuild/compiler/source_freshness/_helpers/sql.py @@ -5,6 +5,9 @@ from collections.abc import Callable from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) from sqlbuild.compiler.source_freshness.constants import ( COLUMN_DATA_VERSION, COLUMN_DATA_VERSION_HASH, @@ -21,7 +24,6 @@ SOURCE_FRESHNESS_TABLE_NAME, ) from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError -from sqlbuild.sql_values.types import StateSqlValueType _REQUIRED_SOURCE_FRESHNESS_COLUMNS: frozenset[str] = frozenset( { @@ -59,29 +61,18 @@ def build_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Build a CREATE TABLE IF NOT EXISTS statement for source freshness state.""" - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, + return render_state_table_create_sql( + qualified_name=build_qualified_table_name( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=SOURCE_FRESHNESS_COLUMNS, + column_types=SOURCE_FRESHNESS_COLUMN_TYPES, + required_columns=_REQUIRED_SOURCE_FRESHNESS_COLUMNS, + render_framework_type=render_framework_type, ) - string_type: str = render_framework_type(FrameworkType.STRING) - timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) - table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" - definitions: list[str] = [] - for column in SOURCE_FRESHNESS_COLUMNS: - column_type: str = ( - timestamp_type - if SOURCE_FRESHNESS_COLUMN_TYPES[column] - in {StateSqlValueType.TIMESTAMP, StateSqlValueType.TEXT_TIMESTAMP} - else string_type - ) - required: str = " NOT NULL" if column in _REQUIRED_SOURCE_FRESHNESS_COLUMNS else "" - definitions.append(f"{column} {column_type}{required}") - return f"CREATE {table_kind} IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" def build_read_latest_sql( diff --git a/src/sqlbuild/compiler/source_freshness/main/_create_table_sql.py b/src/sqlbuild/compiler/source_freshness/main/create_table_sql.py similarity index 93% rename from src/sqlbuild/compiler/source_freshness/main/_create_table_sql.py rename to src/sqlbuild/compiler/source_freshness/main/create_table_sql.py index a8c3f381d9..cd711ea0b0 100644 --- a/src/sqlbuild/compiler/source_freshness/main/_create_table_sql.py +++ b/src/sqlbuild/compiler/source_freshness/main/create_table_sql.py @@ -16,7 +16,6 @@ def build_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Build source freshness table DDL through the public entrypoint.""" @@ -25,5 +24,4 @@ def build_create_table_sql( schema=schema, render_qualified_name=render_qualified_name, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/compiler/source_freshness/main/write.py b/src/sqlbuild/compiler/source_freshness/main/write.py index be61e8623a..6014cdabf2 100644 --- a/src/sqlbuild/compiler/source_freshness/main/write.py +++ b/src/sqlbuild/compiler/source_freshness/main/write.py @@ -22,7 +22,6 @@ def write_source_freshness_records( schema: str, records: tuple[SourceFreshnessRecord, ...], renderers: SourceFreshnessRenderers, - transient: bool = False, ) -> None: """Append source freshness rows, creating the table once if needed.""" @@ -36,7 +35,6 @@ def write_source_freshness_records( schema=schema, render_qualified_name=renderers.render_qualified_name, render_framework_type=renderers.render_framework_type, - transient=transient, ) ) _ = execute(connection=connection, sql=create_sql) diff --git a/src/sqlbuild/executor/audit_results/_helpers/sql.py b/src/sqlbuild/executor/audit_results/_helpers/sql.py index 6944e12702..296d7a1a36 100644 --- a/src/sqlbuild/executor/audit_results/_helpers/sql.py +++ b/src/sqlbuild/executor/audit_results/_helpers/sql.py @@ -95,7 +95,6 @@ def build_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: return render_state_table_create_sql( qualified_name=build_qualified_table_name( @@ -105,7 +104,6 @@ def build_create_table_sql( column_types=AUDIT_RESULT_COLUMN_TYPES, required_columns=_REQUIRED_COLUMNS, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/executor/audit_results/main/create_table_sql.py b/src/sqlbuild/executor/audit_results/main/create_table_sql.py index da6e399892..7329b0efe3 100644 --- a/src/sqlbuild/executor/audit_results/main/create_table_sql.py +++ b/src/sqlbuild/executor/audit_results/main/create_table_sql.py @@ -14,12 +14,10 @@ def build_audit_results_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: return build_create_table_sql( database=database, schema=schema, render_qualified_name=render_qualified_name, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py index 8f9bf5e1da..a3f3a3202e 100644 --- a/src/sqlbuild/executor/janitor_events/_helpers/sql.py +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -49,7 +49,6 @@ def build_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: return render_state_table_create_sql( qualified_name=build_qualified_table_name( @@ -59,7 +58,6 @@ def build_create_table_sql( column_types=JANITOR_EVENT_COLUMN_TYPES, required_columns=JANITOR_EVENT_REQUIRED_COLUMNS, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/executor/janitor_events/main/create_table_sql.py b/src/sqlbuild/executor/janitor_events/main/create_table_sql.py index 4b694023e6..98dc2f708e 100644 --- a/src/sqlbuild/executor/janitor_events/main/create_table_sql.py +++ b/src/sqlbuild/executor/janitor_events/main/create_table_sql.py @@ -14,7 +14,6 @@ def build_janitor_events_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: """Build DDL that creates the janitor audit event table when it is missing.""" @@ -23,5 +22,4 @@ def build_janitor_events_create_table_sql( schema=schema, render_qualified_name=render_qualified_name, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/executor/node_results/_helpers/sql.py b/src/sqlbuild/executor/node_results/_helpers/sql.py index aa1daee92d..be338e043f 100644 --- a/src/sqlbuild/executor/node_results/_helpers/sql.py +++ b/src/sqlbuild/executor/node_results/_helpers/sql.py @@ -5,6 +5,9 @@ from collections.abc import Callable from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( + render_state_table_create_sql, +) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.node_results.constants import ( COLUMN_ERROR_MESSAGE, @@ -25,7 +28,6 @@ ) from sqlbuild.executor.node_results.models import NodeResultQuery, NodeResultRecord from sqlbuild.sql_values.main.render_state_literal import render_state_sql_literal -from sqlbuild.sql_values.types import StateSqlValueType _REQUIRED_NODE_RESULT_COLUMNS: frozenset[str] = frozenset( { @@ -62,27 +64,16 @@ def build_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, + return render_state_table_create_sql( + qualified_name=build_qualified_table_name( + database=database, schema=schema, render_qualified_name=render_qualified_name + ), + columns=NODE_RESULT_COLUMNS, + column_types=NODE_RESULT_COLUMN_TYPES, + required_columns=_REQUIRED_NODE_RESULT_COLUMNS, + render_framework_type=render_framework_type, ) - string_type: str = render_framework_type(FrameworkType.STRING) - timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) - table_kind: str = "TRANSIENT TABLE" if transient else "TABLE" - definitions: list[str] = [] - for column in NODE_RESULT_COLUMNS: - column_type: str = ( - timestamp_type - if NODE_RESULT_COLUMN_TYPES[column] - in {StateSqlValueType.TIMESTAMP, StateSqlValueType.TEXT_TIMESTAMP} - else string_type - ) - required: str = " NOT NULL" if column in _REQUIRED_NODE_RESULT_COLUMNS else "" - definitions.append(f"{column} {column_type}{required}") - return f"CREATE {table_kind} IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" def build_insert_sql( diff --git a/src/sqlbuild/executor/node_results/main/create_table_sql.py b/src/sqlbuild/executor/node_results/main/create_table_sql.py index 1e2ff9acc8..779eabf71b 100644 --- a/src/sqlbuild/executor/node_results/main/create_table_sql.py +++ b/src/sqlbuild/executor/node_results/main/create_table_sql.py @@ -14,12 +14,10 @@ def build_node_results_create_table_sql( schema: str, render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], - transient: bool = False, ) -> str: return build_create_table_sql( database=database, schema=schema, render_qualified_name=render_qualified_name, render_framework_type=render_framework_type, - transient=transient, ) diff --git a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py index 10dc1dbb38..c19f5368ad 100644 --- a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py @@ -91,6 +91,14 @@ def listener_scope(listener: StatementListener) -> Iterator[None]: with statement_listener_scope(listener): yield + @staticmethod + def discard_failed_attempt() -> None: + """Forget a failed attempt deferred to the active statement before an adapter retry.""" + + owner: StatementLifecycle | None = _STATEMENT_LIFECYCLE_OWNER.get() + if owner is not None and owner._execution_owner_key == _current_execution_owner_key(): + owner._pending_failure = None + def __enter__(self) -> StatementLifecycle: execution_owner_key: _ExecutionOwnerKey = _current_execution_owner_key() owner: StatementLifecycle | None = _STATEMENT_LIFECYCLE_OWNER.get() diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py index c0fc8e91cc..3b9efaa8cb 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py @@ -21,3 +21,13 @@ class SchemaEvolutionTestCase: added_column: str expected_target_reads: int expected_merge_fragment: str + + +@dataclass(frozen=True) +class StateTableRetentionTestCase: + """An offline account edition and the time travel its state tables are created with.""" + + description: str + max_retention_days: int + expected_first_retention_days: str + expected_later_retention_days: frozenset[str] diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 491d2b3e30..011b11dff5 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -25,6 +25,7 @@ from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( FakeColumn, FakeRelation, + FakeSnowflakeProgrammingError, OfflineSnowflakeAdapter, RecordingSnowflakeWarehouse, ) @@ -80,6 +81,7 @@ _COPY_SOURCE_PATTERN: re.Pattern[str] = re.compile( r"^\s*SELECT\s+\*\s+FROM\s+" + _NAME.format(group="source") + r"\s*$", re.IGNORECASE ) +_RETENTION_PATTERN: re.Pattern[str] = re.compile(r"DATA_RETENTION_TIME_IN_DAYS = (?P\d+)") _DEFAULT_COLUMNS: tuple[FakeColumn, ...] = (FakeColumn(name="ID", data_type="NUMBER"),) _STATUS_DESCRIPTION: tuple[tuple[str], ...] = (("status",),) @@ -94,6 +96,7 @@ class SimulatedSnowflakeWarehouse(RecordingSnowflakeWarehouse): model_columns: dict[str, tuple[FakeColumn, ...]] = field(default_factory=dict) ddl_events: list[tuple[int, frozenset[_Identity]]] = field(default_factory=list) + max_retention_days: int = 90 def reset(self) -> None: """Forget recorded statements and simulated DDL events, keeping the catalog.""" @@ -109,6 +112,11 @@ def set_model_columns(self, *, model: str, columns: tuple[FakeColumn, ...]) -> N self.model_columns[model.upper()] = columns def _answer(self, *, sql: str, params: tuple[object, ...]) -> tuple[list[tuple[Any, ...]], Any]: + retention: re.Match[str] + for retention in filter(None, (_RETENTION_PATTERN.search(sql),)): + _RETENTION_CHECKS[int(retention.group("days")) > self.max_retention_days]( + retention.group("days") + ) routes: tuple[tuple[re.Pattern[str], _DdlAction], ...] = ( (_CREATE_AS_PATTERN, self._create_as), (_CREATE_DEFINED_PATTERN, self._create_defined), @@ -256,6 +264,24 @@ def _parts(self, name: str) -> _Identity: return full[0], full[1], full[2] +def _accept_retention(days: str) -> None: + del days + + +def _reject_retention(days: str) -> None: + raise FakeSnowflakeProgrammingError( + "001008 (22023): SQL compilation error:\n" + f"invalid value [{days}] for parameter 'DATA_RETENTION_TIME_IN_DAYS'", + errno=1008, + ) + + +_RETENTION_CHECKS: dict[bool, Callable[[str], None]] = { + False: _accept_retention, + True: _reject_retention, +} + + def _name_length(name: str) -> int: return len(name) @@ -317,6 +343,17 @@ def build_offline_snowflake_project( ) +def state_table_creates(build: OfflineBuild) -> tuple[str, ...]: + """Return the CREATE statements the build attempted for SQLBuild state tables.""" + + return tuple( + filter( + lambda statement: _STATE_TABLE_CREATE_PATTERN.match(statement) is not None, + build.statements, + ) + ) + + def information_schema_statements(build: OfflineBuild) -> tuple[str, ...]: """Return build statements that read INFORMATION_SCHEMA, which needs a running warehouse.""" @@ -380,6 +417,9 @@ def _show_schema_identity(match: re.Match[str]) -> _Identity: return match.group("database"), match.group("schema"), "" +_STATE_TABLE_CREATE_PATTERN: re.Pattern[str] = re.compile( + r"^\s*CREATE\s+(?:TRANSIENT\s+)?TABLE\s+IF\s+NOT\s+EXISTS\s+\S*_sqlbuild_", re.IGNORECASE +) _INFORMATION_SCHEMA_PATTERN: re.Pattern[str] = re.compile(r"information_schema", re.IGNORECASE) _SHOW_COLUMNS_RELATION: re.Pattern[str] = re.compile(r"^SHOW COLUMNS IN (?:TABLE|VIEW) (\S+)$") _METADATA_STATEMENTS: tuple[tuple[re.Pattern[str], Callable[[re.Match[str]], _Identity]], ...] = ( diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_state_table_retention.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_state_table_retention.py new file mode 100644 index 0000000000..df943340f3 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_state_table_retention.py @@ -0,0 +1,60 @@ +"""State tables created by an offline Snowflake build are permanent with maximum time travel.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata._test_types import ( + StateTableRetentionTestCase, +) +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata.helpers import ( + OfflineBuild, + SimulatedSnowflakeWarehouse, + build_offline_snowflake_project, + failed_models, + orders_warehouse, + state_table_creates, + write_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StateTableRetentionTestCase( + description="enterprise account keeps 90 days", + max_retention_days=90, + expected_first_retention_days="90", + expected_later_retention_days=frozenset({"90"}), + ), + StateTableRetentionTestCase( + description="standard account retries once and then creates with 1 day", + max_retention_days=1, + expected_first_retention_days="90", + expected_later_retention_days=frozenset({"1"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_account_edition_when_building_then_state_tables_are_permanent_with_max_retention( + test_case: StateTableRetentionTestCase, tmp_path: Path +) -> None: + write_orders_project(project_dir=tmp_path) + warehouse: SimulatedSnowflakeWarehouse = orders_warehouse() + warehouse.max_retention_days = test_case.max_retention_days + + build: OfflineBuild = build_offline_snowflake_project( + project_dir=tmp_path, warehouse=warehouse, max_concurrency=1 + ) + creates: tuple[str, ...] = state_table_creates(build) + + assert failed_models(build) == () + assert len(creates) > 1 + assert all(create.startswith("CREATE TABLE IF NOT EXISTS ") for create in creates) + assert all("._sqlbuild_fingerprints (" in create for create in creates) + assert creates[0].rsplit(" = ", 1)[1] == test_case.expected_first_retention_days + assert { + create.rsplit(" = ", 1)[1] for create in creates[1:] + } == test_case.expected_later_retention_days diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index a436ca076a..13b0b95252 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -125,3 +125,11 @@ class RelationRevokeTestCase: grants: tuple[RelationGrant, ...] destination: str expected_statements: tuple[str, ...] + + +@dataclass(frozen=True) +class AdapterStateTableDdlTestCase: + description: str + adapter: StrictAdapter + render_method: str + expected_create_fragment: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index ce18d26ebb..427104900e 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -3,6 +3,7 @@ from sqlbuild.adapter.contract.models import FunctionInfo, SchemaDiffResult from sqlbuild.adapter.contract.types import TableFreshnessStatus +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.cost.types import CostStatus @@ -225,3 +226,61 @@ class SnowflakeFunctionDiscoveryTestCase: expected_relation: str expected_params: tuple[str, ...] expected_functions: tuple[FunctionInfo, ...] + + +@dataclass(frozen=True) +class SnowflakeStateTableDdlTestCase: + description: str + render_method: str + expected_prefix: str + expected_suffix: str + + +@dataclass(frozen=True) +class SnowflakeStateTableExactDdlTestCase: + description: str + render_method: str + expected_sql: str + + +@dataclass(frozen=True) +class SnowflakeStateTableFallbackTestCase: + description: str + adapter_type: type[SnowflakeAdapter] + rejection_errno: int + rejection_message: str + expected_executed_tables: tuple[str, ...] + expected_executed_retention_days: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeStateTableRetentionErrorTestCase: + description: str + errno: int + message: str + expected_executed_count: int + expected_next_retention_days: str + + +@dataclass(frozen=True) +class SnowflakeNonStateRetentionErrorTestCase: + description: str + sql: str + expected_executed_sql: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeStateTableFallbackLifecycleTestCase: + description: str + rejections: tuple[tuple[tuple[str, ...], Exception], ...] + expected_statement_events: tuple[str, ...] + expected_progress_fail_lines: int + expected_executed_retention_days: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeStateTableSchemaFallbackTestCase: + description: str + rejecting_schema: str + accepting_schema: str + expected_executed: tuple[tuple[str, str], ...] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 3fbdbe68d9..815baefd31 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -3,11 +3,27 @@ from __future__ import annotations import sys +from collections.abc import Callable +from dataclasses import dataclass +from io import StringIO from itertools import cycle, islice from types import ModuleType from typing import Any from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection +from sqlbuild.cli.progress.classes.native_progress_projector import NativeProgressProjector +from sqlbuild.observability import ( + EventDispatcher, + LifecycleEvent, + OperationAttributes, + OperationLifecycle, + ResourceAttemptLifecycle, + dispatcher_scope, + invocation_scope, + run_scope, +) class FakeSnowflakeDescribeCursor: @@ -248,3 +264,135 @@ def connect(**kwargs: object) -> FakeSnowflakeRawConnection: monkeypatch.setitem(sys.modules, "snowflake", snowflake_module) monkeypatch.setitem(sys.modules, "snowflake.connector", connector_module) return captured_kwargs, raw_connection + + +def _raise_rejection(error: Exception) -> None: + raise error + + +def _accept_statement(error: Exception) -> None: + del error + + +_REJECTION_OUTCOMES: dict[bool, Callable[[Exception], None]] = { + True: _raise_rejection, + False: _accept_statement, +} + + +class FakeSnowflakeRejectingConnection: + """Connection double that raises one driver error for statements containing a fragment.""" + + def __init__(self, *, rejected_fragments: tuple[str, ...], error: Exception) -> None: + self._rejected_fragments: tuple[str, ...] = rejected_fragments + self._error: Exception = error + self.executed_sql: list[str] = [] + + def execute(self, sql: str) -> FakeSnowflakeDescribeCursor: + self.executed_sql.append(sql) + _REJECTION_OUTCOMES[all(fragment in sql for fragment in self._rejected_fragments)]( + self._error + ) + return FakeSnowflakeDescribeCursor(description=(("status",),)) + + +class FakeSnowflakeRejectingRawCursor: + """Raw driver cursor that submits every statement and rejects those matching all fragments.""" + + def __init__(self, *, rejections: tuple[tuple[tuple[str, ...], Exception], ...]) -> None: + self._rejections: tuple[tuple[tuple[str, ...], Exception], ...] = rejections + self.sfqid: str | None = None + self.rowcount: int = 0 + self.executed_sql: list[str] = [] + + def execute(self, sql: str, **kwargs: object) -> FakeSnowflakeRejectingRawCursor: + del kwargs + self.executed_sql.append(sql) + self.sfqid = f"01c-query-{len(self.executed_sql)}" + fragments: tuple[str, ...] + error: Exception + for fragments, error in self._rejections: + _REJECTION_OUTCOMES[all(fragment in sql for fragment in fragments)](error) + return self + + def close(self) -> None: + return None + + +class FakeSnowflakeRejectingRawConnection: + """Raw driver connection handing out one shared cursor.""" + + def __init__(self, cursor: FakeSnowflakeRejectingRawCursor) -> None: + self._cursor: FakeSnowflakeRejectingRawCursor = cursor + + def cursor(self) -> FakeSnowflakeRejectingRawCursor: + return self._cursor + + +@dataclass(frozen=True) +class StatementProgressCapture: + """Lifecycle statement events and CLI progress text published around one statement.""" + + statement_events: tuple[str, ...] + statement_fail_lines: tuple[str, ...] + output: str + error: Exception | None + + +def execute_with_statement_progress( + *, adapter: SnowflakeAdapter, raw_cursor: FakeSnowflakeRejectingRawCursor, sql: str +) -> StatementProgressCapture: + """Execute one statement through the real Snowflake cursor under a resource operation.""" + + stream: StringIO = StringIO() + events: list[LifecycleEvent] = [] + dispatcher: EventDispatcher = EventDispatcher() + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + dispatcher.subscribe_lifecycle( + subscriber=NativeProgressProjector(stream=stream, use_color=False).consume, + accepts_opaque=False, + ) + connection: _SnowflakeConnection = _SnowflakeConnection( + FakeSnowflakeRejectingRawConnection(raw_cursor) + ) + error: Exception | None = None + try: + with ( + invocation_scope("inv-state-table-retention"), + run_scope("run-state-table-retention"), + dispatcher_scope(dispatcher), + ResourceAttemptLifecycle( + resource_id="model:orders", resource_kind="table", resource_name="orders" + ), + OperationLifecycle( + operation_kind="warehouse", + operation_name="staging_creation", + attributes=OperationAttributes(phase="create", adapter="snowflake"), + ), + ): + _ = adapter.execute(connection=connection, sql=sql) + except Exception as caught: + error = caught + return StatementProgressCapture( + statement_events=tuple( + filter( + lambda event_type: event_type.startswith("statement_"), + (event.event_type for event in events), + ) + ), + statement_fail_lines=tuple( + filter( + lambda line: line.lstrip().startswith("statement") and "FAIL" in line, + stream.getvalue().splitlines(), + ) + ), + output=stream.getvalue(), + error=error, + ) + + +class CustomInitSnowflakeAdapter(SnowflakeAdapter): + """Custom adapter whose own initializer does not call the base initializer.""" + + def __init__(self) -> None: + self.custom_label: str = "orders" diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py new file mode 100644 index 0000000000..00496491bd --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, cast + +import pytest + +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeNonStateRetentionErrorTestCase, + SnowflakeStateTableDdlTestCase, + SnowflakeStateTableExactDdlTestCase, + SnowflakeStateTableFallbackLifecycleTestCase, + SnowflakeStateTableFallbackTestCase, + SnowflakeStateTableRetentionErrorTestCase, + SnowflakeStateTableSchemaFallbackTestCase, +) +from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( + CustomInitSnowflakeAdapter, + FakeSnowflakeRejectingConnection, + FakeSnowflakeRejectingRawCursor, + StatementProgressCapture, + execute_with_statement_progress, +) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + FakeSnowflakeProgrammingError, +) + +_RETENTION_90: str = "DATA_RETENTION_TIME_IN_DAYS = 90" +_STANDARD_EDITION_REJECTION: str = ( + "001008 (22023): SQL compilation error:\n" + "invalid value [90] for parameter 'DATA_RETENTION_TIME_IN_DAYS'" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStateTableDdlTestCase( + description="fingerprints", + render_method="render_create_fingerprint_table_sql", + expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_fingerprints (", + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + SnowflakeStateTableDdlTestCase( + description="source freshness", + render_method="render_create_source_freshness_table_sql", + expected_prefix=( + "CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_source_freshness (" + ), + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + SnowflakeStateTableDdlTestCase( + description="node results", + render_method="render_create_node_result_table_sql", + expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_node_results (", + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + SnowflakeStateTableDdlTestCase( + description="audit results", + render_method="render_create_audit_result_table_sql", + expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_audit_results (", + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + SnowflakeStateTableDdlTestCase( + description="janitor events", + render_method="render_create_janitor_event_table_sql", + expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_janitor_events (", + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + SnowflakeStateTableDdlTestCase( + description="model migrations", + render_method="render_create_migration_state_table_sql", + expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_migrations (", + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + SnowflakeStateTableDdlTestCase( + description="old-name views", + render_method="render_create_old_name_view_state_table_sql", + expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_old_name_views (", + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + SnowflakeStateTableDdlTestCase( + description="column migrations", + render_method="render_create_column_migration_state_table_sql", + expected_prefix=( + "CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_column_migrations (" + ), + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + SnowflakeStateTableDdlTestCase( + description="microbatches", + render_method="render_create_microbatch_state_table_sql", + expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_microbatches (", + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), + ], + ids=lambda case: case.description, +) +def test_given_state_table_when_rendering_on_snowflake_then_permanent_with_90_day_retention( + test_case: SnowflakeStateTableDdlTestCase, +) -> None: + render: Callable[..., str] = getattr(SnowflakeAdapter(), test_case.render_method) + + sql: str = render(database="analytics", schema="marts") + + assert sql.startswith(test_case.expected_prefix) + assert sql.endswith(test_case.expected_suffix) + assert "TRANSIENT" not in sql + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStateTableExactDdlTestCase( + description="fingerprints", + render_method="render_create_fingerprint_table_sql", + expected_sql=( + "CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_fingerprints (" + "node_type VARCHAR NOT NULL, node_name VARCHAR NOT NULL, target_database VARCHAR, " + "target_schema VARCHAR, target_name VARCHAR, run_id VARCHAR NOT NULL, " + "definition_hash VARCHAR NOT NULL, version_hash VARCHAR NOT NULL, " + "schema_fingerprint VARCHAR NOT NULL, definition_b64 VARCHAR NOT NULL, " + "metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL" + ") DATA_RETENTION_TIME_IN_DAYS = 90" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_state_table_when_rendering_on_snowflake_then_matches_exact_ddl( + test_case: SnowflakeStateTableExactDdlTestCase, +) -> None: + render: Callable[..., str] = getattr(SnowflakeAdapter(), test_case.render_method) + + assert render(database="analytics", schema="marts") == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStateTableFallbackTestCase( + description="standard edition rejects 90 days", + adapter_type=SnowflakeAdapter, + rejection_errno=1008, + rejection_message=_STANDARD_EDITION_REJECTION, + expected_executed_tables=( + "analytics.marts._sqlbuild_fingerprints", + "analytics.marts._sqlbuild_fingerprints", + "analytics.marts._sqlbuild_node_results", + "analytics.marts._sqlbuild_microbatches", + ), + expected_executed_retention_days=("90", "1", "1", "1"), + ), + SnowflakeStateTableFallbackTestCase( + description="custom adapter initializer without super", + adapter_type=CustomInitSnowflakeAdapter, + rejection_errno=1008, + rejection_message=_STANDARD_EDITION_REJECTION, + expected_executed_tables=( + "analytics.marts._sqlbuild_fingerprints", + "analytics.marts._sqlbuild_fingerprints", + "analytics.marts._sqlbuild_node_results", + "analytics.marts._sqlbuild_microbatches", + ), + expected_executed_retention_days=("90", "1", "1", "1"), + ), + ], + ids=lambda case: case.description, +) +def test_given_account_rejecting_90_days_when_creating_state_tables_then_retries_once_with_1( + test_case: SnowflakeStateTableFallbackTestCase, +) -> None: + adapter: SnowflakeAdapter = test_case.adapter_type() + connection: FakeSnowflakeRejectingConnection = FakeSnowflakeRejectingConnection( + rejected_fragments=(_RETENTION_90,), + error=FakeSnowflakeProgrammingError( + test_case.rejection_message, errno=test_case.rejection_errno + ), + ) + fingerprints_sql: str = adapter.render_create_fingerprint_table_sql( + database="analytics", schema="marts" + ) + rendered_before_fallback: str = adapter.render_create_node_result_table_sql( + database="analytics", schema="marts" + ) + + _ = adapter.execute(connection=cast(Any, connection), sql=fingerprints_sql) + _ = adapter.execute(connection=cast(Any, connection), sql=rendered_before_fallback) + _ = adapter.execute( + connection=cast(Any, connection), + sql=adapter.render_create_microbatch_state_table_sql(database="analytics", schema="marts"), + ) + + assert ( + tuple( + sql.removeprefix("CREATE TABLE IF NOT EXISTS ").split(" (", 1)[0] + for sql in connection.executed_sql + ) + == test_case.expected_executed_tables + ) + assert ( + tuple(sql.rsplit(" = ", 1)[1] for sql in connection.executed_sql) + == test_case.expected_executed_retention_days + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStateTableRetentionErrorTestCase( + description="same errno for a different parameter", + errno=1008, + message=( + "001008 (22023): SQL compilation error:\n" + "invalid value [90] for parameter 'MAX_DATA_EXTENSION_TIME_IN_DAYS'" + ), + expected_executed_count=1, + expected_next_retention_days="90", + ), + SnowflakeStateTableRetentionErrorTestCase( + description="retention parameter named by a different error", + errno=3001, + message=( + "003001 (42501): SQL access control error: insufficient privileges to set " + "parameter 'DATA_RETENTION_TIME_IN_DAYS'" + ), + expected_executed_count=1, + expected_next_retention_days="90", + ), + SnowflakeStateTableRetentionErrorTestCase( + description="missing schema", + errno=2003, + message=( + "002003 (02000): SQL compilation error: Schema 'ANALYTICS.MARTS' does not exist" + ), + expected_executed_count=1, + expected_next_retention_days="90", + ), + ], + ids=lambda case: case.description, +) +def test_given_unrelated_error_when_creating_state_table_then_propagates_without_retry( + test_case: SnowflakeStateTableRetentionErrorTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + error: FakeSnowflakeProgrammingError = FakeSnowflakeProgrammingError( + test_case.message, errno=test_case.errno + ) + connection: FakeSnowflakeRejectingConnection = FakeSnowflakeRejectingConnection( + rejected_fragments=(_RETENTION_90,), error=error + ) + sql: str = adapter.render_create_audit_result_table_sql(database="analytics", schema="marts") + + with pytest.raises(FakeSnowflakeProgrammingError) as raised: + _ = adapter.execute(connection=cast(Any, connection), sql=sql) + + assert raised.value is error + assert len(connection.executed_sql) == test_case.expected_executed_count + assert ( + adapter.render_create_audit_result_table_sql(database="analytics", schema="marts").rsplit( + " = ", 1 + )[1] + == test_case.expected_next_retention_days + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeNonStateRetentionErrorTestCase( + description="model retention change", + sql="ALTER TABLE analytics.marts.orders SET DATA_RETENTION_TIME_IN_DAYS = 90", + expected_executed_sql=( + "ALTER TABLE analytics.marts.orders SET DATA_RETENTION_TIME_IN_DAYS = 90", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_retention_rejection_when_statement_is_not_state_table_ddl_then_propagates( + test_case: SnowflakeNonStateRetentionErrorTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + connection: FakeSnowflakeRejectingConnection = FakeSnowflakeRejectingConnection( + rejected_fragments=(_RETENTION_90,), + error=FakeSnowflakeProgrammingError(_STANDARD_EDITION_REJECTION, errno=1008), + ) + + with pytest.raises(FakeSnowflakeProgrammingError): + _ = adapter.execute(connection=cast(Any, connection), sql=test_case.sql) + + assert tuple(connection.executed_sql) == test_case.expected_executed_sql + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStateTableFallbackLifecycleTestCase( + description="retry with 1 day succeeds", + rejections=( + ( + (_RETENTION_90,), + FakeSnowflakeProgrammingError(_STANDARD_EDITION_REJECTION, errno=1008), + ), + ), + expected_statement_events=( + "statement_started", + "statement_submitted", + "statement_completed", + ), + expected_progress_fail_lines=0, + expected_executed_retention_days=("90", "1"), + ) + ], + ids=lambda case: case.description, +) +def test_given_fallback_succeeds_when_executing_through_cursor_then_one_successful_statement( + test_case: SnowflakeStateTableFallbackLifecycleTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + cursor: FakeSnowflakeRejectingRawCursor = FakeSnowflakeRejectingRawCursor( + rejections=test_case.rejections + ) + sql: str = adapter.render_create_fingerprint_table_sql(database="analytics", schema="marts") + + capture: StatementProgressCapture = execute_with_statement_progress( + adapter=adapter, raw_cursor=cursor, sql=sql + ) + + assert capture.error is None + assert capture.statement_events == test_case.expected_statement_events + assert len(capture.statement_fail_lines) == test_case.expected_progress_fail_lines + assert "statement " in capture.output + assert ( + tuple(statement.rsplit(" = ", 1)[1] for statement in cursor.executed_sql) + == test_case.expected_executed_retention_days + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStateTableFallbackLifecycleTestCase( + description="retry with 1 day also fails", + rejections=( + ( + (_RETENTION_90,), + FakeSnowflakeProgrammingError(_STANDARD_EDITION_REJECTION, errno=1008), + ), + ( + ("DATA_RETENTION_TIME_IN_DAYS = 1",), + FakeSnowflakeProgrammingError( + "003001 (42501): SQL access control error: Insufficient privileges", + errno=3001, + ), + ), + ), + expected_statement_events=( + "statement_started", + "statement_submitted", + "statement_failed", + ), + expected_progress_fail_lines=1, + expected_executed_retention_days=("90", "1"), + ) + ], + ids=lambda case: case.description, +) +def test_given_fallback_fails_when_executing_through_cursor_then_final_failure_published_once( + test_case: SnowflakeStateTableFallbackLifecycleTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + cursor: FakeSnowflakeRejectingRawCursor = FakeSnowflakeRejectingRawCursor( + rejections=test_case.rejections + ) + sql: str = adapter.render_create_fingerprint_table_sql(database="analytics", schema="marts") + + capture: StatementProgressCapture = execute_with_statement_progress( + adapter=adapter, raw_cursor=cursor, sql=sql + ) + + assert isinstance(capture.error, FakeSnowflakeProgrammingError) + assert capture.error.errno == 3001 + assert capture.statement_events == test_case.expected_statement_events + assert len(capture.statement_fail_lines) == test_case.expected_progress_fail_lines + assert ( + tuple(statement.rsplit(" = ", 1)[1] for statement in cursor.executed_sql) + == test_case.expected_executed_retention_days + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStateTableSchemaFallbackTestCase( + description="transient schema falls back alone", + rejecting_schema="scratch", + accepting_schema="marts", + expected_executed=( + ("analytics.scratch._sqlbuild_fingerprints", "90"), + ("analytics.scratch._sqlbuild_fingerprints", "1"), + ("analytics.marts._sqlbuild_fingerprints", "90"), + ("analytics.scratch._sqlbuild_node_results", "1"), + ("analytics.marts._sqlbuild_node_results", "90"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_one_schema_rejecting_90_days_when_creating_state_tables_then_other_schema_keeps_90( + test_case: SnowflakeStateTableSchemaFallbackTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + connection: FakeSnowflakeRejectingConnection = FakeSnowflakeRejectingConnection( + rejected_fragments=(f".{test_case.rejecting_schema}.", _RETENTION_90), + error=FakeSnowflakeProgrammingError(_STANDARD_EDITION_REJECTION, errno=1008), + ) + schemas: tuple[str, str] = (test_case.rejecting_schema, test_case.accepting_schema) + + schema: str + for schema in schemas: + _ = adapter.execute( + connection=cast(Any, connection), + sql=adapter.render_create_fingerprint_table_sql(database="analytics", schema=schema), + ) + for schema in schemas: + _ = adapter.execute( + connection=cast(Any, connection), + sql=adapter.render_create_node_result_table_sql(database="analytics", schema=schema), + ) + + assert ( + tuple( + ( + sql.removeprefix("CREATE TABLE IF NOT EXISTS ").split(" (", 1)[0], + sql.rsplit(" = ", 1)[1], + ) + for sql in connection.executed_sql + ) + == test_case.expected_executed + ) diff --git a/tests/unit/src/sqlbuild/adapters/test_rename_column.py b/tests/unit/src/sqlbuild/adapters/test_rename_column.py index 78067bfac1..52bd6eb61b 100644 --- a/tests/unit/src/sqlbuild/adapters/test_rename_column.py +++ b/tests/unit/src/sqlbuild/adapters/test_rename_column.py @@ -135,11 +135,12 @@ def test_given_adapter_when_rendering_column_rename_then_uses_the_engine_stateme ), ), AdapterColumnMigrationStateTableTestCase( - description="snowflake state tables follow the transient state setting", + description="snowflake state tables are permanent with maximum time travel", adapter=SnowflakeAdapter(), expected_fragments=( - "CREATE TRANSIENT TABLE IF NOT EXISTS main._sqlbuild_column_migrations", + "CREATE TABLE IF NOT EXISTS main._sqlbuild_column_migrations", "origin_column", + ") DATA_RETENTION_TIME_IN_DAYS = 90", ), ), AdapterColumnMigrationStateTableTestCase( diff --git a/tests/unit/src/sqlbuild/adapters/test_state_table_ddl.py b/tests/unit/src/sqlbuild/adapters/test_state_table_ddl.py new file mode 100644 index 0000000000..f784d51eab --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/test_state_table_ddl.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from collections.abc import Callable +from itertools import product + +import pytest + +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from tests.unit.src.sqlbuild.adapters._test_types import AdapterStateTableDdlTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterStateTableDdlTestCase( + description=f"{adapter_name} {render_method}", + adapter=adapter, + render_method=render_method, + expected_create_fragment=create_fragment, + ) + for (adapter_name, adapter, create_fragment), render_method in product( + ( + ("duckdb", DuckDbAdapter(), "CREATE TABLE IF NOT EXISTS "), + ("postgres", PostgresAdapter(), "CREATE TABLE IF NOT EXISTS "), + ("bigquery", BigQueryAdapter(), "CREATE TABLE IF NOT EXISTS "), + ("databricks", DatabricksAdapter(), "CREATE TABLE IF NOT EXISTS "), + ("sql server", SqlServerAdapter(), "CREATE TABLE "), + ), + ( + "render_create_fingerprint_table_sql", + "render_create_source_freshness_table_sql", + "render_create_node_result_table_sql", + "render_create_audit_result_table_sql", + "render_create_janitor_event_table_sql", + "render_create_migration_state_table_sql", + "render_create_old_name_view_state_table_sql", + "render_create_column_migration_state_table_sql", + "render_create_microbatch_state_table_sql", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_non_snowflake_adapter_when_rendering_state_table_then_permanent_without_retention( + test_case: AdapterStateTableDdlTestCase, +) -> None: + render: Callable[..., str] = getattr(test_case.adapter, test_case.render_method) + + sql: str = render(database="analytics", schema="marts") + + assert test_case.expected_create_fragment in sql + assert "TRANSIENT" not in sql.upper() + assert "DATA_RETENTION_TIME_IN_DAYS" not in sql.upper() diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index 0fae86699c..dda7c71329 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -12,6 +12,9 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, SqlTestPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason +from sqlbuild.compiler.source_freshness.main.create_table_sql import ( + build_create_table_sql as build_source_freshness_create_table_sql, +) from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, @@ -21,8 +24,6 @@ class RecordingAdapter: - state_tables_transient: bool = False - def __init__(self) -> None: self.insert_count: int = 0 self.executed_sql: list[str] = [] @@ -54,6 +55,14 @@ def render_qualified_name( def render_framework_type(self, framework_type: FrameworkType) -> str: return framework_type.value + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + return build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + def render_create_source_freshness_index_sqls( self, *, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py index 599e836f63..c7c37a087c 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py @@ -17,7 +17,6 @@ class BuildCreateTableSqlTestCase: database: str | None schema: str expected_contains: tuple[str, ...] - transient: bool = False @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py index ece3b5241d..5899b547a8 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py @@ -88,16 +88,6 @@ def test_given_schema_when_building_qualified_name_then_returns_expected( "ts TIMESTAMP NOT NULL", ), ), - BuildCreateTableSqlTestCase( - description="emits transient table when requested", - database=None, - schema="analytics", - transient=True, - expected_contains=( - "CREATE TRANSIENT TABLE IF NOT EXISTS", - f"analytics.{FINGERPRINT_TABLE_NAME}", - ), - ), ], ids=lambda case: case.description, ) @@ -109,7 +99,6 @@ def test_given_schema_when_building_create_table_sql_then_contains_expected_frag schema=test_case.schema, render_qualified_name=RENDER_QUALIFIED_NAME, render_framework_type=RENDER_FRAMEWORK_TYPE, - transient=test_case.transient, ) fragment: str diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/_test_types.py index 6e4b436fc2..dcae4dd320 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/_test_types.py @@ -7,4 +7,3 @@ class BuildSourceFreshnessSqlTestCase: database: str | None schema: str expected_contains: tuple[str, ...] - transient: bool = False diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py index 01de982b3c..08a5bd4908 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py @@ -68,16 +68,6 @@ def test_given_schema_when_building_qualified_name_then_returns_expected( "observed_at TIMESTAMP NOT NULL", ), ), - BuildSourceFreshnessSqlTestCase( - description="emits transient table when requested", - database=None, - schema="analytics", - transient=True, - expected_contains=( - "CREATE TRANSIENT TABLE IF NOT EXISTS", - f"analytics.{SOURCE_FRESHNESS_TABLE_NAME}", - ), - ), ], ids=lambda case: case.description, ) @@ -89,7 +79,6 @@ def test_given_schema_when_building_create_table_sql_then_contains_expected_frag schema=test_case.schema, render_qualified_name=RENDER_QUALIFIED_NAME, render_framework_type=RENDER_FRAMEWORK_TYPE, - transient=test_case.transient, ) fragment: str diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py index 0be0493ac7..f385f71556 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py @@ -80,7 +80,7 @@ def test_given_recorded_column_renames_when_composing_then_old_names_map_to_curr expected_statements=( "CREATE OR REPLACE VIEW analytics.revenue AS SELECT * FROM analytics.daily_revenue", ), - expected_state_table_prefix="CREATE TRANSIENT TABLE IF NOT EXISTS", + expected_state_table_prefix="CREATE TABLE IF NOT EXISTS", expected_transactional=False, ), AdapterOldNameSqlTestCase( @@ -92,7 +92,7 @@ def test_given_recorded_column_renames_when_composing_then_old_names_map_to_curr "CREATE OR REPLACE VIEW analytics.revenue AS " 'SELECT "ORDER_ID", "REVENUE" AS "AMOUNT" FROM analytics.daily_revenue', ), - expected_state_table_prefix="CREATE TRANSIENT TABLE IF NOT EXISTS", + expected_state_table_prefix="CREATE TABLE IF NOT EXISTS", expected_transactional=False, ), AdapterOldNameSqlTestCase( diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 83d4814dae..9bac82061c 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -63,6 +63,18 @@ During a build, checks for whether one relation exists use `SHOW TERSE TABLES` a need a running warehouse. Each build reuses these answers until it changes that relation, for example by replacing a delta table or adding a column, and then reads it again. +## State tables + +SQLBuild records its state in `_sqlbuild_*` tables, such as `_sqlbuild_fingerprints`, +`_sqlbuild_source_freshness`, and `_sqlbuild_microbatches`. On Snowflake, SQLBuild creates every +state table as a permanent table with `DATA_RETENTION_TIME_IN_DAYS = 90`, the maximum time travel, +so the table also has Fail-safe. On editions that allow only 1 day of time travel, Snowflake +rejects 90 and SQLBuild creates the table with 1 day instead. + +The retention is set only in the statement that creates the table. SQLBuild never alters an +existing state table, so a state table that already exists keeps its current table type and +retention. Models are not affected: their table type follows the model `table_type` setting. + ## Shared connections across targets Use multiple targets to reuse one Snowflake connection while selecting different authoritative diff --git a/website/src/content/docs/docs/concepts/planning.mdx b/website/src/content/docs/docs/concepts/planning.mdx index db7f857471..adb21a5f7f 100644 --- a/website/src/content/docs/docs/concepts/planning.mdx +++ b/website/src/content/docs/docs/concepts/planning.mdx @@ -122,6 +122,10 @@ SQLBuild records append-only planning evidence in the warehouse: The planner reads the latest applicable facts and appends new facts after successful work. +On Snowflake, SQLBuild creates its state tables as permanent tables with the maximum time travel: +90 days, or 1 day on editions that allow only 1. The retention is set when SQLBuild creates the +table. See [Snowflake state tables](/docs/concepts/adapters/snowflake#state-tables). + ## Related topics - [Replay decisions](/docs/concepts/planning/replay-decisions) From f737c0e5e60689a3aeb52c103681024374d4b5aa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:18:31 +0100 Subject: [PATCH 2/2] test(compile): change size in the hidden-directory fingerprint case so a same-tick rewrite cannot flake --- .../sqlbuild/cli/compile_reuse/_helpers/test_project_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py index f4867265a1..cb38c6869c 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py @@ -129,7 +129,7 @@ ), ProjectFingerprintTestCase( description="root_hidden_project_directory_change", - change=lambda root: write_file(root / ".orders_rules/limits.txt", "10\n"), + change=lambda root: write_file(root / ".orders_rules/limits.txt", "400\n"), expected_unchanged=False, ), ProjectFingerprintTestCase(