From 354a031f33b2c59a1b55e32924cb20fb680cdd69 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:22:02 +0000 Subject: [PATCH 01/12] perf: drop and replace only the table kinds a build creates Staging and delta tables are created with CREATE OR REPLACE where the adapter replaces, dropped only as tables, kept between microbatch batches that replace them, and counted only when DML reports no row count. A table without an enforced contract skips the staging column read. --- .../contract/classes/duckdb_backed_adapter.py | 3 + .../contract/classes/strict_adapter.py | 25 ++++++ .../run/_helpers/execution/staging.py | 33 ++++++-- .../run/_helpers/materializations/custom.py | 1 + .../_helpers/materializations/incremental.py | 17 ++-- .../_helpers/materializations/microbatch.py | 84 ++++++++++++------- .../run/_helpers/materializations/snapshot.py | 1 + .../run/_helpers/materializations/view.py | 1 + .../_helpers/validation/type_enforcement.py | 7 +- src/sqlbuild/executor/run/main/_execute.py | 18 ++-- src/sqlbuild/executor/run/models.py | 2 + .../run/_helpers/test_lifecycle_progress.py | 2 +- 12 files changed, 135 insertions(+), 59 deletions(-) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 4909d3869c..d79c83879c 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -126,6 +126,9 @@ def get_columns_for_relations( execution_duration_limit_seconds: ClassVar[int | None] = CANCELLABLE_DURATION_LIMIT_SECONDS + def create_table_as_replaces(self) -> bool: + return True + def create_sql_test_output(self, *, connection: Any, sql: str, name: str) -> str | None: """Store a unit test's output in a temporary table; compare inline on an engine bug.""" diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index ae44b4191c..ae849af077 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -134,6 +134,31 @@ def drop_sql_test_output(self, *, connection: Any, relation: str) -> None: del connection, relation + def drop_table( + self, *, connection: Any, destination: str, statement_recorder: StatementRecorder + ) -> None: + """Optional: drop a staging or delta table SQLBuild created, if it exists.""" + + self.drop( + connection=connection, + destination=destination, + if_exists=True, + statement_recorder=statement_recorder, + ) + + def list_relations_with_prefix( + self, *, connection: Any, database: str | None, schemas: tuple[str, ...], prefix: str + ) -> tuple[RelationInfo, ...] | None: + """Optional: relations in ``schemas`` whose names start with ``prefix``, or None.""" + + del connection, database, schemas, prefix + return None + + def create_table_as_replaces(self) -> bool: + """Optional: whether `create_table_as` replaces a table already at its destination.""" + + return False + def render_cursor_operand(self, *, expression: str, cursor_type: str | None) -> str: """Optional: the cursor-column expression compared with cursor bounds and aggregated.""" diff --git a/src/sqlbuild/executor/run/_helpers/execution/staging.py b/src/sqlbuild/executor/run/_helpers/execution/staging.py index 379040407a..9c91e843bd 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/staging.py +++ b/src/sqlbuild/executor/run/_helpers/execution/staging.py @@ -38,13 +38,8 @@ def create_staging_relation( target_kind="staging_relation", ), ) as lifecycle: - adapter.drop( - connection=connection, - destination=staging_qualified, - if_exists=True, - statement_recorder=statement_recorder, - ) - adapter.create_table_as( + create_intermediate_table( + adapter=adapter, connection=connection, destination=staging_qualified, sql=resolved_sql, @@ -53,3 +48,27 @@ def create_staging_relation( ) lifecycle.completed(metadata={"changed_count": 1}) return None + + +def create_intermediate_table( + *, + adapter: BaseAdapter, + connection: Any, + destination: str, + sql: str, + config: dict[str, Any] | None, + statement_recorder: StatementRecorder, +) -> None: + """Create a table only SQLBuild writes, replacing one an interrupted run left behind.""" + + if not adapter.create_table_as_replaces(): + adapter.drop_table( + connection=connection, destination=destination, statement_recorder=statement_recorder + ) + adapter.create_table_as( + connection=connection, + destination=destination, + sql=sql, + config=config, + statement_recorder=statement_recorder, + ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/custom.py b/src/sqlbuild/executor/run/_helpers/materializations/custom.py index d12c469e5a..8a71bd6c93 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/custom.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/custom.py @@ -391,6 +391,7 @@ def _complete_custom_materialization( connection=context.connection, run_id=context.run_id, query_change_tracking=context.query_change_tracking, + fingerprint_writes=context.fingerprint_writes, model_audits=context.model_audits, audit_results=tuple(state.audit_results), ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 380f3272a4..52e6654911 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -42,6 +42,7 @@ build_skipped_result, ) from sqlbuild.executor.run._helpers.execution.schema import inspect_runtime_relation_schema +from sqlbuild.executor.run._helpers.execution.staging import create_intermediate_table from sqlbuild.executor.run._helpers.execution.table_config import model_output_config from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint from sqlbuild.executor.run._helpers.validation.contracts import validate_runtime_contract @@ -388,10 +389,9 @@ def execute_incremental_entry( return post_hook_outcome.failure if post_hook_outcome.skipped: with diagnostics_context(sqlbuild_phase="cleanup", sqlbuild_action_name="drop_delta"): - adapter.drop( + adapter.drop_table( connection=connection, destination=delta_qualified, - if_exists=True, statement_recorder=statement_recorder, ) return build_skipped_result( @@ -410,16 +410,16 @@ def execute_incremental_entry( connection=connection, run_id=context.run_id, query_change_tracking=context.query_change_tracking, + fingerprint_writes=context.fingerprint_writes, model_audits=context.model_audits, audit_results=tuple(audit_results), ) ) with diagnostics_context(sqlbuild_phase="cleanup", sqlbuild_action_name="drop_delta"): - adapter.drop( + adapter.drop_table( connection=connection, destination=delta_qualified, - if_exists=True, statement_recorder=statement_recorder, ) @@ -513,13 +513,8 @@ def _prepare_delta_relation( target_kind="staging_relation", ), ) as lifecycle: - adapter.drop( - connection=connection, - destination=delta_qualified, - if_exists=True, - statement_recorder=statement_recorder, - ) - adapter.create_table_as( + create_intermediate_table( + adapter=adapter, connection=connection, destination=delta_qualified, sql=preparation.resolved_sql, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index acc24fc55f..850714cf33 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -9,6 +9,7 @@ from dataclasses import dataclass, field, replace from datetime import UTC, date, datetime from decimal import Decimal, InvalidOperation +from functools import partial from typing import Any from sqlbuild._native import ( @@ -83,6 +84,7 @@ build_skipped_result, ) from sqlbuild.executor.run._helpers.execution.schema import inspect_runtime_relation_schema +from sqlbuild.executor.run._helpers.execution.staging import create_intermediate_table from sqlbuild.executor.run._helpers.execution.table_config import model_output_config from sqlbuild.executor.run._helpers.materializations.full_refresh import ( promote_full_refresh_rebuild, @@ -241,6 +243,20 @@ class _MicrobatchLifecyclePlan: cursor_end_exclusive: str +class _BatchColumns: + """The delta columns of a model's first batch; later batches stage the same query's output.""" + + def __init__(self) -> None: + self._delta: tuple[ColumnInfo, ...] | None = None + + def delta(self, read: Callable[[], tuple[ColumnInfo, ...]]) -> tuple[ColumnInfo, ...]: + """The first batch's delta columns, read once.""" + + if self._delta is None: + self._delta = read() + return self._delta + + @dataclass(frozen=True) class _MicrobatchBatchPreparation: """Prepared batch state or the failure that prevents target DML.""" @@ -608,6 +624,7 @@ def execute_microbatch_entry( connection=context.connection, run_id=context.run_id, query_change_tracking=context.query_change_tracking, + fingerprint_writes=context.fingerprint_writes, model_audits=context.model_audits, audit_results=tuple(state.audit_results), ) @@ -1052,6 +1069,7 @@ def _execute_microbatch_batches( completed_batches: int = 0 total_rows: int = 0 row_count_known: bool = False + known_columns: _BatchColumns = _BatchColumns() batch: BatchWindow for batch in batches: lifecycle: MicrobatchLifecycle = _microbatch_lifecycle( @@ -1122,6 +1140,7 @@ def _execute_microbatch_batches( window_text=window_text, targets=batch_targets, state=state, + known_columns=known_columns, ) if isinstance(dml_result, ModelExecutionResult): lifecycle.failed(error_code=dml_result.error_code) @@ -1183,6 +1202,8 @@ def _execute_microbatch_batches( window_text=window_text, targets=batch_targets, state=state, + replaced_next=batch is not batches[-1] + and batch_targets.delta_qualified == targets.delta_qualified, ) except BaseException as error: lifecycle.failed(error=error) @@ -2815,13 +2836,8 @@ def _stage_microbatch_delta( target_kind="staging_relation", ), ) as lifecycle: - context.adapter.drop( - connection=context.connection, - destination=targets.delta_qualified, - if_exists=True, - statement_recorder=state.statement_recorder, - ) - context.adapter.create_table_as( + create_intermediate_table( + adapter=context.adapter, connection=context.connection, destination=targets.delta_qualified, sql=batch_sql, @@ -2999,6 +3015,7 @@ def _apply_microbatch_dml( window_text: str, targets: MicrobatchTargets, state: MicrobatchLifecycleState, + known_columns: _BatchColumns, ) -> ModelExecutionResult | int | None: try: with diagnostics_context( @@ -3017,16 +3034,20 @@ def _apply_microbatch_dml( if not is_full_refresh or completed_batches > 0 else () ) - delta_columns: tuple[ColumnInfo, ...] = inspect_runtime_relation_schema( - adapter=context.adapter, - connection=context.connection, - database=targets.target_database, - schema=targets.target_schema, - name=targets.delta_table, + delta_columns: tuple[ColumnInfo, ...] = known_columns.delta( + partial( + inspect_runtime_relation_schema, + adapter=context.adapter, + connection=context.connection, + database=targets.target_database, + schema=targets.target_schema, + name=targets.delta_table, + ) ) _validate_cursor_output_columns(entry=context.entry, delta_columns=delta_columns) - batch_rows: int | None = _count_microbatch_delta_rows(context=context, targets=targets) + batch_rows: int | None if is_full_refresh and completed_batches == 0: + batch_rows = _count_microbatch_delta_rows(context=context, targets=targets) context.adapter.create_table_as( connection=context.connection, destination=targets.target_qualified, @@ -3050,8 +3071,11 @@ def _apply_microbatch_dml( cursor_end=render(value=batch.end), statement_recorder=state.statement_recorder, ) - if reported_rows is not None: - batch_rows = reported_rows + batch_rows = ( + reported_rows + if reported_rows is not None + else _count_microbatch_delta_rows(context=context, targets=targets) + ) except Exception as exc: return build_failed_result( entry=context.entry, @@ -3085,18 +3109,21 @@ def _complete_microbatch_batch( window_text: str, targets: MicrobatchTargets, state: MicrobatchLifecycleState, + replaced_next: bool, ) -> None: - with diagnostics_context( - sqlbuild_phase="cleanup", - sqlbuild_action_name="drop_delta", - sqlbuild_window=window_text, - ): - context.adapter.drop( - connection=context.connection, - destination=targets.delta_qualified, - if_exists=True, - statement_recorder=state.statement_recorder, - ) + """Finish one batch; its delta stays only when the next batch's delta replaces it.""" + + if not (replaced_next and context.adapter.create_table_as_replaces()): + with diagnostics_context( + sqlbuild_phase="cleanup", + sqlbuild_action_name="drop_delta", + sqlbuild_window=window_text, + ): + context.adapter.drop_table( + connection=context.connection, + destination=targets.delta_qualified, + statement_recorder=state.statement_recorder, + ) log_debug_event( logger=_DEBUG_LOGGER, message="", @@ -3140,10 +3167,9 @@ def _prepare_full_refresh_rebuild( statement_recorder=state.statement_recorder, ) elif rebuild_exists: - context.adapter.drop( + context.adapter.drop_table( connection=context.connection, destination=relations.rebuild_qualified, - if_exists=True, statement_recorder=state.statement_recorder, ) except Exception as exc: diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 44964715d8..abefe46ae5 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -358,6 +358,7 @@ def execute_snapshot_entry( # noqa: PLR0915 connection=connection, run_id=context.run_id, query_change_tracking=context.query_change_tracking, + fingerprint_writes=context.fingerprint_writes, model_audits=context.model_audits, audit_results=tuple(audit_results), ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index eca08c28fb..69ea65c384 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -213,6 +213,7 @@ def execute_view_entry( query_change_tracking=query_change_tracking, model_audits=model_audits, audit_results=tuple(audit_results), + fingerprint_writes=context.fingerprint_writes, ) ) diff --git a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py index 41d5599c25..bec5e60b1b 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py +++ b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py @@ -82,11 +82,8 @@ def enforce_types_staged( }, statement_recorder=statement_recorder, ) - adapter.drop( - connection=connection, - destination=staging_qualified, - if_exists=True, - statement_recorder=statement_recorder, + adapter.drop_table( + connection=connection, destination=staging_qualified, statement_recorder=statement_recorder ) adapter.rename( connection=connection, diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 2e1bce918c..7089ac50e8 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -262,12 +262,16 @@ def _staged_lifecycle( with diagnostics_context( sqlbuild_phase="contract", sqlbuild_action_name="validate_staging" ): - staging_columns: tuple[ColumnInfo, ...] = inspect_runtime_relation_schema( - adapter=adapter, - connection=connection, - database=targets.target_database, - schema=targets.target_schema, - name=targets.staging_table, + staging_columns: tuple[ColumnInfo, ...] = ( + () + if not entry.contract_enforced + else inspect_runtime_relation_schema( + adapter=adapter, + connection=connection, + database=targets.target_database, + schema=targets.target_schema, + name=targets.staging_table, + ) ) validate_runtime_contract( entry=entry, @@ -362,6 +366,7 @@ def _staged_lifecycle( connection=connection, run_id=context.run_id, query_change_tracking=context.query_change_tracking, + fingerprint_writes=context.fingerprint_writes, model_audits=context.model_audits, audit_results=tuple(audit_results), ) @@ -490,6 +495,7 @@ def _immediate_lifecycle( connection=connection, run_id=context.run_id, query_change_tracking=context.query_change_tracking, + fingerprint_writes=context.fingerprint_writes, model_audits=context.model_audits, audit_results=tuple(audit_results), ) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index d324f736fb..551539f588 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -41,6 +41,7 @@ from sqlbuild.executor.custom.models import MaterializationResult from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites from sqlbuild.executor.run.types import ( AuditGateMetadataParseFailure, AuditGateReuseReason, @@ -328,6 +329,7 @@ class ModelMaterializationContext: watermark_resolver: WatermarkResolver | None = None enforce_explicit_references: bool = False bound_views: BoundViewGuard = field(default_factory=BoundViewGuard) + fingerprint_writes: FingerprintWrites | None = None @dataclass(frozen=True, kw_only=True, init=False) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py index b434d33fb2..cc4b2e7ef5 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py @@ -35,7 +35,7 @@ def test_given_staging_creation_when_adapter_blocks_then_start_is_already_dispat context: Mock = Mock(adapter=adapter, connection=object()) context.entry.table_type = "transient" context.entry.inferred_columns = () - adapter.drop.side_effect = lambda **_: barrier_events.append(events[-1]) + adapter.create_table_as.side_effect = lambda **_: barrier_events.append(events[-1]) with dispatcher_scope(dispatcher): create_staging_relation( From 18c8e0bcbae33a6798505827e59d7e54e684b144 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:22:03 +0000 Subject: [PATCH 02/12] perf(snowflake): answer relation and retention lookups from run schema listings A run lists a schema with SHOW TABLES and SHOW VIEWS once it looks up a second relation there, and reads retention with SHOW instead of INFORMATION_SCHEMA. Promotion reads the destination again when another session changed it. --- .../classes/relation_metadata_cache.py | 85 +++++- .../classes/run_relation_listings.py | 85 ++++++ src/sqlbuild/adapter/relations/constants.py | 1 + src/sqlbuild/adapter/relations/models.py | 9 + src/sqlbuild/adapter/relations/types.py | 1 + .../snowflake/classes/snowflake_adapter.py | 267 ++++++++++++------ .../run/_helpers/execution/promotion.py | 168 +++++++---- .../build/snowflake_metadata/_test_types.py | 39 +++ .../build/snowflake_metadata/helpers.py | 148 +++++++++- .../test_build_metadata_budget.py | 7 +- .../test_statement_overhead.py | 188 ++++++++++++ .../adapters/snowflake/_test_types.py | 36 ++- .../sqlbuild/adapters/snowflake/helpers.py | 151 +++++++++- .../adapters/snowflake/inspection/helpers.py | 34 ++- .../adapters/snowflake/test_retention.py | 186 +++++++----- .../adapters/snowflake/test_run_listing.py | 45 +++ 16 files changed, 1231 insertions(+), 219 deletions(-) create mode 100644 src/sqlbuild/adapter/relations/classes/run_relation_listings.py create mode 100644 tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_statement_overhead.py create mode 100644 tests/unit/src/sqlbuild/adapters/snowflake/test_run_listing.py diff --git a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py index 588c1ffb73..4e93832592 100644 --- a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py +++ b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py @@ -11,8 +11,14 @@ relation_name_key, statement_metadata_effect, ) -from sqlbuild.adapter.relations.models import StatementMetadataEffect -from sqlbuild.adapter.relations.types import RelationCacheKey, RelationCacheVersion +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.relations.constants import SCHEMA_LISTING_LOOKUP_THRESHOLD +from sqlbuild.adapter.relations.models import RunSchemaListing, StatementMetadataEffect +from sqlbuild.adapter.relations.types import ( + RelationCacheKey, + RelationCacheVersion, + SchemaListingScope, +) class RelationMetadataCache: @@ -26,6 +32,9 @@ def __init__(self, *, transactional_ddl: bool) -> None: self._global_version: int = 0 self._pending_names: dict[int, set[str]] = {} self._pending_all: set[int] = set() + self._listings: dict[SchemaListingScope, RunSchemaListing] = {} + self._schema_lookups: dict[SchemaListingScope, int] = {} + self._listing_locks: dict[SchemaListingScope, threading.Lock] = {} def lookup[ValueT]( self, @@ -40,6 +49,77 @@ def lookup[ValueT]( return cast(ValueT, self._lookup(key=(kind, database, schema, name), read=read)) + def remember( + self, *, kind: str, database: str | None, schema: str | None, name: str, value: object + ) -> None: + """Record an answer this run has just established, such as a relation it dropped.""" + + with self._lock: + self._entries[(kind, database, schema, name)] = value + + def schema_relation( + self, + *, + scope: SchemaListingScope, + name: str, + stored_name: str, + read_listing: Callable[[], dict[str, RelationInfo]], + ) -> tuple[bool, RelationInfo | None]: + """Answer ``(answered, relation)`` from the schema listing made at its second lookup.""" + + listing: RunSchemaListing | None + with self._lock: + listing = self._listings.get(scope) + if listing is None: + lookups: int = self._schema_lookups.get(scope, 0) + 1 + self._schema_lookups[scope] = lookups + if lookups < SCHEMA_LISTING_LOOKUP_THRESHOLD: + return False, None + listing_lock: threading.Lock = self._listing_locks.setdefault(scope, threading.Lock()) + if listing is None: + with listing_lock: + with self._lock: + listing = self._listings.get(scope) + if listing is None: + listing = self.refresh_schema_listing(scope=scope, read_listing=read_listing) + with self._lock: + if not self._listed_fresh(listing=listing, name=name): + return False, None + return True, listing.relations.get(stored_name) + + def refresh_schema_listing( + self, *, scope: SchemaListingScope, read_listing: Callable[[], dict[str, RelationInfo]] + ) -> RunSchemaListing: + """List the schema again; every relation in it is current as of this listing.""" + + with self._lock: + global_version: int = self._global_version + name_versions: dict[str, int] = dict(self._name_versions) + listing: RunSchemaListing = RunSchemaListing( + relations=read_listing(), global_version=global_version, name_versions=name_versions + ) + with self._lock: + if self._global_version == global_version: + self._listings[scope] = listing + return listing + + def listed_relation( + self, *, scope: SchemaListingScope, name: str, stored_name: str + ) -> tuple[bool, RelationInfo | None]: + """Answer from an existing listing of the schema only, without listing it.""" + + with self._lock: + listing: RunSchemaListing | None = self._listings.get(scope) + if listing is None or not self._listed_fresh(listing=listing, name=name): + return False, None + return True, listing.relations.get(stored_name) + + def _listed_fresh(self, *, listing: RunSchemaListing, name: str) -> bool: + name_key: str = relation_name_key(name) + return listing.global_version == self._global_version and listing.name_versions.get( + name_key, 0 + ) == self._name_versions.get(name_key, 0) + def invalidate_relations(self, *, qualified_names: tuple[str, ...]) -> None: """Evict relations changed outside SQL statements, such as by a warehouse copy job.""" @@ -106,3 +186,4 @@ def _invalidate_names(self, *, names: frozenset[str]) -> None: def _invalidate_all(self) -> None: self._global_version += 1 self._entries.clear() + self._listings.clear() diff --git a/src/sqlbuild/adapter/relations/classes/run_relation_listings.py b/src/sqlbuild/adapter/relations/classes/run_relation_listings.py new file mode 100644 index 0000000000..b36ddd67a8 --- /dev/null +++ b/src/sqlbuild/adapter/relations/classes/run_relation_listings.py @@ -0,0 +1,85 @@ +"""Relation answers shared through the open run's relation metadata cache.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.relations._helpers.relation_metadata_context import ( + current_relation_metadata_cache, +) +from sqlbuild.adapter.relations.classes.relation_metadata_cache import RelationMetadataCache +from sqlbuild.adapter.relations.constants import RELATION_EXISTS_LOOKUP, RELATION_TYPE_LOOKUP +from sqlbuild.adapter.relations.types import SchemaListingScope + + +class RunRelationListings: + """Schema listings and known relation states of the run open for an adapter, if any.""" + + @staticmethod + def schema_relation( + *, + adapter: Any, + scope: SchemaListingScope, + name: str, + stored_name: str, + read_listing: Callable[[], dict[str, RelationInfo]], + ) -> tuple[bool, RelationInfo | None]: + """Whether the run's schema listing answers ``name``, and the relation it lists.""" + + cache: RelationMetadataCache | None = current_relation_metadata_cache(adapter=adapter) + if cache is None: + return False, None + return cache.schema_relation( + scope=scope, name=name, stored_name=stored_name, read_listing=read_listing + ) + + @staticmethod + def listed_relation( + *, adapter: Any, scope: SchemaListingScope, name: str, stored_name: str + ) -> tuple[bool, RelationInfo | None]: + """Whether a current listing answers ``name``, and the relation it lists; never lists.""" + + cache: RelationMetadataCache | None = current_relation_metadata_cache(adapter=adapter) + if cache is None: + return False, None + return cache.listed_relation(scope=scope, name=name, stored_name=stored_name) + + @staticmethod + def refresh_schema_listing( + *, + adapter: Any, + scope: SchemaListingScope, + read_listing: Callable[[], dict[str, RelationInfo]], + ) -> dict[str, RelationInfo]: + """Read the schema's relations, keeping them for the run's later lookups.""" + + cache: RelationMetadataCache | None = current_relation_metadata_cache(adapter=adapter) + if cache is None: + return read_listing() + return cache.refresh_schema_listing(scope=scope, read_listing=read_listing).relations + + @staticmethod + def remember_absent( + *, adapter: Any, relations: tuple[tuple[str | None, str | None, str], ...] + ) -> None: + """Answer later existence and type lookups of ``relations`` as absent until SQL runs.""" + + cache: RelationMetadataCache | None = current_relation_metadata_cache(adapter=adapter) + if cache is None: + return + database: str | None + schema: str | None + name: str + for database, schema, name in relations: + cache.remember( + kind=RELATION_EXISTS_LOOKUP, + database=database, + schema=schema, + name=name, + value=False, + ) + cache.remember( + kind=RELATION_TYPE_LOOKUP, database=database, schema=schema, name=name, value=None + ) diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 9d1557c487..bc8d6175ff 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -6,4 +6,5 @@ RELATION_COLUMNS_LOOKUP: str = "columns" RELATION_EXISTS_LOOKUP: str = "exists" RELATION_TYPE_LOOKUP: str = "type" +SCHEMA_LISTING_LOOKUP_THRESHOLD: int = 2 UNKNOWN_RELATION_READER: str = "unknown user" diff --git a/src/sqlbuild/adapter/relations/models.py b/src/sqlbuild/adapter/relations/models.py index 40ed29e19e..2cf2f8af24 100644 --- a/src/sqlbuild/adapter/relations/models.py +++ b/src/sqlbuild/adapter/relations/models.py @@ -51,3 +51,12 @@ class StatementMetadataEffect: relation_names: frozenset[str] = frozenset() invalidates_all: bool = False ends_transaction: bool = False + + +@dataclass(frozen=True) +class RunSchemaListing: + """One schema's relations as a build listed them, and the cache versions at that moment.""" + + relations: dict[str, RelationInfo] + global_version: int + name_versions: dict[str, int] diff --git a/src/sqlbuild/adapter/relations/types.py b/src/sqlbuild/adapter/relations/types.py index 30d9bf395c..cc3f0cf0d4 100644 --- a/src/sqlbuild/adapter/relations/types.py +++ b/src/sqlbuild/adapter/relations/types.py @@ -11,6 +11,7 @@ type RelationCacheKey = tuple[str, str | None, str | None, str] type RelationCacheVersion = tuple[int, int] +type SchemaListingScope = tuple[str | None, str] class RelationLocation(Protocol): diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index fb44ee1b78..74157a4ec1 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -9,6 +9,7 @@ from dataclasses import replace from datetime import date, datetime from decimal import Decimal +from functools import partial from pathlib import Path from typing import Any, ClassVar, cast @@ -100,6 +101,7 @@ TablePromotionMode, ) from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog +from sqlbuild.adapter.relations.classes.run_relation_listings import RunRelationListings from sqlbuild.adapter.relations.constants import INSPECTION_IN_LIST_LIMIT from sqlbuild.adapter.relations.main.active_inspection_catalog import active_inspection_catalog from sqlbuild.adapter.relations.main.cached_view_definition import cached_view_definition @@ -114,6 +116,7 @@ SchemaColumnListing, SchemaRelationListing, ) +from sqlbuild.adapter.relations.types import SchemaListingScope from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( conditional_result_nullability, ) @@ -174,6 +177,7 @@ _STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" _METADATA_INSPECTION_CONCURRENCY: int = 8 _SHOW_RESULT_LIMIT: int = 10_000 +_SHOW_OBJECT_RELATION_TYPES: dict[str, str] = {"TABLE": "base table", "VIEW": "view"} _COST_LOGIN_TIMEOUT_SECONDS: int = 5 _COST_NETWORK_TIMEOUT_SECONDS: int = 15 _SHOW_COLUMN_KIND: str = "COLUMN" @@ -214,39 +218,14 @@ class SnowflakeAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): ) def inspect_retention(self, *, connection: Any, request: RetentionRequest) -> RetentionState: - self._validate_relation_retention_request(request=request) - clauses: list[str] = ["table_name = %s", "table_schema = %s"] - params: list[str] = [ - self._information_schema_identifier(str(request.name)), - self._information_schema_identifier(request.schema), + return self.inspect_retentions(connection=connection, requests=(request,))[ + request.request_id ] - if request.database is not None: - clauses.append("table_catalog = %s") - params.append(self._information_schema_identifier(request.database)) - cursor: Any = connection.cursor() - try: - cursor.execute( - "SELECT retention_time, is_transient FROM " - + self._information_schema_relation(database=request.database, name="tables") - + " WHERE " - + " AND ".join(clauses), - tuple(params), - ) - row: tuple[Any, ...] | None = cursor.fetchone() - finally: - cursor.close() - if row is None: - raise AdapterUserError( - message=f"Snowflake retention metadata not found for {request.name}" - ) - return self._relation_retention_state( - request=request, retention_time=row[0], is_transient_value=row[1] - ) def inspect_retentions( self, *, connection: Any, requests: tuple[RetentionRequest, ...] ) -> dict[str, RetentionState]: - """Inspect relation retention with one metadata query per database and schema.""" + """Inspect relation retention with SHOW TABLES, which needs no running warehouse.""" grouped: dict[tuple[str | None, str], list[RetentionRequest]] = {} for request in requests: @@ -254,45 +233,112 @@ def inspect_retentions( grouped.setdefault((request.database, request.schema), []).append(request) states: dict[str, RetentionState] = {} for (database, schema), group in grouped.items(): - names: tuple[str, ...] = tuple( - dict.fromkeys( - self._information_schema_identifier(str(request.name)) for request in group - ) + relations: dict[str, RelationInfo | None] = self._retention_relations( + connection=connection, database=database, schema=schema, group=tuple(group) ) - clauses: list[str] = [ - f"table_name IN ({', '.join(['%s'] * len(names))})", - "table_schema = %s", - ] - params: list[str] = [*names, self._information_schema_identifier(schema)] - if database is not None: - clauses.append("table_catalog = %s") - params.append(self._information_schema_identifier(database)) - cursor: Any = connection.cursor() - try: - cursor.execute( - "SELECT table_name, retention_time, is_transient FROM " - + self._information_schema_relation(database=database, name="tables") - + " WHERE " - + " AND ".join(clauses), - tuple(params), - ) - rows: list[tuple[Any, ...]] = cursor.fetchall() - finally: - cursor.close() - rows_by_name: dict[str, tuple[Any, ...]] = {str(row[0]): row for row in rows} for request in group: - row: tuple[Any, ...] | None = rows_by_name.get( + relation: RelationInfo | None = relations.get( self._information_schema_identifier(str(request.name)) ) - if row is None: + state: RetentionState | None = ( + None + if relation is None + else self.retention_state_from_relation(request=request, relation=relation) + ) + if state is None: raise AdapterUserError( message=f"Snowflake retention metadata not found for {request.name}" ) - states[request.request_id] = self._relation_retention_state( - request=request, retention_time=row[1], is_transient_value=row[2] - ) + states[request.request_id] = state return states + def _retention_relations( + self, + *, + connection: Any, + database: str | None, + schema: str, + group: tuple[RetentionRequest, ...], + ) -> dict[str, RelationInfo | None]: + """The listed relation of each request, keyed by stored name.""" + + scope: SchemaListingScope = self._listing_scope( + connection=connection, database=database, schema=schema + ) + relations: dict[str, RelationInfo | None] = {} + unanswered: list[str] = [] + for request in group: + stored_name: str = self._information_schema_identifier(str(request.name)) + answered, relation = RunRelationListings.listed_relation( + adapter=self, scope=scope, name=str(request.name), stored_name=stored_name + ) + if answered: + relations[stored_name] = relation + else: + unanswered.append(stored_name) + if not unanswered: + return relations + if len(set(unanswered)) == 1 and SHOW_LIKE_ESCAPE_CHARACTER not in unanswered[0]: + relations[unanswered[0]] = self._show_table( + connection=connection, database=database, schema=schema, stored_name=unanswered[0] + ) + return relations + listed: dict[str, RelationInfo] = RunRelationListings.refresh_schema_listing( + adapter=self, + scope=scope, + read_listing=partial( + self._schema_relations, connection=connection, database=database, schema=schema + ), + ) + for stored_name in unanswered: + relations[stored_name] = listed.get(stored_name) + return relations + + def _show_table( + self, *, connection: Any, database: str | None, schema: str, stored_name: str + ) -> RelationInfo | None: + """One table as SHOW TABLES lists it, or None when the schema holds no such table.""" + + show_database: str = self._show_database( + connection=connection, database=database, schema=schema + ) + scope: str = self._show_schema_scope( + connection=connection, database=database, schema=schema + ) + rows: list[dict[str, object]] = self._fetch_schema_show_rows( + connection=connection, + query=f"SHOW TABLES LIKE '{show_like_pattern(stored_name)}' IN SCHEMA {scope}", + show_database=show_database, + schema=schema, + ) + row: dict[str, object] + for row in rows: + if str(row.get("name")) == stored_name: + return listed_relation_from_show_table(row=row, database=database).relation + return None + + def _schema_relations( + self, *, connection: Any, database: str | None, schema: str + ) -> dict[str, RelationInfo]: + """Every table and view in one schema, keyed by stored name.""" + + listing: SchemaRelationListing = self.read_schema_relation_listing( + connection=connection, database=database, schema=schema + ) + return {entry.stored_name: entry.relation for entry in listing.entries} + + def _listing_scope( + self, *, connection: Any, database: str | None, schema: str + ) -> SchemaListingScope: + """The stored database and schema names that key a build's listing of one schema.""" + + return ( + self._information_schema_identifier( + self._show_database(connection=connection, database=database, schema=schema) + ), + self._information_schema_identifier(schema), + ) + def retention_state_from_relation( self, *, request: RetentionRequest, relation: RelationInfo ) -> RetentionState | None: @@ -307,20 +353,6 @@ def retention_state_from_relation( is_transient=relation.is_transient, ) - @staticmethod - def _relation_retention_state( - *, request: RetentionRequest, retention_time: Any, is_transient_value: Any - ) -> RetentionState: - is_transient: bool = str(is_transient_value).upper() == TRUE_METADATA_VALUE - return RetentionState( - request_id=request.request_id, - scope=request.scope, - configured_days=None, - effective_days=int(retention_time), - relation_kind="TRANSIENT" if is_transient else "PERMANENT", - is_transient=is_transient, - ) - def render_retention_changes( self, *, request: RetentionRequest, state: RetentionState | None = None ) -> tuple[RenderedRetentionChange, ...]: @@ -1094,8 +1126,7 @@ def render_delete_insert( sql=f"SELECT * FROM {staged}", columns=columns, ) - drop_staged: tuple[str, ...] = self.render_drop(destination=staged, if_exists=True) - return (*create_staged, delete_sql, *insert_stmts, *drop_staged) + return (*create_staged, delete_sql, *insert_stmts, f"DROP TABLE IF EXISTS {staged}") def render_delete_insert_cursor( self, @@ -1272,6 +1303,54 @@ def drop( for stmt in statements: self.execute(connection=connection, sql=stmt) + def drop_table( + self, *, connection: Any, destination: str, statement_recorder: StatementRecorder + ) -> None: + statement: str = f"DROP TABLE IF EXISTS {destination}" + statement_recorder.record(statement) + self.execute(connection=connection, sql=statement) + + def create_table_as_replaces(self) -> bool: + return True + + def list_relations_with_prefix( + self, *, connection: Any, database: str | None, schemas: tuple[str, ...], prefix: str + ) -> tuple[RelationInfo, ...] | None: + """One SHOW TERSE OBJECTS for a single schema; None for several or past its row limit.""" + + if len(schemas) != 1: + return None + schema: str = schemas[0] + stored_prefix: str = self._information_schema_identifier(prefix) + scope: str = self._show_schema_scope( + connection=connection, database=database, schema=schema + ) + rows: list[dict[str, object]] = self._fetch_schema_show_rows( + connection=connection, + query=( + f"SHOW TERSE OBJECTS LIKE '{show_like_pattern(stored_prefix)}%' " + f"IN SCHEMA {scope} LIMIT {_SHOW_RESULT_LIMIT}" + ), + show_database=self._show_database( + connection=connection, database=database, schema=schema + ), + schema=schema, + ) + if len(rows) >= _SHOW_RESULT_LIMIT: + return None + return tuple( + RelationInfo( + database=database, + schema=str(row.get("schema_name") or schema).lower(), + name=str(row["name"]).lower(), + relation_type=_SHOW_OBJECT_RELATION_TYPES.get( + str(row.get("kind") or "").upper(), str(row.get("kind") or "").lower() + ), + ) + for row in rows + if str(row["name"]).startswith(stored_prefix) + ) + def drop_view( self, *, @@ -1400,14 +1479,12 @@ def append( sql: str, columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, - ) -> None: + ) -> int | None: statements: tuple[str, ...] = self.render_append( destination=destination, sql=sql, columns=columns ) statement_recorder.record_many(statements) - stmt: str - for stmt in statements: - self.execute(connection=connection, sql=stmt) + return self._inserted_rows(connection=connection, statements=statements) def delete_insert( self, @@ -1418,16 +1495,14 @@ def delete_insert( unique_key: str | tuple[str, ...], columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, - ) -> None: + ) -> int | None: keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key statements: tuple[str, ...] = self.render_delete_insert( destination=destination, sql=sql, unique_key=keys, columns=columns ) statement_recorder.record_many(statements) with self.transaction(connection): - stmt: str - for stmt in statements: - self.execute(connection=connection, sql=stmt) + return self._inserted_rows(connection=connection, statements=statements) def delete_insert_cursor( self, @@ -1441,7 +1516,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, - ) -> None: + ) -> int | None: statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, sql=sql, @@ -1453,9 +1528,18 @@ def delete_insert_cursor( ) statement_recorder.record_many(statements) with self.transaction(connection): - stmt: str - for stmt in statements: - self.execute(connection=connection, sql=stmt) + return self._inserted_rows(connection=connection, statements=statements) + + def _inserted_rows(self, *, connection: Any, statements: tuple[str, ...]) -> int | None: + """Run DML statements; the rows the INSERT among them wrote, as Snowflake reports them.""" + + inserted: int | None = None + stmt: str + for stmt in statements: + cursor: Any = self.execute(connection=connection, sql=stmt) + if stmt.startswith("INSERT"): + inserted = self.affected_row_count(cursor=cursor) + return inserted def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" @@ -1966,6 +2050,17 @@ def relation_exists( connection=connection, database=database, schema=schema ) stored_name: str = self._information_schema_identifier(name) + answered, listed = RunRelationListings.schema_relation( + adapter=self, + scope=self._listing_scope(connection=connection, database=database, schema=schema), + name=name, + stored_name=stored_name, + read_listing=partial( + self._schema_relations, connection=connection, database=database, schema=schema + ), + ) + if answered: + return listed is not None pattern: str = show_like_pattern(stored_name) kind: str for kind in ("TABLES", "VIEWS"): diff --git a/src/sqlbuild/executor/run/_helpers/execution/promotion.py b/src/sqlbuild/executor/run/_helpers/execution/promotion.py index edf070a67b..2a44e0b6a2 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/promotion.py +++ b/src/sqlbuild/executor/run/_helpers/execution/promotion.py @@ -10,6 +10,7 @@ from sqlbuild.adapter.contract.types import PromotionStrategy, RelationType from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type +from sqlbuild.adapter.relations.main.invalidate_cached_relations import invalidate_cached_relations from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run.models import BoundViewGuard from sqlbuild.runtime.observability.classes.operation_lifecycle import ( @@ -54,63 +55,57 @@ def promote_relation_to_destination( schema=destination_schema, name=destination_name, ) - strategy: str = ( - RENAME_OPERATION_STRATEGY if not existing else str(adapter.default_promotion_strategy()) - ) + strategy: str = _promotion_strategy(adapter=adapter, existing=existing) terminal_attributes: OperationAttributes | None = ( OperationAttributes(strategy=strategy) if strategy in OPERATION_STRATEGIES else None ) + relations: tuple[str, str] = (origin_relation, destination_relation) try: - if strategy == PromotionStrategy.ATOMIC_SWAP: - with guard.transaction(): - adapter.swap( - connection=connection, - left=destination_relation, - right=origin_relation, - statement_recorder=statement_recorder, - ) - _ = guard.rebind() - adapter.drop( - connection=connection, - destination=origin_relation, - if_exists=True, - statement_recorder=statement_recorder, - ) - elif strategy == PromotionStrategy.ATOMIC_REPLACE: - with guard.transaction(): - adapter.replace_table_from_relation( - connection=connection, - destination=destination_relation, - origin=origin_relation, - statement_recorder=statement_recorder, - ) - _ = guard.rebind() - adapter.drop( + _promote( + adapter=adapter, + connection=connection, + strategy=strategy, + relations=relations, + guard=guard, + statement_recorder=statement_recorder, + ) + except Exception as error: + if isinstance(error, ExecutorInputError): + lifecycle.failed(error=error, attributes=terminal_attributes) + raise + live_existing: bool = _live_existence( + adapter=adapter, + connection=connection, + destination=(destination_database, destination_schema, destination_name), + destination_relation=destination_relation, + ) + if live_existing != existing: + strategy = _promotion_strategy(adapter=adapter, existing=live_existing) + terminal_attributes = ( + OperationAttributes(strategy=strategy) + if strategy in OPERATION_STRATEGIES + else None + ) + try: + _promote( + adapter=adapter, connection=connection, - destination=origin_relation, - if_exists=True, + strategy=strategy, + relations=relations, + guard=guard, statement_recorder=statement_recorder, ) - elif strategy == RENAME_OPERATION_STRATEGY: - adapter.rename( - connection=connection, - origin=origin_relation, - destination=destination_relation, - statement_recorder=statement_recorder, - ) - else: - raise ExecutorInputError(f"Unsupported promotion strategy: {strategy}") - except Exception as error: - if ( - isinstance(error, ExecutorInputError) - or not existing - or not _dropped_replaced_view( - adapter=adapter, - connection=connection, - destination=(destination_database, destination_schema, destination_name), - destination_relation=destination_relation, - statement_recorder=statement_recorder, - ) + except Exception as retry_error: + lifecycle.failed(error=retry_error, attributes=terminal_attributes) + raise + lifecycle.completed(metadata={"changed_count": 1}, attributes=terminal_attributes) + return + if not existing or not _dropped_replaced_view( + adapter=adapter, + connection=connection, + destination=(destination_database, destination_schema, destination_name), + destination_relation=destination_relation, + statement_recorder=statement_recorder, ): lifecycle.failed(error=error, attributes=terminal_attributes) raise @@ -127,6 +122,79 @@ def promote_relation_to_destination( lifecycle.completed(metadata={"changed_count": 1}, attributes=terminal_attributes) +def _promotion_strategy(*, adapter: BaseAdapter, existing: bool) -> str: + """Rename into an absent destination; otherwise the adapter's replacement strategy.""" + + return RENAME_OPERATION_STRATEGY if not existing else str(adapter.default_promotion_strategy()) + + +def _promote( + *, + adapter: BaseAdapter, + connection: Any, + strategy: str, + relations: tuple[str, str], + guard: BoundViewGuard, + statement_recorder: StatementRecorder, +) -> None: + """Move the staged ``origin`` into ``destination`` with one promotion strategy.""" + + origin_relation, destination_relation = relations + if strategy == PromotionStrategy.ATOMIC_SWAP: + with guard.transaction(): + adapter.swap( + connection=connection, + left=destination_relation, + right=origin_relation, + statement_recorder=statement_recorder, + ) + _ = guard.rebind() + adapter.drop_table( + connection=connection, + destination=origin_relation, + statement_recorder=statement_recorder, + ) + elif strategy == PromotionStrategy.ATOMIC_REPLACE: + with guard.transaction(): + adapter.replace_table_from_relation( + connection=connection, + destination=destination_relation, + origin=origin_relation, + statement_recorder=statement_recorder, + ) + _ = guard.rebind() + adapter.drop_table( + connection=connection, + destination=origin_relation, + statement_recorder=statement_recorder, + ) + elif strategy == RENAME_OPERATION_STRATEGY: + adapter.rename( + connection=connection, + origin=origin_relation, + destination=destination_relation, + statement_recorder=statement_recorder, + ) + else: + raise ExecutorInputError(f"Unsupported promotion strategy: {strategy}") + + +def _live_existence( + *, + adapter: BaseAdapter, + connection: Any, + destination: tuple[str | None, str | None, str], + destination_relation: str, +) -> bool: + """Read the destination's existence again, past what this build remembered of it.""" + + invalidate_cached_relations(adapter=adapter, qualified_names=(destination_relation,)) + database, schema, name = destination + return cached_relation_exists( + adapter=adapter, connection=connection, database=database, schema=schema, name=name + ) + + def _dropped_replaced_view( *, adapter: BaseAdapter, diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py index 460df0d0ad..f57317ade7 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py @@ -1,7 +1,12 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata.helpers import ( + SimulatedSnowflakeWarehouse, +) + @dataclass(frozen=True) class BuildMetadataBudgetTestCase: @@ -41,3 +46,37 @@ class SourceBindingCaseTestCase: select: str expected_error: str | None = None expected_models: tuple[str, ...] = () + + +@dataclass(frozen=True) +class MaterializationSwitchTestCase: + """A model's materialization across builds and the relation type each build leaves.""" + + description: str + materializations: tuple[str, ...] + expected_table_types: tuple[str, ...] + expected_owned_view_drops: tuple[str, ...] = () + + +@dataclass(frozen=True) +class RetentionLookupTestCase: + """Many managed-retention tables in one schema and the lookups their build may issue.""" + + description: str + table_count: int + retention_days: int + expected_first_alters: int + expected_max_table_lookups: int + expected_max_listings: int + expected_information_schema: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ExternalChangeTestCase: + """A change another session makes to a model's relation while a build promotes it.""" + + description: str + builds_before: int + change: Callable[[SimulatedSnowflakeWarehouse], None] + before_statement: str + expected_table_types: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index d4484ccc70..49fa0a5cdf 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -47,7 +47,7 @@ re.IGNORECASE | re.DOTALL, ) _DROP_PATTERN: re.Pattern[str] = re.compile( - r"^\s*DROP\s+(?:TABLE|VIEW)\s+(?:IF\s+EXISTS\s+)?" + _NAME.format(group="name") + r"\s*$", + r"^\s*DROP\s+(?PTABLE|VIEW)\s+(?:IF\s+EXISTS\s+)?" + _NAME.format(group="name") + r"\s*$", re.IGNORECASE, ) _RENAME_PATTERN: re.Pattern[str] = re.compile( @@ -72,6 +72,12 @@ + r'\s+ADD\s+COLUMN\s+"?(?P[A-Za-z0-9_]+)"?\s+(?P.+?)\s*$', re.IGNORECASE, ) +_SET_RETENTION_PATTERN: re.Pattern[str] = re.compile( + r"^\s*ALTER\s+TABLE\s+" + + _NAME.format(group="name") + + r"\s+SET\s+DATA_RETENTION_TIME_IN_DAYS\s*=\s*(?P\d+)\s*$", + re.IGNORECASE, +) _CREATE_SCHEMA_PATTERN: re.Pattern[str] = re.compile( r"^\s*CREATE\s+SCHEMA\s+IF\s+NOT\s+EXISTS\s+" + _NAME.format(group="name") + r"\s*$", re.IGNORECASE, @@ -100,6 +106,10 @@ class SimulatedSnowflakeWarehouse(RecordingSnowflakeWarehouse): model_columns: dict[str, tuple[FakeColumn, ...]] = field(default_factory=dict) ddl_events: list[tuple[int, frozenset[_Identity]]] = field(default_factory=list) max_retention_days: int = 90 + # Changes another session makes just before a matching statement runs. + external_changes: list[ + tuple[re.Pattern[str], Callable[[SimulatedSnowflakeWarehouse], None]] + ] = field(default_factory=list) def reset(self) -> None: """Forget recorded statements and simulated DDL events, keeping the catalog.""" @@ -115,6 +125,12 @@ def set_model_columns(self, *, model: str, columns: tuple[FakeColumn, ...]) -> N self.model_columns[model.upper()] = columns def _answer(self, *, sql: str, params: tuple[object, ...]) -> tuple[list[tuple[Any, ...]], Any]: + external: tuple[re.Pattern[str], Callable[[SimulatedSnowflakeWarehouse], None]] + for external in tuple( + filter(lambda item: item[0].search(sql) is not None, self.external_changes) + ): + self.external_changes.remove(external) + external[1](self) retention: re.Match[str] for retention in filter(None, (_RETENTION_PATTERN.search(sql),)): _RETENTION_CHECKS[int(retention.group("days")) > self.max_retention_days]( @@ -127,6 +143,7 @@ def _answer(self, *, sql: str, params: tuple[object, ...]) -> tuple[list[tuple[A (_RENAME_PATTERN, self._rename), (_SWAP_PATTERN, self._swap), (_ADD_COLUMN_PATTERN, self._add_column), + (_SET_RETENTION_PATTERN, self._set_retention), (_CREATE_SCHEMA_PATTERN, self._create_schema), ) pattern: re.Pattern[str] @@ -155,6 +172,7 @@ def _answer(self, *, sql: str, params: tuple[object, ...]) -> tuple[list[tuple[A def _create_as(self, match: re.Match[str]) -> frozenset[_Identity]: identity: _Identity = self._parts(match.group("name")) + self._require_kind(identity=identity, kind=match.group("kind"), error=_already_exists) copied: tuple[re.Match[str], ...] = tuple( filter(None, (_COPY_SOURCE_PATTERN.match(match.group("query")),)) ) @@ -201,6 +219,7 @@ def _create_defined(self, match: re.Match[str]) -> frozenset[_Identity]: def _drop(self, match: re.Match[str]) -> frozenset[_Identity]: identity: _Identity = self._parts(match.group("name")) + self._require_kind(identity=identity, kind=match.group("kind"), error=_wrong_type) self.replace_relations( relations=tuple(filter(lambda fake: _identity(fake) != identity, self.relations)) ) @@ -209,6 +228,9 @@ def _drop(self, match: re.Match[str]) -> frozenset[_Identity]: def _rename(self, match: re.Match[str]) -> frozenset[_Identity]: origin: _Identity = self._parts(match.group("name")) destination: _Identity = self._parts(match.group("other")) + fake: FakeRelation + for fake in filter(lambda candidate: _identity(candidate) == destination, self.relations): + _already_exists(destination[2], ("TABLE", "VIEW")[fake.table_type == "VIEW"]) self.replace_relations( relations=tuple( (fake, _moved(fake=fake, identity=destination))[_identity(fake) == origin] @@ -220,6 +242,15 @@ def _rename(self, match: re.Match[str]) -> frozenset[_Identity]: def _swap(self, match: re.Match[str]) -> frozenset[_Identity]: left: _Identity = self._parts(match.group("name")) right: _Identity = self._parts(match.group("other")) + identity: _Identity + for identity in (left, right): + missing: _Identity + for missing in filter( + lambda wanted: all(_identity(fake) != wanted for fake in self.relations), + (identity,), + ): + _does_not_exist(missing[2]) + self._require_kind(identity=identity, kind="TABLE", error=_wrong_type) targets: dict[_Identity, _Identity] = {left: right, right: left} self.replace_relations( relations=tuple( @@ -240,6 +271,34 @@ def _add_column(self, match: re.Match[str]) -> frozenset[_Identity]: ) return frozenset({identity}) + def _set_retention(self, match: re.Match[str]) -> frozenset[_Identity]: + identity: _Identity = self._parts(match.group("name")) + days: int = int(match.group("days")) + self.replace_relations( + relations=tuple( + (fake, replace(fake, retention_time=days))[_identity(fake) == identity] + for fake in self.relations + ) + ) + return frozenset({identity}) + + def _require_kind( + self, *, identity: _Identity, kind: str, error: Callable[[str, str], None] + ) -> None: + """Fail like Snowflake when the object at ``identity`` is not of ``kind``.""" + + found: str + for found in filter( + lambda listed: listed != kind.upper(), + ( + ("TABLE", "VIEW")[fake.table_type == "VIEW"] + for fake in filter( + lambda candidate: _identity(candidate) == identity, self.relations + ) + ), + ): + error(identity[2], found) + def _create_schema(self, match: re.Match[str]) -> frozenset[_Identity]: parts: tuple[str, ...] = tuple( part.strip('"').upper() for part in match.group("name").split(".") @@ -267,6 +326,29 @@ def _parts(self, name: str) -> _Identity: return full[0], full[1], full[2] +def _does_not_exist(name: str) -> None: + raise FakeSnowflakeProgrammingError( + f"002003 (02000): SQL compilation error:\nTable '{name}' does not exist or not authorized.", + errno=2003, + ) + + +def _already_exists(name: str, found: str) -> None: + raise FakeSnowflakeProgrammingError( + f"002002 (42710): SQL compilation error:\nObject '{name}' already exists as {found}", + errno=2002, + ) + + +def _wrong_type(name: str, found: str) -> None: + wanted: str = ("VIEW", "TABLE")[found == "VIEW"] + raise FakeSnowflakeProgrammingError( + f"002203 (02000): SQL compilation error: Object found is of type '{found}', " + f"not specified type '{wanted}'. ({name})", + errno=2203, + ) + + def _accept_retention(days: str) -> None: del days @@ -617,3 +699,67 @@ def failed_models(build: OfflineBuild) -> tuple[str, ...]: lambda result: result.status != ExecutionStatus.SUCCESS, build.result.model_results ) ) + + +def relation_types(warehouse: SimulatedSnowflakeWarehouse, name: str) -> tuple[str, ...]: + """The table types of the warehouse's relations stored as ``name``.""" + + return tuple( + relation.table_type + for relation in filter(lambda candidate: candidate.name == name, warehouse.relations) + ) + + +def retention_by_prefix(warehouse: SimulatedSnowflakeWarehouse, prefix: str) -> dict[str, int]: + """The retention of each relation whose stored name starts with ``prefix``.""" + + return { + relation.name: relation.retention_time + for relation in filter( + lambda candidate: candidate.name.startswith(prefix), warehouse.relations + ) + } + + +def matching_statements(statements: tuple[str, ...], pattern: re.Pattern[str]) -> tuple[str, ...]: + """The statements ``pattern`` matches at their start.""" + + return tuple(filter(lambda statement: pattern.match(statement) is not None, statements)) + + +def drop_external_relation(warehouse: SimulatedSnowflakeWarehouse, name: str) -> None: + """Drop a relation the way another session would, outside the build's statements.""" + + warehouse.replace_relations( + relations=tuple(filter(lambda relation: relation.name != name, warehouse.relations)) + ) + + +def create_external_table(warehouse: SimulatedSnowflakeWarehouse, name: str) -> None: + """Create a table in the marts schema the way another session would.""" + + warehouse.replace_relations( + relations=( + *warehouse.relations, + FakeRelation(database="ANALYTICS", schema="MARTS", name=name), + ) + ) + + +def write_model_copies(*, project_dir: Path, count: int, retention_days: int) -> None: + """Add ``count`` permanent table copies of the orders table with managed retention.""" + + project: Path = project_dir / "sqlbuild_project.toml" + _ = project.write_text( + project.read_text(encoding="utf-8") + + '\n[materialization_defaults.table]\ntable_type = "permanent"\n' + + f'time_travel_retention = "{retention_days}d"\n', + encoding="utf-8", + ) + index: int + for index in range(count): + _ = (project_dir / "models" / "marts" / f"order_copy_{index}.sql").write_text( + f"MODEL (description 'Copy {index} of the orders.', materialized table);\n\n" + 'SELECT order_id, ordered_at, amount FROM __ref("orders_table")\n', + encoding="utf-8", + ) diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_build_metadata_budget.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_build_metadata_budget.py index d71f563a96..138590f3a3 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_build_metadata_budget.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_build_metadata_budget.py @@ -29,16 +29,13 @@ from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import FakeColumn _FIRST_BUILD_COLUMN_READS: dict[str, int] = { - "ANALYTICS.MARTS.ORDERS_TABLE__STAGING": 1, - "ANALYTICS.MARTS.ORDERS_MERGE__STAGING": 1, - "ANALYTICS.MARTS.ORDERS_DAILY__DELTA": 3, + "ANALYTICS.MARTS.ORDERS_DAILY__DELTA": 1, "ANALYTICS.MARTS.__SQB_REBUILD__ORDERS_DAILY": 1, } _SECOND_BUILD_COLUMN_READS: dict[str, int] = { - "ANALYTICS.MARTS.ORDERS_TABLE__STAGING": 1, "ANALYTICS.MARTS.ORDERS_MERGE__DELTA": 1, "ANALYTICS.MARTS.ORDERS_MERGE": 1, - "ANALYTICS.MARTS.ORDERS_DAILY__DELTA": 2, + "ANALYTICS.MARTS.ORDERS_DAILY__DELTA": 1, "ANALYTICS.MARTS.ORDERS_DAILY": 1, } diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_statement_overhead.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_statement_overhead.py new file mode 100644 index 0000000000..6cfed89015 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_statement_overhead.py @@ -0,0 +1,188 @@ +"""Housekeeping statements of an offline Snowflake build, and the cases each one guards. + +The simulated warehouse fails like Snowflake when a statement names an object of the other kind, +so a build that drops or replaces only the object kind it created is checked against the same +errors a real account would raise. +""" + +from __future__ import annotations + +import re +from functools import partial +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata._test_types import ( + ExternalChangeTestCase, + MaterializationSwitchTestCase, + RetentionLookupTestCase, +) +from tests.integration.src.sqlbuild.executor.build.snowflake_metadata.helpers import ( + OfflineBuild, + SimulatedSnowflakeWarehouse, + build_offline_snowflake_project, + create_external_table, + drop_external_relation, + failed_models, + information_schema_statements, + matching_statements, + orders_warehouse, + relation_types, + retention_by_prefix, + write_model_copies, + write_orders_project, +) + +_RETENTION_ALTER: re.Pattern[str] = re.compile( + r"^ALTER TABLE \S+ SET DATA_RETENTION_TIME_IN_DAYS = (?P\d+)$" +) +_TABLE_LOOKUP: re.Pattern[str] = re.compile(r"^SHOW TABLES LIKE ") +_SCHEMA_LISTING: re.Pattern[str] = re.compile(r"^SHOW (?:TABLES|VIEWS) IN SCHEMA ") +_OWNED_VIEW_DROP: re.Pattern[str] = re.compile(r"^DROP VIEW IF EXISTS \S+__(?:staging|delta)$") + + +@pytest.mark.parametrize( + "test_case", + [ + MaterializationSwitchTestCase( + description="a view becomes a table and then a view again", + materializations=("view", "table", "view"), + expected_table_types=("VIEW", "BASE TABLE", "VIEW"), + ), + MaterializationSwitchTestCase( + description="a table becomes a view and then a table again", + materializations=("table", "view", "table"), + expected_table_types=("BASE TABLE", "VIEW", "BASE TABLE"), + ), + ], + ids=lambda case: case.description, +) +def test_given_materialization_switch_when_building_then_each_kind_replaces_the_other( + test_case: MaterializationSwitchTestCase, tmp_path: Path +) -> None: + write_orders_project(project_dir=tmp_path) + warehouse: SimulatedSnowflakeWarehouse = orders_warehouse() + model: Path = tmp_path / "models" / "marts" / "orders_view.sql" + source: str = model.read_text(encoding="utf-8") + table_types: list[tuple[str, ...]] = [] + builds: list[OfflineBuild] = [] + + materialization: str + for materialization in test_case.materializations: + _ = model.write_text( + re.sub(r"materialized \w+", f"materialized {materialization}", source), + encoding="utf-8", + ) + builds.append( + build_offline_snowflake_project( + project_dir=tmp_path, warehouse=warehouse, max_concurrency=1 + ) + ) + table_types.append(relation_types(warehouse, "ORDERS_VIEW")) + + assert [failed_models(build) for build in builds] == [()] * len(builds) + assert tuple(table_types) == tuple((kind,) for kind in test_case.expected_table_types) + assert tuple(matching_statements(build.statements, _OWNED_VIEW_DROP) for build in builds) == ( + test_case.expected_owned_view_drops, + ) * len(builds) + + +@pytest.mark.parametrize( + "test_case", + [ + RetentionLookupTestCase( + description="six tables in one schema keep their retention without INFORMATION_SCHEMA", + table_count=6, + retention_days=3, + expected_first_alters=6, + expected_max_table_lookups=7, + expected_max_listings=4, + ), + ], + ids=lambda case: case.description, +) +def test_given_many_tables_in_one_schema_when_building_then_retention_is_read_with_show( + test_case: RetentionLookupTestCase, tmp_path: Path +) -> None: + write_orders_project(project_dir=tmp_path) + write_model_copies( + project_dir=tmp_path, count=test_case.table_count, retention_days=test_case.retention_days + ) + warehouse: SimulatedSnowflakeWarehouse = orders_warehouse() + + first: OfflineBuild = build_offline_snowflake_project( + project_dir=tmp_path, warehouse=warehouse, max_concurrency=4 + ) + second: OfflineBuild = build_offline_snowflake_project( + project_dir=tmp_path, warehouse=warehouse, max_concurrency=4 + ) + + copies: dict[str, int] = retention_by_prefix(warehouse, "ORDER_COPY_") + assert (failed_models(first), failed_models(second)) == ((), ()) + assert (information_schema_statements(first), information_schema_statements(second)) == ( + test_case.expected_information_schema, + test_case.expected_information_schema, + ) + assert copies == dict.fromkeys(copies, test_case.retention_days) + assert len(copies) == test_case.table_count + assert len(matching_statements(first.statements, _RETENTION_ALTER)) >= ( + test_case.expected_first_alters + ) + assert max( + len(matching_statements(build.statements, _TABLE_LOOKUP)) for build in (first, second) + ) <= (test_case.expected_max_table_lookups) + assert max( + len(matching_statements(build.statements, _SCHEMA_LISTING)) for build in (first, second) + ) <= (test_case.expected_max_listings) + + +@pytest.mark.parametrize( + "test_case", + [ + ExternalChangeTestCase( + description="another session drops the table the build is about to swap", + builds_before=1, + change=partial(drop_external_relation, name="ORDERS_TABLE"), + before_statement=r"^ALTER TABLE \S+ SWAP WITH ", + expected_table_types=("BASE TABLE",), + ), + ExternalChangeTestCase( + description="another session creates the table the build is about to rename into", + builds_before=0, + change=partial(create_external_table, name="ORDERS_TABLE"), + before_statement=r"^ALTER TABLE \S+__staging RENAME TO ", + expected_table_types=("BASE TABLE",), + ), + ], + ids=lambda case: case.description, +) +def test_given_external_change_before_promotion_when_building_then_promotion_reads_it_again( + test_case: ExternalChangeTestCase, tmp_path: Path +) -> None: + write_orders_project(project_dir=tmp_path) + warehouse: SimulatedSnowflakeWarehouse = orders_warehouse() + earlier: tuple[OfflineBuild, ...] = tuple( + build_offline_snowflake_project( + project_dir=tmp_path, warehouse=warehouse, max_concurrency=1 + ) + for _ in range(test_case.builds_before) + ) + warehouse.external_changes.append((re.compile(test_case.before_statement), test_case.change)) + model: Path = tmp_path / "models" / "marts" / "orders_table.sql" + _ = model.write_text( + model.read_text(encoding="utf-8").replace("SELECT ", "SELECT 1 AS revision, ", 1), + encoding="utf-8", + ) + + build: OfflineBuild = build_offline_snowflake_project( + project_dir=tmp_path, warehouse=warehouse, max_concurrency=1 + ) + + assert tuple(failed_models(done) for done in (*earlier, build)) == ((),) * (len(earlier) + 1) + assert warehouse.external_changes == [] + assert relation_types(warehouse, "ORDERS_TABLE") == test_case.expected_table_types + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 35713fd376..8d8e943b24 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -7,18 +7,28 @@ from sqlbuild.adapter.contract.types import TableFreshnessStatus from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.types import CostStatus +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import FakeRelation @dataclass(frozen=True) class SnowflakeRetentionTestCase: description: str desired_days: int - observed_row: tuple[object, ...] + relation: FakeRelation expected_days: int expected_kind: str expected_sql: str +@dataclass(frozen=True) +class SnowflakeShowRetentionTestCase: + description: str + relations: tuple[FakeRelation, ...] + steps: tuple[tuple[str, str, str], ...] + in_build: bool + expected_steps: tuple[tuple[int | None, tuple[str, ...]], ...] + + @dataclass(frozen=True) class SnowflakeTableTypeDdlTestCase: description: str @@ -207,8 +217,8 @@ class SnowflakeQualifiedColumnInspectionTestCase: class SnowflakeBatchedRetentionTestCase: description: str requests: tuple[tuple[str, str], ...] - schema_rows: tuple[list[tuple[object, ...]], ...] - expected_query_schemas: tuple[str, ...] + relations: tuple[FakeRelation, ...] + expected_queries: tuple[str, ...] expected_days: dict[str, int] @@ -216,7 +226,7 @@ class SnowflakeBatchedRetentionTestCase: class SnowflakeBatchedRetentionErrorTestCase: description: str requests: tuple[tuple[str, str], ...] - rows: list[tuple[object, ...]] + relations: tuple[FakeRelation, ...] expected_error: str @@ -382,3 +392,21 @@ class SnowflakeRelationUsageChainsUnfollowedTestCase: message: str expected_usages: tuple[tuple[str, tuple[str, ...], int], ...] expected_notices: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakePrefixListingTestCase: + description: str + relations: tuple[FakeRelation, ...] + schema: str + prefix: str + expected_relations: tuple[tuple[str, str], ...] + expected_statements: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeRunListingTestCase: + description: str + relations: tuple[FakeRelation, ...] + lookups: tuple[str, ...] + expected_listings: tuple[int, int] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index f3424538b0..4882cc8bc5 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -4,6 +4,7 @@ import sys from collections.abc import Callable +from contextlib import nullcontext from dataclasses import dataclass from io import StringIO from itertools import cycle, islice @@ -11,7 +12,16 @@ from typing import Any from sqlbuild._native import NativeProgressProjector -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationUsageReport +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + RelationInfo, + RelationUsageReport, + RetentionRequest, +) +from sqlbuild.adapter.contract.types import RetentionScope +from sqlbuild.adapter.relations.main.relation_metadata_cache_scope import ( + relation_metadata_cache_scope, +) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection from sqlbuild.observability import ( @@ -24,6 +34,11 @@ invocation_scope, run_scope, ) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + FakeRelation, + OfflineSnowflakeAdapter, + RecordingSnowflakeWarehouse, +) class FakeSnowflakeDescribeCursor: @@ -525,3 +540,137 @@ def read_snowflake_usage( relations=relations, window_seconds=window_seconds, ) + + +def retention_relation( + schema: str, + name: str, + *, + retention: int, + transient: bool = False, + table_type: str = "BASE TABLE", +) -> FakeRelation: + """One synthetic relation in the ANALYTICS database with its retention.""" + + return FakeRelation( + database="ANALYTICS", + schema=schema, + name=name, + table_type=table_type, + is_transient=transient, + retention_time=retention, + ) + + +def retention_request(*, schema: str, name: str, desired_days: int = 7) -> RetentionRequest: + """A relation retention request in the analytics database, identified by its name.""" + + return RetentionRequest( + request_id=name, + scope=RetentionScope.RELATION, + database="analytics", + schema=schema, + name=name, + desired_days=desired_days, + ) + + +def show_statement(sql: str) -> str: + """A SHOW statement without its schema scope after LIKE, or its row limit after IN SCHEMA.""" + + return (sql.split(" LIMIT ")[0], sql.split(" IN SCHEMA ")[0])[" LIKE " in sql] + + +def _lookup_exists( + *, adapter: OfflineSnowflakeAdapter, connection: Any, schema: str, name: str +) -> None: + _ = adapter.relation_exists( + connection=connection, database="analytics", schema=schema, name=name + ) + + +def _lookup_retention( + *, adapter: OfflineSnowflakeAdapter, connection: Any, schema: str, name: str +) -> int | None: + return adapter.inspect_retention( + connection=connection, request=retention_request(schema=schema, name=name) + ).effective_days + + +def _replace_table( + *, adapter: OfflineSnowflakeAdapter, connection: Any, schema: str, name: str +) -> None: + _ = adapter.execute( + connection=connection, sql=f"CREATE OR REPLACE TABLE analytics.{schema}.{name} (id INT)" + ) + + +_LOOKUP_STEPS: dict[str, Callable[..., int | None]] = { + "exists": _lookup_exists, + "retention": _lookup_retention, + "replace": _replace_table, +} + + +def run_metadata_steps( + *, + relations: tuple[FakeRelation, ...], + steps: tuple[tuple[str, str, str], ...], + in_build: bool, +) -> tuple[tuple[int | None, tuple[str, ...]], ...]: + """Run each (kind, schema, name) step; return its retention days and SHOW statements.""" + + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse(relations=relations) + adapter: OfflineSnowflakeAdapter = OfflineSnowflakeAdapter(warehouse=warehouse) + connection: Any = adapter.connect({}) + results: list[tuple[int | None, tuple[str, ...]]] = [] + with (nullcontext(), relation_metadata_cache_scope(adapter=adapter))[in_build]: + kind: str + schema: str + name: str + for kind, schema, name in steps: + warehouse.reset() + days: int | None = _LOOKUP_STEPS[kind]( + adapter=adapter, connection=connection, schema=schema, name=name + ) + results.append( + ( + days, + tuple(show_statement(query.sql) for query in warehouse.queries), + ) + ) + return tuple(results) + + +def relation_existence( + *, relations: tuple[FakeRelation, ...], lookups: tuple[str, ...], in_build: bool +) -> tuple[tuple[bool, ...], int]: + """Each name's existence in the MART schema, and how many whole-schema listings were read.""" + + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse(relations=relations) + adapter: OfflineSnowflakeAdapter = OfflineSnowflakeAdapter(warehouse=warehouse) + connection: Any = adapter.connect({}) + with (nullcontext(), relation_metadata_cache_scope(adapter=adapter))[in_build]: + answers: tuple[bool, ...] = tuple( + adapter.relation_exists( + connection=connection, database="analytics", schema="mart", name=name + ) + for name in lookups + ) + listings: int = len( + tuple( + filter( + lambda sql: " IN SCHEMA " in sql and " LIKE " not in sql, warehouse.attempted_sql + ) + ) + ) + return answers, listings + + +def prefix_listing_statements(warehouse: RecordingSnowflakeWarehouse) -> tuple[str, ...]: + """The SHOW TERSE OBJECTS statements the warehouse received, without their schema scope.""" + + return tuple( + sql.split(" IN SCHEMA ")[0] + for sql in filter(lambda sql: sql.startswith("SHOW TERSE OBJECTS"), warehouse.attempted_sql) + ) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py index 7d8cb469e7..e049c88eac 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py @@ -36,7 +36,7 @@ r"^\s*SHOW\s+COLUMNS\s+IN\s+(?:TABLE|VIEW)\s+(?P\S+)\s*$", re.IGNORECASE ) _SHOW_SCHEMA_RELATIONS_PATTERN: re.Pattern[str] = re.compile( - r"^SHOW (?PTERSE )?(?PTABLES|VIEWS)(?: LIKE '(?P(?:[^']|'')*)')? " + r"^SHOW (?PTERSE )?(?PTABLES|VIEWS|OBJECTS)(?: LIKE '(?P(?:[^']|'')*)')? " r"IN SCHEMA (?P\S+)" r"(?: LIMIT (?P\d+)(?: FROM '(?P(?:[^']|'')*)')?)?$" ) @@ -134,6 +134,7 @@ "autoincrement", ) _VIEW_TABLE_TYPES: frozenset[str] = frozenset({"VIEW", "MATERIALIZED VIEW"}) +_OBJECT_KINDS: dict[str, str] = {"BASE TABLE": "TABLE", "VIEW": "VIEW"} _TIMESTAMP_TYPES: frozenset[str] = frozenset( {"TIMESTAMP_NTZ", "TIMESTAMP_LTZ", "TIMESTAMP_TZ", "TIME"} ) @@ -432,12 +433,13 @@ def _answer_show_schema_relations( del params match: re.Match[str] = next(_SHOW_SCHEMA_RELATIONS_PATTERN.finditer(sql)) wants_views: bool = match.group("kind") == "VIEWS" + wants_objects: bool = match.group("kind") == "OBJECTS" after: str = (match.group("after") or "").replace("''", "'") like: re.Pattern[str] = _like_pattern(match.group("like") or "%") selected: list[FakeRelation] = sorted( filter( lambda fake: ( - (fake.table_type in _VIEW_TABLE_TYPES) is wants_views + (wants_objects or (fake.table_type in _VIEW_TABLE_TYPES) is wants_views) and fake.name > after and like.fullmatch(fake.name) is not None ), @@ -448,13 +450,19 @@ def _answer_show_schema_relations( limit: int = int(match.group("limit") or SHOW_RESULT_CAP) terse: bool = match.group("terse") is not None row_builder: Callable[[FakeRelation], tuple[Any, ...]] = ( - (_show_table_row, _show_view_row)[wants_views], - _show_terse_row, - )[terse] + ( + (_show_table_row, _show_view_row)[wants_views], + _show_terse_row, + )[terse], + _show_object_row, + )[wants_objects] fields: tuple[str, ...] = ( - (_SHOW_TABLE_FIELDS, _SHOW_VIEW_FIELDS)[wants_views], + ( + (_SHOW_TABLE_FIELDS, _SHOW_VIEW_FIELDS)[wants_views], + _SHOW_TERSE_FIELDS, + )[terse], _SHOW_TERSE_FIELDS, - )[terse] + )[wants_objects] return [row_builder(fake) for fake in selected[:limit]], tuple((f,) for f in fields) def _answer_show_schema_columns( @@ -603,6 +611,18 @@ def _show_terse_row(relation: FakeRelation) -> tuple[Any, ...]: ) +def _show_object_row(relation: FakeRelation) -> tuple[Any, ...]: + """A SHOW TERSE OBJECTS row, whose kind names the object type as Snowflake spells it.""" + + return ( + _FIXED_CREATED_AT, + relation.name, + _OBJECT_KINDS.get(relation.table_type, relation.table_type.replace(" ", "_")), + relation.database, + relation.schema, + ) + + def _show_view_row(relation: FakeRelation) -> tuple[Any, ...]: return ( _FIXED_CREATED_AT, diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_retention.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_retention.py index 1bee1c90e3..3772cbdf95 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_retention.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_retention.py @@ -1,5 +1,3 @@ -from typing import Any, cast - import pytest from sqlbuild.adapter.contract.exceptions import AdapterUserError @@ -8,17 +6,27 @@ RetentionRequest, RetentionState, ) -from sqlbuild.adapter.contract.types import RetentionChangePhase, RetentionScope -from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapter.contract.types import RetentionChangePhase from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeBatchedRetentionErrorTestCase, SnowflakeBatchedRetentionTestCase, SnowflakeRetentionTestCase, + SnowflakeShowRetentionTestCase, ) from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( - FakeSnowflakeMetadataConnection, - FakeSnowflakeMetadataCursor, - FakeSnowflakeMetadataSequenceConnection, + retention_relation, + retention_request, + run_metadata_steps, + show_statement, +) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + OfflineSnowflakeAdapter, + RecordingSnowflakeWarehouse, +) + +_MART_LISTING: tuple[str, ...] = ( + 'SHOW TABLES IN SCHEMA "ANALYTICS"."MART"', + 'SHOW VIEWS IN SCHEMA "ANALYTICS"."MART"', ) @@ -28,10 +36,10 @@ SnowflakeRetentionTestCase( description="transient table retention is inspected and altered without conversion", desired_days=7, - observed_row=(3, "YES"), + relation=retention_relation("MART", "RESULTS", retention=3, transient=True), expected_days=3, expected_kind="TRANSIENT", - expected_sql=("ALTER TABLE analytics.mart.results SET DATA_RETENTION_TIME_IN_DAYS = 7"), + expected_sql="ALTER TABLE analytics.mart.results SET DATA_RETENTION_TIME_IN_DAYS = 7", ) ], ids=lambda case: case.description, @@ -39,20 +47,15 @@ def test_given_snowflake_relation_when_managing_retention_then_observes_and_renders_alter( test_case: SnowflakeRetentionTestCase, ) -> None: - adapter: SnowflakeAdapter = SnowflakeAdapter() - cursor: FakeSnowflakeMetadataCursor = FakeSnowflakeMetadataCursor(row=test_case.observed_row) - connection: FakeSnowflakeMetadataConnection = FakeSnowflakeMetadataConnection(cursor) - request: RetentionRequest = RetentionRequest( - request_id="model.results", - scope=RetentionScope.RELATION, - database="analytics", - schema="mart", - name="results", - desired_days=test_case.desired_days, + adapter: OfflineSnowflakeAdapter = OfflineSnowflakeAdapter( + warehouse=RecordingSnowflakeWarehouse(relations=(test_case.relation,)) + ) + request: RetentionRequest = retention_request( + schema="mart", name="results", desired_days=test_case.desired_days ) state: RetentionState = adapter.inspect_retention( - connection=cast(Any, connection), request=request + connection=adapter.connect({}), request=request ) changes: tuple[RenderedRetentionChange, ...] = adapter.render_retention_changes(request=request) @@ -61,21 +64,96 @@ def test_given_snowflake_relation_when_managing_retention_then_observes_and_rend assert state.is_transient is True assert changes[0].phase == RetentionChangePhase.ALTER assert changes[0].statements == (test_case.expected_sql,) - assert cursor.executed_sql is not None - assert "retention_time, is_transient" in cursor.executed_sql + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeShowRetentionTestCase( + description="one table outside a build is read alone, without a warehouse", + relations=(retention_relation("MART", "RESULTS", retention=3, transient=True),), + steps=(("retention", "mart", "results"),), + in_build=False, + expected_steps=((3, ("SHOW TABLES LIKE 'RESULTS'",)),), + ), + SnowflakeShowRetentionTestCase( + description="a lowercase name finds its stored table, not a LIKE-wildcard sibling", + relations=( + retention_relation("MART", "ORDERSXLINES", retention=5), + retention_relation("MART", "ORDERS_LINES", retention=2), + ), + steps=(("retention", "mart", "orders_lines"),), + in_build=False, + expected_steps=((2, ("SHOW TABLES LIKE 'ORDERS_LINES'",)),), + ), + SnowflakeShowRetentionTestCase( + description="a build answers retention from the listing its existence checks made", + relations=( + retention_relation("MART", "ORDERS", retention=7), + retention_relation("MART", "CUSTOMERS", retention=1), + retention_relation("MART", "PRODUCTS", retention=0, transient=True), + ), + steps=( + ("exists", "mart", "orders"), + ("exists", "mart", "customers"), + ("retention", "mart", "products"), + ("retention", "mart", "orders"), + ), + in_build=True, + expected_steps=( + (None, ("SHOW TERSE TABLES LIKE 'ORDERS'",)), + (None, _MART_LISTING), + (0, ()), + (7, ()), + ), + ), + SnowflakeShowRetentionTestCase( + description="a table the build replaced after the listing is read again alone", + relations=( + retention_relation("MART", "ORDERS", retention=7), + retention_relation("MART", "CUSTOMERS", retention=1), + ), + steps=( + ("exists", "mart", "orders"), + ("exists", "mart", "customers"), + ("replace", "mart", "orders"), + ("retention", "mart", "customers"), + ("retention", "mart", "orders"), + ), + in_build=True, + expected_steps=( + (None, ("SHOW TERSE TABLES LIKE 'ORDERS'",)), + (None, _MART_LISTING), + (None, ("CREATE OR REPLACE TABLE analytics.mart.orders (id INT)",)), + (1, ()), + (7, ("SHOW TABLES LIKE 'ORDERS'",)), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_relations_when_inspecting_retention_then_show_tables_answers_each_lookup( + test_case: SnowflakeShowRetentionTestCase, +) -> None: + steps: tuple[tuple[int | None, tuple[str, ...]], ...] = run_metadata_steps( + relations=test_case.relations, steps=test_case.steps, in_build=test_case.in_build + ) + + assert steps == test_case.expected_steps @pytest.mark.parametrize( "test_case", [ SnowflakeBatchedRetentionTestCase( - description="three tables in two schemas take one metadata query per schema", + description="three tables in two schemas list a schema once or read one table alone", requests=(("mart", "orders"), ("mart", "customers"), ("staging", "stg_orders")), - schema_rows=( - [("ORDERS", 7, "NO"), ("CUSTOMERS", 1, "NO")], - [("STG_ORDERS", 0, "YES")], + relations=( + retention_relation("MART", "ORDERS", retention=7), + retention_relation("MART", "CUSTOMERS", retention=1), + retention_relation("STAGING", "STG_ORDERS", retention=0, transient=True), ), - expected_query_schemas=("MART", "STAGING"), + expected_queries=(*_MART_LISTING, "SHOW TABLES LIKE 'STG_ORDERS'"), expected_days={"orders": 7, "customers": 1, "stg_orders": 0}, ), ], @@ -84,35 +162,20 @@ def test_given_snowflake_relation_when_managing_retention_then_observes_and_rend def test_given_many_relations_when_inspecting_retention_then_queries_once_per_schema( test_case: SnowflakeBatchedRetentionTestCase, ) -> None: - adapter: SnowflakeAdapter = SnowflakeAdapter() - cursors: tuple[FakeSnowflakeMetadataCursor, ...] = tuple( - FakeSnowflakeMetadataCursor(rows=rows) for rows in test_case.schema_rows - ) - connection: FakeSnowflakeMetadataSequenceConnection = FakeSnowflakeMetadataSequenceConnection( - cursors + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse( + relations=test_case.relations ) + adapter: OfflineSnowflakeAdapter = OfflineSnowflakeAdapter(warehouse=warehouse) requests: tuple[RetentionRequest, ...] = tuple( - RetentionRequest( - request_id=name, - scope=RetentionScope.RELATION, - database="analytics", - schema=schema, - name=name, - desired_days=7, - ) - for schema, name in test_case.requests + retention_request(schema=schema, name=name) for schema, name in test_case.requests ) states: dict[str, RetentionState] = adapter.inspect_retentions( - connection=cast(Any, connection), requests=requests + connection=adapter.connect({}), requests=requests ) - assert ( - tuple( - cast(tuple[object, ...], cursor.executed_params)[-2] - for cursor in connection.returned_cursors - ) - == test_case.expected_query_schemas + assert tuple(show_statement(query.sql) for query in warehouse.queries) == ( + test_case.expected_queries ) assert {request_id: state.effective_days for request_id, state in states.items()} == ( test_case.expected_days @@ -125,30 +188,27 @@ def test_given_many_relations_when_inspecting_retention_then_queries_once_per_sc SnowflakeBatchedRetentionErrorTestCase( description="a table missing from metadata fails like single inspection", requests=(("mart", "orders"), ("mart", "customers")), - rows=[("ORDERS", 7, "NO")], + relations=(retention_relation("MART", "ORDERS", retention=7),), expected_error="Snowflake retention metadata not found for customers", ), + SnowflakeBatchedRetentionErrorTestCase( + description="a view in place of the table has no retention to manage", + requests=(("mart", "orders"),), + relations=(retention_relation("MART", "ORDERS", retention=7, table_type="VIEW"),), + expected_error="Snowflake retention metadata not found for orders", + ), ], ids=lambda case: case.description, ) def test_given_missing_relation_when_batch_inspecting_retention_then_raises( test_case: SnowflakeBatchedRetentionErrorTestCase, ) -> None: - adapter: SnowflakeAdapter = SnowflakeAdapter() - connection: FakeSnowflakeMetadataConnection = FakeSnowflakeMetadataConnection( - FakeSnowflakeMetadataCursor(rows=test_case.rows) + adapter: OfflineSnowflakeAdapter = OfflineSnowflakeAdapter( + warehouse=RecordingSnowflakeWarehouse(relations=test_case.relations) ) requests: tuple[RetentionRequest, ...] = tuple( - RetentionRequest( - request_id=name, - scope=RetentionScope.RELATION, - database="analytics", - schema=schema, - name=name, - desired_days=7, - ) - for schema, name in test_case.requests + retention_request(schema=schema, name=name) for schema, name in test_case.requests ) with pytest.raises(AdapterUserError, match=test_case.expected_error): - adapter.inspect_retentions(connection=cast(Any, connection), requests=requests) + _ = adapter.inspect_retentions(connection=adapter.connect({}), requests=requests) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_run_listing.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_run_listing.py new file mode 100644 index 0000000000..541850a728 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_run_listing.py @@ -0,0 +1,45 @@ +import pytest + +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import SnowflakeRunListingTestCase +from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( + relation_existence, + retention_relation, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeRunListingTestCase( + description="unquoted, case-sensitive and wildcard-like names", + relations=( + retention_relation("MART", "ORDERS", retention=1), + retention_relation("MART", "Orders_Mixed", retention=1), + retention_relation("MART", "ORDERSXLINES", retention=1), + retention_relation("MART", "CUSTOMERS", retention=1, table_type="VIEW"), + ), + lookups=( + "orders", + "orders_mixed", + "Orders_Mixed", + "orders_lines", + "customers", + "products", + ), + expected_listings=(0, 2), + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_listing_when_checking_existence_then_answers_match_single_lookups( + test_case: SnowflakeRunListingTestCase, +) -> None: + single, single_listings = relation_existence( + relations=test_case.relations, lookups=test_case.lookups, in_build=False + ) + listed, build_listings = relation_existence( + relations=test_case.relations, lookups=test_case.lookups, in_build=True + ) + + assert listed == single + assert (single_listings, build_listings) == test_case.expected_listings From 2e1d6e953ad584259e81348e07db4ca38b679ef6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:22:04 +0000 Subject: [PATCH 03/12] perf: append fingerprint rows only when the identity changes A build sets up each fingerprint table once and skips rows identical to the latest one, except in projects with model migration history, where rename matching compares rows with rename events. --- .../_helpers/fingerprints/storage_io.rs | 26 +++-- .../_helpers/planner_records/constants.rs | 1 + .../planner_strategy/model_entries.rs | 11 +- src/sqlbuild/_native.pyi | 5 +- .../executor/build/classes/build_scheduler.py | 5 + .../executor/diff/_helpers/query_artifacts.py | 2 +- .../executor/functions/main/_execute.py | 2 +- .../python_nodes/_helpers/fingerprinting.py | 2 +- .../executor/review/_helpers/marker.py | 2 +- .../run/_helpers/reuse/fingerprinting.py | 46 ++++++-- .../run/classes/fingerprint_writes.py | 105 ++++++++++++++++++ src/sqlbuild/executor/run/types.py | 3 + .../executor/seed/_helpers/fingerprinting.py | 2 +- .../cli/commands/main/janitor/helpers.py | 6 +- .../sqlbuild/adapters/bigquery/test_client.py | 2 +- .../adapters/databricks/test_client.py | 2 +- .../main/model_migrations/_test_types.py | 9 ++ .../commands/main/model_migrations/helpers.py | 11 ++ .../test_automatic_migrations.py | 33 ++++++ .../compiler/fingerprints/main/helpers.py | 2 +- .../fingerprints/main/test_read_write.py | 8 +- .../_helpers/test_warehouse_snapshot.py | 2 +- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- .../test_state_table_retention.py | 6 +- .../main/shared/_helpers/test_sql.py | 2 +- .../compiler/fingerprints/main/test_read.py | 2 +- .../compiler/fingerprints/main/test_write.py | 4 +- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- .../executor/run/classes/_test_types.py | 7 ++ .../run/classes/test_fingerprint_writes.py | 63 +++++++++++ 30 files changed, 328 insertions(+), 47 deletions(-) create mode 100644 src/sqlbuild/executor/run/classes/fingerprint_writes.py create mode 100644 tests/unit/src/sqlbuild/executor/run/classes/test_fingerprint_writes.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 7f99a3849c..3a92672f90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -354,21 +354,30 @@ pub(crate) fn read_latest_fingerprints<'py>( fingerprint_set_from_rows(py, &schema, &qualified_name, &rows) } -/// Append one fingerprint row, retrying transient concurrent-writer conflicts. +/// Append one fingerprint row, retrying transient concurrent-writer conflicts. With `ensure_table` false the table is assumed to exist and is set up only after the insert fails; returns whether the write set up the table. #[pyfunction] #[pyo3(signature = (**request))] -fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> { +fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult { let request = Request::new("write_fingerprint", request)?; let py: Python<'py> = request.py(); let fingerprint: Bound<'py, PyAny> = request.get("fingerprint")?; + let ensure_table: bool = match present(&request, "ensure_table")? { + Some(value) => value.is_truthy()?, + None => true, + }; let parts: WriteParts<'py> = WriteParts::read(&request)?; let (location, writer) = (parts.location(), parts.writer()); + let insert: String = + fingerprint_insert_sql(py, &qualified_table_name(&location)?, &fingerprint)?; + if !ensure_table { + match execute_sql(&writer, &insert) { + Ok(_) => return Ok(false), + Err(error) if error.is_instance_of::(py) => {} + Err(error) => return Err(error), + } + } let mut sqls: Vec = setup_sqls(&location, &writer)?; - sqls.push(fingerprint_insert_sql( - py, - &qualified_table_name(&location)?, - &fingerprint, - )?); + sqls.push(insert); execute_with_retry(&writer, &sqls, &|error| { let mut details: Vec> = vec![ native_field(&fingerprint, "node_type")?, @@ -376,7 +385,8 @@ fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> ]; details.extend(error_details(py, error)?); Ok((SINGLE_RETRY_MESSAGE, details)) - }) + })?; + Ok(true) } /// The batch warning details: row count, the first row's identity, then the error. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 1f7d0a8eed..296e235af1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -644,6 +644,7 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("pre_hooks", FieldDefault::None), FieldSpec::new("post_hooks", FieldDefault::None), FieldSpec::new("previous_query_sql", FieldDefault::None), + FieldSpec::new("previous_fingerprint", FieldDefault::None), FieldSpec::new("fingerprint_metadata_json", FieldDefault::None), FieldSpec::new("previous_metadata_json", FieldDefault::None), FieldSpec::new("fingerprint_version_hash", FieldDefault::None), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 45faefc0a7..8b735dfe96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -691,11 +691,20 @@ pub(crate) fn assemble_model_plan_entry<'py>( entry.set_item( "previous_query_sql", if fingerprint.is_none() { - fingerprint + fingerprint.clone() } else { native_field(&fingerprint, "definition")? }, )?; + let skippable: bool = native_field(snapshot, "migration_state_schemas")?.len()? == 0; + entry.set_item( + "previous_fingerprint", + if skippable { + fingerprint + } else { + py.None().into_bound(py) + }, + )?; entry.set_item("schema_actions", schema_actions)?; custom_fields( &values.0, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index eaf5e10e58..5a1ecd12de 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4146,7 +4146,8 @@ def write_fingerprint( render_framework_type: Callable[[FrameworkType], str], render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, -) -> None: ... + ensure_table: bool = True, +) -> bool: ... def write_fingerprints( *, connection: Any, @@ -13811,6 +13812,7 @@ class ModelPlanEntry(NativeRecord): pre_hooks: object post_hooks: object previous_query_sql: str | None + previous_fingerprint: Fingerprint | None fingerprint_metadata_json: str | None previous_metadata_json: str | None fingerprint_version_hash: str | None @@ -13897,6 +13899,7 @@ class ModelPlanEntry(NativeRecord): pre_hooks: object = ..., post_hooks: object = ..., previous_query_sql: str | None = ..., + previous_fingerprint: Fingerprint | None = ..., fingerprint_metadata_json: str | None = ..., previous_metadata_json: str | None = ..., fingerprint_version_hash: str | None = ..., diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index c4d3805ffc..06203a8fcd 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -102,6 +102,7 @@ ) from sqlbuild.executor.migrations.main._complete_old_name_views import complete_old_name_views from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites from sqlbuild.executor.run.classes.runtime_watermark_resolver import RuntimeWatermarkResolver from sqlbuild.executor.run.models import ( BatchWindow, @@ -194,6 +195,9 @@ def __init__( self._promotion_mode: TablePromotionMode = runtime.promotion_mode self._run_id: str = runtime.run_id self._runtime_dir: Path = runtime.runtime_dir + self._fingerprint_writes: FingerprintWrites = FingerprintWrites( + skips_identical=not plan.migration_entries + ) self._query_change_tracking: bool = ( True if runtime.query_change_tracking is None else runtime.query_change_tracking ) @@ -1210,6 +1214,7 @@ def _execute_model_node( on_batch_complete=self._on_batch_complete, watermark_resolver=self._watermark_resolver, bound_views=bound_views, + fingerprint_writes=self._fingerprint_writes, ), promotion_mode=self._promotion_mode, snapshots=self._snapshots, diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index eea80a0fa1..d8d0c9f088 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -97,7 +97,7 @@ def materialize_query_diff_artifact( sort_keys=True, separators=(",", ":"), ) - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=artifact.database, diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index ed4d2c305f..a95ec0cd3b 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -174,7 +174,7 @@ def _try_write_function_fingerprint( metadata_json="{}", ts=datetime.now(tz=UTC), ) - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=entry.fingerprint_destination.database, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py index bb4aa6a85d..edc76da34c 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py @@ -34,7 +34,7 @@ def try_write_python_node_identity_fingerprint( ) return try: - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=database, diff --git a/src/sqlbuild/executor/review/_helpers/marker.py b/src/sqlbuild/executor/review/_helpers/marker.py index 31e2b8b560..ee87afbfbf 100644 --- a/src/sqlbuild/executor/review/_helpers/marker.py +++ b/src/sqlbuild/executor/review/_helpers/marker.py @@ -31,7 +31,7 @@ def write_review_marker( sort_keys=True, separators=(",", ":"), ) - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=marker.marker_database, diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index b32c74fb52..e084b6ab5d 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +from contextlib import AbstractContextManager, nullcontext from datetime import UTC, datetime from typing import Any @@ -20,6 +21,10 @@ model_fingerprint_metadata_with_audit_gate, model_fingerprint_metadata_with_migration_fingerprint, ) +from sqlbuild.executor.run.classes.fingerprint_writes import ( + FingerprintTableLocation, + FingerprintWrites, +) def try_write_fingerprint( @@ -31,8 +36,9 @@ def try_write_fingerprint( query_change_tracking: bool, model_audits: tuple[AuditPlanEntry, ...] = (), audit_results: tuple[AuditExecutionResult, ...] = (), + fingerprint_writes: FingerprintWrites | None = None, ) -> tuple[str, ...]: - """Attempt best-effort fingerprint write, returning any warnings.""" + """Attempt a best-effort fingerprint write unless the latest row matches, returning warnings.""" if not query_change_tracking: return () @@ -70,18 +76,34 @@ def try_write_fingerprint( metadata_json=metadata_json, ts=datetime.now(tz=UTC), ) - execute_fn: Any = adapter.execute - write_fingerprint( - connection=connection, - execute=execute_fn, - database=entry.destination.database, - schema=target_schema, - fingerprint=fingerprint, - render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - render_create_table_sql=adapter.render_create_fingerprint_table_sql, - render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, + location: FingerprintTableLocation = (entry.destination.database, target_schema) + if fingerprint_writes is not None and not fingerprint_writes.records( + location=location, previous=entry.previous_fingerprint, current=fingerprint + ): + return () + setting_up: AbstractContextManager[bool] = ( + nullcontext(True) + if fingerprint_writes is None + else fingerprint_writes.setting_up(location=location) ) + with setting_up as requested_setup: + execute_fn: Any = adapter.execute + set_up: bool = write_fingerprint( + ensure_table=requested_setup, + connection=connection, + execute=execute_fn, + database=entry.destination.database, + schema=target_schema, + fingerprint=fingerprint, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, + ) + if fingerprint_writes is not None: + fingerprint_writes.written( + location=location, requested_setup=requested_setup, set_up=set_up + ) except Exception as exc: return ( f"fingerprint write failed for '{entry.name}'; " diff --git a/src/sqlbuild/executor/run/classes/fingerprint_writes.py b/src/sqlbuild/executor/run/classes/fingerprint_writes.py new file mode 100644 index 0000000000..1df3a6df0a --- /dev/null +++ b/src/sqlbuild/executor/run/classes/fingerprint_writes.py @@ -0,0 +1,105 @@ +"""The fingerprint tables and recorded identities one build writes through.""" + +from __future__ import annotations + +import json +import threading +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlbuild._native import AUDIT_GATE_METADATA_KEY, Fingerprint +from sqlbuild.executor.run.types import FingerprintTableLocation + +_AUDIT_GATE_RUN_KEY: str = "run_id" + + +class FingerprintWrites: + """Set up each fingerprint table once per build and skip rows identical to the latest one.""" + + def __init__(self, *, skips_identical: bool = True) -> None: + """Skip identical rows unless the build applies migrations, which rename matching needs.""" + + self._skips_identical: bool = skips_identical + self._lock: threading.Lock = threading.Lock() + self._ready: set[FingerprintTableLocation] = set() + self._recreated: set[FingerprintTableLocation] = set() + self._setup_locks: dict[FingerprintTableLocation, threading.Lock] = {} + + def records( + self, + *, + location: FingerprintTableLocation, + previous: Fingerprint | None, + current: Fingerprint, + ) -> bool: + """Whether ``current`` must be appended: the latest row differs, or the table was lost.""" + + with self._lock: + if location in self._recreated: + return True + return ( + not self._skips_identical + or previous is None + or _identity(previous) != _identity(current) + ) + + @contextmanager + def setting_up(self, *, location: FingerprintTableLocation) -> Iterator[bool]: + """Yield whether to set up the table; writers wait while another sets it up.""" + + with self._lock: + ready: bool = location in self._ready + setup_lock: threading.Lock = self._setup_locks.setdefault(location, threading.Lock()) + if ready: + yield False + return + with setup_lock: + with self._lock: + requested: bool = location not in self._ready + yield requested + + def written( + self, *, location: FingerprintTableLocation, requested_setup: bool, set_up: bool + ) -> None: + """Note a write; a table set up only because its insert failed was lost mid-run.""" + + with self._lock: + if set_up and not requested_setup: + self._recreated.add(location) + self._ready.add(location) + + +def _identity(fingerprint: Fingerprint) -> tuple[object, ...]: + """Everything a fingerprint row records except when and by which run it was written.""" + + return ( + fingerprint.node_type, + fingerprint.node_name, + fingerprint.target_database, + fingerprint.target_schema, + fingerprint.target_name, + fingerprint.definition_hash, + fingerprint.version_hash, + fingerprint.schema_fingerprint, + fingerprint.definition, + _metadata_identity(fingerprint.metadata_json), + ) + + +def _metadata_identity(metadata_json: str) -> object: + """Metadata without the audit gate's run reference; unparseable metadata compares as text.""" + + try: + metadata: object = json.loads(metadata_json) + except json.JSONDecodeError: + return metadata_json + if isinstance(metadata, dict): + gate: object = metadata.get(AUDIT_GATE_METADATA_KEY) + if isinstance(gate, dict): + metadata = { + **metadata, + AUDIT_GATE_METADATA_KEY: { + key: value for key, value in gate.items() if key != _AUDIT_GATE_RUN_KEY + }, + } + return json.dumps(metadata, sort_keys=True) diff --git a/src/sqlbuild/executor/run/types.py b/src/sqlbuild/executor/run/types.py index eee3e57897..d7ad567e24 100644 --- a/src/sqlbuild/executor/run/types.py +++ b/src/sqlbuild/executor/run/types.py @@ -96,3 +96,6 @@ class AuditGateMetadataParseFailure(StrEnum): NON_DICT = "non-dict payload" MISSING_FIELD = "missing field" WRONG_TYPE = "wrong type" + + +type FingerprintTableLocation = tuple[str | None, str] diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index 3bb9f4842d..a103b8bb6b 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -48,7 +48,7 @@ def try_write_seed_fingerprint( metadata_json=seed_entry.fingerprint_metadata_json, ts=datetime.now(tz=UTC), ) - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=seed_entry.destination.database, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index ff77f40f60..6c2047da66 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -83,7 +83,7 @@ def create_janitor_demo_relations(*, db_path: Path) -> None: connection.execute("CREATE TABLE janitor_untracked_extra AS SELECT 1 AS id") connection.execute("CREATE TABLE partition_state AS SELECT 1 AS id") connection.execute("CREATE TABLE _sqlbuild_microbatches AS SELECT 1 AS id") - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=lambda *, connection, sql: connection.execute(sql), database=None, @@ -103,7 +103,7 @@ def create_janitor_demo_relations(*, db_path: Path) -> None: render_qualified_name=DuckDbAdapter().render_qualified_name, render_framework_type=DuckDbAdapter().render_framework_type, ) - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=lambda *, connection, sql: connection.execute(sql), database=None, @@ -157,7 +157,7 @@ def create_direct_state_history(*, db_path: Path, schema: str = "main") -> None: ("run_002", 12), ("run_003", 13), ): - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=lambda *, connection, sql: connection.execute(sql), database=None, diff --git a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py index 0bc8c859f7..d6f2b98fe8 100644 --- a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py @@ -889,7 +889,7 @@ def test_given_fingerprint_row_when_written_to_bigquery_then_base64_sql_round_tr ts=datetime(2026, 5, 4, 12, 0, tzinfo=UTC), ) - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=bigquery_project, diff --git a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py index af1f8ae885..730f605721 100644 --- a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py @@ -788,7 +788,7 @@ def test_given_fingerprint_row_when_written_to_databricks_then_base64_sql_round_ ts=datetime(2026, 5, 4, 12, 0, tzinfo=UTC), ) - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=databricks_catalog, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index c690e2e4e3..42aa220ac6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -154,3 +154,12 @@ class FingerprintStateReadTestCase: expected_reads: int expected_rows_read: Callable[[int], int] expected_migrations: tuple[tuple[str | None, str, str, str], ...] + + +@dataclass(frozen=True) +class FingerprintHistoryTestCase: + """Fingerprint rows unchanged builds append before and after a project has rename history.""" + + description: str + expected_rows_without_history: int + expected_min_rows_after_history: int diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 98ac4033b8..2cb80efbe6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -842,3 +842,14 @@ def declared_rename_models() -> dict[str, str]: "orders_enriched": enriched_view_sql(upstream=DESTINATION_MODEL, alias="o"), "daily_order_totals": daily_totals_sql(upstream="orders_enriched", cte="orders_enriched"), } + + +def model_fingerprint_rows(*, project_dir: Path) -> int: + """Count the model fingerprint rows recorded in the main schema.""" + + return int( + query( + project_dir=project_dir, + sql="SELECT COUNT(*) FROM main._sqlbuild_fingerprints WHERE node_type = 'model'", + )[0][0] + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py index 912a5e1ab5..6cac8cd3c3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py @@ -9,6 +9,7 @@ from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( AutomaticMigrationTestCase, + FingerprintHistoryTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( CliRun, @@ -23,6 +24,7 @@ load_raw_orders, migration_events, model_entry, + model_fingerprint_rows, order_ids, original_order_models, plan_json, @@ -453,5 +455,36 @@ def test_given_fingerprinted_upstream_when_retrying_failed_downstream_then_histo assert tuple(sorted(migration_events(project_dir=tmp_path))) == test_case.expected_events +@pytest.mark.parametrize( + "test_case", + [ + FingerprintHistoryTestCase( + description="rename history keeps every rebuilt incremental model recording", + expected_rows_without_history=0, + expected_min_rows_after_history=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_migration_history_when_rebuilding_unchanged_models_then_every_build_records( + test_case: FingerprintHistoryTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + build_original_order_models(project_dir=tmp_path, capsys=capsys) + before_history: int = model_fingerprint_rows(project_dir=tmp_path) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + unchanged_without_history: int = model_fingerprint_rows(project_dir=tmp_path) + write_project(project_dir=tmp_path, models=renamed_order_models()) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + after_migration: int = model_fingerprint_rows(project_dir=tmp_path) + + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + assert unchanged_without_history - before_history == test_case.expected_rows_without_history + assert migration_events(project_dir=tmp_path) != () + assert model_fingerprint_rows(project_dir=tmp_path) - after_migration >= ( + test_case.expected_min_rows_after_history + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py index 09488d9cc3..536ccaaebc 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py @@ -23,7 +23,7 @@ def write_fingerprint_from_own_connection( connection: duckdb.DuckDBPyConnection = duckdb.connect(str(db_path)) barrier.wait() try: - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=None, diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py index effd6e49d8..577698c6a8 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py @@ -256,7 +256,7 @@ def test_given_fingerprints_when_writing_and_reading_then_returns_expected( ) -> None: fp: Fingerprint for fp in test_case.fingerprints: - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=execute, database=test_case.database, @@ -496,7 +496,7 @@ def test_given_no_table_when_writing_then_creates_table( connection: Any, execute: Any, ) -> None: - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=execute, database=test_case.database, @@ -606,7 +606,7 @@ def test_given_multiple_fingerprints_when_reading_then_resolves_latest( ) -> None: fp: Fingerprint for fp in test_case.fingerprints: - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=execute, database=test_case.database, @@ -782,7 +782,7 @@ def test_given_fingerprint_history_when_pruning_then_keeps_latest_versions_per_n ) -> None: fp: Fingerprint for fp in test_case.fingerprints: - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=execute, database=test_case.database, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 5488189925..e77bcf29ae 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -237,7 +237,7 @@ def test_given_warehouse_state_when_gathering_snapshot_then_returns_expected( fp_entry: tuple[str, Fingerprint] for fp_entry in test_case.fingerprints_to_write: - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=execute, database=None, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 2cda67eeaf..5db6a59333 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -325,7 +325,7 @@ def write_previous_function_fingerprints( for function_name in test_case.previous_function_bodies: function: CompiledFunctionView = function_map[function_name] fingerprint_sql: str = build_compiled_function_fingerprint_sql(function) - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=function.destination.database, diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_state_table_retention.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_state_table_retention.py index df943340f3..d495a9a22f 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_state_table_retention.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_state_table_retention.py @@ -24,10 +24,10 @@ "test_case", [ StateTableRetentionTestCase( - description="enterprise account keeps 90 days", + description="enterprise account keeps 90 days, set up once per build", max_retention_days=90, expected_first_retention_days="90", - expected_later_retention_days=frozenset({"90"}), + expected_later_retention_days=frozenset(), ), StateTableRetentionTestCase( description="standard account retries once and then creates with 1 day", @@ -51,7 +51,7 @@ def test_given_account_edition_when_building_then_state_tables_are_permanent_wit creates: tuple[str, ...] = state_table_creates(build) assert failed_models(build) == () - assert len(creates) > 1 + assert len(creates) == 1 + len(test_case.expected_later_retention_days) assert all(create.startswith("CREATE TABLE IF NOT EXISTS ") for create in creates) assert all("._sqlbuild_fingerprints (" in create for create in creates) assert creates[0].rsplit(" = ", 1)[1] == test_case.expected_first_retention_days diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py index 92cfc7dc16..9bd5975e08 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py @@ -215,7 +215,7 @@ def test_given_fingerprint_values_when_writing_then_insert_sql_contains_expected test_case: BuildInsertSqlTestCase, ) -> None: executed: list[str] = [] - write_fingerprint( + _ = write_fingerprint( connection=object(), execute=lambda *, connection, sql: executed.append(sql), database=test_case.database, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py index aa2738c008..926011ac44 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py @@ -160,7 +160,7 @@ def test_given_index_renderer_when_writing_fingerprint_then_executes_index_befor adapter: DuckDbAdapter = DuckDbAdapter() execute: FakeFingerprintWriteExecute = FakeFingerprintWriteExecute() - write_fingerprint( + _ = write_fingerprint( connection=object(), execute=execute, database=None, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py index be01fb9f6e..d5e0f9dfee 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py @@ -153,7 +153,7 @@ def test_given_transient_write_conflicts_when_writing_fingerprint_then_retries_u sleeper: RecordingSleeper = RecordingSleeper() monkeypatch.setattr("time.sleep", sleeper) - write_fingerprint( + _ = write_fingerprint( connection=object(), execute=execute, database=None, @@ -198,7 +198,7 @@ def test_given_persistent_write_conflicts_when_writing_fingerprint_then_raises_a monkeypatch.setattr("time.sleep", sleeper) with pytest.raises(RuntimeError) as error_info: - write_fingerprint( + _ = write_fingerprint( connection=object(), execute=execute, database=None, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 672fc0c2b4..ba62c9e9da 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -102,7 +102,7 @@ def create_protected_model_state(*, adapter: DuckDbAdapter, connection: Any) -> adapter.execute(connection=connection, sql="CREATE TABLE main.order_history (id INTEGER)") query_sql: str = build_protected_schema_replay_project().models[0].query_sql - write_fingerprint( + _ = write_fingerprint( connection=connection, execute=adapter.execute, database=None, diff --git a/tests/unit/src/sqlbuild/executor/run/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/run/classes/_test_types.py index 65d43d08cc..56e737ee16 100644 --- a/tests/unit/src/sqlbuild/executor/run/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/classes/_test_types.py @@ -5,3 +5,10 @@ class RuntimeWatermarkResolverTestCase: description: str expected_query_count: int + + +@dataclass(frozen=True) +class FingerprintSetupTestCase: + description: str + first_writer_finishes: bool + expected_requests: tuple[bool, bool] diff --git a/tests/unit/src/sqlbuild/executor/run/classes/test_fingerprint_writes.py b/tests/unit/src/sqlbuild/executor/run/classes/test_fingerprint_writes.py new file mode 100644 index 0000000000..9eaf98108f --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/run/classes/test_fingerprint_writes.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import threading +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor + +import pytest + +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites +from sqlbuild.executor.run.types import FingerprintTableLocation +from tests.unit.src.sqlbuild.executor.run.classes._test_types import FingerprintSetupTestCase + +_LOCATION: FingerprintTableLocation = ("analytics", "orders") + + +@pytest.mark.parametrize( + "test_case", + [ + FingerprintSetupTestCase( + description="a writer arriving during setup waits and then skips the setup", + first_writer_finishes=True, + expected_requests=(True, False), + ), + FingerprintSetupTestCase( + description="a writer arriving during a failed setup sets the table up itself", + first_writer_finishes=False, + expected_requests=(True, True), + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_writers_when_setting_up_a_table_then_only_one_sets_it_up_at_a_time( + test_case: FingerprintSetupTestCase, +) -> None: + writes: FingerprintWrites = FingerprintWrites() + first_entered: threading.Event = threading.Event() + second_started: threading.Event = threading.Event() + requests: list[bool] = [] + finish_actions: dict[bool, Callable[[bool], None]] = { + True: lambda requested: writes.written( + location=_LOCATION, requested_setup=requested, set_up=requested + ), + False: lambda _requested: None, + } + + def first_writer() -> None: + with writes.setting_up(location=_LOCATION) as requested: + requests.append(requested) + first_entered.set() + _ = second_started.wait(timeout=5) + finish_actions[test_case.first_writer_finishes](requested) + + def second_writer() -> None: + _ = first_entered.wait(timeout=5) + second_started.set() + with writes.setting_up(location=_LOCATION) as requested: + requests.append(requested) + + with ThreadPoolExecutor(max_workers=2) as pool: + futures: list[Future[None]] = [pool.submit(first_writer), pool.submit(second_writer)] + _ = [future.result(timeout=10) for future in futures] + + assert tuple(requests) == test_case.expected_requests From fcf8b2ef24ddc21fc0172fcf994c1a3655ccd6fc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:22:05 +0000 Subject: [PATCH 04/12] perf: list scenario namespaces by prefix and reuse cleanup results Scenario cleanup lists the namespace with one SHOW TERSE OBJECTS on Snowflake, drops each relation by its listed kind, and answers the scenario's later existence checks for relations it confirmed absent. --- .../executor/pipeline/_helpers/scenario.py | 20 +++-- .../scenario/_helpers/lifecycle/cleanup.py | 54 +++++++++++-- .../executor/scenario/main/_cleanup.py | 21 ++++- src/sqlbuild/executor/scenario/models.py | 1 + .../adapters/snowflake/test_prefix_listing.py | 80 +++++++++++++++++++ .../executor/scenario/main/_test_types.py | 2 + .../executor/scenario/main/helpers.py | 22 +++++ .../executor/scenario/main/test_cleanup.py | 16 ++++ 8 files changed, 200 insertions(+), 16 deletions(-) create mode 100644 tests/unit/src/sqlbuild/adapters/snowflake/test_prefix_listing.py diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index fdc81cfee4..372658a443 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -32,6 +32,9 @@ from sqlbuild._native import build_cli_scenario_plan as build_scenario_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.adapter.relations.main.relation_metadata_cache_scope import ( + relation_metadata_cache_scope, +) from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, @@ -161,13 +164,16 @@ def run_scenario_test_pipeline( ), running=running, ) - with _scenario_target_connections( - adapter=adapter, - connection_config=connection_config, - connection_hooks=connection_hooks, - connection_count=worker_count, - abandoned=scheduling.interrupts.abandoned, - ) as connection_pool: + with ( + relation_metadata_cache_scope(adapter=adapter), + _scenario_target_connections( + adapter=adapter, + connection_config=connection_config, + connection_hooks=connection_hooks, + connection_count=worker_count, + abandoned=scheduling.interrupts.abandoned, + ) as connection_pool, + ): options: ScenarioRunOptions = ScenarioRunOptions( promotion_mode=resolve_promotion_mode( settings=pipeline_result.project.settings, adapter=adapter diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py index 7988d10312..979c659d01 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import os from collections import defaultdict from dataclasses import replace from typing import Any @@ -36,6 +37,8 @@ _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") +_MINIMUM_LISTING_PREFIX: int = 8 + def collect_scenario_cleanup_targets( *, @@ -178,12 +181,32 @@ def existing_scenario_cleanup_targets( database: str | None names: set[str] for database, names in names_by_database.items(): + database_targets: tuple[ScenarioCleanupTarget, ...] = tuple( + target for target in listed_targets if target.database == database + ) + prefix: str = os.path.commonprefix( + [str(target.name).lower() for target in database_targets] + ) + relations: tuple[RelationInfo, ...] | None = ( + adapter.list_relations_with_prefix( + connection=connection, + database=database, + schemas=tuple(sorted({str(target.schema) for target in database_targets})), + prefix=prefix, + ) + if len(prefix) >= _MINIMUM_LISTING_PREFIX + else None + ) relation: RelationInfo - for relation in adapter.list_relations( - connection=connection, - database=database, - schemas=tuple(sorted(schemas_by_database[database])), - names=tuple(sorted(names)), + for relation in ( + relations + if relations is not None + else adapter.list_relations( + connection=connection, + database=database, + schemas=tuple(sorted(schemas_by_database[database])), + names=tuple(sorted(names)), + ) ): exact_types[(database, relation.schema, relation.name)] = relation.relation_type folded_types[ @@ -212,19 +235,38 @@ def existing_scenario_cleanup_targets( ) if relation_type is None: continue + listed_type: RelationType = normalize_relation_type(relation_type) existing.append( replace( target, materialization_type=( MaterializationType.VIEW - if normalize_relation_type(relation_type) == RelationType.VIEW + if listed_type == RelationType.VIEW else MaterializationType.TABLE ), + listed=listed_type != RelationType.OTHER, ) ) return tuple(existing) +def confirmed_absent_targets( + *, + targets: tuple[ScenarioCleanupTarget, ...], + dropped: tuple[ScenarioCleanupTarget, ...], + adapter: BaseAdapter, +) -> tuple[tuple[str | None, str | None, str], ...]: + """Locations a successful cleanup left empty: listed absent, or dropped by their listed kind.""" + + unconfirmed: set[str] = {target.target_relation for target in dropped if not target.listed} + return tuple( + (target.database, target.schema, str(target.name)) + for target in targets + if _authoritatively_listable(target=target, adapter=adapter) + and target.target_relation not in unconfirmed + ) + + def _authoritatively_listable(*, target: ScenarioCleanupTarget, adapter: BaseAdapter) -> bool: return ( target.name is not None diff --git a/src/sqlbuild/executor/scenario/main/_cleanup.py b/src/sqlbuild/executor/scenario/main/_cleanup.py index 291cd7ea0d..5253b6e441 100644 --- a/src/sqlbuild/executor/scenario/main/_cleanup.py +++ b/src/sqlbuild/executor/scenario/main/_cleanup.py @@ -7,9 +7,11 @@ from sqlbuild._native import MaterializationType, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.relations.classes.run_relation_listings import RunRelationListings from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.scenario._helpers.lifecycle.cleanup import ( collect_scenario_cleanup_targets, + confirmed_absent_targets, existing_scenario_cleanup_targets, ) from sqlbuild.executor.scenario.constants import SCENARIO_EXEC_CLEANUP_FAILED @@ -53,10 +55,11 @@ def _execute_scenario_cleanup( ) try: - drop_target: ScenarioCleanupTarget - for drop_target in existing_scenario_cleanup_targets( + dropped: tuple[ScenarioCleanupTarget, ...] = existing_scenario_cleanup_targets( targets=cleanup_targets, adapter=adapter, connection=connection - ): + ) + drop_target: ScenarioCleanupTarget + for drop_target in dropped: with diagnostics_context( sqlbuild_phase="scenario_cleanup", sqlbuild_action_name="drop_relation", @@ -71,6 +74,12 @@ def _execute_scenario_cleanup( if_exists=True, statement_recorder=statement_recorder, ) + elif drop_target.listed: + adapter.drop_table( + connection=connection, + destination=drop_target.target_relation, + statement_recorder=statement_recorder, + ) else: adapter.drop( connection=connection, @@ -90,6 +99,12 @@ def _execute_scenario_cleanup( error_message=str(exc), ) + RunRelationListings.remember_absent( + adapter=adapter, + relations=confirmed_absent_targets( + targets=cleanup_targets, dropped=dropped, adapter=adapter + ), + ) return ScenarioCleanupExecutionResult( scenario_name=scenario_plan.name, status=ExecutionStatus.SUCCESS, diff --git a/src/sqlbuild/executor/scenario/models.py b/src/sqlbuild/executor/scenario/models.py index 80563767e6..f208128e9b 100644 --- a/src/sqlbuild/executor/scenario/models.py +++ b/src/sqlbuild/executor/scenario/models.py @@ -59,6 +59,7 @@ class ScenarioCleanupTarget: database: str | None = None schema: str | None = None name: str | None = None + listed: bool = False @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_prefix_listing.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_prefix_listing.py new file mode 100644 index 0000000000..3dee97813e --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_prefix_listing.py @@ -0,0 +1,80 @@ +import pytest + +from sqlbuild.adapter.contract.models import RelationInfo +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakePrefixListingTestCase, +) +from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( + prefix_listing_statements, + retention_relation, +) +from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( + OfflineSnowflakeAdapter, + RecordingSnowflakeWarehouse, +) + +_LISTING: tuple[str, ...] = ("SHOW TERSE OBJECTS LIKE '__SQB_AB12__%'",) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakePrefixListingTestCase( + description="tables and views under the prefix, in one SHOW TERSE OBJECTS", + relations=( + retention_relation("TESTS", "__SQB_AB12__SOURCE__ORDERS", retention=1), + retention_relation( + "TESTS", "__SQB_AB12__MODEL__ORDER_TOTALS", retention=1, table_type="VIEW" + ), + retention_relation("TESTS", "__SQB_CD34__SOURCE__ORDERS", retention=1), + retention_relation("TESTS", "ORDERS", retention=1), + ), + schema="tests", + prefix="__sqb_ab12__", + expected_relations=( + ("__sqb_ab12__model__order_totals", "view"), + ("__sqb_ab12__source__orders", "base table"), + ), + expected_statements=_LISTING, + ), + SnowflakePrefixListingTestCase( + description="a name LIKE matches only through its underscores is left out", + relations=( + retention_relation("TESTS", "__SQB_AB12__SOURCE__ORDERS", retention=1), + retention_relation("TESTS", "X_SQB_AB12__ORDERS", retention=1), + ), + schema="tests", + prefix="__sqb_ab12__", + expected_relations=(("__sqb_ab12__source__orders", "base table"),), + expected_statements=_LISTING, + ), + SnowflakePrefixListingTestCase( + description="a schema that does not exist lists nothing", + relations=(retention_relation("TESTS", "__SQB_AB12__SOURCE__ORDERS", retention=1),), + schema="missing", + prefix="__sqb_ab12__", + expected_relations=(), + expected_statements=_LISTING, + ), + ], + ids=lambda case: case.description, +) +def test_given_scenario_namespace_when_listing_by_prefix_then_one_show_finds_its_relations( + test_case: SnowflakePrefixListingTestCase, +) -> None: + warehouse: RecordingSnowflakeWarehouse = RecordingSnowflakeWarehouse( + relations=test_case.relations + ) + adapter: OfflineSnowflakeAdapter = OfflineSnowflakeAdapter(warehouse=warehouse) + + relations: tuple[RelationInfo, ...] | None = adapter.list_relations_with_prefix( + connection=adapter.connect({}), + database="analytics", + schemas=(test_case.schema,), + prefix=test_case.prefix, + ) + + assert tuple( + sorted((relation.name, relation.relation_type) for relation in relations or ()) + ) == (test_case.expected_relations) + assert prefix_listing_statements(warehouse) == test_case.expected_statements diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/_test_types.py b/tests/unit/src/sqlbuild/executor/scenario/main/_test_types.py index 5e2b17a923..09e641c911 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/_test_types.py @@ -80,6 +80,8 @@ class ScenarioCatalogCleanupTestCase: authoritative: bool = True catalog_schema: str = "scenario_schema" expected_listing_count: int = 1 + prefix_listing: bool = False + expected_prefix_listings: int = 0 @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py index a97f517c5a..24332d22e7 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py @@ -65,8 +65,11 @@ def __init__( interrupt_on_create: bool = False, authoritative: bool = True, catalog_schema: str = "scenario_schema", + prefix_listing: bool = False, ) -> None: super().__init__() + self.prefix_listing: bool = prefix_listing + self.prefix_listings: list[str] = [] self.relation_types: dict[str, str] = relation_types self.fail_listing: bool = fail_listing self.authoritative: bool = authoritative @@ -103,6 +106,25 @@ def list_relations( for name in sorted(listed_names) ) + def list_relations_with_prefix( + self, *, connection: Any, database: str | None, schemas: tuple[str, ...], prefix: str + ) -> tuple[RelationInfo, ...] | None: + del connection + self.prefix_listings.extend((prefix,) * self.prefix_listing) + listed: tuple[RelationInfo, ...] = tuple( + RelationInfo( + database=database, + schema=schemas[0], + name=name.lower(), + relation_type=relation_type, + ) + for name, relation_type in filter( + lambda item: item[0].lower().startswith(prefix.lower()), + sorted(self.relation_types.items()), + ) + ) + return (None, listed)[self.prefix_listing] + def _execute(self, connection: object, sql: str) -> object: _CATALOG_CREATE_STRATEGIES[self.interrupt_on_create and sql.startswith("CREATE ")]() return super()._execute(connection, sql) diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py index 805788c1eb..00aa1713f9 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py @@ -222,6 +222,20 @@ def test_given_unmocked_project_seed_when_cleaning_up_then_drops_seed_target( fail_listing=True, expected_drop_sql=_ALL_PLANNED_DROPS, ), + ScenarioCatalogCleanupTestCase( + description="lists only the scenario's namespace when the adapter can", + relation_types={ + "__SQB_51B385AEBE20__SOURCE__RAW__ORDERS": "BASE TABLE", + "__SQB_51B385AEBE20__MODEL__DAILY_REVENUE": "VIEW", + }, + prefix_listing=True, + expected_listing_count=0, + expected_prefix_listings=1, + expected_drop_sql=( + f"DROP TABLE IF EXISTS {_CATALOG_PREFIX}__source__raw__orders", + f"DROP VIEW IF EXISTS {_CATALOG_PREFIX}__model__daily_revenue", + ), + ), ], ids=lambda case: case.description, ) @@ -233,6 +247,7 @@ def test_given_catalog_state_when_cleaning_up_then_drops_only_existing_relations fail_listing=test_case.fail_listing, authoritative=test_case.authoritative, catalog_schema=test_case.catalog_schema, + prefix_listing=test_case.prefix_listing, ) result: ScenarioCleanupExecutionResult = execute_scenario_cleanup( @@ -243,6 +258,7 @@ def test_given_catalog_state_when_cleaning_up_then_drops_only_existing_relations assert result.status == ExecutionStatus.SUCCESS assert adapter.listing_count == test_case.expected_listing_count + assert len(adapter.prefix_listings) == test_case.expected_prefix_listings assert executed_drop_sql(adapter) == test_case.expected_drop_sql From 3216d477ad1322415b8d4aa2b2f8ae85c62ed01b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:22:06 +0000 Subject: [PATCH 05/12] test: count housekeeping statements through the CLI and document run metadata reads --- .../sqlbuild/references/docs/cli/build.md | 2 +- .../docs/concepts/adapters/snowflake.md | 17 +- .../main/statement_counts/__init__.py | 1 + .../main/statement_counts/_test_types.py | 64 +++ .../commands/main/statement_counts/helpers.py | 397 ++++++++++++++++++ .../statement_counts/test_guarded_cases.py | 225 ++++++++++ .../statement_counts/test_statement_budget.py | 113 +++++ website/src/content/docs/docs/cli/build.mdx | 2 +- .../docs/docs/concepts/adapters/snowflake.mdx | 17 +- 9 files changed, 828 insertions(+), 10 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/statement_counts/__init__.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/statement_counts/_test_types.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/statement_counts/helpers.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_guarded_cases.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_statement_budget.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index b034a137af..7d2eee3bf4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -168,7 +168,7 @@ JSON `status` is `failed`. ## Fingerprints -After a successful build, SQLBuild writes version identities to `_sqlbuild_fingerprints` in the target schema. These are used on subsequent runs to detect changes and skip unchanged work. See [Planning and Change Detection](../concepts/planning.md) for details. +After a successful build, SQLBuild writes version identities to `_sqlbuild_fingerprints` in the target schema. These are used on subsequent runs to detect changes and skip unchanged work. A build writes a row only for a resource whose identity differs from its latest row, so a build that changes no definitions writes none. In a project with model migration history, every build writes a row for each resource it builds. See [Planning and Change Detection](../concepts/planning.md) for details. ## Runtime artifacts diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index 39c68946f3..d82303fc86 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -72,10 +72,19 @@ Run `sqb plan -v` to see each metadata query with its elapsed time and row count ## Build metadata -During a build, checks for whether one relation exists use `SHOW TERSE TABLES` and -`SHOW TERSE VIEWS`, and reads of one relation's columns use `SHOW COLUMNS`, so they also don't -need a running warehouse. Each build reuses these answers until it changes that relation, for -example by replacing a delta table or adding a column, and then reads it again. +Builds and scenario runs read relation metadata with `SHOW` commands too, so these reads don't +need a running warehouse either: + +| Read | Statement | +| -- | -- | +| Whether one relation exists | `SHOW TERSE TABLES` and `SHOW TERSE VIEWS` | +| A second relation in the same schema | `SHOW TABLES` and `SHOW VIEWS` for the whole schema, once | +| A table's time travel retention | `SHOW TABLES LIKE`, or the schema listing | +| One relation's columns | `SHOW COLUMNS` | +| A scenario's leftover relations | `SHOW TERSE OBJECTS` for the scenario's name prefix | + +Each run reuses these answers until it changes that relation, for example by replacing a delta +table or adding a column, and then reads it again. ## State tables diff --git a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/__init__.py b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/__init__.py new file mode 100644 index 0000000000..385f63290a --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/__init__.py @@ -0,0 +1 @@ +"""Integration tests counting the warehouse statements real CLI commands issue.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/_test_types.py new file mode 100644 index 0000000000..2cdbc42eb8 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/_test_types.py @@ -0,0 +1,64 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class StatementStep: + """One CLI command in a sequence and the housekeeping statements it issues, by kind.""" + + description: str + args: tuple[str, ...] + new_orders: int + expected_counts: dict[str, int] + + +@dataclass(frozen=True) +class StatementBudgetTestCase: + """A sequence of commands against one project, each with its housekeeping budget.""" + + description: str + steps: tuple[StatementStep, ...] + expected_recreated_after_drop: tuple[str, ...] = () + + +@dataclass(frozen=True) +class FingerprintWriteTestCase: + """An edit between two builds and the fingerprint statements the second build issues.""" + + description: str + edited_models: tuple[str, ...] + expected_setups: int + expected_rows: dict[str, int] + expected_later_inserts: int = 0 + + +@dataclass(frozen=True) +class FingerprintLossTestCase: + """A fingerprint table dropped by a model's post hook in the middle of a build.""" + + description: str + edited_models: tuple[str, ...] + hooked_model: str + expected_setups: int + expected_rows_after_loss: dict[str, int] + expected_models_after_rebuild: tuple[str, ...] + + +@dataclass(frozen=True) +class LeftoverRelationTestCase: + """A relation another run left at a name SQLBuild builds through, and the build's result.""" + + description: str + leftover_sql: str + expected_rows: dict[str, int] + expected_intermediates: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ScenarioLeftoverTestCase: + """What an earlier scenario run, or a user, left in the scenario namespace.""" + + description: str + retained_runs: int + planted_sql: tuple[str, ...] + expected_kinds: tuple[tuple[str, str], ...] + expected_after_cleanup: tuple[tuple[str, str], ...] = () diff --git a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/helpers.py new file mode 100644 index 0000000000..24f133d795 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/helpers.py @@ -0,0 +1,397 @@ +"""Count the statements real CLI commands send, over an orders project with every model kind.""" + +from __future__ import annotations + +import re +import threading +from collections import Counter +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import TypeGuard + +import duckdb +import pytest + +from sqlbuild._native import run_cli as main +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle + +_NAME: str = r'(?P(?:"[^"]*"|[A-Za-z0-9_$]+)(?:\.(?:"[^"]*"|[A-Za-z0-9_$]+)){0,2})' +_KINDS: tuple[tuple[re.Pattern[str], str], ...] = ( + (re.compile(r"^SHOW\s+COLUMNS\s+IN\s+(?:TABLE|VIEW)\s+" + _NAME, re.I), "SHOW COLUMNS {role}"), + (re.compile(r"^SHOW\s+COLUMNS\s+IN\s+SCHEMA\b", re.I), "SHOW COLUMNS IN SCHEMA"), + ( + re.compile(r"^SHOW\s+TERSE\s+(?PTABLES|VIEWS)\s+LIKE\b", re.I), + "SHOW TERSE {kind} LIKE", + ), + (re.compile(r"^SHOW\s+(?PTABLES|VIEWS)\s+LIKE\b", re.I), "SHOW {kind} LIKE"), + ( + re.compile(r"^SHOW\s+(?:TERSE\s+)?(?PTABLES|VIEWS)\s+IN\b", re.I), + "SHOW {kind} IN SCHEMA", + ), + (re.compile(r"^SHOW\s+(?P\w+)", re.I), "SHOW {kind}"), + (re.compile(r"\binformation_schema\.(?P\w+)", re.I), "INFORMATION_SCHEMA {kind}"), + ( + re.compile(r"^DROP\s+(?PTABLE|VIEW)\s+(?:IF\s+EXISTS\s+)?" + _NAME, re.I), + "DROP {kind} {role}", + ), + ( + re.compile( + r"^CREATE\s+(?:OR\s+REPLACE\s+)?(?:TRANSIENT\s+|TEMPORARY\s+|TEMP\s+)?" + r"(?PTABLE|VIEW)\s+(?PIF\s+NOT\s+EXISTS\s+)?" + _NAME, + re.I, + ), + "CREATE {kind} {role}", + ), + ( + re.compile(r"^ALTER\s+\w+\s+" + _NAME + r"\s+(?PSWAP|RENAME|SET|ADD)\b", re.I), + "ALTER {kind} {role}", + ), + (re.compile(r"^INSERT\s+INTO\s+" + _NAME, re.I), "INSERT {role}"), + (re.compile(r"^DELETE\s+FROM\s+" + _NAME, re.I), "DELETE {role}"), + (re.compile(r"^MERGE\s+INTO\s+" + _NAME, re.I), "MERGE {role}"), + (re.compile(r"^UPDATE\s+" + _NAME, re.I), "UPDATE {role}"), + (re.compile(r"^SELECT\s+COUNT\(\*\)\s+FROM\s+" + _NAME, re.I), "SELECT COUNT {role}"), + (re.compile(r"^(?PBEGIN|COMMIT|ROLLBACK)\b", re.I), "{kind}"), + (re.compile(r"^(?:SELECT|WITH)\b", re.I), "SELECT"), + (re.compile(r"^(?P\w+)", re.I), "{kind}"), + (re.compile(r"^"), "OTHER"), +) +_ROLE_RULES: tuple[tuple[Callable[[str], bool], Callable[[str], str]], ...] = ( + (lambda last: last.startswith("_sqlbuild_"), lambda last: last), + (lambda last: "__sqb_rebuild__" in last, lambda _last: "rebuild"), + (lambda last: last.endswith("__delta"), lambda _last: "delta"), + (lambda last: last.endswith("__staging"), lambda _last: "staging"), + (lambda _last: True, lambda _last: "relation"), +) +_LEADING_NOISE: re.Pattern[str] = re.compile(r"^(?:\s+|--[^\n]*\n|/\*.*?\*/)+", re.S) + + +def relation_role(name: str) -> str: + """The part a relation plays: a SQLBuild state table, a build intermediate, or a relation.""" + + last: str = name.split(".")[-1].strip('"').lower() + return next(rule[1](last) for rule in filter(lambda rule: rule[0](last), _ROLE_RULES)) + + +def _has_match(pair: tuple[re.Match[str] | None, str]) -> TypeGuard[tuple[re.Match[str], str]]: + return pair[0] is not None + + +def statement_kind(sql: str) -> str: + """Classify one statement by its leading keywords and the role of the relation it names.""" + + text: str = _LEADING_NOISE.sub("", sql) + matched: tuple[re.Match[str], str] = next( + filter(_has_match, ((pattern.search(text), template) for pattern, template in _KINDS)) + ) + template: str = matched[1] + groups: dict[str, str | None] = matched[0].groupdict() + kind: str = (groups.get("kind") or "").upper() + ("", " IF NOT EXISTS")[ + bool(groups.get("exists")) + ] + return template.format(kind=kind, role=relation_role(groups.get("name") or "")).strip() + + +def kind_counts(statements: tuple[str, ...] | list[str]) -> dict[str, int]: + """Count statements by kind.""" + + return dict(sorted(Counter(map(statement_kind, statements)).items())) + + +@contextmanager +def recorded_statements() -> Iterator[list[str]]: + """Collect every finished outermost statement any thread of the command sends.""" + + statements: list[str] = [] + lock: threading.Lock = threading.Lock() + + def record(*, sql: str) -> None: + with lock: + statements.append(sql) + + with StatementLifecycle.listener_scope(record): + yield statements + + +def run_counted( + *, project_dir: Path, args: tuple[str, ...], capsys: pytest.CaptureFixture[str] +) -> tuple[int, str, dict[str, int], tuple[str, ...]]: + """Run one real CLI command and return its exit code, output, statement kinds and statements.""" + + with recorded_statements() as statements: + exit_code: int = main(["--project-dir", str(project_dir), "--no-color", *args]) + recorded: tuple[str, ...] = tuple(statements) + return exit_code, "".join(capsys.readouterr()), kind_counts(recorded), recorded + + +def write_files(*, project_dir: Path, files: dict[str, str]) -> None: + """Write project files relative to ``project_dir``.""" + + relative: str + content: str + for relative, content in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(content, encoding="utf-8") + + +def matching(statements: tuple[str, ...], predicate: Callable[[str], bool]) -> tuple[str, ...]: + """The statements one predicate selects, in issue order.""" + + return tuple(filter(predicate, statements)) + + +ORDERS_DATABASE: str = "orders.duckdb" +_RAW_ORDERS_SQL: str = ( + "CREATE OR REPLACE TABLE main.raw_orders AS SELECT * FROM (VALUES " + "(1, TIMESTAMP '2026-01-01 08:00:00', 10.00), " + "(2, TIMESTAMP '2026-01-02 09:00:00', 20.00), " + "(3, TIMESTAMP '2026-01-03 10:00:00', 30.00)" + ") AS raw(order_id, ordered_at, amount)" +) +_MODEL_HEADER: str = "MODEL (description 'Test model {name}.',\n{config});\n\n" +_ORDERS_SELECT: str = 'SELECT order_id, ordered_at, amount FROM __source("raw_orders")\n' +ORDERS_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders_shop"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + f'database = "{ORDERS_DATABASE}"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n" + " table: raw_orders\n" + ), + "models/orders_table.sql": _MODEL_HEADER.format( + name="orders_table", + config=(" materialized table,\n columns (order_id (audits [not_null])),\n"), + ) + + _ORDERS_SELECT, + "models/orders_view.sql": _MODEL_HEADER.format( + name="orders_view", config=" materialized view,\n" + ) + + 'SELECT order_id, amount FROM __ref("orders_table")\n', + "models/orders_merge.sql": _MODEL_HEADER.format( + name="orders_merge", + config=( + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [order_id],\n" + " cursor ordered_at,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (\n raw_orders ordered_at,\n ),\n" + ), + ) + + _ORDERS_SELECT, + "models/orders_daily.sql": _MODEL_HEADER.format( + name="orders_daily", + config=( + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n" + " cursor ordered_at,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_start '2026-01-01',\n" + " cursor_watermark_mode all,\n" + " cursor_inputs (\n" + " raw_orders (column ordered_at, roles [filter, watermark]),\n" + " ),\n" + " batch_size 1d,\n" + ), + ) + + _ORDERS_SELECT, + "tests/unit/test_orders_table.sql": ( + "TEST ();\n\n" + "WITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, TIMESTAMP '2026-01-01 08:00:00' AS ordered_at, " + "10.00 AS amount\n" + "),\n" + "__expected__orders_table AS (\n" + " SELECT 1 AS order_id, TIMESTAMP '2026-01-01 08:00:00' AS ordered_at, " + "10.00 AS amount\n" + ")\n" + ), + "tests/unit/test_orders_view_chain.sql": ( + "TEST ();\n\n" + "WITH\n" + "__source__raw_orders AS (\n" + " SELECT 2 AS order_id, TIMESTAMP '2026-01-02 09:00:00' AS ordered_at, " + "20.00 AS amount\n" + "),\n" + "__expected__orders_view AS (\n" + " SELECT 2 AS order_id, 20.00 AS amount\n" + ")\n" + ), + "tests/scenarios/orders_flow.sql": ( + "SCENARIO (description 'Orders flow through the table and view.');\n\n" + "WITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, TIMESTAMP '2026-01-01 08:00:00' AS ordered_at, " + "10.00 AS amount\n" + "),\n" + "__expected__orders_view AS (\n" + " SELECT 1 AS order_id, 10.00 AS amount\n" + ")\n" + "SELECT 1\n" + ), +} + + +NEW_ORDER_SQL: str = ( + "INSERT INTO main.raw_orders VALUES (4, TIMESTAMP '2026-01-04 11:00:00', 40.00)" +) + + +def write_orders_project(*, project_dir: Path) -> Path: + """Write the orders project and its raw source table.""" + + write_files(project_dir=project_dir, files=ORDERS_FILES) + with duckdb.connect(str(project_dir / ORDERS_DATABASE)) as connection: + _ = connection.execute(_RAW_ORDERS_SQL) + return project_dir + + +def query_rows(project_dir: Path, sql: str) -> list[tuple[object, ...]]: + """Rows one read-only query returns from the orders database.""" + + with duckdb.connect(str(project_dir / ORDERS_DATABASE), read_only=True) as connection: + return connection.execute(sql).fetchall() + + +def execute_sql(project_dir: Path, sql: str) -> None: + """Run statements against the orders database, as another session would.""" + + with duckdb.connect(str(project_dir / ORDERS_DATABASE)) as connection: + _ = connection.execute(sql) + + +def fingerprint_rows(project_dir: Path) -> dict[str, int]: + """Recorded fingerprint rows per model.""" + + return { + str(name): int(str(count)) + for name, count in query_rows( + project_dir, + "SELECT node_name, COUNT(*) FROM main._sqlbuild_fingerprints " + "WHERE node_type = 'model' GROUP BY node_name ORDER BY node_name", + ) + } + + +def row_counts(project_dir: Path, names: tuple[str, ...]) -> dict[str, int]: + """The row count of each named relation in the main schema.""" + + return { + name: int(str(query_rows(project_dir, f"SELECT COUNT(*) FROM main.{name}")[0][0])) + for name in names + } + + +def intermediate_relations(project_dir: Path) -> tuple[str, ...]: + """Delta and staging relations left in the main schema.""" + + return tuple( + str(row[0]) + for row in query_rows( + project_dir, + "SELECT table_name FROM information_schema.tables " + "WHERE table_name LIKE '%\\_\\_delta' ESCAPE '\\' " + "OR table_name LIKE '%\\_\\_staging' ESCAPE '\\' ORDER BY table_name", + ) + ) + + +def scenario_relation_kinds(project_dir: Path) -> tuple[tuple[str, str], ...]: + """Each scenario-owned relation's name suffix after its namespace, with its table type.""" + + return tuple( + (str(name).split("__", 3)[-1], str(kind)) + for name, kind in query_rows( + project_dir, + "SELECT table_name, table_type FROM information_schema.tables " + "WHERE table_name LIKE '\\_\\_sqb\\_%' ESCAPE '\\' ORDER BY table_name", + ) + ) + + +def scenario_relation_name(project_dir: Path, suffix: str) -> str: + """The qualified name of the scenario-owned relation ending in ``suffix``.""" + + return "main." + next( + str(row[0]) + for row in query_rows( + project_dir, + "SELECT table_name FROM information_schema.tables " + f"WHERE table_name LIKE '\\_\\_sqb\\_%{suffix}' ESCAPE '\\'", + ) + ) + + +def revise_model(project_dir: Path, name: str) -> None: + """Change one model's definition by adding a constant column.""" + + model: Path = project_dir / "models" / f"{name}.sql" + _ = model.write_text( + model.read_text(encoding="utf-8").replace("SELECT ", "SELECT 1 AS revision, ", 1), + encoding="utf-8", + ) + + +def add_post_hook(project_dir: Path, name: str, sql: str) -> str: + """Give one model an inline SQL post hook, returning its original definition.""" + + model: Path = project_dir / "models" / f"{name}.sql" + original: str = model.read_text(encoding="utf-8") + _ = model.write_text( + re.sub( + r"(materialized \w+,\n)", + lambda match: f'{match.group(1)} post_hooks [\n inline_sql("{sql}"),\n ],\n', + original, + ), + encoding="utf-8", + ) + return original + + +def run_step( + *, + project_dir: Path, + args: tuple[str, ...], + new_orders: int, + capsys: pytest.CaptureFixture[str], +) -> tuple[int, str, dict[str, int], tuple[str, ...]]: + """Add ``new_orders`` raw orders, then run one real CLI command and count its statements.""" + + sql: str + for sql in (NEW_ORDER_SQL,) * new_orders: + execute_sql(project_dir, sql) + return run_counted(project_dir=project_dir, args=args, capsys=capsys) + + +def budgeted(counts: dict[str, int], prefixes: tuple[str, ...]) -> dict[str, int]: + """The statement counts whose kind starts with one of ``prefixes``.""" + + return dict(filter(lambda item: item[0].startswith(prefixes), counts.items())) + + +def recreated_after_drop(statements: tuple[str, ...]) -> tuple[str, ...]: + """Drops immediately followed by a CREATE of the same table, which replacing makes redundant.""" + + return tuple( + pair[0] + for pair in filter( + lambda pair: ( + statement_kind(pair[0]).startswith("DROP TABLE") + and statement_kind(pair[1]).startswith("CREATE TABLE") + and pair[0].split()[-1] in pair[1] + ), + zip(statements, statements[1:], strict=False), + ) + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_guarded_cases.py b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_guarded_cases.py new file mode 100644 index 0000000000..c89d38306a --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_guarded_cases.py @@ -0,0 +1,225 @@ +"""Real CLI builds of the cases the removed housekeeping statements used to guard.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.statement_counts._test_types import ( + FingerprintLossTestCase, + FingerprintWriteTestCase, + LeftoverRelationTestCase, + ScenarioLeftoverTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.statement_counts.helpers import ( + add_post_hook, + execute_sql, + fingerprint_rows, + intermediate_relations, + revise_model, + row_counts, + run_counted, + scenario_relation_kinds, + scenario_relation_name, + write_orders_project, +) + +_FINGERPRINT_SETUP: str = "CREATE TABLE IF NOT EXISTS _sqlbuild_fingerprints" +_FINGERPRINT_INSERT: str = "INSERT _sqlbuild_fingerprints" +_ALL_MODELS: tuple[str, ...] = ("orders_daily", "orders_merge", "orders_table", "orders_view") +_RETAINED: tuple[tuple[str, str], ...] = ( + ("orders_table", "BASE TABLE"), + ("orders_view", "VIEW"), + ("raw_orders", "BASE TABLE"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + FingerprintWriteTestCase( + description="an unchanged build appends no row and sets up no table", + edited_models=(), + expected_setups=0, + expected_rows=dict.fromkeys(_ALL_MODELS, 1), + ), + FingerprintWriteTestCase( + description="a changed model records its new identity once, as it builds", + edited_models=("orders_view",), + expected_setups=1, + expected_rows={**dict.fromkeys(_ALL_MODELS, 1), "orders_view": 2}, + ), + ], + ids=lambda case: case.description, +) +def test_given_second_build_when_identities_are_unchanged_then_only_changed_models_append( + test_case: FingerprintWriteTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + project_dir: Path = write_orders_project(project_dir=tmp_path) + first_code, first_output, _, _ = run_counted( + project_dir=project_dir, args=("build",), capsys=capsys + ) + name: str + for name in test_case.edited_models: + revise_model(project_dir, name) + + code, output, counts, _ = run_counted(project_dir=project_dir, args=("build",), capsys=capsys) + third_code, third_output, third_counts, _ = run_counted( + project_dir=project_dir, args=("build",), capsys=capsys + ) + + assert (first_code, code, third_code) == (0, 0, 0), first_output + output + third_output + assert counts.get(_FINGERPRINT_SETUP, 0) == test_case.expected_setups + assert counts.get(_FINGERPRINT_INSERT, 0) == test_case.expected_setups + assert third_counts.get(_FINGERPRINT_INSERT, 0) == test_case.expected_later_inserts + assert fingerprint_rows(project_dir) == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + FingerprintLossTestCase( + description="a view's post hook drops the table after the table model recorded", + edited_models=("orders_table",), + hooked_model="orders_view", + expected_setups=2, + expected_rows_after_loss={"orders_view": 1}, + expected_models_after_rebuild=_ALL_MODELS, + ), + ], + ids=lambda case: case.description, +) +def test_given_fingerprint_table_dropped_mid_build_when_building_then_later_models_record( + test_case: FingerprintLossTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + project_dir: Path = write_orders_project(project_dir=tmp_path) + _ = (project_dir / "sqlbuild_project.toml").write_text( + (project_dir / "sqlbuild_project.toml").read_text(encoding="utf-8") + + "\n[settings]\nconcurrency = 1\n", + encoding="utf-8", + ) + first_code, first_output, _, _ = run_counted( + project_dir=project_dir, args=("build",), capsys=capsys + ) + name: str + for name in test_case.edited_models: + revise_model(project_dir, name) + original: str = add_post_hook( + project_dir, test_case.hooked_model, "DROP TABLE main._sqlbuild_fingerprints" + ) + + code, output, counts, _ = run_counted(project_dir=project_dir, args=("build",), capsys=capsys) + after_loss: dict[str, int] = fingerprint_rows(project_dir) + _ = (project_dir / "models" / f"{test_case.hooked_model}.sql").write_text( + original, encoding="utf-8" + ) + third_code, third_output, _, _ = run_counted( + project_dir=project_dir, args=("build",), capsys=capsys + ) + + assert (first_code, code, third_code) == (0, 0, 0), first_output + output + third_output + assert "fingerprint write" not in output.lower() + assert counts.get(_FINGERPRINT_SETUP, 0) == test_case.expected_setups + assert after_loss == test_case.expected_rows_after_loss + assert tuple(fingerprint_rows(project_dir)) == test_case.expected_models_after_rebuild + + +@pytest.mark.parametrize( + "test_case", + [ + LeftoverRelationTestCase( + description="a staging table with other columns left by an interrupted rebuild", + leftover_sql="CREATE TABLE main.orders_table__staging AS SELECT 'stale' AS note", + expected_rows={"orders_table": 3, "orders_view": 3}, + ), + LeftoverRelationTestCase( + description="a delta table with other columns left by an interrupted incremental build", + leftover_sql="CREATE TABLE main.orders_merge__delta AS SELECT 'stale' AS note", + expected_rows={"orders_merge": 3}, + ), + LeftoverRelationTestCase( + description="a microbatch delta left by a failed batch", + leftover_sql=( + "CREATE TABLE main.orders_daily__delta AS " + "SELECT 99 AS order_id, TIMESTAMP '2026-01-02 00:00:00' AS ordered_at, " + "1.00 AS amount" + ), + expected_rows={"orders_daily": 3}, + ), + ], + ids=lambda case: case.description, +) +def test_given_leftover_intermediate_when_building_then_it_is_replaced( + test_case: LeftoverRelationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + project_dir: Path = write_orders_project(project_dir=tmp_path) + first_code, first_output, _, _ = run_counted( + project_dir=project_dir, args=("build",), capsys=capsys + ) + execute_sql(project_dir, test_case.leftover_sql) + + code, output, _, _ = run_counted(project_dir=project_dir, args=("build",), capsys=capsys) + + assert (first_code, code) == (0, 0), first_output + output + assert row_counts(project_dir, tuple(test_case.expected_rows)) == test_case.expected_rows + assert intermediate_relations(project_dir) == test_case.expected_intermediates + + +@pytest.mark.parametrize( + "test_case", + [ + ScenarioLeftoverTestCase( + description="a retained run's relations are dropped and rebuilt", + retained_runs=1, + planted_sql=(), + expected_kinds=_RETAINED, + ), + ScenarioLeftoverTestCase( + description="a view at a scenario table's name is replaced by the table", + retained_runs=1, + planted_sql=("DROP TABLE {table}; CREATE VIEW {table} AS SELECT 'stale' AS note",), + expected_kinds=_RETAINED, + ), + ScenarioLeftoverTestCase( + description="a first run in an empty namespace", + retained_runs=0, + planted_sql=(), + expected_kinds=_RETAINED, + ), + ], + ids=lambda case: case.description, +) +def test_given_scenario_leftovers_when_testing_again_then_the_scenario_passes( + test_case: ScenarioLeftoverTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + project_dir: Path = write_orders_project(project_dir=tmp_path) + retain: tuple[str, ...] = ("scenario", "test", "--retain") + earlier: tuple[tuple[int, str, dict[str, int], tuple[str, ...]], ...] = tuple( + run_counted(project_dir=project_dir, args=retain, capsys=capsys) + for _ in range(test_case.retained_runs) + ) + sql: str + for sql in test_case.planted_sql: + execute_sql( + project_dir, + sql.format( + table=scenario_relation_name(project_dir, "\\_\\_model\\_\\_orders\\_table") + ), + ) + + code, output, _, _ = run_counted(project_dir=project_dir, args=retain, capsys=capsys) + rerun: tuple[tuple[str, str], ...] = scenario_relation_kinds(project_dir) + final_code, final_output, _, _ = run_counted( + project_dir=project_dir, args=("scenario", "test"), capsys=capsys + ) + + assert tuple(run[0] for run in earlier) == (0,) * test_case.retained_runs + assert (code, final_code) == (0, 0), output + final_output + assert "PASS=1" in output + assert rerun == test_case.expected_kinds + assert scenario_relation_kinds(project_dir) == test_case.expected_after_cleanup + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_statement_budget.py b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_statement_budget.py new file mode 100644 index 0000000000..8460cb562a --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_statement_budget.py @@ -0,0 +1,113 @@ +"""Housekeeping statements real CLI commands issue on DuckDB, counted by kind. + +Model work is not budgeted; the kinds below are the per-model and per-batch housekeeping that a +remote warehouse pays a round trip for: drops of build intermediates, fingerprint writes, delta +row counts, and column reads. +""" + +from __future__ import annotations + +from itertools import chain +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.statement_counts._test_types import ( + StatementBudgetTestCase, + StatementStep, +) +from tests.integration.src.sqlbuild.cli.commands.main.statement_counts.helpers import ( + budgeted, + recreated_after_drop, + run_step, + write_orders_project, +) + +_BUDGETED_PREFIXES: tuple[str, ...] = ( + "DROP ", + "SELECT COUNT ", + "INFORMATION_SCHEMA ", + "CREATE TABLE IF NOT EXISTS _sqlbuild_fingerprints", + "INSERT _sqlbuild_fingerprints", +) +_FIRST_BUILD: dict[str, int] = { + "CREATE TABLE IF NOT EXISTS _sqlbuild_fingerprints": 1, + "DROP TABLE delta": 1, + "DROP TABLE relation": 2, + "INFORMATION_SCHEMA COLUMNS": 4, + "INFORMATION_SCHEMA TABLES": 7, + "INSERT _sqlbuild_fingerprints": 4, + "SELECT COUNT delta": 1, +} +_UNCHANGED_BUILD: dict[str, int] = { + "DROP TABLE delta": 2, + "DROP TABLE relation": 2, + "DROP TABLE staging": 1, + "INFORMATION_SCHEMA COLUMNS": 6, + "INFORMATION_SCHEMA TABLES": 3, +} + + +@pytest.mark.parametrize( + "test_case", + [ + StatementBudgetTestCase( + description="table, view, merge and microbatch models, then an audit, tests and a scenario", + steps=( + StatementStep("first build", ("build",), 0, _FIRST_BUILD), + StatementStep( + "a new order", + ("build",), + 1, + _UNCHANGED_BUILD, + ), + StatementStep( + "nothing changed", + ("build",), + 0, + _UNCHANGED_BUILD, + ), + StatementStep("audit", ("audit",), 0, {}), + StatementStep("unit tests", ("test",), 0, {"DROP TABLE relation": 2}), + StatementStep( + "scenario", + ("scenario", "test"), + 0, + { + "DROP TABLE relation": 2, + "DROP VIEW relation": 1, + "INFORMATION_SCHEMA COLUMNS": 1, + "INFORMATION_SCHEMA TABLES": 2, + }, + ), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_orders_project_when_running_commands_then_housekeeping_stays_within_budget( + test_case: StatementBudgetTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = write_orders_project(project_dir=tmp_path) + + runs: tuple[tuple[int, str, dict[str, int], tuple[str, ...]], ...] = tuple( + run_step(project_dir=project_dir, args=step.args, new_orders=step.new_orders, capsys=capsys) + for step in test_case.steps + ) + + assert [ + (step.description, run[0]) for step, run in zip(test_case.steps, runs, strict=True) + ] == [(step.description, 0) for step in test_case.steps], "".join(run[1] for run in runs) + assert [budgeted(run[2], _BUDGETED_PREFIXES) for run in runs] == [ + step.expected_counts for step in test_case.steps + ] + assert ( + tuple(chain.from_iterable(recreated_after_drop(run[3]) for run in runs)) + == test_case.expected_recreated_after_drop + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 47e5cad318..8640a900ea 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -154,7 +154,7 @@ JSON `status` is `failed`. ## Fingerprints -After a successful build, SQLBuild writes version identities to `_sqlbuild_fingerprints` in the target schema. These are used on subsequent runs to detect changes and skip unchanged work. See [Planning and Change Detection](/docs/concepts/planning) for details. +After a successful build, SQLBuild writes version identities to `_sqlbuild_fingerprints` in the target schema. These are used on subsequent runs to detect changes and skip unchanged work. A build writes a row only for a resource whose identity differs from its latest row, so a build that changes no definitions writes none. In a project with model migration history, every build writes a row for each resource it builds. See [Planning and Change Detection](/docs/concepts/planning) for details. ## Runtime artifacts diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 829697736e..2d07f75c78 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -58,10 +58,19 @@ Run `sqb plan -v` to see each metadata query with its elapsed time and row count ## Build metadata -During a build, checks for whether one relation exists use `SHOW TERSE TABLES` and -`SHOW TERSE VIEWS`, and reads of one relation's columns use `SHOW COLUMNS`, so they also don't -need a running warehouse. Each build reuses these answers until it changes that relation, for -example by replacing a delta table or adding a column, and then reads it again. +Builds and scenario runs read relation metadata with `SHOW` commands too, so these reads don't +need a running warehouse either: + +| Read | Statement | +| -- | -- | +| Whether one relation exists | `SHOW TERSE TABLES` and `SHOW TERSE VIEWS` | +| A second relation in the same schema | `SHOW TABLES` and `SHOW VIEWS` for the whole schema, once | +| A table's time travel retention | `SHOW TABLES LIKE`, or the schema listing | +| One relation's columns | `SHOW COLUMNS` | +| A scenario's leftover relations | `SHOW TERSE OBJECTS` for the scenario's name prefix | + +Each run reuses these answers until it changes that relation, for example by replacing a delta +table or adding a column, and then reads it again. ## State tables From 1cf9a78857db7db2ececcd9867ac2b2f63964be5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:59:27 +0000 Subject: [PATCH 06/12] feat(compile): write a Rules-failed compile's SQL to target/compiled-failed/ A compile whose Rules fail writes every model's SQL to target/compiled-failed/, laid out like target/compiled/, with the not-buildable header on failing models, and never touches target/compiled/. Each failing compile replaces the folder as a whole and the next passing compile deletes it, so target/compiled/ always holds exactly the last passing compile. The report names each failing model's file. The record of written model files is gone. Compile reuse, delta compiles and delta verification cover the new folder: a failing delta writes the models it did not rerun from the stored compile, and a passing delta after a failed compile brings target/compiled/ up to date. --- .../src/project_reuse/_helpers/target.rs | 41 ++- .../src/project_reuse/constants.rs | 2 + .../src/cli_text/main/format_compile_text.rs | 3 + crates/sqlbuild-core/src/cli_text/models.rs | 2 + .../src/cli_text/tests/test_cli_text.rs | 65 +++-- .../_helpers/compile_command/report.rs | 28 +- .../_helpers/compile_command/reuse_attempt.rs | 5 + .../compile_command/reuse_recording.rs | 37 +++ .../_helpers/compile_command/target_writer.rs | 262 ++++++++++-------- .../bindings/_helpers/compile_delta/splice.rs | 8 +- .../bindings/_helpers/compile_delta/verify.rs | 36 ++- .../_helpers/compile_driver/command.rs | 32 ++- .../_helpers/compile_outputs/reuse.rs | 20 +- .../bindings/_helpers/records/constants.rs | 2 + scripts/cold_compile_performance/constants.py | 1 + .../main/semantic_compile_fingerprint.py | 29 +- .../_helpers/measure.py | 24 +- .../compile_performance_ratio/constants.py | 3 +- .../_helpers/goldens/goldens.py | 29 +- .../_helpers/running/execution.py | 2 + scripts/compiler_differential/constants.py | 1 + scripts/compiler_differential/models.py | 1 + .../sqlbuild/references/docs/cli/compile.md | 30 +- .../docs/concepts/macros/sql-macros.md | 2 +- src/sqlbuild/_native.pyi | 4 + .../cli/commands/main/compile/helpers.py | 13 +- .../compiler/failure/rule-custom-finding.json | 3 +- ...e-unused-cte-after-variable-and-macro.json | 3 +- .../compiler/failure/rule-unused-cte.json | 3 +- .../sqlbuild/cli/commands/main/_test_types.py | 13 +- .../src/sqlbuild/cli/commands/main/helpers.py | 7 + .../main/test_compile_prepared_artifacts.py | 11 +- .../main/test_compile_rule_failures.py | 240 +++++++++++++--- .../sqlbuild/cli/commands/main/test_rules.py | 2 +- .../_helpers/goldens/test_compiler_goldens.py | 23 ++ website/src/content/docs/docs/cli/compile.mdx | 30 +- .../docs/docs/concepts/macros/sql-macros.mdx | 2 +- 37 files changed, 756 insertions(+), 263 deletions(-) diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs index 649a4737ae..21e66f3c93 100644 --- a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use crate::project_reuse::_helpers::files::file_digest; +use crate::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use crate::project_reuse::models::{CompileRecord, RecordedArtifact, ReuseRules}; use crate::project_snapshot::main::stamp_path::stamp_path; use crate::project_snapshot::models::PathStamp; @@ -20,10 +21,18 @@ fn compiled_root(project_dir: &Path, rules: &ReuseRules) -> PathBuf { project_dir.join(&rules.compiled_root) } -/// Stat every compiled artifact plus `extra_paths`, keyed by path. +/// The failed-compile directory beside the compiled directory. +fn failed_root(project_dir: &Path, rules: &ReuseRules) -> PathBuf { + compiled_root(project_dir, rules).with_file_name(FAILED_COMPILED_DIRECTORY_NAME) +} + +/// Stat every compiled and failed-compile artifact plus `extra_paths`, keyed by path. fn snapshot_target_files(project_dir: &Path, extra_paths: &[&str], rules: &ReuseRules) -> Stamps { let mut stamps: Stamps = BTreeMap::new(); - let mut pending: Vec = vec![compiled_root(project_dir, rules)]; + let mut pending: Vec = vec![ + compiled_root(project_dir, rules), + failed_root(project_dir, rules), + ]; while let Some(directory) = pending.pop() { let Ok(entries) = std::fs::read_dir(&directory) else { continue; @@ -58,21 +67,29 @@ fn target_stamps(project_dir: &Path, paths: &[&str], tree: bool, rules: &ReuseRu if !tree { return paths.iter().copied().filter_map(keyed_stamp).collect(); } - let prefix: String = compiled_prefix(project_dir, rules); + let prefixes: [String; 2] = tree_prefixes(project_dir, rules); let extra: Vec<&str> = paths .iter() .copied() - .filter(|path| !path.starts_with(prefix.as_str())) + .filter(|path| !in_trees(path, &prefixes)) .collect(); snapshot_target_files(project_dir, &extra, rules) } -fn compiled_prefix(project_dir: &Path, rules: &ReuseRules) -> String { - format!( - "{}{}", - compiled_root(project_dir, rules).to_string_lossy(), - std::path::MAIN_SEPARATOR - ) +/// The compiled and failed-compile directories, each with a trailing separator. +fn tree_prefixes(project_dir: &Path, rules: &ReuseRules) -> [String; 2] { + [ + compiled_root(project_dir, rules), + failed_root(project_dir, rules), + ] + .map(|root| format!("{}{}", root.to_string_lossy(), std::path::MAIN_SEPARATOR)) +} + +/// Whether `path` is under one of the `prefixes` directories. +fn in_trees(path: &str, prefixes: &[String]) -> bool { + prefixes + .iter() + .any(|prefix| path.starts_with(prefix.as_str())) } /// Whether `stamp` shows what this compile left there, and whether that read its content. @@ -123,13 +140,13 @@ pub(crate) fn verified_target_files( since_ns: i64, rules: &ReuseRules, ) -> Option> { - let prefix: String = compiled_prefix(project_dir, rules); + let prefixes: [String; 2] = tree_prefixes(project_dir, rules); let dag_path: Option<&str> = record.dag_path.as_deref(); let written: bool = record.artifacts_written; let expected: BTreeMap<&str, &RecordedArtifact> = record .artifacts .iter() - .filter(|(path, _)| path.starts_with(prefix.as_str()) || Some(path.as_str()) == dag_path) + .filter(|(path, _)| in_trees(path, &prefixes) || Some(path.as_str()) == dag_path) .map(|(path, artifact)| (path.as_str(), artifact)) .collect(); let paths: Vec<&str> = expected.keys().copied().collect(); diff --git a/crates/sqlbuild-cache/src/project_reuse/constants.rs b/crates/sqlbuild-cache/src/project_reuse/constants.rs index 17ffaad603..c88b3fd5c4 100644 --- a/crates/sqlbuild-cache/src/project_reuse/constants.rs +++ b/crates/sqlbuild-cache/src/project_reuse/constants.rs @@ -8,3 +8,5 @@ pub const REUSE_IMAGE_SUFFIX: &str = "image"; pub const REUSE_MAX_STORED_IMAGES: usize = 2; /// Prefix of digests computed natively, so they never equal a Python digest. pub const NATIVE_DIGEST_PREFIX: &str = "b3:"; +/// The sibling of the compiled directory that holds a compile whose Rules failed. +pub const FAILED_COMPILED_DIRECTORY_NAME: &str = "compiled-failed"; diff --git a/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs b/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs index 87f9e1d43e..3c00f2252f 100644 --- a/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs +++ b/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs @@ -83,6 +83,9 @@ pub fn format_compile_text(report: &CompileTextReport) -> String { "Project compiled", &summary, )); + for path in &report.failed_model_sql { + lines.push(format!(" {} {path}", style.muted("Failed SQL:"))); + } lines.push(format!( " {} {}/", style.muted("Wrote:"), diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index a7edb05016..6d22f84d96 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -232,6 +232,8 @@ pub struct CompileTextReport { pub warning_count: usize, /// The compiled SQL directory relative to `target/`'s parent, without a trailing slash. pub compiled_dir: String, + /// Each failing model's SQL file under `compiled_dir` when Rules failed, relative like it. + pub failed_model_sql: Vec, pub manifest: bool, pub lineage: LineageNote, } diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 60b9f89cc3..f4f6cfd9a4 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -104,27 +104,52 @@ fn given_partial_reasons_when_building_notice_then_groups_by_reason() { #[test] fn given_report_when_formatting_then_lists_models_and_summary() { - let test_cases = [CompileTextTestCase { - description: "one plain model with a manifest and unavailable lineage", - report: CompileTextReport { - style: CliStyle { use_color: false }, - models: vec![ModelLine { - name: "orders".to_owned(), - failed: false, - column_count: 2, - }], - semantic_notice: None, - diagnostics: Vec::new(), - seed_count: 1, - function_count: 0, - error_count: 0, - warning_count: 0, - compiled_dir: "target/compiled".to_owned(), - manifest: true, - lineage: LineageNote::Unavailable, + let test_cases = [ + CompileTextTestCase { + description: "one plain model with a manifest and unavailable lineage", + report: CompileTextReport { + style: CliStyle { use_color: false }, + models: vec![ModelLine { + name: "orders".to_owned(), + failed: false, + column_count: 2, + }], + semantic_notice: None, + diagnostics: Vec::new(), + seed_count: 1, + function_count: 0, + error_count: 0, + warning_count: 0, + compiled_dir: "target/compiled".to_owned(), + failed_model_sql: Vec::new(), + manifest: true, + lineage: LineageNote::Unavailable, + }, + expected_text: "\nCompile ready 1 model\n\n\u{2514}\u{2500}\u{2500} orders OK 2 columns\n\n\u{2713} Project compiled 1 model, 1 seed, 0 functions, 0 errors, 0 warnings\n Wrote: target/compiled/\n Wrote: target/manifest.json\n Column lineage: unavailable\n", }, - expected_text: "\nCompile ready 1 model\n\n\u{2514}\u{2500}\u{2500} orders OK 2 columns\n\n\u{2713} Project compiled 1 model, 1 seed, 0 functions, 0 errors, 0 warnings\n Wrote: target/compiled/\n Wrote: target/manifest.json\n Column lineage: unavailable\n", - }]; + CompileTextTestCase { + description: "a compile whose Rules failed names each failing model's SQL", + report: CompileTextReport { + style: CliStyle { use_color: false }, + models: vec![ModelLine { + name: "orders".to_owned(), + failed: true, + column_count: 2, + }], + semantic_notice: None, + diagnostics: Vec::new(), + seed_count: 0, + function_count: 0, + error_count: 1, + warning_count: 0, + compiled_dir: "target/compiled-failed".to_owned(), + failed_model_sql: vec!["target/compiled-failed/models/orders.sql".to_owned()], + manifest: false, + lineage: LineageNote::Silent, + }, + expected_text: "\nCompile ready 1 model\n\n\u{2514}\u{2500}\u{2500} orders FAIL 2 columns\n\n\u{2717} Project compiled 1 model, 0 seeds, 0 functions, 1 error, 0 warnings\n Failed SQL: target/compiled-failed/models/orders.sql\n Wrote: target/compiled-failed/\n", + }, + ]; for test_case in test_cases { assert_eq!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 9b8cd39722..017d0f13ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -5,6 +5,7 @@ use pyo3::types::{ PyDict, PyDictMethods, PyList, PySet, PySetMethods, PyString, PyTuple, PyTupleMethods, }; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use sqlbuild_core::cli_text::main::format_compile_text::format_compile_text as compile_text; use sqlbuild_core::cli_text::main::semantic_coverage_notice::semantic_coverage_notice as notice_text; use sqlbuild_core::cli_text::models::{ @@ -182,11 +183,15 @@ impl<'py> ReportInputs<'py> { coverage_notice(&self.semantic_coverage()?, reader) } - /// `written.target_dir / "compiled"` relative to its `target` directory, POSIX style. + /// `written.target_dir / "compiled"`, or `"compiled-failed"` when Rules failed, relative to its `target` directory, POSIX style. fn compiled_dir(&self) -> PyResult> { let py: Python<'py> = self.py(); - let path: Bound<'py, PyAny> = - native_field(&self.written, "target_dir")?.div(COMPILED_DIRECTORY)?; + let directory: &str = if native_field(&self.written, "failed_compile")?.is_truthy()? { + FAILED_COMPILED_DIRECTORY_NAME + } else { + COMPILED_DIRECTORY + }; + let path: Bound<'py, PyAny> = native_field(&self.written, "target_dir")?.div(directory)?; let parts: Bound<'py, PyTuple> = native_field(&path, "parts")?.downcast_into()?; for (index, part) in parts.iter().enumerate() { if part.eq(TARGET_DIRECTORY)? { @@ -198,6 +203,16 @@ impl<'py> ReportInputs<'py> { } path.call_method0("as_posix") } + + /// Each failing model's SQL file under `compiled_dir`, relative like it, when Rules failed. + fn failed_model_sql(&self) -> PyResult> { + let directory: String = self.compiled_dir()?.extract()?; + Ok(native_field(&self.written, "failed_model_sql")? + .extract::>()? + .into_iter() + .map(|relative| format!("{directory}/{relative}")) + .collect()) + } } /// `int(value)` of a location field, or `None`. @@ -392,6 +407,7 @@ pub(crate) fn format_compile_text<'py>( error_count: inputs.severity_count(ERROR_SEVERITY)?, warning_count: inputs.severity_count(WARNING_SEVERITY)?, compiled_dir: reader.text(&inputs.compiled_dir()?)?, + failed_model_sql: inputs.failed_model_sql()?, manifest: inputs.manifest, lineage, }; @@ -1036,6 +1052,12 @@ fn compile_json<'py>( ), ], )?; + if native_field(&inputs.written, "failed_compile")?.is_truthy()? { + artifacts.set_item( + "failed_model_sql", + PyList::new(py, inputs.failed_model_sql()?)?, + )?; + } let report: Bound<'_, PyDict> = dict( py, vec![ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 445eda2c01..9d6db0071b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -8,6 +8,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; @@ -554,6 +555,7 @@ fn compiled_with_delta<'py>( .join(TARGET_DIRECTORY_NAME) .join(COMPILED_DIRECTORY_NAME), )?; + let failed_root: PathBuf = compiled_root.with_file_name(FAILED_COMPILED_DIRECTORY_NAME); match delta { DeltaOutcome::Finished(finished) if !verifying => { keep_delta_claim(); @@ -595,10 +597,13 @@ fn compiled_with_delta<'py>( raised: None, compiled: verify::compiled_digests(&compiled_root), compiled_root: compiled_root.to_string_lossy().into_owned(), + failed: verify::compiled_digests(&failed_root), + failed_root: failed_root.to_string_lossy().into_owned(), }), DeltaOutcome::Raised(error) => verify::hold_expected(verify::Expected { raised: Some(error.to_string()), compiled_root: compiled_root.to_string_lossy().into_owned(), + failed_root: failed_root.to_string_lossy().into_owned(), ..verify::Expected::default() }), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 400b3c3faf..ee4456afa7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -1,6 +1,7 @@ //! What one full compile printed and left in each artifact, and storing it for compile reuse. use sqlbuild_cache::project_reuse::types::ImageSections; +use sqlbuild_cache::project_snapshot::main::stamp_path::stamp_path; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Mutex, PoisonError}; @@ -82,6 +83,14 @@ impl CompileArtifactWrites { } impl CompileArtifactWrites { + /// Note a file this compile left as an earlier compile wrote it, unless it already recorded the path. + pub(crate) fn keep_unrecorded(&self, path: String, (size, mtime_ns): (u64, i64)) { + let _ = self + .locked() + .entry(path) + .or_insert(RecordedArtifact::Kept { size, mtime_ns }); + } + /// Every recorded artifact's absolute path, with its digest when this compile wrote it. pub(crate) fn written_digests(&self) -> HashMap> { self.locked() @@ -202,6 +211,34 @@ pub(crate) fn artifact_kept( Ok(()) } +/// Note every file under `root` this compile did not record as kept as an earlier compile left it. +pub(crate) fn artifact_tree_kept( + py: Python<'_>, + recorder: ArtifactRecorder<'_>, + root: &Path, +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + let mut pending: Vec = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + let Ok(entries) = std::fs::read_dir(&directory) else { + continue; + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + if entry.file_type().is_ok_and(|kind| kind.is_dir()) { + pending.push(path); + } else if let Some(stamp) = stamp_path(&path) { + recorder + .get() + .keep_unrecorded(absolute_path(py, &path)?, (stamp.size, stamp.mtime_ns)); + } + } + } + Ok(()) +} + /// Note that one staged artifact was published at another path. pub(crate) fn artifact_moved( recorder: ArtifactRecorder<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 11fe1c157d..8e6b4900cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -1,6 +1,6 @@ //! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; @@ -12,6 +12,7 @@ use pyo3::{IntoPyObjectExt, pyclass, pyfunction, wrap_pyfunction}; use sqlbuild_cache::artifacts::main::new_sql_test_record::new_sql_test_record; use sqlbuild_cache::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; use sqlbuild_cache::artifacts::models::{SqlTestArtifactRecord, SqlTestRecordMiss}; +use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::ExecutorHost; @@ -22,7 +23,8 @@ use crate::bindings::_helpers::boundary::python_calls::{ use crate::bindings::_helpers::compile_command::artifact_batch::ArtifactBatch; use crate::bindings::_helpers::compile_command::constants::RACY_WINDOW_NS; use crate::bindings::_helpers::compile_command::reuse_recording::{ - ArtifactRecorder, CompileArtifactWrites, artifact_kept, artifact_moved, artifact_tree_moved, + ArtifactRecorder, CompileArtifactWrites, artifact_kept, artifact_moved, artifact_tree_kept, + artifact_tree_moved, }; use crate::bindings::_helpers::compile_command::test_artifact_cache::{ IdentityContext, artifact_identity_payload, artifact_record_key_payload, identity_context, @@ -58,8 +60,6 @@ const TESTS_DIR: &str = "tests"; const MANIFEST_FILE: &str = "manifest.json"; /// The compile cache file stamping the compiled tree the last stale-file walk left. const COMPILED_TREE_STAMP_FILE: &str = "compiled-tree.bin"; -/// The model files compiles wrote under `target/compiled/`, one per line, relative to it. -const COMPILED_MODELS_FILE: &str = "compiled-models.txt"; const SQL_FILE_SUFFIX: &str = ".sql"; const SQL_TEST_CACHE_LOGGER: &str = "sqlbuild.compile.reuse"; @@ -324,15 +324,8 @@ fn write_planned_target<'py>( managed.push(path); } batch.flush(py)?; - let managed: Vec = extracted(&managed)?; - record_compiled_models( - &target_dir.extract::()?, - &compiled.extract::()?, - &managed, - record_mode(remove_stale_files), - )?; if remove_stale_files { - remove_stale(target_dir, managed, None)?; + remove_stale(target_dir, extracted(&managed)?, None)?; } write_manifest(target_dir, manifest, None)?; written_target( @@ -736,15 +729,9 @@ pub(crate) fn write_static_target<'py>( })(); let (test_paths, diagnostics) = batch.settle(py, queued)?; managed.extend(test_paths); - let mut paths: Vec = managed_models; - paths.extend(extracted(&managed)?); - record_compiled_models( - &target_dir.extract::()?, - &compiled_dir, - &paths, - record_mode(remove_stale_files), - )?; if remove_stale_files { + let mut paths: Vec = managed_models; + paths.extend(extracted(&managed)?); let cache_dir: Bound<'py, PyAny> = native_field(project, "compile_cache_dir")?; let cache_dir: Option = if cache_dir.is_none() { None @@ -822,23 +809,23 @@ fn rule_error_lines( Ok(lines) } -/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. An earlier manifest is removed, since it no longer describes `target/`, and so is a model file an earlier compile wrote for a model the project no longer has. Every other artifact stays as the last compile left it, for the next passing compile to publish or remove. +/// After Rules failed, write every model's compiled SQL to `target/compiled-failed/` and remove every other file there, so it holds exactly the expansion these Rules checked. A model with Rule errors starts with a header marking it not buildable and listing them. `target/compiled/` keeps the last passing compile; only an earlier manifest is removed, since it no longer describes `target/`. Returns the failing models' files, relative to `target/compiled-failed/` and sorted. pub(crate) fn write_rule_failed_models<'py>( target_dir: &Bound<'py, PyAny>, (project, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), diagnostics: &Bound<'py, PyAny>, -) -> PyResult<()> { + recorder: ArtifactRecorder<'_>, +) -> PyResult> { let py: Python<'py> = target_dir.py(); let errors: HashMap> = rule_error_lines(diagnostics, project_dir)?; let _ = prepared_target(target_dir)?; - let manifest: PathBuf = target_dir.extract::()?.join(MANIFEST_FILE); - match std::fs::remove_file(&manifest) { + let target: PathBuf = target_dir.extract()?; + match std::fs::remove_file(target.join(MANIFEST_FILE)) { Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error.into()), _ => {} } - let compiled_dir: PathBuf = joined(target_dir, &[COMPILED_DIR])?.extract()?; - let mut batch: ArtifactBatch = ArtifactBatch::new(true, None); - let mut written: Vec = Vec::new(); + let failed_dir: PathBuf = target.join(FAILED_COMPILED_DIRECTORY_NAME); + let mut models: Vec<(String, String)> = Vec::new(); for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; let (relative_path, query_sql): (String, String) = @@ -851,96 +838,161 @@ pub(crate) fn write_rule_failed_models<'py>( native_field(&model, "query_sql")?.extract()?, ) }; + models.push((relative_path, query_sql)); + } + if let Some(injection) = crate::bindings::_helpers::compile_context::delta::injection() { + models.extend(models_not_rerun( + &target, + &models, + &injection.project_model_paths, + )?); + } + let mut batch: ArtifactBatch = ArtifactBatch::new(true, recorder); + let mut failing: Vec = Vec::new(); + let mut written: Vec = Vec::new(); + for (relative_path, query_sql) in models { + let path: PathBuf = native_model_output_path(&failed_dir, &relative_path); let sql: String = match errors.get(&relative_path) { - Some(lines) => format!("{RULE_FAILED_HEADER}\n{}\n{query_sql}", lines.join("\n")), + Some(lines) => { + failing.push(failed_relative(&failed_dir, &path)); + format!("{RULE_FAILED_HEADER}\n{}\n{query_sql}", lines.join("\n")) + } None => query_sql, }; - let path: PathBuf = native_model_output_path(&compiled_dir, &relative_path); - batch.queue_sql_at(py, path.clone(), &sql)?; - written.push(path); + written.push(path.clone()); + batch.queue_sql_at(py, path, &sql)?; } batch.flush(py)?; - record_compiled_models( - &target_dir.extract::()?, - &compiled_dir, - &written, - ModelRecord::RemoveRetired, - ) + let _ = remove_stale_artifacts(py, failed_dir, written, None)?; + failing.sort_unstable(); + Ok(failing) } -fn record_mode(removes_stale_files: bool) -> ModelRecord { - if removes_stale_files { - ModelRecord::Replace - } else { - ModelRecord::Extend +/// After a passing delta compile whose stored compile failed its Rules, bring `target/compiled/` up to that compile: write each model the delta did not rerun from `target/compiled-failed/` without its header, remove model files the project no longer has, and record every other compiled file as kept. +pub(crate) fn restore_models_after_failed_compile<'py>( + target_dir: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + recorder: ArtifactRecorder<'_>, +) -> PyResult<()> { + let py: Python<'py> = target_dir.py(); + let Some(injection) = crate::bindings::_helpers::compile_context::delta::injection() else { + return Ok(()); + }; + let target: PathBuf = target_dir.extract()?; + if !target.join(FAILED_COMPILED_DIRECTORY_NAME).is_dir() { + return Ok(()); } + let mut rerun: Vec<(String, String)> = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let relative_path: String = if let Ok(native) = model.cast::() { + native.get().output_texts(py)?.0 + } else { + path_posix_text(&native_field(&model, "relative_path")?)? + }; + rerun.push((relative_path, String::new())); + } + let compiled_dir: PathBuf = target.join(COMPILED_DIR); + let mut batch: ArtifactBatch = ArtifactBatch::new(true, recorder); + for (relative_path, query_sql) in + models_not_rerun(&target, &rerun, &injection.project_model_paths)? + { + batch.queue_sql_at( + py, + native_model_output_path(&compiled_dir, &relative_path), + &query_sql, + )?; + } + batch.flush(py)?; + let managed: Vec = injection + .project_model_paths + .iter() + .map(|relative_path| native_model_output_path(&compiled_dir, relative_path)) + .collect(); + let _ = remove_stale_artifacts(py, compiled_dir.join(MODELS_DIR), managed, None)?; + artifact_tree_kept(py, recorder, &compiled_dir) } -/// How a compile updates the record of the model files SQLBuild wrote under `target/compiled/`. -#[derive(Clone, Copy, PartialEq, Eq)] -enum ModelRecord { - /// A passing compile that removed stale files: the record is exactly what it wrote. - Replace, - /// A passing compile that kept the files already there: the record gains what it wrote. - Extend, - /// A compile whose Rules failed: other recorded models are retired, so their files are removed. - RemoveRetired, -} - -/// Update the record of written model files, one path per line relative to `compiled_dir`. -fn record_compiled_models( +/// In a delta compile, the compiled SQL of each project model it did not rerun, unchanged since the stored compile: read from `target/compiled-failed/` without its header when that compile failed its Rules, otherwise from `target/compiled/`. +fn models_not_rerun( target: &Path, - compiled_dir: &Path, - written: &[PathBuf], - mode: ModelRecord, -) -> PyResult<()> { - let record: PathBuf = target.join(COMPILED_MODELS_FILE); - let previous: String = match std::fs::read_to_string(&record) { - Ok(text) => text, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), - Err(error) => return Err(error.into()), + rerun: &[(String, String)], + project_model_paths: &[String], +) -> PyResult> { + let rerun: HashSet<&str> = rerun.iter().map(|(path, _)| path.as_str()).collect(); + let failed_dir: PathBuf = target.join(FAILED_COMPILED_DIRECTORY_NAME); + let stored_failed: bool = failed_dir.is_dir(); + let stored_dir: PathBuf = if stored_failed { + failed_dir + } else { + target.join(COMPILED_DIR) }; - let models_dir: PathBuf = compiled_dir.join(MODELS_DIR); - let mut lines: BTreeSet = BTreeSet::new(); - for path in written { - match path.strip_prefix(&models_dir) { - Ok(relative) => { - let _ = lines.insert(format!( - "{MODELS_DIR}/{}", - relative.to_string_lossy().replace('\\', "/") - )); - } - Err(_not_a_model) => continue, - } - } - for line in previous.lines().filter(|line| !line.is_empty()) { - if lines.contains(line) { + let mut models: Vec<(String, String)> = Vec::new(); + for relative_path in project_model_paths { + if rerun.contains(relative_path.as_str()) { continue; } - match mode { - ModelRecord::Replace => {} - ModelRecord::Extend => { - let _ = lines.insert(line.to_owned()); - } - ModelRecord::RemoveRetired => { - let path: PathBuf = compiled_dir.join(line); - if !path.starts_with(&models_dir) { - continue; - } - match std::fs::remove_file(&path) { - Err(error) if error.kind() != std::io::ErrorKind::NotFound => { - return Err(error.into()); - } - _ => {} - } - } + let path: PathBuf = native_model_output_path(&stored_dir, relative_path); + let Ok(text) = std::fs::read_to_string(&path) else { + return Err( + crate::bindings::_helpers::compile_context::delta::delta_unsupported( + "the stored compile's SQL for a model it did not rerun is unreadable", + ), + ); + }; + let text: String = if cfg!(windows) { + text.replace("\r\n", "\n") + } else { + text + }; + let query_sql: String = if stored_failed { + without_failure_header(&text, relative_path).to_owned() + } else { + text + }; + models.push((relative_path.clone(), query_sql)); + } + Ok(models) +} + +/// `text` without the not-buildable header and the Rule error lines `rule_error_lines` wrote for `relative_path`. +fn without_failure_header<'a>(text: &'a str, relative_path: &str) -> &'a str { + let Some(mut rest) = text + .strip_prefix(RULE_FAILED_HEADER) + .and_then(|rest| rest.strip_prefix('\n')) + else { + return text; + }; + let location: String = format!("at {relative_path}"); + while let Some((line, after)) = rest.split_once('\n') { + let is_error_line: bool = line + .strip_prefix("-- ") + .and_then(|line| line.split_once(' ')) + .is_some_and(|(_, line)| line.starts_with(location.as_str())); + if !is_error_line { + break; } + rest = after; } - let contents: String = lines.iter().map(|line| format!("{line}\n")).collect(); - if contents != previous { - std::fs::write(&record, contents)?; + rest +} + +/// `path` relative to `failed_dir`, with `/` separators. +fn failed_relative(failed_dir: &Path, path: &Path) -> String { + match path.strip_prefix(failed_dir) { + Ok(relative) => relative + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/"), + Err(_outside) => path.to_string_lossy().into_owned(), + } +} + +/// Delete `target/compiled-failed/`, which only the latest compile whose Rules failed owns. +pub(crate) fn remove_failed_compile(target: &Path) -> PyResult<()> { + match std::fs::remove_dir_all(target.join(FAILED_COMPILED_DIRECTORY_NAME)) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error.into()), + _ => Ok(()), } - Ok(()) } /// Plan uncached SQL tests and report their errors without writing artifacts or cache. @@ -1002,16 +1054,6 @@ pub(crate) fn publish_staged_target<'py>( remove_stale(target_dir, extracted(&managed)?, None)?; } } - let written: Vec = expected - .iter() - .map(|relative| compiled.join(relative)) - .collect(); - record_compiled_models( - &target_dir.extract::()?, - &compiled, - &written, - ModelRecord::Replace, - )?; write_manifest(target_dir, manifest, recorder)?; written_target( target_dir, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs index bd71d20971..2085227242 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/splice.rs @@ -211,9 +211,11 @@ pub(crate) fn spliced_report( let mut cursor: usize = 0; for ((key, base_span), (_, delta_span)) in base_members.iter().zip(&delta_members) { let replacement: String = match key.as_str() { - "has_errors" | "diagnostics" | "semantic_checks_partial" | "compile_timings" => { - text(delta, delta_span).to_owned() - } + "has_errors" + | "diagnostics" + | "semantic_checks_partial" + | "compile_timings" + | "artifacts" => text(delta, delta_span).to_owned(), "summary" => spliced_summary(base, *base_span, delta, *delta_span) .ok_or("the summaries do not line up")?, "resources" => spliced_resources((base, *base_span), (delta, *delta_span), scope) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs index 5ab8ccb6e9..5bc7c4197a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/verify.rs @@ -27,6 +27,9 @@ pub(crate) struct Expected { pub(crate) compiled: BTreeMap, pub(crate) compiled_root: String, pub(crate) artifacts_written: bool, + /// Every file under `target/compiled-failed/` after the delta, with its native digest. + pub(crate) failed: BTreeMap, + pub(crate) failed_root: String, } /// What the full compile produced. @@ -176,18 +179,34 @@ pub(crate) fn differences(raised: Option) -> Vec { expected.artifacts_written, actual.artifacts_written )); } - if !actual.artifacts_written { - return found; + found.extend(tree_differences( + (&expected.failed_root, &expected.failed), + &actual.written, + )); + if actual.artifacts_written { + found.extend(tree_differences( + (&expected.compiled_root, &expected.compiled), + &actual.written, + )); } - let prefix: &str = expected.compiled_root.as_str(); - let full_paths: BTreeMap<&str, &Option> = actual - .written + found.truncate(50); + found +} + +/// Where the files the full compile wrote under `root` differ from what the delta left there. +fn tree_differences( + (root, expected): (&str, &BTreeMap), + written: &HashMap>, +) -> Vec { + let mut found: Vec = Vec::new(); + let prefix: String = format!("{root}{}", std::path::MAIN_SEPARATOR); + let full_paths: BTreeMap<&str, &Option> = written .iter() - .filter(|(path, _)| path.starts_with(prefix)) + .filter(|(path, _)| path.starts_with(prefix.as_str())) .map(|(path, digest)| (path.as_str(), digest)) .collect(); for (path, digest) in &full_paths { - match (expected.compiled.get(*path), digest) { + match (expected.get(*path), digest) { (None, _) => found.push(format!("the delta left no artifact at {path}")), (Some(left), Some(right)) if left != right => { found.push(format!("artifact differs: {path}")); @@ -195,14 +214,13 @@ pub(crate) fn differences(raised: Option) -> Vec { _ => {} } } - for path in expected.compiled.keys() { + for path in expected.keys() { if !full_paths.contains_key(path.as_str()) { found.push(format!( "the delta left an artifact the full compile does not: {path}" )); } } - found.truncate(50); found } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 6b7a54e1b5..23ce9c2f2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -12,6 +12,7 @@ use sqlbuild_host::compile_context::types::CompileTimingsHandle; use sqlbuild_rules::custom_host_pool::constants::{ CUSTOM_RULES_GUARD_KEY, CUSTOM_RULES_INVOCATIONS_KEY, CUSTOM_RULES_MONITOR_KEY, }; +use std::path::PathBuf; use std::rc::Rc; use crate::bindings::_helpers::boundary::python_calls::{ @@ -26,7 +27,9 @@ use crate::bindings::_helpers::compile_command::reuse_recording::write_reusable_ use crate::bindings::_helpers::compile_command::reuse_recording::{ CompileArtifactWrites, RecordedCompileOutput, compile_reuse_timings, new_artifact_recorder, }; -use crate::bindings::_helpers::compile_command::target_writer::write_rule_failed_models; +use crate::bindings::_helpers::compile_command::target_writer::{ + remove_failed_compile, restore_models_after_failed_compile, write_rule_failed_models, +}; use crate::bindings::_helpers::compile_context::input_reads::record_compile_input_reads; use crate::bindings::_helpers::compile_driver::analysis::analyze; use crate::bindings::_helpers::compile_driver::phases::{ @@ -359,16 +362,37 @@ fn compiled_with_status<'py>( (prepared_artifacts, recorder), status, )?; - if rules_failed && !skip_write { - write_rule_failed_models( + let written: Bound<'py, PyAny> = if skip_write { + written + } else if rules_failed { + let failing: Vec = write_rule_failed_models( &target_dir(&project_dir)?, ( &native_field(&native_field(&analysis, "graph")?, "project")?, &project_dir, ), &native_field(&analysis, "diagnostics")?, + recorder, )?; - } + replaced( + &written, + &[ + ( + "failed_compile", + PyBool::new(py, true).to_owned().into_any(), + ), + ("failed_model_sql", PyTuple::new(py, failing)?.into_any()), + ], + )? + } else { + restore_models_after_failed_compile( + &target_dir(&project_dir)?, + &native_field(&native_field(&analysis, "graph")?, "project")?, + recorder, + )?; + remove_failed_compile(&target_dir(&project_dir)?.extract::()?)?; + written + }; let timings: Bound<'py, PyDict> = timings_ms( (&analysis, write_ms), (&recorders.detailed_timings, reuse_timings), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index e07ab58f5f..c929ba3394 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -10,7 +10,9 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{FromPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; use sqlbuild_cache::digest::main::hex_digest::hex_digest; -use sqlbuild_cache::project_reuse::constants::{NATIVE_DIGEST_PREFIX, REUSE_STORE_SUFFIX}; +use sqlbuild_cache::project_reuse::constants::{ + FAILED_COMPILED_DIRECTORY_NAME, NATIVE_DIGEST_PREFIX, REUSE_STORE_SUFFIX, +}; use sqlbuild_cache::project_reuse::main::check_reuse::check_reuse; use sqlbuild_cache::project_reuse::main::pending_digest_bytes::pending_digest_bytes; use sqlbuild_cache::project_reuse::main::record_reuse::record_reuse; @@ -228,9 +230,23 @@ impl NativeReuseAttempt { image: crate::bindings::_helpers::compile_context::delta::take_pending_image(), vouched_target_stamps: None, }; - if let Some(merge) = + if let Some(mut merge) = crate::bindings::_helpers::compile_context::delta::take_record_merge() { + let failed: String = format!( + "{}{}", + self.attempt + .lock() + .map_err(poisoned)? + .project_dir + .join(&rules.compiled_root) + .with_file_name(FAILED_COMPILED_DIRECTORY_NAME) + .to_string_lossy(), + std::path::MAIN_SEPARATOR + ); + merge + .kept + .retain(|stamp| !stamp.relative_path.starts_with(failed.as_str())); for stamp in &merge.kept { compile .artifacts diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 848655d07e..d8e3df098d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1024,6 +1024,8 @@ const WRITTEN_TARGET: RecordSpec = RecordSpec { FieldSpec::required("test_count"), FieldSpec::required("target_dir"), FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("failed_compile", FieldDefault::False), + FieldSpec::new("failed_model_sql", FieldDefault::EmptyTuple), ], }; const SKILL_MAINTENANCE_RESULT: RecordSpec = RecordSpec { diff --git a/scripts/cold_compile_performance/constants.py b/scripts/cold_compile_performance/constants.py index ab5452b22a..7ed6e83829 100644 --- a/scripts/cold_compile_performance/constants.py +++ b/scripts/cold_compile_performance/constants.py @@ -20,3 +20,4 @@ DENSE_DEEP_CTE_PERCENTILE: int = 97 DENSE_MEDIUM_CTE_PERCENTILE: int = 80 DENSE_ID_COLUMN: str = "id" +FAILED_COMPILED_DIRECTORY: str = "compiled-failed" diff --git a/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py b/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py index 8a2adcb219..91172b10ab 100644 --- a/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py +++ b/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py @@ -8,18 +8,35 @@ import orjson +from scripts.cold_compile_performance.constants import FAILED_COMPILED_DIRECTORY + def semantic_compile_fingerprint(*, payload: dict[str, object], compiled_dir: Path) -> str: - """Hash deterministic semantic output and every compiled artifact.""" + """Hash deterministic semantic output and every compiled and failed-compile artifact.""" semantic_payload: dict[str, object] = dict(payload) semantic_payload.pop("compile_timings", None) semantic_payload.pop("compiler_engine", None) semantic_payload.pop("version", None) fingerprint: Any = hashlib.sha256(orjson.dumps(semantic_payload, option=orjson.OPT_SORT_KEYS)) - for artifact_path in sorted(path for path in compiled_dir.rglob("*") if path.is_file()): - fingerprint.update(str(artifact_path.relative_to(compiled_dir)).encode("utf-8")) - fingerprint.update(b"\0") - fingerprint.update(artifact_path.read_bytes()) - fingerprint.update(b"\0") + fingerprint.update(_tree_bytes(root=compiled_dir, relative_to=compiled_dir)) + fingerprint.update( + _tree_bytes( + root=compiled_dir.with_name(FAILED_COMPILED_DIRECTORY), relative_to=compiled_dir.parent + ) + ) return str(fingerprint.hexdigest()) + + +def _tree_bytes(*, root: Path, relative_to: Path) -> bytes: + parts: list[bytes] = [] + for artifact_path in sorted(path for path in root.rglob("*") if path.is_file()): + parts.extend( + ( + str(artifact_path.relative_to(relative_to)).encode("utf-8"), + b"\0", + artifact_path.read_bytes(), + b"\0", + ) + ) + return b"".join(parts) diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 32b4209acc..05add8ed2a 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -25,7 +25,7 @@ COLD_MODE, COMPILE_ENTRY, COMPILE_TIMINGS_KEY, - COMPILED_DIRECTORY, + COMPILED_TREES, DENSE_KIND, EDIT_MODE, ERROR_TAIL_CHARACTERS, @@ -41,6 +41,7 @@ MODE_TITLES, PYTHONPATH_KEY, REPORTED_PHASES, + TARGET_DIRECTORY, UNCACHED_MODE, WARM_MODE, ) @@ -247,14 +248,19 @@ def check_cache_use(*, run: CompileRun, mode: str, models: int) -> None: def compiled_tree(*, project_dir: Path) -> dict[str, bytes]: - """Every compiled artifact's bytes by its path under the compiled directory.""" - - root: Path = project_dir / COMPILED_DIRECTORY - return { - path.relative_to(root).as_posix(): path.read_bytes() - for path in sorted(root.rglob("*")) - if path.is_file() - } + """Every compiled and failed-compile artifact's bytes by its path under `target/`.""" + + target: Path = project_dir / TARGET_DIRECTORY + artifacts: dict[str, bytes] = {} + for tree in COMPILED_TREES: + artifacts.update( + { + path.relative_to(target).as_posix(): path.read_bytes() + for path in sorted((target / tree).rglob("*")) + if path.is_file() + } + ) + return artifacts def check_matches_uncached( diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index eb7a87d241..550ce619f3 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -44,7 +44,8 @@ PYTHONPATH_KEY: str = "PYTHONPATH" PHASE_NOISE_FLOOR_MS: float = 50.0 COMPILE_TIMINGS_KEY: str = "compile_timings" -COMPILED_DIRECTORY: str = "target/compiled" +TARGET_DIRECTORY: str = "target" +COMPILED_TREES: tuple[str, ...] = ("compiled", "compiled-failed") UNCACHED_MODE: str = "uncached" ERROR_TAIL_CHARACTERS: int = 2000 FRESH_SOURCE_SHARE: float = 713 / 3000 diff --git a/scripts/compiler_differential/_helpers/goldens/goldens.py b/scripts/compiler_differential/_helpers/goldens/goldens.py index 4bf6581bad..d9d5f5a671 100644 --- a/scripts/compiler_differential/_helpers/goldens/goldens.py +++ b/scripts/compiler_differential/_helpers/goldens/goldens.py @@ -119,21 +119,30 @@ def golden_path(*, golden_dir: Path, project: str) -> Path | None: def golden_payload( *, run: ProjectRun, masked_paths: tuple[tuple[str, str], ...] ) -> dict[str, object]: - """Return a run's diagnostics, compiled SQL and slimmed manifest with run noise masked.""" + """Return a run's diagnostics, compiled and failed-compile SQL and slimmed manifest, masked.""" - return { + payload: dict[str, object] = { "commands": [ _command(outcome=outcome, masked_paths=masked_paths) for outcome in run.outcomes ], - "compiled": { - path: _lines( - text=_masked( - text=contents.decode("utf-8", "surrogateescape"), masked_paths=masked_paths - ) + "compiled": _tree(files=run.compiled, masked_paths=masked_paths), + } + if run.compiled_failed: + payload["compiled_failed"] = _tree(files=run.compiled_failed, masked_paths=masked_paths) + payload["manifest"] = _manifest(text=run.manifest, masked_paths=masked_paths) + return payload + + +def _tree( + *, files: dict[str, bytes], masked_paths: tuple[tuple[str, str], ...] +) -> dict[str, list[str]]: + return { + path: _lines( + text=_masked( + text=contents.decode("utf-8", "surrogateescape"), masked_paths=masked_paths ) - for path, contents in sorted(run.compiled.items()) - }, - "manifest": _manifest(text=run.manifest, masked_paths=masked_paths), + ) + for path, contents in sorted(files.items()) } diff --git a/scripts/compiler_differential/_helpers/running/execution.py b/scripts/compiler_differential/_helpers/running/execution.py index 4f7d5a5a28..302a49dfff 100644 --- a/scripts/compiler_differential/_helpers/running/execution.py +++ b/scripts/compiler_differential/_helpers/running/execution.py @@ -13,6 +13,7 @@ COMPILED_DIRECTORY, EXCLUDED_ENVIRONMENT_KEYS, EXCLUDED_ENVIRONMENT_PREFIX, + FAILED_COMPILED_DIRECTORY, GENERATOR_ENVIRONMENT, MANIFEST_FILE, PROJECT_DIRECTORY, @@ -73,6 +74,7 @@ def run_project( compiled=_compiled_tree(project_dir / COMPILED_DIRECTORY), manifest=_optional_text(project_dir / MANIFEST_FILE), captures=_captures(capture_root), + compiled_failed=_compiled_tree(project_dir / FAILED_COMPILED_DIRECTORY), ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 94ccae6a9c..085040bf50 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -41,6 +41,7 @@ PROJECT_DIRECTORY: str = "project" CAPTURES_DIRECTORY: str = "captures" COMPILED_DIRECTORY: str = "target/compiled" +FAILED_COMPILED_DIRECTORY: str = "target/compiled-failed" MANIFEST_FILE: str = "target/manifest.json" ELAPSED_TIME_PATTERN: re.Pattern[str] = re.compile(r"\b\d+(?:\.\d+)?\s?m?s\b") ELAPSED_TIME_MASK: str = "" diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index a928324435..daa8870758 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -59,6 +59,7 @@ class ProjectRun: compiled: dict[str, bytes] manifest: str | None captures: dict[str, dict[str, Path]] = field(default_factory=dict) + compiled_failed: dict[str, bytes] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 1a4af4ad95..b8f3aae804 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -83,8 +83,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb 5. **Contracts.** Checks declared columns against the inferred query output. 6. **Rules.** Runs the selected built-in Rules, then the selected custom Rules. 7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. When they fail, it - writes only the models' SQL, so you can read what the Rules checked; see - [When Rules fail](#when-rules-fail). + writes the models' SQL to `target/compiled-failed/` instead, so you can read what the Rules + checked; see [When Rules fail](#when-rules-fail). Rules that need inferred types, such as `SQBRCONTRACT105`, are skipped when SQL analysis is off, and compile prints one note naming them. @@ -95,12 +95,21 @@ configuration, use [`sqb rules run`](rules.md). ### When Rules fail -A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a -`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each -model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros -generated, and the compiled file of a model you removed from the project is deleted. Other -compiled files stay as the last passing compile left them. A model with Rule errors starts with a -header that marks it not buildable and lists its findings: +A Rule finding exits `1` and blocks the manifest, the DAG file and the other artifacts. A +`target/manifest.json` from an earlier compile is removed, since it no longer matches. +`target/compiled/` keeps exactly what the last passing compile wrote. + +Each model's expanded SQL goes to `target/compiled-failed/`, laid out like `target/compiled/`, so +you can read the SQL that macros generated. The report names the file of each model with Rule +errors: + +``` +✗ Project compiled 2 models, 0 seeds, 0 functions, 1 error, 0 warnings + Failed SQL: target/compiled-failed/models/open_orders.sql + Wrote: target/compiled-failed/ +``` + +A model with Rule errors starts with a header that marks it not buildable and lists its findings: ```sql -- sqb: compile failed for this model; not buildable @@ -108,7 +117,8 @@ header that marks it not buildable and lists its findings: SELECT o.order_id, status FROM __ref("orders") AS o ``` -The next compile that passes writes the files again without the header. +`target/compiled-failed/` holds only the latest failing compile: each failing compile replaces it +as a whole, and the next passing compile deletes it. ## Compiling a selection @@ -194,7 +204,7 @@ sqb compile --json | `diagnostics` | Every diagnostic, with its source location | | `compile_timings` | Timings and cache counters; see below | | `lineage_mode` | The lineage mode used | -| `artifacts` | Paths of the files written | +| `artifacts` | Paths of the files written. When the Rules fail, `compiled_sql_dir` is `target/compiled-failed/` and `failed_model_sql` lists each failing model's file | If compile stops before it can build the report, for example on a model header that doesn't parse, `--json` still prints one document with `command`, `has_errors: true`, `stopped: true` and diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index 28bf4feea9..ab7cff180e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -516,7 +516,7 @@ A `* REPLACE` item takes its source column's type rather than the call's. - [Rules](../rules.md) and [column lineage](../column-lineage.md) read the expanded SQL, so they see through every call. A Rule finding inside an expansion is reported at the call, marked `(in SQL generated by @sql.name(...))`. The model's expanded SQL is in - `target/compiled/models/`, also when the Rules fail; see + `target/compiled/models/`, or in `target/compiled-failed/models/` when the Rules fail; see [When Rules fail](../../cli/compile.md#when-rules-fail). - Editing a macro recompiles its callers. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5a1ecd12de..2bd38fd736 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -7345,6 +7345,8 @@ class WrittenTarget(NativeRecord): test_count: int target_dir: Path diagnostics: tuple[CompilerDiagnostic, ...] + failed_compile: bool + failed_model_sql: tuple[str, ...] def __init__( self, model_count: int, @@ -7354,6 +7356,8 @@ class WrittenTarget(NativeRecord): test_count: int, target_dir: Path, diagnostics: tuple[CompilerDiagnostic, ...] = ..., + failed_compile: bool = ..., + failed_model_sql: tuple[str, ...] = ..., ) -> None: ... class SkillMaintenanceResult(NativeRecord): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index a3838018fe..2c40057a26 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3688,13 +3688,14 @@ def touch_back(path: Path, *, seconds: int) -> None: def compiled_artifacts(*, project_dir: Path) -> dict[str, bytes]: - """Return every compiled SQL artifact by path relative to target/compiled.""" + """Return every compiled and failed-compile SQL artifact by path relative to target/.""" - compiled_dir: Path = project_dir / "target" / "compiled" - return { - path.relative_to(compiled_dir).as_posix(): path.read_bytes() - for path in sorted(compiled_dir.rglob("*.sql")) - } + target_dir: Path = project_dir / "target" + paths: list[Path] = [ + *sorted((target_dir / "compiled").rglob("*.sql")), + *sorted((target_dir / "compiled-failed").rglob("*.sql")), + ] + return {path.relative_to(target_dir).as_posix(): path.read_bytes() for path in paths} def compile_in_process_reused( diff --git a/tests/goldens/compiler/failure/rule-custom-finding.json b/tests/goldens/compiler/failure/rule-custom-finding.json index 1f2b7f5aa9..b79f012d73 100644 --- a/tests/goldens/compiler/failure/rule-custom-finding.json +++ b/tests/goldens/compiler/failure/rule-custom-finding.json @@ -22,7 +22,8 @@ ] } ], - "compiled": { + "compiled": {}, + "compiled_failed": { "models/marts/customer_totals.sql": [ "SELECT customer_id, SUM(amount) AS total_amount", "FROM __ref(\"stg_orders\")", diff --git a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json index 4c5cbf2d49..19fd87eae7 100644 --- a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json +++ b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json @@ -25,7 +25,8 @@ ] } ], - "compiled": { + "compiled": {}, + "compiled_failed": { "models/marts/customer_totals.sql": [ "-- sqb: compile failed for this model; not buildable", "-- SQBRSQL005 at models/marts/customer_totals.sql:5:117: CTE is unreachable from the final query", diff --git a/tests/goldens/compiler/failure/rule-unused-cte.json b/tests/goldens/compiler/failure/rule-unused-cte.json index 6eb544614e..7146a6cfca 100644 --- a/tests/goldens/compiler/failure/rule-unused-cte.json +++ b/tests/goldens/compiler/failure/rule-unused-cte.json @@ -25,7 +25,8 @@ ] } ], - "compiled": { + "compiled": {}, + "compiled_failed": { "models/marts/customer_totals.sql": [ "-- sqb: compile failed for this model; not buildable", "-- SQBRSQL005 at models/marts/customer_totals.sql:5:6: CTE is unreachable from the final query", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index d5825b942e..37eaa64ebc 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1334,8 +1334,19 @@ class RuleFailureArtifactsTestCase: description: str open_orders_sql: str expected_exit_code: int - expected_open_orders: str + expected_compiled: dict[str, bytes] + expected_failed: dict[str, bytes] expected_manifest: bool + expected_output_fragment: str + + +@dataclass(frozen=True) +class RuleFailureSequenceTestCase: + description: str + environment: dict[str, str] + extra_args: tuple[str, ...] + expected_rewarm_reused: bool + expected_minimum_verified_deltas: int @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 0a3476eb66..41c72480f9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -57,6 +57,13 @@ ) +def artifact_tree(root: Path) -> dict[str, bytes]: + """Return each file under root by its POSIX path relative to root, or nothing when missing.""" + + files: list[Path] = sorted(filter(Path.is_file, root.rglob("*"))) + return {path.relative_to(root).as_posix(): path.read_bytes() for path in files} + + def staging_directories(target_dir: Path) -> list[str]: """Return hidden entries left in target_dir, such as abandoned artifact staging.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py index a31ce8a271..240e547810 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py @@ -234,12 +234,12 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu failed: dict[str, object] = json.loads(capsys.readouterr().out) assert failed["has_errors"] is True changed: Path = compiled / "models" / "orders_000.sql" - assert {path: path.read_bytes() for path in before} == { - **before, - changed: b"-- sqb: compile failed for this model; not buildable\n" + assert {path: path.read_bytes() for path in before} == before + assert (tmp_path / "target" / "compiled-failed" / "models" / "orders_000.sql").read_bytes() == ( + b"-- sqb: compile failed for this model; not buildable\n" b"-- XSQBRARCH001 at models/orders_000.sql:1:1: Model rejected\n" - b"SELECT CAST(2 AS INTEGER) AS order_id\n", - } + b"SELECT CAST(2 AS INTEGER) AS order_id\n" + ) assert stale.exists() assert staging_directories(tmp_path / "target") == [] @@ -248,6 +248,7 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu final: dict[str, object] = json.loads(capsys.readouterr().out) assert final["has_errors"] is False assert not stale.exists() + assert not (tmp_path / "target" / "compiled-failed").exists() for path in modified_times.keys() - {changed}: assert path.stat().st_mtime_ns == modified_times[path] assert path.read_bytes() == before[path] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py index b39902a37e..82e5392af8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -6,11 +6,17 @@ import pytest +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR, DELTA_COMPILE_VERIFY_ENV_VAR from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RuleFailureArtifactsTestCase, + RuleFailureSequenceTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + artifact_tree, + open_orders_model, + write_project_files, ) -from tests.integration.src.sqlbuild.cli.commands.main.helpers import open_orders_model _PROJECT: str = ( 'name = "shop"\n' @@ -28,50 +34,80 @@ 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' "SELECT 1 AS order_id, 'open' AS status\n" ) -_HEADER: str = "-- sqb: compile failed for this model; not buildable\n" +_CUSTOMERS: str = ( + 'MODEL (\n description "One row per customer",\n materialized table,\n);\n\n' + "SELECT 1 AS customer_id\n" +) +_ORDERS_SQL: bytes = b"SELECT 1 AS order_id, 'open' AS status\n" +_PASSING: str = 'SELECT o.order_id, o.status FROM __ref("orders") AS o' +_FAILING: str = 'SELECT o.order_id, status FROM __ref("orders") AS o' +_FAILED_OPEN_ORDERS: bytes = ( + b"-- sqb: compile failed for this model; not buildable\n" + b"-- SQBRSQL028 at models/open_orders.sql:6:20: " + b"Single-relation query mixes qualified and unqualified columns\n" + b'SELECT o.order_id, status FROM __ref("orders") AS o\n' +) +_PASSING_OPEN_ORDERS: bytes = b'SELECT o.order_id, o.status FROM __ref("orders") AS o\n' +_REUSED_LINE: str = "reused the previous compile" +_VERIFIED_LINE: str = "delta compile verified" @pytest.mark.parametrize( "test_case", [ RuleFailureArtifactsTestCase( - description="a model failing a Rule is written with a header listing its findings", - open_orders_sql='SELECT o.order_id, status FROM __ref("orders") AS o', - expected_exit_code=1, - expected_open_orders=( - _HEADER + "-- SQBRSQL028 at models/open_orders.sql:6:20: " - "Single-relation query mixes qualified and unqualified columns\n" - 'SELECT o.order_id, status FROM __ref("orders") AS o\n' + description=( + "a failing compile writes compiled-failed/ with a header on the failing model " + "and leaves compiled/ as the last passing compile wrote it" ), + open_orders_sql=_FAILING, + expected_exit_code=1, + expected_compiled={ + "models/open_orders.sql": _PASSING_OPEN_ORDERS, + "models/orders.sql": _ORDERS_SQL, + "models/retired_orders.sql": _ORDERS_SQL, + }, + expected_failed={ + "models/open_orders.sql": _FAILED_OPEN_ORDERS, + "models/orders.sql": _ORDERS_SQL, + }, expected_manifest=False, + expected_output_fragment=( + "Failed SQL: target/compiled-failed/models/open_orders.sql\n" + " Wrote: target/compiled-failed/\n" + ), ), RuleFailureArtifactsTestCase( - description="a passing compile writes the same model without the header", - open_orders_sql='SELECT o.order_id, o.status FROM __ref("orders") AS o', + description="a passing compile replaces compiled/ and writes no compiled-failed/", + open_orders_sql=_PASSING.replace("o.status", "o.status AS order_status"), expected_exit_code=0, - expected_open_orders='SELECT o.order_id, o.status FROM __ref("orders") AS o\n', + expected_compiled={ + "models/open_orders.sql": _PASSING_OPEN_ORDERS.replace( + b"o.status", b"o.status AS order_status" + ), + "models/orders.sql": _ORDERS_SQL, + }, + expected_failed={}, expected_manifest=True, + expected_output_fragment="Wrote: target/compiled/\n", ), ], ids=lambda case: case.description, ) -def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failure_header( +def test_given_rule_findings_when_compiling_then_failed_sql_is_written_beside_compiled( test_case: RuleFailureArtifactsTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - files: dict[str, str] = { - "sqlbuild_project.toml": _PROJECT, - "models/orders.sql": _ORDERS, - "models/open_orders.sql": open_orders_model( - 'SELECT o.order_id, o.status FROM __ref("orders") AS o' - ), - "models/retired_orders.sql": _ORDERS, - } - for relative, contents in files.items(): - path: Path = tmp_path / relative - path.parent.mkdir(parents=True, exist_ok=True) - _ = path.write_text(contents, encoding="utf-8") + write_project_files( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": _PROJECT, + "models/orders.sql": _ORDERS, + "models/open_orders.sql": open_orders_model(_PASSING), + "models/retired_orders.sql": _ORDERS, + }, + ) compile_args: list[str] = [ "--no-color", "--project-dir", @@ -79,23 +115,155 @@ def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failu "compile", "--manifest", ] - _ = main(compile_args) - _ = (tmp_path / "models" / "open_orders.sql").write_text( - open_orders_model(test_case.open_orders_sql), encoding="utf-8" - ) + assert main(compile_args) == 0 + _ = capsys.readouterr() assert (tmp_path / "target" / "manifest.json").exists() + write_project_files( + project_dir=tmp_path, + files={"models/open_orders.sql": open_orders_model(test_case.open_orders_sql)}, + ) (tmp_path / "models" / "retired_orders.sql").unlink() exit_code: int = main(compile_args) output: str = "".join(capsys.readouterr()) - compiled: Path = tmp_path / "target" / "compiled" / "models" + failed: Path = tmp_path / "target" / "compiled-failed" assert exit_code == test_case.expected_exit_code, output - assert (compiled / "open_orders.sql").read_text(encoding="utf-8") == ( - test_case.expected_open_orders + assert test_case.expected_output_fragment in output + assert artifact_tree(tmp_path / "target" / "compiled") == test_case.expected_compiled + assert artifact_tree(failed) == test_case.expected_failed + assert failed.exists() is bool(test_case.expected_failed) + assert (tmp_path / "target" / "manifest.json").exists() is test_case.expected_manifest + assert not (tmp_path / "target" / "compiled-models.txt").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + RuleFailureSequenceTestCase( + description="with the compile cache", + environment={COMPILE_REUSE_DISABLE_ENV_VAR: "0"}, + extra_args=(), + expected_rewarm_reused=True, + expected_minimum_verified_deltas=0, + ), + RuleFailureSequenceTestCase( + description="with each delta compile checked against a full compile", + environment={COMPILE_REUSE_DISABLE_ENV_VAR: "0", DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + extra_args=(), + expected_rewarm_reused=True, + expected_minimum_verified_deltas=5, + ), + RuleFailureSequenceTestCase( + description="without the compile cache", + environment={}, + extra_args=("--no-cache",), + expected_rewarm_reused=False, + expected_minimum_verified_deltas=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_and_passing_compiles_when_alternating_then_compiled_failed_follows_the_latest( + test_case: RuleFailureSequenceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + for name, value in test_case.environment.items(): + monkeypatch.setenv(name, value) + write_project_files( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": _PROJECT, + "models/customers.sql": _CUSTOMERS, + "models/orders.sql": _ORDERS, + "models/open_orders.sql": open_orders_model(_PASSING), + }, ) - assert (compiled / "orders.sql").read_text(encoding="utf-8") == ( - "SELECT 1 AS order_id, 'open' AS status\n" + compiled: Path = tmp_path / "target" / "compiled" + failed: Path = tmp_path / "target" / "compiled-failed" + compile_args: list[str] = [ + "--no-color", + "--project-dir", + str(tmp_path), + "compile", + *test_case.extra_args, + ] + stderr_lines: list[str] = [] + + def compiled_once(expected_exit_code: int) -> str: + exit_code: int = main(compile_args) + stdout, stderr = capsys.readouterr() + assert exit_code == expected_exit_code, stdout + stderr + stderr_lines.extend(stderr.splitlines()) + return stderr + + def rewarmed(expected_exit_code: int) -> None: + stderr: str = compiled_once(expected_exit_code) + assert (_REUSED_LINE in stderr) is test_case.expected_rewarm_reused, stderr + + def edited(files: dict[str, str]) -> None: + write_project_files(project_dir=tmp_path, files=files) + + compiled_once(0) + first_pass: dict[str, bytes] = artifact_tree(compiled) + failing: dict[str, bytes] = { + "models/customers.sql": b"SELECT 1 AS customer_id\n", + "models/open_orders.sql": _FAILED_OPEN_ORDERS, + "models/orders.sql": _ORDERS_SQL, + } + assert sorted(first_pass) == [ + "models/customers.sql", + "models/open_orders.sql", + "models/orders.sql", + ] + assert not failed.exists() + + edited( + {"models/open_orders.sql": open_orders_model(_FAILING), "models/new_orders.sql": _ORDERS} ) - assert (tmp_path / "target" / "manifest.json").exists() is test_case.expected_manifest - assert sorted(path.name for path in compiled.iterdir()) == ["open_orders.sql", "orders.sql"] + compiled_once(1) + assert artifact_tree(compiled) == first_pass + assert artifact_tree(failed) == {**failing, "models/new_orders.sql": _ORDERS_SQL} + + (tmp_path / "models" / "new_orders.sql").unlink() + compiled_once(1) + rewarmed(1) + assert artifact_tree(compiled) == first_pass + assert artifact_tree(failed) == failing + + edited({"models/open_orders.sql": open_orders_model(_PASSING)}) + compiled_once(0) + assert not failed.exists() + assert artifact_tree(compiled) == first_pass + + edited({"models/open_orders.sql": open_orders_model(_FAILING)}) + compiled_once(1) + assert artifact_tree(failed) == failing + assert artifact_tree(compiled) == first_pass + + edited({"models/customers.sql": _CUSTOMERS.replace("1 AS", "2 AS")}) + compiled_once(1) + rewarmed(1) + assert artifact_tree(failed) == { + **failing, + "models/customers.sql": b"SELECT 2 AS customer_id\n", + } + assert artifact_tree(compiled) == first_pass + + edited({"models/open_orders.sql": open_orders_model(_PASSING)}) + compiled_once(0) + rewarmed(0) + assert not failed.exists() + assert artifact_tree(compiled) == { + **first_pass, + "models/customers.sql": b"SELECT 2 AS customer_id\n", + } + + edited({"models/customers.sql": _CUSTOMERS}) + compiled_once(0) + rewarmed(0) + assert not failed.exists() + assert artifact_tree(compiled) == first_pass + assert stderr_lines.count(_VERIFIED_LINE) >= test_case.expected_minimum_verified_deltas diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 44e099df62..2a89641171 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -1441,7 +1441,7 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: assert cold["compile_timings"]["rule_cache_hits"] == 0 assert warm["compile_timings"]["rule_cache_hits"] >= 1 assert ( - (tmp_path / "target" / "compiled" / "models" / "orders.sql") + (tmp_path / "target" / "compiled-failed" / "models" / "orders.sql") .read_text(encoding="utf-8") .startswith("-- sqb: compile failed for this model; not buildable\n-- XSQBRARCH001 at ") ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py b/tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py index b87eefc4e5..9b1ab86750 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py +++ b/tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import replace from pathlib import Path import pytest @@ -116,6 +117,28 @@ def test_given_corpus_project_when_locating_its_golden_then_only_golden_corpora_ "manifest": None, }, ), + GoldenPayloadTestCase( + description="failed_compile_sql_is_kept_beside_the_compiled_sql", + run=replace( + machine_error_run(), + compiled_failed={"models/orders.sql": b"-- sqb: not buildable\nSELECT 1\n"}, + ), + masked_paths=(), + expected_payload={ + "commands": [ + { + "label": "plan", + "exit_code": 1, + "diagnostics": ["error[C214]: no manifest", ""], + } + ], + "compiled": {}, + "compiled_failed": { + "models/orders.sql": ["-- sqb: not buildable", "SELECT 1", ""], + }, + "manifest": None, + }, + ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 3fbe67637a..b018135bbc 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -67,8 +67,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb 5. **Contracts.** Checks declared columns against the inferred query output. 6. **Rules.** Runs the selected built-in Rules, then the selected custom Rules. 7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. When they fail, it - writes only the models' SQL, so you can read what the Rules checked; see - [When Rules fail](#when-rules-fail). + writes the models' SQL to `target/compiled-failed/` instead, so you can read what the Rules + checked; see [When Rules fail](#when-rules-fail). Rules that need inferred types, such as `SQBRCONTRACT105`, are skipped when SQL analysis is off, and compile prints one note naming them. @@ -79,12 +79,21 @@ configuration, use [`sqb rules run`](/docs/cli/rules). ### When Rules fail -A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a -`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each -model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros -generated, and the compiled file of a model you removed from the project is deleted. Other -compiled files stay as the last passing compile left them. A model with Rule errors starts with a -header that marks it not buildable and lists its findings: +A Rule finding exits `1` and blocks the manifest, the DAG file and the other artifacts. A +`target/manifest.json` from an earlier compile is removed, since it no longer matches. +`target/compiled/` keeps exactly what the last passing compile wrote. + +Each model's expanded SQL goes to `target/compiled-failed/`, laid out like `target/compiled/`, so +you can read the SQL that macros generated. The report names the file of each model with Rule +errors: + +``` +✗ Project compiled 2 models, 0 seeds, 0 functions, 1 error, 0 warnings + Failed SQL: target/compiled-failed/models/open_orders.sql + Wrote: target/compiled-failed/ +``` + +A model with Rule errors starts with a header that marks it not buildable and lists its findings: ```sql -- sqb: compile failed for this model; not buildable @@ -92,7 +101,8 @@ header that marks it not buildable and lists its findings: SELECT o.order_id, status FROM __ref("orders") AS o ``` -The next compile that passes writes the files again without the header. +`target/compiled-failed/` holds only the latest failing compile: each failing compile replaces it +as a whole, and the next passing compile deletes it. ## Compiling a selection @@ -178,7 +188,7 @@ sqb compile --json | `diagnostics` | Every diagnostic, with its source location | | `compile_timings` | Timings and cache counters; see below | | `lineage_mode` | The lineage mode used | -| `artifacts` | Paths of the files written | +| `artifacts` | Paths of the files written. When the Rules fail, `compiled_sql_dir` is `target/compiled-failed/` and `failed_model_sql` lists each failing model's file | If compile stops before it can build the report, for example on a model header that doesn't parse, `--json` still prints one document with `command`, `has_errors: true`, `stopped: true` and diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 7948e1daeb..921565ae55 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -501,7 +501,7 @@ A `* REPLACE` item takes its source column's type rather than the call's. - [Rules](/docs/concepts/rules) and [column lineage](/docs/concepts/column-lineage) read the expanded SQL, so they see through every call. A Rule finding inside an expansion is reported at the call, marked `(in SQL generated by @sql.name(...))`. The model's expanded SQL is in - `target/compiled/models/`, also when the Rules fail; see + `target/compiled/models/`, or in `target/compiled-failed/models/` when the Rules fail; see [When Rules fail](/docs/cli/compile#when-rules-fail). - Editing a macro recompiles its callers. From 83bdf3be7f2fe8530329bab72274be58370ceb98 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:37:53 +0000 Subject: [PATCH 07/12] fix(compile): compile in full after a compile whose Rules failed A compile whose Rules fail writes no SQL tests, functions or audits, so a delta on top of it left those files as an older passing compile wrote them. The next compile after a Rules failure now compiles in full, and the delta paths that restored models from the failed compile's SQL are removed. Delta tests whose stored project failed its Rules by accident now give their custom Rules a harness case, or turn off the built-in Rules that cannot parse PIVOT models, so they keep exercising deltas. --- .../compile_command/reuse_recording.rs | 37 ----- .../_helpers/compile_command/target_writer.rs | 89 +----------- .../bindings/_helpers/compile_delta/driver.rs | 12 ++ .../_helpers/compile_driver/command.rs | 7 +- .../_helpers/compile_outputs/reuse.rs | 20 +-- .../delta_compile_series/_helpers/outcomes.py | 3 + scripts/delta_compile_series/constants.py | 1 + .../main/compile/test_compile_delta.py | 48 ++++++- .../sqlbuild/cli/commands/main/_test_types.py | 8 ++ .../src/sqlbuild/cli/commands/main/helpers.py | 6 + .../main/test_compile_rule_failures.py | 127 +++++++++++++++++- 11 files changed, 207 insertions(+), 151 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index ee4456afa7..400b3c3faf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -1,7 +1,6 @@ //! What one full compile printed and left in each artifact, and storing it for compile reuse. use sqlbuild_cache::project_reuse::types::ImageSections; -use sqlbuild_cache::project_snapshot::main::stamp_path::stamp_path; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Mutex, PoisonError}; @@ -83,14 +82,6 @@ impl CompileArtifactWrites { } impl CompileArtifactWrites { - /// Note a file this compile left as an earlier compile wrote it, unless it already recorded the path. - pub(crate) fn keep_unrecorded(&self, path: String, (size, mtime_ns): (u64, i64)) { - let _ = self - .locked() - .entry(path) - .or_insert(RecordedArtifact::Kept { size, mtime_ns }); - } - /// Every recorded artifact's absolute path, with its digest when this compile wrote it. pub(crate) fn written_digests(&self) -> HashMap> { self.locked() @@ -211,34 +202,6 @@ pub(crate) fn artifact_kept( Ok(()) } -/// Note every file under `root` this compile did not record as kept as an earlier compile left it. -pub(crate) fn artifact_tree_kept( - py: Python<'_>, - recorder: ArtifactRecorder<'_>, - root: &Path, -) -> PyResult<()> { - let Some(recorder) = recorder else { - return Ok(()); - }; - let mut pending: Vec = vec![root.to_path_buf()]; - while let Some(directory) = pending.pop() { - let Ok(entries) = std::fs::read_dir(&directory) else { - continue; - }; - for entry in entries.flatten() { - let path: PathBuf = entry.path(); - if entry.file_type().is_ok_and(|kind| kind.is_dir()) { - pending.push(path); - } else if let Some(stamp) = stamp_path(&path) { - recorder - .get() - .keep_unrecorded(absolute_path(py, &path)?, (stamp.size, stamp.mtime_ns)); - } - } - } - Ok(()) -} - /// Note that one staged artifact was published at another path. pub(crate) fn artifact_moved( recorder: ArtifactRecorder<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 8e6b4900cb..6045989c26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -23,8 +23,7 @@ use crate::bindings::_helpers::boundary::python_calls::{ use crate::bindings::_helpers::compile_command::artifact_batch::ArtifactBatch; use crate::bindings::_helpers::compile_command::constants::RACY_WINDOW_NS; use crate::bindings::_helpers::compile_command::reuse_recording::{ - ArtifactRecorder, CompileArtifactWrites, artifact_kept, artifact_moved, artifact_tree_kept, - artifact_tree_moved, + ArtifactRecorder, CompileArtifactWrites, artifact_kept, artifact_moved, artifact_tree_moved, }; use crate::bindings::_helpers::compile_command::test_artifact_cache::{ IdentityContext, artifact_identity_payload, artifact_record_key_payload, identity_context, @@ -868,71 +867,20 @@ pub(crate) fn write_rule_failed_models<'py>( Ok(failing) } -/// After a passing delta compile whose stored compile failed its Rules, bring `target/compiled/` up to that compile: write each model the delta did not rerun from `target/compiled-failed/` without its header, remove model files the project no longer has, and record every other compiled file as kept. -pub(crate) fn restore_models_after_failed_compile<'py>( - target_dir: &Bound<'py, PyAny>, - project: &Bound<'py, PyAny>, - recorder: ArtifactRecorder<'_>, -) -> PyResult<()> { - let py: Python<'py> = target_dir.py(); - let Some(injection) = crate::bindings::_helpers::compile_context::delta::injection() else { - return Ok(()); - }; - let target: PathBuf = target_dir.extract()?; - if !target.join(FAILED_COMPILED_DIRECTORY_NAME).is_dir() { - return Ok(()); - } - let mut rerun: Vec<(String, String)> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { - let model: Bound<'py, PyAny> = model?; - let relative_path: String = if let Ok(native) = model.cast::() { - native.get().output_texts(py)?.0 - } else { - path_posix_text(&native_field(&model, "relative_path")?)? - }; - rerun.push((relative_path, String::new())); - } - let compiled_dir: PathBuf = target.join(COMPILED_DIR); - let mut batch: ArtifactBatch = ArtifactBatch::new(true, recorder); - for (relative_path, query_sql) in - models_not_rerun(&target, &rerun, &injection.project_model_paths)? - { - batch.queue_sql_at( - py, - native_model_output_path(&compiled_dir, &relative_path), - &query_sql, - )?; - } - batch.flush(py)?; - let managed: Vec = injection - .project_model_paths - .iter() - .map(|relative_path| native_model_output_path(&compiled_dir, relative_path)) - .collect(); - let _ = remove_stale_artifacts(py, compiled_dir.join(MODELS_DIR), managed, None)?; - artifact_tree_kept(py, recorder, &compiled_dir) -} - -/// In a delta compile, the compiled SQL of each project model it did not rerun, unchanged since the stored compile: read from `target/compiled-failed/` without its header when that compile failed its Rules, otherwise from `target/compiled/`. +/// In a delta compile, which runs only after a passing compile, the compiled SQL of each project model it did not rerun, unchanged in `target/compiled/` since that compile. fn models_not_rerun( target: &Path, rerun: &[(String, String)], project_model_paths: &[String], ) -> PyResult> { let rerun: HashSet<&str> = rerun.iter().map(|(path, _)| path.as_str()).collect(); - let failed_dir: PathBuf = target.join(FAILED_COMPILED_DIRECTORY_NAME); - let stored_failed: bool = failed_dir.is_dir(); - let stored_dir: PathBuf = if stored_failed { - failed_dir - } else { - target.join(COMPILED_DIR) - }; + let compiled_dir: PathBuf = target.join(COMPILED_DIR); let mut models: Vec<(String, String)> = Vec::new(); for relative_path in project_model_paths { if rerun.contains(relative_path.as_str()) { continue; } - let path: PathBuf = native_model_output_path(&stored_dir, relative_path); + let path: PathBuf = native_model_output_path(&compiled_dir, relative_path); let Ok(text) = std::fs::read_to_string(&path) else { return Err( crate::bindings::_helpers::compile_context::delta::delta_unsupported( @@ -940,43 +888,16 @@ fn models_not_rerun( ), ); }; - let text: String = if cfg!(windows) { + let query_sql: String = if cfg!(windows) { text.replace("\r\n", "\n") } else { text }; - let query_sql: String = if stored_failed { - without_failure_header(&text, relative_path).to_owned() - } else { - text - }; models.push((relative_path.clone(), query_sql)); } Ok(models) } -/// `text` without the not-buildable header and the Rule error lines `rule_error_lines` wrote for `relative_path`. -fn without_failure_header<'a>(text: &'a str, relative_path: &str) -> &'a str { - let Some(mut rest) = text - .strip_prefix(RULE_FAILED_HEADER) - .and_then(|rest| rest.strip_prefix('\n')) - else { - return text; - }; - let location: String = format!("at {relative_path}"); - while let Some((line, after)) = rest.split_once('\n') { - let is_error_line: bool = line - .strip_prefix("-- ") - .and_then(|line| line.split_once(' ')) - .is_some_and(|(_, line)| line.starts_with(location.as_str())); - if !is_error_line { - break; - } - rest = after; - } - rest -} - /// `path` relative to `failed_dir`, with `/` separators. fn failed_relative(failed_dir: &Path, path: &Path) -> String { match path.strip_prefix(failed_dir) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs index 1c779e83ed..8d3821e387 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/driver.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyStringMethods; use sqlbuild_analysis::assembly::analysis_session::models::RelationTables; +use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME; use sqlbuild_cache::project_reuse::main::read_delta_base::read_delta_base; use sqlbuild_cache::project_reuse::main::read_delta_output::read_delta_output; use sqlbuild_cache::project_reuse::models::{DeltaBase, DeltaCandidate, ProjectFileChanges}; @@ -14,6 +15,7 @@ use sqlbuild_cache::project_reuse::types::ImageSections; use crate::bindings::_helpers::build_progress::progress::{ NativeProgressProjector, current_projector, }; +use crate::bindings::_helpers::compile_command::constants::TARGET_DIRECTORY_NAME; use crate::bindings::_helpers::compile_command::reuse_attempt::reuse_debug; use crate::bindings::_helpers::compile_command::reuse_recording::{ RecordedCompileOutput, print_to, @@ -73,6 +75,8 @@ const UNCOMPILED_MODEL: &str = "the delta compile reported a model it did not co const CANDIDATES_LOGGED: usize = 5; const UNREADABLE_IMAGE: &str = "the project image cannot be read"; const TOO_MANY_MODELS: &str = "too many models would compile again"; +/// A compile whose Rules failed writes no SQL tests, functions or audits, so the next compile is full. +const AFTER_FAILED_RULES: &str = "the previous compile failed its Rules"; /// The attribute holding a Python enum member's value. const ENUM_VALUE: &str = "value"; /// The lineage mode a delta compile reproduces; richer modes compile in full. @@ -514,6 +518,14 @@ pub(crate) fn run<'py>( if let Some(reason) = ineligible(request)? { return fallback(reason); } + let project_dir: PathBuf = native_field(attempt, "project_dir")?.extract()?; + if project_dir + .join(TARGET_DIRECTORY_NAME) + .join(FAILED_COMPILED_DIRECTORY_NAME) + .is_dir() + { + return fallback(AFTER_FAILED_RULES); + } let py: Python<'py> = request.py(); let started: std::time::Instant = std::time::Instant::now(); let prepared: Prepared = match prepared(request, attempt)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 23ce9c2f2a..d48526ba63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::compile_command::reuse_recording::{ CompileArtifactWrites, RecordedCompileOutput, compile_reuse_timings, new_artifact_recorder, }; use crate::bindings::_helpers::compile_command::target_writer::{ - remove_failed_compile, restore_models_after_failed_compile, write_rule_failed_models, + remove_failed_compile, write_rule_failed_models, }; use crate::bindings::_helpers::compile_context::input_reads::record_compile_input_reads; use crate::bindings::_helpers::compile_driver::analysis::analyze; @@ -385,11 +385,6 @@ fn compiled_with_status<'py>( ], )? } else { - restore_models_after_failed_compile( - &target_dir(&project_dir)?, - &native_field(&native_field(&analysis, "graph")?, "project")?, - recorder, - )?; remove_failed_compile(&target_dir(&project_dir)?.extract::()?)?; written }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index c929ba3394..e07ab58f5f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -10,9 +10,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{FromPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; use sqlbuild_cache::digest::main::hex_digest::hex_digest; -use sqlbuild_cache::project_reuse::constants::{ - FAILED_COMPILED_DIRECTORY_NAME, NATIVE_DIGEST_PREFIX, REUSE_STORE_SUFFIX, -}; +use sqlbuild_cache::project_reuse::constants::{NATIVE_DIGEST_PREFIX, REUSE_STORE_SUFFIX}; use sqlbuild_cache::project_reuse::main::check_reuse::check_reuse; use sqlbuild_cache::project_reuse::main::pending_digest_bytes::pending_digest_bytes; use sqlbuild_cache::project_reuse::main::record_reuse::record_reuse; @@ -230,23 +228,9 @@ impl NativeReuseAttempt { image: crate::bindings::_helpers::compile_context::delta::take_pending_image(), vouched_target_stamps: None, }; - if let Some(mut merge) = + if let Some(merge) = crate::bindings::_helpers::compile_context::delta::take_record_merge() { - let failed: String = format!( - "{}{}", - self.attempt - .lock() - .map_err(poisoned)? - .project_dir - .join(&rules.compiled_root) - .with_file_name(FAILED_COMPILED_DIRECTORY_NAME) - .to_string_lossy(), - std::path::MAIN_SEPARATOR - ); - merge - .kept - .retain(|stamp| !stamp.relative_path.starts_with(failed.as_str())); for stamp in &merge.kept { compile .artifacts diff --git a/scripts/delta_compile_series/_helpers/outcomes.py b/scripts/delta_compile_series/_helpers/outcomes.py index 7f69434af1..d47cb8aed8 100644 --- a/scripts/delta_compile_series/_helpers/outcomes.py +++ b/scripts/delta_compile_series/_helpers/outcomes.py @@ -7,6 +7,7 @@ from pathlib import Path from scripts.delta_compile_series.constants import ( + AFTER_FAILED_RULES, COMPILE_REUSE_HIT_LINE, COMPILE_TIMINGS_PATTERN, DELTA_COMPILE_LINE, @@ -47,6 +48,8 @@ def judged( reason: str | None = fallback_reason(outcome=outcome) if others_edited and reason is not None and STALE_PROBLEMS in reason: return False, None + if after_failure and reason == AFTER_FAILED_RULES: + return False, None if edit.fallback_reason is None: verified: bool = DELTA_VERIFIED_LINE in outcome.stderr return verified, None if verified else f"fell back: {reason}" diff --git a/scripts/delta_compile_series/constants.py b/scripts/delta_compile_series/constants.py index d82dbe34cc..edafd2c76e 100644 --- a/scripts/delta_compile_series/constants.py +++ b/scripts/delta_compile_series/constants.py @@ -18,6 +18,7 @@ DELTA_WEIGHT: int = 3 FALLBACK_WEIGHT: int = 1 STALE_PROBLEMS: str = "reports problems in files the edit leaves alone" +AFTER_FAILED_RULES: str = "the previous compile failed its Rules" PROJECT_CONFIG: str = "sqlbuild_project.toml" RANDOM_VALUE_LIMIT: int = 10**6 FULL_COPY_SUFFIX: str = "_full_copy" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index a9fb903033..52d64bb4aa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -146,6 +146,20 @@ def order_quality(): "MODEL (description", "MODEL (audit_factories [order_quality], description", ) +_RULE_HARNESS: str = """import pytest +from sqlbuild.rules.testing import RuleCase, evaluate_rule +from rules.{module} import {function} + +CASES = ( + RuleCase(description="plain model", source="MODEL (); SELECT 1 AS id", expected_finding_count=0), +) + + +@pytest.mark.parametrize("test_case", CASES, ids=lambda case: case.description) +def test_given_model_when_checking_then_reports_expected_findings(test_case): + result = evaluate_rule(rule={function}, test_case=test_case) + assert result.finding_count == test_case.expected_finding_count +""" _MODEL_SQL_RULE: tuple[str, str] = ( "rules/order_sql.py", """from sqlbuild.rules import Finding, Model, RuleContext, rule @@ -178,6 +192,21 @@ def order_folder_siblings(*, model: Model, ctx: RuleContext) -> list[Finding]: return [ctx.finding(subject=model)] if len(siblings) < 2 else [] """, ) +# Each custom Rule needs a harness case, or the stored compile fails its Rules and the next is full. +_MODEL_SQL_RULE_CASES: tuple[str, str] = ( + "tests/test_rule_order_sql.py", + _RULE_HARNESS.format(module="order_sql", function="order_todo_markers"), +) +_FOLDER_RULE_CASES: tuple[str, str] = ( + "tests/test_rule_order_folders.py", + _RULE_HARNESS.format(module="order_folders", function="order_folder_siblings"), +) +# Built-in Rules cannot parse PIVOT models, which would fail the stored compile's Rules. +_BUILT_IN_RULES_OFF: tuple[str, str, str] = ( + "sqlbuild_project.toml", + 'select = ["SQBR", "XSQBR"]', + 'select = ["XSQBR"]', +) _RULE_READ_MORE: str = "delta compile fell back: a custom rule read more than the changed models" _FAMILY_TYPE_EDIT: tuple[str, str, str] = (_CATEGORY_PIVOT_PATH, "type DOUBLE", "type VARCHAR") _FAMILIES_CHANGED: str = "delta compile fell back: a model's dynamic column families changed" @@ -354,6 +383,7 @@ def test_given_stored_compile_when_compiling_an_edit_then_it_prints_what_a_full_ DeltaCompileProjectShapeTestCase( description="a model with dynamic column families", files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + stored_edits=(_BUILT_IN_RULES_OFF,), edits=(_BODY_EDIT,), expected_delta_line=True, ), @@ -366,19 +396,20 @@ def test_given_stored_compile_when_compiling_an_edit_then_it_prints_what_a_full_ ), DeltaCompileProjectShapeTestCase( description="a model-subject custom rule reading the edited model's SQL", - files=(_MODEL_SQL_RULE,), + files=(_MODEL_SQL_RULE, _MODEL_SQL_RULE_CASES), edits=(_BODY_EDIT,), expected_delta_line=True, ), DeltaCompileProjectShapeTestCase( description="a model-subject custom rule listing models under a folder", - files=(_FOLDER_RULE,), + files=(_FOLDER_RULE, _FOLDER_RULE_CASES), edits=(_BODY_EDIT,), expected_delta_line=False, ), DeltaCompileProjectShapeTestCase( description="an edit to dynamic column families compiles in full", files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + stored_edits=(_BUILT_IN_RULES_OFF,), edits=(_FAMILY_TYPE_EDIT,), expected_delta_line=False, ), @@ -419,8 +450,14 @@ def test_given_project_shape_when_compiling_an_edit_then_it_prints_what_a_full_c [ DeltaCompileVerifyTestCase( description="every shape together matches a full compile", - files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT, _ORDER_QUALITY_FACTORY, _MODEL_SQL_RULE), - stored_edits=(_ATTACH_ORDER_QUALITY,), + files=( + _CATEGORY_AMOUNTS, + _CATEGORY_PIVOT, + _ORDER_QUALITY_FACTORY, + _MODEL_SQL_RULE, + _MODEL_SQL_RULE_CASES, + ), + stored_edits=(_ATTACH_ORDER_QUALITY, _BUILT_IN_RULES_OFF), edits=(_BODY_EDIT,), env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, expected_stderr_line=DELTA_VERIFIED_LINE, @@ -428,13 +465,14 @@ def test_given_project_shape_when_compiling_an_edit_then_it_prints_what_a_full_c DeltaCompileVerifyTestCase( description="an edit to dynamic column families names why it fell back", files=(_CATEGORY_AMOUNTS, _CATEGORY_PIVOT), + stored_edits=(_BUILT_IN_RULES_OFF,), edits=(_FAMILY_TYPE_EDIT,), env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, expected_stderr_line=_FAMILIES_CHANGED, ), DeltaCompileVerifyTestCase( description="a rule listing models under a folder names why it fell back", - files=(_FOLDER_RULE,), + files=(_FOLDER_RULE, _FOLDER_RULE_CASES), edits=(_BODY_EDIT,), env={DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, expected_stderr_line=_RULE_READ_MORE, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 37eaa64ebc..3f3103c468 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1349,6 +1349,14 @@ class RuleFailureSequenceTestCase: expected_minimum_verified_deltas: int +@dataclass(frozen=True) +class RuleFailureStaleArtifactTestCase: + description: str + environment: dict[str, str] + extra_args: tuple[str, ...] + expected_after_failure_fragment: str + + @dataclass(frozen=True) class SignalledScenarioTestCase: description: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 41c72480f9..9b0960d5c6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -64,6 +64,12 @@ def artifact_tree(root: Path) -> dict[str, bytes]: return {path.relative_to(root).as_posix(): path.read_bytes() for path in files} +def changed_artifacts(before: dict[str, bytes], after: dict[str, bytes]) -> list[str]: + """Return the sorted paths whose bytes differ between two artifact trees, or exist in one.""" + + return sorted(filter(lambda path: before.get(path) != after.get(path), before | after)) + + def staging_directories(target_dir: Path) -> list[str]: """Return hidden entries left in target_dir, such as abandoned artifact staging.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py index 82e5392af8..1d9b0464f1 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -11,9 +11,11 @@ from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RuleFailureArtifactsTestCase, RuleFailureSequenceTestCase, + RuleFailureStaleArtifactTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( artifact_tree, + changed_artifacts, open_orders_model, write_project_files, ) @@ -52,6 +54,31 @@ _VERIFIED_LINE: str = "delta compile verified" +_TOTALS: str = ( + 'MODEL (\n description "Order totals",\n materialized table,\n);\n\n' + 'SELECT o.order_id, o.order_id * 10 AS total FROM __ref("orders") AS o\n' +) +_TOTALS_TEST: str = ( + 'TEST (name "order_totals_case");\n' + "WITH\n" + "__ref__orders AS (SELECT 1 AS order_id, 'open' AS status),\n" + "__expected__order_totals AS (SELECT 1 AS order_id, 10 AS total)\n" + "SELECT 1\n" +) +_LARGE_FUNCTION: str = ( + "FUNCTION (description 'Whether a total is large.',\n" + " arguments (total INTEGER),\n" + " returns BOOLEAN\n" + ");\n\n" + "total > 100" +) +_TOTALS_AUDIT: str = ( + 'AUDIT ();\n\nSELECT t.order_id FROM __ref("order_totals") AS t ' + 'JOIN __ref("orders") AS o ON o.order_id = t.order_id WHERE t.total < 0' +) +_AFTER_FAILURE_LINE: str = "delta compile fell back: the previous compile failed its Rules" + + @pytest.mark.parametrize( "test_case", [ @@ -152,7 +179,7 @@ def test_given_rule_findings_when_compiling_then_failed_sql_is_written_beside_co environment={COMPILE_REUSE_DISABLE_ENV_VAR: "0", DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, extra_args=(), expected_rewarm_reused=True, - expected_minimum_verified_deltas=5, + expected_minimum_verified_deltas=2, ), RuleFailureSequenceTestCase( description="without the compile cache", @@ -267,3 +294,101 @@ def edited(files: dict[str, str]) -> None: assert not failed.exists() assert artifact_tree(compiled) == first_pass assert stderr_lines.count(_VERIFIED_LINE) >= test_case.expected_minimum_verified_deltas + + +@pytest.mark.parametrize( + "test_case", + [ + RuleFailureStaleArtifactTestCase( + description="with the compile cache", + environment={COMPILE_REUSE_DISABLE_ENV_VAR: "0"}, + extra_args=(), + expected_after_failure_fragment="", + ), + RuleFailureStaleArtifactTestCase( + description="with each delta compile checked against a full compile", + environment={COMPILE_REUSE_DISABLE_ENV_VAR: "0", DELTA_COMPILE_VERIFY_ENV_VAR: "1"}, + extra_args=(), + expected_after_failure_fragment=_AFTER_FAILURE_LINE, + ), + RuleFailureStaleArtifactTestCase( + description="without the compile cache", + environment={}, + extra_args=("--no-cache",), + expected_after_failure_fragment="", + ), + ], + ids=lambda case: case.description, +) +def test_given_resources_edited_while_rules_fail_when_the_fix_passes_then_compiled_matches_a_full_compile( + test_case: RuleFailureStaleArtifactTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + for name, value in test_case.environment.items(): + monkeypatch.setenv(name, value) + project_dir: Path = tmp_path / "project" + files: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT, + "models/orders.sql": _ORDERS, + "models/order_totals.sql": _TOTALS, + "models/open_orders.sql": open_orders_model(_PASSING), + "tests/unit/test_order_totals.sql": _TOTALS_TEST, + "functions/sql/is_large.sql": _LARGE_FUNCTION, + "audits/singular/positive_totals.sql": _TOTALS_AUDIT, + } + write_project_files(project_dir=project_dir, files=files) + compile_args: list[str] = ["--project-dir", str(project_dir), "compile", *test_case.extra_args] + compiled_dir: Path = project_dir / "target" / "compiled" + + def fixed_after_failing(name: str, edits: dict[str, str]) -> tuple[str, dict[str, bytes]]: + write_project_files( + project_dir=project_dir, + files={**edits, "models/open_orders.sql": open_orders_model(_FAILING)}, + ) + assert main(compile_args) == 1 + _ = capsys.readouterr() + write_project_files( + project_dir=project_dir, files={"models/open_orders.sql": open_orders_model(_PASSING)} + ) + exit_code: int = main(compile_args) + stdout, stderr = capsys.readouterr() + assert exit_code == 0, stdout + stderr + reference_dir: Path = tmp_path / name + files.update(edits) + write_project_files(project_dir=reference_dir, files=files) + assert main(["--project-dir", str(reference_dir), "compile", "--no-cache"]) == 0 + _ = capsys.readouterr() + assert artifact_tree(compiled_dir) == artifact_tree(reference_dir / "target" / "compiled") + assert not (project_dir / "target" / "compiled-failed").exists() + return stderr, artifact_tree(compiled_dir) + + assert main(compile_args) == 0 + passing: dict[str, bytes] = artifact_tree(compiled_dir) + test_edit_stderr, after_test_edit = fixed_after_failing( + "after_test_edit", + { + "tests/unit/test_order_totals.sql": _TOTALS_TEST.replace("10 AS total", "20 AS total"), + "functions/sql/is_large.sql": _LARGE_FUNCTION.replace("> 100", "> 200"), + }, + ) + model_edit_stderr, after_model_edit = fixed_after_failing( + "after_model_edit", + { + "models/order_totals.sql": _TOTALS.replace("* 10", "* 20"), + "audits/singular/positive_totals.sql": _TOTALS_AUDIT.replace("< 0", "<= 0"), + }, + ) + + assert test_case.expected_after_failure_fragment in test_edit_stderr + assert test_case.expected_after_failure_fragment in model_edit_stderr + assert changed_artifacts(passing, after_test_edit) == [ + "functions/sql/is_large.sql", + "tests/order_totals/order_totals_case.sql", + ] + assert changed_artifacts(after_test_edit, after_model_edit) == [ + "audits/singular/positive_totals.sql", + "models/order_totals.sql", + "tests/order_totals/order_totals_case.sql", + ] From 5cea495d0dd037136c9b35785f635aa76201cf2c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:13:14 +0000 Subject: [PATCH 08/12] fix(compile): report no manifest when the Rules fail A compile whose Rules fail removes target/manifest.json, but with --manifest its report still said it wrote one. The text and JSON reports now leave the manifest out after a Rules failure. --- .../_helpers/compile_command/report.rs | 7 +++- .../sqlbuild/cli/commands/main/_test_types.py | 7 ++++ .../main/test_compile_rule_failures.py | 38 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 017d0f13ff..ef027c023e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -60,6 +60,7 @@ struct ReportInputs<'py> { graph: Bound<'py, PyAny>, project: Bound<'py, PyAny>, written: Bound<'py, PyAny>, + /// A manifest was requested and written, which a compile whose Rules failed never does. manifest: bool, lineage: Bound<'py, PyAny>, diagnostics: Vec>, @@ -72,11 +73,13 @@ impl<'py> ReportInputs<'py> { fn read(arguments: &KeywordArguments<'py>) -> PyResult { let graph: Bound<'py, PyAny> = arguments.value("graph")?; let lineage_mode: Bound<'py, PyAny> = arguments.value("lineage_mode")?; + let written: Bound<'py, PyAny> = arguments.value("written")?; + let failed_compile: bool = native_field(&written, "failed_compile")?.is_truthy()?; Ok(Self { project: native_field(&graph, "project")?, graph, - written: arguments.value("written")?, - manifest: arguments.flag("manifest")?, + manifest: arguments.flag("manifest")? && !failed_compile, + written, lineage: arguments.value("lineage")?, diagnostics: arguments .value("diagnostics")? diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 3f3103c468..d858fb7f28 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1363,3 +1363,10 @@ class SignalledScenarioTestCase: sent_signal: signal.Signals expected_exit_code: int expected_last_line: str + + +@dataclass(frozen=True) +class RuleFailureManifestTestCase: + description: str + compile_args: tuple[str, ...] + expected_artifacts: dict[str, object] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py index 1d9b0464f1..4faa1eb01e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path import pytest @@ -10,6 +11,7 @@ from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RuleFailureArtifactsTestCase, + RuleFailureManifestTestCase, RuleFailureSequenceTestCase, RuleFailureStaleArtifactTestCase, ) @@ -161,9 +163,45 @@ def test_given_rule_findings_when_compiling_then_failed_sql_is_written_beside_co assert artifact_tree(failed) == test_case.expected_failed assert failed.exists() is bool(test_case.expected_failed) assert (tmp_path / "target" / "manifest.json").exists() is test_case.expected_manifest + assert ("Wrote: target/manifest.json" in output) is test_case.expected_manifest assert not (tmp_path / "target" / "compiled-models.txt").exists() +@pytest.mark.parametrize( + "test_case", + [ + RuleFailureManifestTestCase( + description="JSON names the failed SQL and no manifest", + compile_args=("--manifest", "--json"), + expected_artifacts={ + "compiled_sql_dir": "target/compiled-failed/", + "manifest": None, + "failed_model_sql": ["target/compiled-failed/models/open_orders.sql"], + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_manifest_requested_when_rules_fail_then_json_reports_no_manifest( + test_case: RuleFailureManifestTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project_files( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": _PROJECT, + "models/orders.sql": _ORDERS, + "models/open_orders.sql": open_orders_model(_FAILING), + }, + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "compile", *test_case.compile_args]) + + payload: dict[str, object] = json.loads(capsys.readouterr().out) + assert exit_code == 1 + assert payload["artifacts"] == test_case.expected_artifacts + assert not (tmp_path / "target" / "manifest.json").exists() + + @pytest.mark.parametrize( "test_case", [ From 354d9c8abbf13cfed7fa3113bd883c4fe4e2f58a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:53:14 +0000 Subject: [PATCH 09/12] perf(snowflake): keep relation metadata across session-only statements A per-model query tag hook such as ALTER SESSION SET QUERY_TAG, and USE WAREHOUSE, change no relation and no name resolution, but they invalidated every cached listing, so the next lookup listed the whole schema again. Setting or resetting one parameter known not to affect name resolution, and changing the warehouse, now leave the cache alone. Other session parameters, USE of a database, schema or role, and ambiguous statements such as CALL or EXECUTE IMMEDIATE still invalidate it. --- .../src/ddl_effects/_helpers/relations.rs | 71 ++++++++++++-- .../src/ddl_effects/constants.rs | 13 +++ .../src/ddl_effects/tests/test_ddl_effects.rs | 98 +++++++++++++++++++ .../build/snowflake_metadata/_test_types.py | 2 + .../build/snowflake_metadata/helpers.py | 9 +- .../test_statement_overhead.py | 17 +++- 6 files changed, 200 insertions(+), 10 deletions(-) diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs index c33a15d955..8f177f7e74 100644 --- a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs @@ -11,9 +11,10 @@ use crate::ddl_effects::_helpers::matching::{ }; use crate::ddl_effects::_helpers::names::{UnquotedPart, part_name_key, qualified_name}; use crate::ddl_effects::constants::{ - ALTER_FOLLOWING_WORDS, COPY_INTO, CREATE_FOLLOWING_WORDS, CREATE_SCHEMA_IF_NOT_EXISTS, - DROP_FOLLOWING_WORDS, FROM_WORD, IF_EXISTS, IF_NOT_EXISTS, INTO, NEUTRAL_OBJECT_KINDS, - NEUTRAL_OBJECT_LEADS, NEUTRAL_OBJECT_MODIFIERS, OR_REPLACE, QUERY_STATEMENTS, + ALTER_FOLLOWING_WORDS, ALTER_SESSION_SET, ALTER_SESSION_UNSET, COMPUTE_ONLY_STATEMENTS, + COPY_INTO, CREATE_FOLLOWING_WORDS, CREATE_SCHEMA_IF_NOT_EXISTS, DROP_FOLLOWING_WORDS, + FROM_WORD, IF_EXISTS, IF_NOT_EXISTS, INTO, NEUTRAL_OBJECT_KINDS, NEUTRAL_OBJECT_LEADS, + NEUTRAL_OBJECT_MODIFIERS, NEUTRAL_SESSION_PARAMETERS, OR_REPLACE, QUERY_STATEMENTS, READ_OR_DATA_STATEMENTS, RELATION_DDL_STATEMENTS, RELATION_KIND_MODIFIERS, RELATION_KINDS, RENAMING_ALTERS, STATEMENT_SEPARATOR, TRANSACTION_END_STATEMENTS, }; @@ -85,9 +86,10 @@ fn select_into_relation(text: &str) -> Option> { .find_map(|(index, _)| named_relation(text, whitespace(text, index + INTO.len())?)) } -fn has_into_keyword(text: &str) -> bool { +/// Whether `keyword` appears in `text` as a whole word. +fn has_keyword(text: &str, keyword: &str) -> bool { text.char_indices().any(|(index, _)| { - word(text, index, INTO) + word(text, index, keyword) .is_some_and(|end| word_boundary(text, index) && word_boundary(text, end)) }) } @@ -144,7 +146,7 @@ fn named_effect(names: BTreeSet) -> StatementMetadataEffect { } fn select_into_effect(statement: &str) -> StatementMetadataEffect { - if !has_into_keyword(statement) { + if !has_keyword(statement, INTO) { return StatementMetadataEffect::default(); } unambiguous_names(Form::SelectInto, select_into_relation(statement)) @@ -190,6 +192,60 @@ fn is_neutral_object(statement: &str) -> bool { bounded_phrase(statement, kind_start, NEUTRAL_OBJECT_KINDS).is_some() } +/// The end of the session parameter value at `start`: a backslash-free literal or a bare word. +fn session_value_end(text: &str, start: usize) -> Option { + let rest: &str = &text[start..]; + let Some(quoted) = rest.strip_prefix('\'') else { + let length: usize = rest + .chars() + .take_while(|character| !python_isspace(*character)) + .map(char::len_utf8) + .sum(); + return (length > 0 && !rest[..length].contains('\'')).then_some(start + length); + }; + let mut position: usize = 0; + loop { + let close: usize = quoted[position..].find('\'')? + position; + if quoted[position..close].contains('\\') { + return None; + } + if !quoted[close + 1..].starts_with('\'') { + return Some(start + 1 + close + 1); + } + position = close + 2; + } +} + +/// Whether the statement only sets or resets one session parameter that name resolution ignores. +fn sets_neutral_session_parameter(statement: &str) -> bool { + if let Some(end) = phrase(statement, 0, ALTER_SESSION_UNSET) { + return whitespace(statement, end) + .and_then(|start| bounded_phrase(statement, start, NEUTRAL_SESSION_PARAMETERS)) + .is_some_and(|end| only_whitespace(statement, end)); + } + let Some(start) = + phrase(statement, 0, ALTER_SESSION_SET).and_then(|end| whitespace(statement, end)) + else { + return false; + }; + let Some(after_name) = bounded_phrase(statement, start, NEUTRAL_SESSION_PARAMETERS) else { + return false; + }; + let equals: usize = skip_whitespace(statement, after_name); + if !statement[equals..].starts_with('=') { + return false; + } + session_value_end(statement, skip_whitespace(statement, equals + 1)) + .is_some_and(|end| only_whitespace(statement, end)) +} + +/// Whether the statement changes only session state that relations and name resolution ignore. +fn is_session_only(statement: &str) -> bool { + bounded_phrase(statement, 0, COMPUTE_ONLY_STATEMENTS) + .is_some_and(|end| whitespace(statement, end).is_some()) + || sets_neutral_session_parameter(statement) +} + /// The effect of one reading of the statement's code text; `None` code changes everything. pub(crate) fn code_effect(code: Option<&str>) -> StatementMetadataEffect { let Some(code) = code else { @@ -217,6 +273,9 @@ pub(crate) fn code_effect(code: Option<&str>) -> StatementMetadataEffect { ..StatementMetadataEffect::default() }; } + if is_session_only(statement) { + return StatementMetadataEffect::default(); + } if !RELATION_DDL_STATEMENTS.contains(&keyword.as_str()) { return all_relations(); } diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs b/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs index 5c5c822933..9576778176 100644 --- a/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs +++ b/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs @@ -31,6 +31,19 @@ pub(crate) const READ_OR_DATA_STATEMENTS: &[&str] = &[ "RM", "ANALYZE", ]; +/// Statements that change only the compute later statements run on. +pub(crate) const COMPUTE_ONLY_STATEMENTS: &[&[&str]] = &[&["USE", "WAREHOUSE"]]; +/// `ALTER SESSION SET` before the parameter it sets. +pub(crate) const ALTER_SESSION_SET: &[&str] = &["ALTER", "SESSION", "SET"]; +/// `ALTER SESSION UNSET` before the parameter it resets. +pub(crate) const ALTER_SESSION_UNSET: &[&str] = &["ALTER", "SESSION", "UNSET"]; +/// Session parameters that change neither relations nor how their names resolve. +pub(crate) const NEUTRAL_SESSION_PARAMETERS: &[&[&str]] = &[ + &["QUERY_TAG"], + &["TIMEZONE"], + &["STATEMENT_TIMEOUT_IN_SECONDS"], + &["USE_CACHED_RESULT"], +]; /// Statements that end the current transaction. pub(crate) const TRANSACTION_END_STATEMENTS: &[&str] = &["COMMIT", "ROLLBACK", "END", "ABORT"]; /// Statements whose relation name is read from the statement. diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs index 761ce6a14a..d75b029d7f 100644 --- a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs @@ -121,6 +121,104 @@ fn given_executed_statement_when_classifying_then_reports_possible_metadata_chan expected_invalidates_all: true, expected_ends_transaction: false, }, + StatementEffectTestCase { + description: "query tag for the session", + sql: "ALTER SESSION SET QUERY_TAG = 'orders'", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "session setting reset", + sql: "ALTER SESSION UNSET QUERY_TAG", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "warehouse for the session", + sql: "USE WAREHOUSE transform_wh", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "search path changes how unqualified names resolve", + sql: "ALTER SESSION SET SEARCH_PATH = 'marts, public'", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "identifier case setting changes how names resolve", + sql: "ALTER SESSION SET QUOTED_IDENTIFIERS_IGNORE_CASE = TRUE", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "query tag followed by another parameter", + sql: "ALTER SESSION SET QUERY_TAG = 'orders' SEARCH_PATH = 'marts'", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "query tag with an escaped quote", + sql: "ALTER SESSION SET QUERY_TAG = 'order''s run'", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "statement timeout for the session", + sql: "ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS = 3600", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "schema for the session changes unqualified names", + sql: "USE SCHEMA analytics.marts", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "database for the session changes unqualified names", + sql: "USE DATABASE analytics", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "role for the session changes visible relations", + sql: "USE ROLE transformer", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "bare use names a database", + sql: "USE analytics", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "dynamic sql is ambiguous", + sql: "EXECUTE IMMEDIATE 'DROP TABLE orders'", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "scripting block is ambiguous", + sql: "BEGIN DROP TABLE orders; END", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, StatementEffectTestCase { description: "commit ends the transaction", sql: "COMMIT;", diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py index f57317ade7..3df1107d3b 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py @@ -68,6 +68,8 @@ class RetentionLookupTestCase: expected_first_alters: int expected_max_table_lookups: int expected_max_listings: int + header_options: str = "" + expected_query_tags: int = 0 expected_information_schema: tuple[str, ...] = () diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 49fa0a5cdf..71fce24b0a 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -746,8 +746,10 @@ def create_external_table(warehouse: SimulatedSnowflakeWarehouse, name: str) -> ) -def write_model_copies(*, project_dir: Path, count: int, retention_days: int) -> None: - """Add ``count`` permanent table copies of the orders table with managed retention.""" +def write_model_copies( + *, project_dir: Path, count: int, retention_days: int, header_options: str = "" +) -> None: + """Add ``count`` permanent table copies of the orders table, with ``header_options``.""" project: Path = project_dir / "sqlbuild_project.toml" _ = project.write_text( @@ -759,7 +761,8 @@ def write_model_copies(*, project_dir: Path, count: int, retention_days: int) -> index: int for index in range(count): _ = (project_dir / "models" / "marts" / f"order_copy_{index}.sql").write_text( - f"MODEL (description 'Copy {index} of the orders.', materialized table);\n\n" + f"MODEL ({header_options}description 'Copy {index} of the orders.', " + "materialized table);\n\n" 'SELECT order_id, ordered_at, amount FROM __ref("orders_table")\n', encoding="utf-8", ) diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_statement_overhead.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_statement_overhead.py index 6cfed89015..c517180c94 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_statement_overhead.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_statement_overhead.py @@ -39,6 +39,7 @@ ) _TABLE_LOOKUP: re.Pattern[str] = re.compile(r"^SHOW TABLES LIKE ") _SCHEMA_LISTING: re.Pattern[str] = re.compile(r"^SHOW (?:TABLES|VIEWS) IN SCHEMA ") +_QUERY_TAG: re.Pattern[str] = re.compile(r"^ALTER SESSION SET QUERY_TAG") _OWNED_VIEW_DROP: re.Pattern[str] = re.compile(r"^DROP VIEW IF EXISTS \S+__(?:staging|delta)$") @@ -99,6 +100,16 @@ def test_given_materialization_switch_when_building_then_each_kind_replaces_the_ expected_max_table_lookups=7, expected_max_listings=4, ), + RetentionLookupTestCase( + description="a query tag set before each table keeps the schema listings", + table_count=6, + retention_days=3, + header_options=("pre_hooks [inline_sql(\"ALTER SESSION SET QUERY_TAG = 'orders'\")], "), + expected_first_alters=6, + expected_max_table_lookups=7, + expected_max_listings=4, + expected_query_tags=6, + ), ], ids=lambda case: case.description, ) @@ -107,7 +118,10 @@ def test_given_many_tables_in_one_schema_when_building_then_retention_is_read_wi ) -> None: write_orders_project(project_dir=tmp_path) write_model_copies( - project_dir=tmp_path, count=test_case.table_count, retention_days=test_case.retention_days + project_dir=tmp_path, + count=test_case.table_count, + retention_days=test_case.retention_days, + header_options=test_case.header_options, ) warehouse: SimulatedSnowflakeWarehouse = orders_warehouse() @@ -135,6 +149,7 @@ def test_given_many_tables_in_one_schema_when_building_then_retention_is_read_wi assert max( len(matching_statements(build.statements, _SCHEMA_LISTING)) for build in (first, second) ) <= (test_case.expected_max_listings) + assert len(matching_statements(first.statements, _QUERY_TAG)) == (test_case.expected_query_tags) @pytest.mark.parametrize( From fe59152b8614c5e851bd02825192a163d5d6ab2a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:07:28 +0000 Subject: [PATCH 10/12] fix(snowflake): clear relation metadata when a session setting lists several parameters An unquoted session parameter value ended only at whitespace, so a statement setting a neutral parameter followed by a comma and one that changes name resolution kept the cache. An unquoted value now ends before a comma or an equals sign, and such a statement clears the cache. --- .../src/ddl_effects/_helpers/relations.rs | 5 +++-- .../src/ddl_effects/tests/test_ddl_effects.rs | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs index 8f177f7e74..7937e89afe 100644 --- a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs @@ -192,7 +192,7 @@ fn is_neutral_object(statement: &str) -> bool { bounded_phrase(statement, kind_start, NEUTRAL_OBJECT_KINDS).is_some() } -/// The end of the session parameter value at `start`: a backslash-free literal or a bare word. +/// The end of the session parameter value at `start`: a backslash-free literal or one bare word. fn session_value_end(text: &str, start: usize) -> Option { let rest: &str = &text[start..]; let Some(quoted) = rest.strip_prefix('\'') else { @@ -201,7 +201,8 @@ fn session_value_end(text: &str, start: usize) -> Option { .take_while(|character| !python_isspace(*character)) .map(char::len_utf8) .sum(); - return (length > 0 && !rest[..length].contains('\'')).then_some(start + length); + let value: &str = &rest[..length]; + return (length > 0 && !value.contains(['\'', ',', '='])).then_some(start + length); }; let mut position: usize = 0; loop { diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs index d75b029d7f..7784b468b2 100644 --- a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs @@ -170,6 +170,27 @@ fn given_executed_statement_when_classifying_then_reports_possible_metadata_chan expected_invalidates_all: false, expected_ends_transaction: false, }, + StatementEffectTestCase { + description: "statement timeout listed before an identifier case setting", + sql: "ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS=10,QUOTED_IDENTIFIERS_IGNORE_CASE=TRUE", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "cached result setting listed before an identifier case setting", + sql: "ALTER SESSION SET USE_CACHED_RESULT=FALSE,QUOTED_IDENTIFIERS_IGNORE_CASE=TRUE", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "statement timeout without spaces around the equals sign", + sql: "ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS=10", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, StatementEffectTestCase { description: "statement timeout for the session", sql: "ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS = 3600", From 34d6b90f8a72e82e276ec0ab82b4f3252bdbe197 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:07:30 +0000 Subject: [PATCH 11/12] docs: name a Rules-failed previous compile as a reason to compile in full --- .../.agents/skills/sqlbuild/references/docs/cli/compile.md | 5 +++-- website/src/content/docs/docs/cli/compile.mdx | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index b8f3aae804..d3988debd7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -316,8 +316,9 @@ Compile runs in full instead when: a model-subject Rule must run again for a model that was not edited (for example because it reads an edited upstream file), or reads more for an edited model than that model's SQL, columns, contract and grain, project files and the source list; -- the previous compile reported errors or warnings, other than Rule findings, in files the edit - didn't change; +- the previous compile failed its Rules, since that compile writes no SQL tests, functions or + audits to `target/compiled/`, or it reported other errors or warnings in files the edit didn't + change; - the previous compile used the other output format (`--json` or text), or ran with delta compiles off; - the edit changes which models or sources a macro reads through its arguments, changes a model's diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index b018135bbc..86a02c0d07 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -300,8 +300,9 @@ Compile runs in full instead when: a model-subject Rule must run again for a model that was not edited (for example because it reads an edited upstream file), or reads more for an edited model than that model's SQL, columns, contract and grain, project files and the source list; -- the previous compile reported errors or warnings, other than Rule findings, in files the edit - didn't change; +- the previous compile failed its Rules, since that compile writes no SQL tests, functions or + audits to `target/compiled/`, or it reported other errors or warnings in files the edit didn't + change; - the previous compile used the other output format (`--json` or text), or ran with delta compiles off; - the edit changes which models or sources a macro reads through its arguments, changes a model's From 18270dc45b3645847720cfc47653731d60b2898d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:49:31 +0000 Subject: [PATCH 12/12] test: order the mid-build fingerprint loss case through dependencies The view whose post hook drops the fingerprint table now reads every other model, so they all record before the drop and the expected rows no longer depend on which independent model the scheduler runs first. --- .../commands/main/statement_counts/_test_types.py | 1 + .../cli/commands/main/statement_counts/helpers.py | 12 ++++++++++++ .../main/statement_counts/test_guarded_cases.py | 5 ++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/_test_types.py index 2cdbc42eb8..c7f2b78454 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/_test_types.py @@ -38,6 +38,7 @@ class FingerprintLossTestCase: description: str edited_models: tuple[str, ...] hooked_model: str + hooked_model_reads: tuple[str, ...] expected_setups: int expected_rows_after_loss: dict[str, int] expected_models_after_rebuild: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/helpers.py index 24f133d795..13c53d30a5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/helpers.py @@ -360,6 +360,18 @@ def add_post_hook(project_dir: Path, name: str, sql: str) -> str: return original +def add_upstream_reads(project_dir: Path, name: str, upstream: tuple[str, ...]) -> None: + """Make one model read ``upstream`` without changing its rows, so it builds after them.""" + + model: Path = project_dir / "models" / f"{name}.sql" + filters: str = " AND ".join( + f'NOT EXISTS (SELECT 1 FROM __ref("{reference}") WHERE FALSE)' for reference in upstream + ) + _ = model.write_text( + model.read_text(encoding="utf-8").rstrip("\n") + f" WHERE {filters}\n", encoding="utf-8" + ) + + def run_step( *, project_dir: Path, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_guarded_cases.py b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_guarded_cases.py index c89d38306a..e985e12248 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_guarded_cases.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/statement_counts/test_guarded_cases.py @@ -14,6 +14,7 @@ ) from tests.integration.src.sqlbuild.cli.commands.main.statement_counts.helpers import ( add_post_hook, + add_upstream_reads, execute_sql, fingerprint_rows, intermediate_relations, @@ -80,9 +81,10 @@ def test_given_second_build_when_identities_are_unchanged_then_only_changed_mode "test_case", [ FingerprintLossTestCase( - description="a view's post hook drops the table after the table model recorded", + description="a view's post hook drops the table after every model it reads recorded", edited_models=("orders_table",), hooked_model="orders_view", + hooked_model_reads=("orders_daily", "orders_merge"), expected_setups=2, expected_rows_after_loss={"orders_view": 1}, expected_models_after_rebuild=_ALL_MODELS, @@ -108,6 +110,7 @@ def test_given_fingerprint_table_dropped_mid_build_when_building_then_later_mode original: str = add_post_hook( project_dir, test_case.hooked_model, "DROP TABLE main._sqlbuild_fingerprints" ) + add_upstream_reads(project_dir, test_case.hooked_model, test_case.hooked_model_reads) code, output, counts, _ = run_counted(project_dir=project_dir, args=("build",), capsys=capsys) after_loss: dict[str, int] = fingerprint_rows(project_dir)