From 891262ecddc98e71959765f4abb8eade9cfb76c9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:58:36 +0100 Subject: [PATCH 01/12] fix(planner)!: decide each model's replay from its own change only Remove upstream backfill cascades. A model's backfill now comes only from its own change and its own replay_on_change, so a first-run, renamed or replayed upstream no longer forces a full or bounded replay of downstream incremental models. A changed function counts as an effective query change of the models that call it directly. Those models plan as function_changed with the cause named and apply their own replay_on_change; models further downstream continue forward. The function-level replay_on_change header is removed. A model that sets full_refresh false now refuses (S203) any planned full rebuild other than its own first run or its own query change with replay_on_change full. A FULL backfill on a non-microbatch model with cursor intrinsics reports the FULL backfill and its cause instead of an unresolved-bounds error. Plan text drops the Upstream changed group and adds a Function changed group; plan JSON drops the cascade field and upstream_changed reason and adds changed_functions to direct callers. BREAKING CHANGE: backfills no longer cascade from upstream models; each model replays only for its own change per its own replay_on_change. The function-level replay_on_change header is removed. Plan JSON drops the cascade field and the upstream_changed reason, and adds changed_functions on direct callers of changed functions and query_changed on renamed models whose query changed. New planner error S203 refuses full rebuilds of full_refresh false models not requested by their own configuration. --- src/sqlbuild/cli/output/_helpers/plan_json.py | 24 +- src/sqlbuild/cli/output/_helpers/plan_text.py | 145 +------- .../compile/_helpers/assembly/project.py | 1 - .../compile/_helpers/attachment/functions.py | 34 -- src/sqlbuild/compiler/compile/models.py | 2 - .../planner/_helpers/changes/actions.py | 32 ++ .../planner/_helpers/changes/detect.py | 69 ++-- .../planner/_helpers/changes/metadata.py | 43 ++- .../planner/_helpers/identity/functions.py | 28 +- .../planner/_helpers/output/plan_entry.py | 59 ++-- .../planner/_helpers/output/plan_output.py | 8 +- .../planner/_helpers/output/strategy.py | 58 ++-- .../planner/_helpers/planning/full_refresh.py | 43 +++ .../planner/_helpers/pruning/cascade.py | 311 ----------------- .../_helpers/pruning/selection_staleness.py | 41 +-- .../planner/_helpers/resolve/resolve.py | 37 +- src/sqlbuild/compiler/planner/constants.py | 2 + .../planner/main/execution/execution.py | 4 +- .../identity/_version_identity_metadata.py | 3 +- src/sqlbuild/compiler/planner/models.py | 36 +- src/sqlbuild/compiler/planner/types.py | 2 +- .../cli/commands/main/build/test_build.py | 3 +- .../main/build/test_query_propagation.py | 12 +- .../cli/commands/main/plan/_test_types.py | 1 + .../cli/commands/main/plan/helpers.py | 4 +- .../commands/main/plan/test_diamond_plan.py | 12 +- .../test_identity_handover.py | 3 - .../compiler/planner/main/_test_types.py | 5 +- .../sqlbuild/compiler/planner/main/helpers.py | 1 - .../planner/main/test_build_execution_plan.py | 129 +------ .../sqlbuild/cli/output/main/plan/helpers.py | 8 +- .../cli/output/main/plan/test_plan_text.py | 123 +------ .../output/main/plan_json/test_plan_json.py | 29 +- .../sqlbuild/compiler/compile/_test_types.py | 1 - .../sqlbuild/compiler/compile/test_main.py | 12 - .../compiler/planner/_helpers/_test_types.py | 27 -- .../planner/_helpers/changes/_test_helpers.py | 25 ++ .../planner/_helpers/changes/_test_types.py | 11 + .../planner/_helpers/changes/test_detect.py | 99 +++++- .../compiler/planner/_helpers/helpers.py | 37 +- .../planner/_helpers/planning/_test_types.py | 9 + .../planner/_helpers/planning/helpers.py | 18 + .../_helpers/planning/test_full_refresh.py | 121 +++++++ .../resolve/test_cursor_intrinsics.py | 4 +- .../compiler/planner/_helpers/test_cascade.py | 317 ------------------ .../_helpers/test_function_fingerprints.py | 57 +--- 46 files changed, 629 insertions(+), 1421 deletions(-) create mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/actions.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/pruning/cascade.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cascade.py diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index d35920e7dd..68b99a06d4 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -13,7 +13,6 @@ from sqlbuild.compiler.migrations.types import MigrationDecision from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( - CascadeResult, FunctionPlanEntry, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, @@ -26,7 +25,7 @@ SeedPlanEntry, SourceLoadPlanEntry, ) -from sqlbuild.compiler.planner.types import IncrementalMode, MaterializationType, PlanReason +from sqlbuild.compiler.planner.types import IncrementalMode, MaterializationType from sqlbuild.compiler.python_nodes.types import PythonIdentityStatus from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token @@ -112,15 +111,12 @@ def format_plan_json( def _serialize_model_entry(entry: ModelPlanEntry) -> dict[str, object]: """Serialize one ModelPlanEntry for plan JSON output.""" - effective_reason: PlanReason = ( - PlanReason.UPSTREAM_CHANGED if entry.cascade is not None else entry.reason - ) model: dict[str, object] = { "name": entry.name, "relative_path": str(entry.relative_path), "materialization_type": entry.materialization_type.value, "action": entry.action.value, - "reason": effective_reason.value, + "reason": entry.reason.value, "expected_version_hash": entry.fingerprint_version_hash, "built_version_hash": entry.previous_version_hash, "built_version_present": entry.previous_version_hash is not None, @@ -191,8 +187,8 @@ def _serialize_model_entry(entry: ModelPlanEntry) -> dict[str, object]: "duration": entry.backfill.duration, } - if entry.cascade is not None: - model["cascade"] = _serialize_cascade(entry.cascade) + if entry.changed_functions: + model["changed_functions"] = list(entry.changed_functions) if entry.destination.qualified_name is not None: model["qualified_name"] = entry.destination.qualified_name @@ -248,18 +244,6 @@ def _model_identity_status(entry: ModelPlanEntry) -> str: return "stale" -def _serialize_cascade(cascade: CascadeResult) -> dict[str, object]: - """Serialize a CascadeResult.""" - - result: dict[str, object] = { - "effective_action": cascade.effective_action.value, - "effective_duration": cascade.effective_duration, - "root_cause": cascade.root_cause, - "cause_count": len(cascade.causes), - } - return result - - def _serialize_seed_entry(entry: SeedPlanEntry) -> dict[str, object]: """Serialize one SeedPlanEntry.""" diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index d54e3c6433..842342a806 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -26,7 +26,6 @@ format_pre_build_work, ) from sqlbuild.compiler.planner.models import ( - CascadeResult, ColumnRenameHint, CursorBounds, FunctionPlanEntry, @@ -72,6 +71,7 @@ _REASON_GROUP_ORDER: tuple[PlanReason, ...] = ( PlanReason.FULL_REFRESH, PlanReason.QUERY_CHANGED, + PlanReason.FUNCTION_CHANGED, PlanReason.CONFIG_CHANGED, PlanReason.SCHEMA_CHANGED, PlanReason.FIRST_RUN, @@ -81,6 +81,7 @@ _REASON_GROUP_LABELS: dict[PlanReason, str] = { PlanReason.FULL_REFRESH: "Full refresh", PlanReason.QUERY_CHANGED: "Query changed", + PlanReason.FUNCTION_CHANGED: "Function changed", PlanReason.CONFIG_CHANGED: "Config changed", PlanReason.SCHEMA_CHANGED: "Schema changed", PlanReason.FIRST_RUN: "First run", @@ -206,10 +207,7 @@ def format_plan( ) normal: list[ModelPlanEntry] = _collect_normal(active) - cascade: list[ModelPlanEntry] = _collect_upstream_changed(active) - groups: dict[PlanReason, list[ModelPlanEntry]] = _group_by_reason( - entries=active, cascade_entries=cascade - ) + groups: dict[PlanReason, list[ModelPlanEntry]] = _group_by_reason(entries=active) lines = _format_changed_functions( lines=lines, @@ -243,25 +241,6 @@ def format_plan( options=resolved_display_options, ) - if cascade: - lines.append("") - lines.append(resolved_section_header_style(f"Upstream changed ({len(cascade)})")) - entry_c: ModelPlanEntry - visible_cascade: Sequence[ModelPlanEntry] = visible_entries( - entries=cascade, options=resolved_display_options - ) - for entry_c in visible_cascade: - lines = _format_upstream_changed_entry( - lines=lines, entry=entry_c, name_column_width=name_column_width - ) - lines = append_overflow_line( - lines=lines, - total_count=len(cascade), - visible_count=len(visible_cascade), - indent=" ", - options=resolved_display_options, - ) - if normal: lines.append("") lines = _format_routine_models_section( @@ -854,39 +833,18 @@ def _collect_normal(entries: list[ModelPlanEntry]) -> list[ModelPlanEntry]: for entry in entries: if entry.reason not in (PlanReason.NO_CHANGE, PlanReason.NORMAL_INCREMENTAL): continue - if entry.cascade is not None: - continue result.append(entry) return result -def _collect_upstream_changed(entries: list[ModelPlanEntry]) -> list[ModelPlanEntry]: - """Collect entries where cascade upgraded the effective window beyond own backfill.""" +def _group_by_reason(*, entries: list[ModelPlanEntry]) -> dict[PlanReason, list[ModelPlanEntry]]: + """Group entries by reason, excluding normal/no-change entries.""" - result: list[ModelPlanEntry] = [] - entry: ModelPlanEntry - for entry in entries: - if entry.cascade is None: - continue - result.append(entry) - return result - - -def _group_by_reason( - *, - entries: list[ModelPlanEntry], - cascade_entries: list[ModelPlanEntry], -) -> dict[PlanReason, list[ModelPlanEntry]]: - """Group entries by reason, excluding normal/no-change and cascade entries.""" - - cascade_names: frozenset[str] = frozenset(e.name for e in cascade_entries) groups: dict[PlanReason, list[ModelPlanEntry]] = {} entry: ModelPlanEntry for entry in entries: if entry.reason in (PlanReason.NO_CHANGE, PlanReason.NORMAL_INCREMENTAL): continue - if entry.name in cascade_names: - continue groups.setdefault(entry.reason, []).append(entry) return groups @@ -957,44 +915,14 @@ def _format_detail_entry( show_range=reason != PlanReason.FULL_REFRESH, ) lines = _append_policy_line(lines=lines, entry=entry) + if entry.changed_functions: + lines.append(f" cause: function {', '.join(entry.changed_functions)} changed") lines = _append_schema_diff(lines=lines, entry=entry) lines = _append_config_diff(lines=lines, entry=entry) lines = _append_query_diff(lines=lines, entry=entry) return lines -def _format_upstream_changed_entry( - *, lines: list[str], entry: ModelPlanEntry, name_column_width: int -) -> list[str]: - """Format a per-model entry in the Upstream changed group.""" - - cascade: CascadeResult | None = entry.cascade - action: str = _cascade_action_text(cascade) - if entry.action == PlanAction.CREATE_VIEW: - action = "recreate view" - elif entry.action == PlanAction.CUSTOM: - materialization: str = entry.custom_materialization_name or "custom materialization" - action = f"run {materialization}" - action_text: str = CliStyle(use_color=True).accent(action) - lines.append( - _format_name_value_line( - name=entry.name, - value=action_text, - name_column_width=name_column_width, - dim_value=False, - ) - ) - lines = _append_cursor_detail( - lines=lines, - entry=entry, - show_range=cascade is None or cascade.effective_action != BackfillAction.FULL, - ) - if cascade is not None and cascade.root_cause is not None: - cause_desc: str = _cascade_cause_description(cascade) - lines.append(f" cause: {cause_desc}") - return lines - - def _append_cursor_detail( *, lines: list[str], entry: ModelPlanEntry, show_range: bool = True ) -> list[str]: @@ -1141,59 +1069,6 @@ def _action_text(entry: ModelPlanEntry) -> str: return "continue forward" -def _cascade_action_text(cascade: CascadeResult | None) -> str: - """Action text for an upstream-changed entry.""" - - if cascade is None: - return "" - if cascade.effective_action == BackfillAction.FULL: - return "full rebuild" - if ( - cascade.effective_action == BackfillAction.BOUNDED - and cascade.effective_duration is not None - ): - return f"rebuild last {cascade.effective_duration}" - return "continue forward" - - -def _cascade_cause_description(cascade: CascadeResult) -> str: - """Format the cause line content for an upstream-changed entry.""" - - root: str = cascade.root_cause or "unknown" - if cascade.root_reason is not None: - reason_text: str = _plan_reason_text(cascade.root_reason) - if reason_text: - return f"{root} ({reason_text})" - if cascade.effective_action == BackfillAction.FULL: - return f"{root} (full)" - if ( - cascade.effective_action == BackfillAction.BOUNDED - and cascade.effective_duration is not None - ): - return f"{root} ({cascade.effective_duration})" - return root - - -def _plan_reason_text(reason: PlanReason) -> str: - """Format a plan reason for cascade cause output.""" - - if reason == PlanReason.QUERY_CHANGED: - return "query changed" - if reason == PlanReason.FUNCTION_CHANGED: - return "function changed" - if reason == PlanReason.CONFIG_CHANGED: - return "config changed" - if reason == PlanReason.SCHEMA_CHANGED: - return "schema changed" - if reason == PlanReason.FIRST_RUN: - return "first run" - if reason == PlanReason.RENAMED: - return "renamed" - if reason == PlanReason.FULL_REFRESH: - return "full refresh" - return "" - - def _schema_change_suffix(entry: ModelPlanEntry) -> str: """Short suffix describing schema changes for inline display.""" @@ -1391,12 +1266,6 @@ def _format_function_entry( elif function_entry.reason == PlanReason.FULL_REFRESH: lines.append(" reason: full refresh") elif function_entry.reason == PlanReason.QUERY_CHANGED: - if function_entry.backfill.action != BackfillAction.FORWARD_ONLY: - duration: str = function_entry.backfill.duration or "full" - policy_value: str = _backfill_value( - action=function_entry.backfill.action, duration=duration - ) - lines.append(f" policy: replay_on_change={policy_value}") if function_entry.previous_query_sql is not None: style: CliStyle = CliStyle(use_color=True) lines.append(style.label(" query diff:")) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 1c0d270c76..96bf3c844f 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -1683,7 +1683,6 @@ def _assemble_compiled_function( runtime_version=function_input.runtime_version, entry_point=function_input.entry_point, packages=function_input.packages, - replay_on_change=function_input.replay_on_change, tags=function_input.tags, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index c42fa581e4..309c614fc8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -255,13 +255,6 @@ def build_sql_function_inputs( fingerprint_schema=function_schema, fingerprint_logical_database=function_logical_database, fingerprint_logical_schema=function_logical_schema, - replay_on_change=_parse_optional_function_header( - header_values=header_values, - key="replay_on_change", - effective_vars=effective_vars, - relative_path=function_file.relative_path, - language="SQL", - ), tags=_parse_function_tags( header_values=header_values, relative_path=function_file.relative_path, @@ -442,13 +435,6 @@ def _build_python_function_input( runtime_version=runtime_version, entry_point=entry_point, packages=packages, - replay_on_change=_parse_optional_function_header( - header_values=header_values, - key="replay_on_change", - effective_vars=effective_vars, - relative_path=python_function_file.relative_path, - language="Python", - ), tags=_parse_function_tags( header_values=header_values, relative_path=python_function_file.relative_path, @@ -586,26 +572,6 @@ def _parse_required_string_header( return raw_value.strip() -def _parse_optional_function_header( - *, - header_values: dict[str, object], - key: str, - effective_vars: dict[str, object], - relative_path: Path, - language: str, -) -> str | None: - raw_value: object | None = header_values.get(key) - if raw_value is None: - return None - if not isinstance(raw_value, str) or not raw_value.strip(): - raise CompileInputError(f"{language} function file {relative_path} {key} must be a string") - return _expand_function_header_value( - raw_value=raw_value.strip(), - effective_vars=effective_vars, - context_label=f"{language} function {relative_path} {key}", - ) - - def _parse_python_packages(*, raw_packages: object | None, relative_path: Path) -> tuple[str, ...]: if raw_packages is None: return () diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index cbe21e788e..b8cf103dd3 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -718,7 +718,6 @@ class CompileSqlFunctionInput: runtime_version: str | None = None entry_point: str | None = None packages: tuple[str, ...] = field(default_factory=tuple) - replay_on_change: str | None = None declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) tags: tuple[str, ...] = field(default_factory=tuple) @@ -958,7 +957,6 @@ class CompiledFunction: runtime_version: str | None = None entry_point: str | None = None packages: tuple[str, ...] = field(default_factory=tuple) - replay_on_change: str | None = None tags: tuple[str, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/actions.py b/src/sqlbuild/compiler/planner/_helpers/changes/actions.py new file mode 100644 index 0000000000..2e870519c2 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/changes/actions.py @@ -0,0 +1,32 @@ +"""Per-model action resolution from each model's own change.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.planner.models import ( + ChangeDetectionResult, + PlannerChangeResults, + PlannerResolvedActions, + PlannerScope, + ResolvedModelAction, +) + + +def resolve_model_actions( + *, + scope: PlannerScope, + changes: PlannerChangeResults, +) -> PlannerResolvedActions: + """Resolve each selected model's backfill from its own change and replay_on_change only.""" + + resolved: dict[str, ResolvedModelAction] = {} + key: CompiledObjectKey + for key in scope.execution_order: + if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: + continue + change: ChangeDetectionResult | None = changes.models.get(key.name) + if key.name not in scope.models_by_name or change is None: + continue + resolved[key.name] = ResolvedModelAction(change=change, backfill=change.backfill) + return PlannerResolvedActions(models=resolved) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index f6fd3f3d1e..550a8dd3c2 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +from collections.abc import Callable from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import ( @@ -15,7 +16,11 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.metadata import version_identity_metadata_payload +from sqlbuild.compiler.planner._helpers.changes.metadata import ( + changed_local_function_names, + non_function_identity_metadata_payload, + version_identity_metadata_payload, +) from sqlbuild.compiler.planner._helpers.changes.policy import ( pick_more_aggressive, resolve_replay_on_change, @@ -42,7 +47,7 @@ SchemaFinding, WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind, PlanReason +from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql @@ -77,21 +82,14 @@ def detect_changes( fingerprint_sql=fingerprint_sql, ) continue - function_reason: PlanReason - function_backfill: BackfillResult - function_reason, function_backfill = detect_function_change( - function=function, - fingerprint_sql=fingerprint_sql, - snapshot=snapshot, - query_change_tracking=project.settings.query_change_tracking, - full_refresh=full_refresh, - ) - if full_refresh: - function_backfill = BackfillResult(action=BackfillAction.FORWARD_ONLY) function_changes[function.name] = FunctionChangeResult( fingerprint_sql=fingerprint_sql, - reason=function_reason, - backfill=function_backfill, + reason=detect_function_change( + fingerprint_sql=fingerprint_sql, + fingerprint=snapshot.fingerprints.functions.get(function.name), + query_change_tracking=project.settings.query_change_tracking, + full_refresh=full_refresh, + ), ) return PlannerChangeResults(models=model_changes, functions=function_changes) @@ -194,12 +192,26 @@ def detect_model_changes( ) query_changed: bool = False - config_changed: bool = ( - fingerprint is not None - and fingerprint.metadata_json != EMPTY_FINGERPRINT_METADATA_JSON - and version_identity_metadata_payload(metadata_json) - != version_identity_metadata_payload(fingerprint.metadata_json) + recorded_metadata_json: str | None = ( + fingerprint.metadata_json + if fingerprint is not None and fingerprint.metadata_json != EMPTY_FINGERPRINT_METADATA_JSON + else None ) + changed_functions: tuple[str, ...] = ( + changed_local_function_names( + metadata_json=metadata_json, previous_metadata_json=recorded_metadata_json + ) + if query_change_tracking and recorded_metadata_json is not None + else () + ) + identity_payload: Callable[[str | None], object] = ( + non_function_identity_metadata_payload + if changed_functions + else version_identity_metadata_payload + ) + config_changed: bool = recorded_metadata_json is not None and identity_payload( + metadata_json + ) != identity_payload(recorded_metadata_json) replay_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) if query_change_tracking and fingerprint is not None: debug_logger: logging.Logger = logging.getLogger("sqlbuild.planner.changes") @@ -224,11 +236,10 @@ def detect_model_changes( ) log_sql(logger=debug_logger, sql=model.query_sql, action="compiled_query") log_sql(logger=debug_logger, sql=fingerprint.definition, action="fingerprint_definition") - if query_changed: - raw_policy: str | None = get_config_str( - values=model.config.values, key="replay_on_change" - ) - replay_backfill = resolve_replay_on_change(replay_on_change=raw_policy) + if query_changed or changed_functions: + replay_backfill = resolve_replay_on_change( + replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") + ) schema_findings: tuple[SchemaFinding, ...] = () schema_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) @@ -252,14 +263,17 @@ def detect_model_changes( dialect=snapshot.column_dialect, ) if schema_findings: - raw_policy = get_config_str(values=model.config.values, key="replay_on_change") - schema_backfill = resolve_replay_on_change(replay_on_change=raw_policy) + schema_backfill = resolve_replay_on_change( + replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") + ) backfill: BackfillResult = pick_more_aggressive(a=replay_backfill, b=schema_backfill) change_kind: ChangeKind if query_changed: change_kind = ChangeKind.QUERY_CHANGED + elif changed_functions: + change_kind = ChangeKind.FUNCTION_CHANGED elif schema_findings: change_kind = ChangeKind.SCHEMA_CHANGED elif config_changed: @@ -278,6 +292,7 @@ def detect_model_changes( previous_version_hash=fingerprint.version_hash if fingerprint is not None else None, schema_findings=schema_findings, backfill=backfill, + changed_functions=changed_functions, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py index f6952a740a..f76b1c2d02 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py @@ -5,7 +5,10 @@ import json from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY -from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY +from sqlbuild.compiler.planner.constants import ( + LOCAL_FUNCTION_HASHES_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, +) def version_identity_metadata_payload(metadata_json: str | None) -> object: @@ -23,3 +26,41 @@ def version_identity_metadata_payload(metadata_json: str | None) -> object: identity_payload.pop(AUDIT_GATE_METADATA_KEY, None) identity_payload.pop(MIGRATION_FINGERPRINT_METADATA_KEY, None) return identity_payload + + +def non_function_identity_metadata_payload(metadata_json: str | None) -> object: + """Return the version identity metadata payload without called-function hashes.""" + + payload: object = version_identity_metadata_payload(metadata_json) + if not isinstance(payload, dict): + return payload + return { + key: value for key, value in payload.items() if key != LOCAL_FUNCTION_HASHES_METADATA_KEY + } + + +def changed_local_function_names( + *, metadata_json: str, previous_metadata_json: str +) -> tuple[str, ...]: + """Return directly called functions whose recorded definition hash changed.""" + + current: dict[str, object] = _local_function_hashes(metadata_json) + previous: dict[str, object] = _local_function_hashes(previous_metadata_json) + return tuple( + sorted( + name + for name, function_hash in current.items() + if name in previous and previous[name] != function_hash + ) + ) + + +def _local_function_hashes(metadata_json: str) -> dict[str, object]: + try: + payload: object = json.loads(metadata_json) + except json.JSONDecodeError: + return {} + if not isinstance(payload, dict): + return {} + hashes: object = payload.get(LOCAL_FUNCTION_HASHES_METADATA_KEY) + return {str(name): value for name, value in hashes.items()} if isinstance(hashes, dict) else {} diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py index 29ca640fb0..2b81e411d7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py @@ -9,9 +9,7 @@ ) from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.changes.policy import resolve_replay_on_change -from sqlbuild.compiler.planner.models import BackfillResult, WarehouseSnapshot -from sqlbuild.compiler.planner.types import BackfillAction, PlanReason +from sqlbuild.compiler.planner.types import PlanReason def build_compiled_function_fingerprint_sql(function: CompiledFunction) -> str: @@ -31,28 +29,20 @@ def build_compiled_function_fingerprint_sql(function: CompiledFunction) -> str: def detect_function_change( *, - function: CompiledFunction, fingerprint_sql: str, - snapshot: WarehouseSnapshot, + fingerprint: Fingerprint | None, query_change_tracking: bool, full_refresh: bool, -) -> tuple[PlanReason, BackfillResult]: - """Resolve function definition changes and downstream backfill policy.""" +) -> PlanReason: + """Resolve why a function is redeployed; callers decide their own replay.""" if full_refresh: - return PlanReason.FULL_REFRESH, BackfillResult(action=BackfillAction.FULL) - fingerprint: Fingerprint | None = snapshot.fingerprints.functions.get(function.name) + return PlanReason.FULL_REFRESH if fingerprint is None: - return PlanReason.FIRST_RUN, resolve_replay_on_change( - replay_on_change=function.replay_on_change - ) - if not query_change_tracking: - return PlanReason.NO_CHANGE, BackfillResult(action=BackfillAction.FORWARD_ONLY) - if compute_query_hash(fingerprint_sql) != fingerprint.definition_hash: - return PlanReason.QUERY_CHANGED, resolve_replay_on_change( - replay_on_change=function.replay_on_change - ) - return PlanReason.NO_CHANGE, BackfillResult(action=BackfillAction.FORWARD_ONLY) + return PlanReason.FIRST_RUN + if query_change_tracking and compute_query_hash(fingerprint_sql) != fingerprint.definition_hash: + return PlanReason.QUERY_CHANGED + return PlanReason.NO_CHANGE def build_function_fingerprint_sql( diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index ffeb2240a4..ad6a54aea9 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -45,7 +45,11 @@ resolve_model_plan_action, resolve_schema_actions, ) -from sqlbuild.compiler.planner._helpers.planning.full_refresh import resolve_model_full_refresh +from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( + check_full_rebuild_allowed, + describe_full_rebuild_cause, + resolve_model_full_refresh, +) from sqlbuild.compiler.planner._helpers.resolve.cursor import ( compute_cursor_bounds, normalize_cursor_snapshot_grain, @@ -327,7 +331,6 @@ def plan_model( query_change_tracking: bool, full_refresh: bool, cursor_overrides: CursorOverridePair, - backfill_override: BackfillResult | None = None, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, ) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: """Detect changes and build a plan entry and warnings for a single model.""" @@ -349,7 +352,6 @@ def plan_model( full_refresh=full_refresh, cursor_overrides=cursor_overrides, change_result=change_result, - backfill_override=backfill_override, external_sql_reference_resolver=external_sql_reference_resolver, ) @@ -366,7 +368,7 @@ def build_plan_entries( full_refresh: bool, build_inputs: PlanEntryBuildInputs | None = None, ) -> PlannerModelEntryResults: - """Build model plan entries from snapshot and cascade-resolved actions.""" + """Build model plan entries from snapshot and per-model resolved actions.""" inputs: PlanEntryBuildInputs = ( build_inputs if build_inputs is not None else PlanEntryBuildInputs() @@ -424,9 +426,6 @@ def build_plan_entries( start_cursor_override=start_cursor_override, end_cursor_override=end_cursor_override, ) - backfill_override: BackfillResult | None = ( - resolved.backfill if resolved.backfill != resolved.change.backfill else None - ) entry: ModelPlanEntry entry_warnings: tuple[PlanWarning, ...] effective_full_refresh: bool = resolve_model_full_refresh( @@ -458,11 +457,8 @@ def build_plan_entries( end_cursor_override=resolved_end, ), change_result=resolved.change, - backfill_override=backfill_override, external_sql_reference_resolver=project.external_sql_reference_resolver, ) - if resolved.cascade is not None: - entry = replace(entry, cascade=resolved.cascade) if entry.name in source_freshness_blocked_model_names: entry = replace( entry, @@ -679,22 +675,15 @@ def plan_model_from_change( full_refresh: bool, cursor_overrides: CursorOverridePair, change_result: ChangeDetectionResult, - backfill_override: BackfillResult | None = None, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, ) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: """Build a model plan entry from a resolved change result.""" - if backfill_override is not None: - change_result = replace(change_result, backfill=backfill_override) - start_cursor_override: str | None = cursor_overrides.start_cursor_override end_cursor_override: str | None = cursor_overrides.end_cursor_override backfill: BackfillResult = ( BackfillResult(action=BackfillAction.FULL) if full_refresh else change_result.backfill ) - suppress_runtime_cursor_bounds: bool = ( - backfill_override is not None and backfill_override.action == BackfillAction.FULL - ) cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") validate_source_cursor_input_columns( model=model, @@ -716,6 +705,15 @@ def plan_model_from_change( replaces_relation: bool = replaces_incremental_relation( materialization_type=materialization_type, action=action ) + full_rebuild_cause: str | None = ( + describe_full_rebuild_cause(change_result=change_result, full_refresh=full_refresh) + if replaces_relation + else None + ) + if full_rebuild_cause is not None and not full_refresh: + check_full_rebuild_allowed( + model=model, change_result=change_result, full_rebuild_cause=full_rebuild_cause + ) resolved_sql: str = resolve_model_sql( adapter=adapter, @@ -725,9 +723,9 @@ def plan_model_from_change( backfill=backfill, full_refresh=full_refresh, cursor_overrides=cursor_overrides, - suppress_runtime_cursor_bounds=suppress_runtime_cursor_bounds, external_sql_reference_resolver=external_sql_reference_resolver, replaces_relation=replaces_relation, + full_rebuild_cause=full_rebuild_cause, ) on_schema_change: OnSchemaChange | None = _get_on_schema_change(model) @@ -769,24 +767,22 @@ def plan_model_from_change( effective_future_cursor_config, effective_start_cursor_config = _resolve_cursor_safety_configs( model=model, context=context ) - cursor_input_relations: tuple[CursorInputRelation, ...] = () validate_capped_watermark_inputs( model=model, models_by_name=context.models_by_name, functions_by_name=context.functions_by_name, ) - if not suppress_runtime_cursor_bounds: - cursor_input_relations = _build_cursor_input_relations( - model=model, - adapter=adapter, - model_locations=context.model_locations, - models_by_name=context.models_by_name, - functions_by_name=context.functions_by_name, - seed_locations=context.seed_locations, - source_map=context.source_map, - cursor_column=cursor_column, - runtime_cursor_producer_names=context.runtime_cursor_producer_names, - ) + cursor_input_relations: tuple[CursorInputRelation, ...] = _build_cursor_input_relations( + model=model, + adapter=adapter, + model_locations=context.model_locations, + models_by_name=context.models_by_name, + functions_by_name=context.functions_by_name, + seed_locations=context.seed_locations, + source_map=context.source_map, + cursor_column=cursor_column, + runtime_cursor_producer_names=context.runtime_cursor_producer_names, + ) runtime_owned_cursor_bounds: bool = _has_runtime_owned_cursor_watermarks( cursor_input_relations ) and not (start_cursor_override is not None and end_cursor_override is not None) @@ -956,6 +952,7 @@ def plan_model_from_change( schema_actions=schema_actions, schema_findings=change_result.schema_findings, backfill=backfill, + changed_functions=change_result.changed_functions, custom_materialization_name=custom.name, custom_config=custom.config, custom_placeholders=custom.placeholders, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index 209a82aaa6..a6320c8265 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -2,8 +2,6 @@ from __future__ import annotations -from dataclasses import replace - from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledAudit, @@ -43,7 +41,6 @@ ) from sqlbuild.compiler.planner.models import ( AuditPlanEntry, - BackfillResult, FunctionChangeResult, FunctionPlanEntry, ModelPlanEntry, @@ -367,7 +364,6 @@ def _build_function_entries( else None ), reason=function_change.reason, - backfill=function_change.backfill, ) ) return entries @@ -381,11 +377,10 @@ def plan_function( fingerprint_query_sql: str, previous_query_sql: str | None = None, reason: PlanReason = PlanReason.NO_CHANGE, - backfill: BackfillResult | None = None, ) -> FunctionPlanEntry: """Build one canonical function entry from resolved relation inputs.""" - entry: FunctionPlanEntry = FunctionPlanEntry( + return FunctionPlanEntry( key=function.key, name=function.name, relative_path=function.relative_path, @@ -413,7 +408,6 @@ def plan_function( previous_query_sql=previous_query_sql, reason=reason, ) - return entry if backfill is None else replace(entry, backfill=backfill) def build_selected_audit_entries( diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index e4b80ada11..ff28242a79 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -28,6 +28,17 @@ ) _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS +_OWN_CHANGE_REASONS: dict[ChangeKind, PlanReason] = { + ChangeKind.FIRST_RUN: PlanReason.FIRST_RUN, + ChangeKind.QUERY_CHANGED: PlanReason.QUERY_CHANGED, + ChangeKind.FUNCTION_CHANGED: PlanReason.FUNCTION_CHANGED, + ChangeKind.CONFIG_CHANGED: PlanReason.CONFIG_CHANGED, + ChangeKind.SCHEMA_CHANGED: PlanReason.SCHEMA_CHANGED, +} +_VIEW_CHANGE_REASONS: dict[ChangeKind, PlanReason] = { + **_OWN_CHANGE_REASONS, + ChangeKind.RENAMED: PlanReason.RENAMED, +} def resolve_model_plan_action( @@ -198,15 +209,20 @@ def build_model_warnings( if ( materialization_type == MaterializationType.INCREMENTAL - and change_result.query_changed + and (change_result.query_changed or change_result.changed_functions) and change_result.backfill.action == BackfillAction.FORWARD_ONLY ): + changed_subject: str = ( + "the model's SQL changed" + if change_result.query_changed + else f"function {', '.join(change_result.changed_functions)} changed" + ) warnings.append( PlanWarning( model_name=model_name, severity=WarningSeverity.WARNING, message=( - f"{model_name}: the model's SQL changed, so already-built dates keep " + f"{model_name}: {changed_subject}, so already-built dates keep " "their old results - only new dates will use the new query. To apply " "the change to existing data, choose how much to reprocess: " "replay_on_change full (rebuild every date with the new SQL), " @@ -251,15 +267,7 @@ def _custom_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) if full_refresh: return PlanReason.FULL_REFRESH - if change_result.change_kind == ChangeKind.FIRST_RUN: - return PlanReason.FIRST_RUN - if change_result.change_kind == ChangeKind.QUERY_CHANGED: - return PlanReason.QUERY_CHANGED - if change_result.change_kind == ChangeKind.CONFIG_CHANGED: - return PlanReason.CONFIG_CHANGED - if change_result.change_kind == ChangeKind.SCHEMA_CHANGED: - return PlanReason.SCHEMA_CHANGED - return PlanReason.NO_CHANGE + return _OWN_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NO_CHANGE) def _view_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> PlanReason: @@ -267,17 +275,7 @@ def _view_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> if full_refresh: return PlanReason.FULL_REFRESH - if change_result.change_kind == ChangeKind.FIRST_RUN: - return PlanReason.FIRST_RUN - if change_result.change_kind == ChangeKind.RENAMED: - return PlanReason.RENAMED - if change_result.change_kind == ChangeKind.QUERY_CHANGED: - return PlanReason.QUERY_CHANGED - if change_result.change_kind == ChangeKind.CONFIG_CHANGED: - return PlanReason.CONFIG_CHANGED - if change_result.change_kind == ChangeKind.SCHEMA_CHANGED: - return PlanReason.SCHEMA_CHANGED - return PlanReason.NO_CHANGE + return _VIEW_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NO_CHANGE) def _snapshot_reason(*, change_result: ChangeDetectionResult, full_refresh: bool) -> PlanReason: @@ -285,15 +283,7 @@ def _snapshot_reason(*, change_result: ChangeDetectionResult, full_refresh: bool if full_refresh: return PlanReason.FULL_REFRESH - if change_result.change_kind == ChangeKind.FIRST_RUN: - return PlanReason.FIRST_RUN - if change_result.change_kind == ChangeKind.QUERY_CHANGED: - return PlanReason.QUERY_CHANGED - if change_result.change_kind == ChangeKind.CONFIG_CHANGED: - return PlanReason.CONFIG_CHANGED - if change_result.change_kind == ChangeKind.SCHEMA_CHANGED: - return PlanReason.SCHEMA_CHANGED - return PlanReason.NORMAL_INCREMENTAL + return _OWN_CHANGE_REASONS.get(change_result.change_kind, PlanReason.NORMAL_INCREMENTAL) def _backfill_reason(change_result: ChangeDetectionResult) -> PlanReason: @@ -301,6 +291,8 @@ def _backfill_reason(change_result: ChangeDetectionResult) -> PlanReason: if change_result.query_changed: return PlanReason.QUERY_CHANGED + if change_result.changed_functions: + return PlanReason.FUNCTION_CHANGED if change_result.config_changed: return PlanReason.CONFIG_CHANGED if change_result.schema_findings: @@ -315,6 +307,8 @@ def _table_action( if change_result.change_kind == ChangeKind.QUERY_CHANGED: return PlanAction.CREATE_TABLE, PlanReason.QUERY_CHANGED + if change_result.change_kind == ChangeKind.FUNCTION_CHANGED: + return PlanAction.CREATE_TABLE, PlanReason.FUNCTION_CHANGED if change_result.change_kind == ChangeKind.CONFIG_CHANGED: return PlanAction.CREATE_TABLE, PlanReason.CONFIG_CHANGED if change_result.change_kind == ChangeKind.SCHEMA_CHANGED: @@ -339,6 +333,8 @@ def _incremental_action( reason: PlanReason if change_result.change_kind == ChangeKind.QUERY_CHANGED: reason = PlanReason.QUERY_CHANGED + elif change_result.change_kind == ChangeKind.FUNCTION_CHANGED: + reason = PlanReason.FUNCTION_CHANGED elif change_result.change_kind == ChangeKind.CONFIG_CHANGED: reason = PlanReason.CONFIG_CHANGED elif change_result.change_kind == ChangeKind.SCHEMA_CHANGED: diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py b/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py index fa4523fc05..55a5a21a52 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py @@ -3,6 +3,10 @@ from __future__ import annotations from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.planner.constants import FULL_REFRESH_DISABLED_REBUILD_CODE +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.planner.models import ChangeDetectionResult +from sqlbuild.compiler.planner.types import ChangeKind def resolve_model_full_refresh(*, model: CompiledModel, cli_full_refresh: bool) -> bool: @@ -22,3 +26,42 @@ def effectively_full_refreshed_model_names( for model in project.models if resolve_model_full_refresh(model=model, cli_full_refresh=cli_full_refresh) ) + + +def describe_full_rebuild_cause(*, change_result: ChangeDetectionResult, full_refresh: bool) -> str: + """Describe why an incremental model is planned as a full rebuild.""" + + if full_refresh: + return "--full-refresh" + if change_result.change_kind == ChangeKind.FIRST_RUN: + return "first run" + if change_result.change_kind == ChangeKind.RENAMED: + return "rename" + if change_result.query_changed: + return "query changed with replay_on_change full" + if change_result.changed_functions: + names: str = ", ".join(change_result.changed_functions) + return f"function {names} changed with replay_on_change full" + if change_result.schema_findings: + return "schema changed with replay_on_change full" + return "full backfill" + + +def check_full_rebuild_allowed( + *, model: CompiledModel, change_result: ChangeDetectionResult, full_rebuild_cause: str +) -> None: + """Refuse a full rebuild of a full_refresh false model unless its own SQL or first run asks.""" + + if model.config.values.get("full_refresh") is not False: + return + if change_result.change_kind == ChangeKind.FIRST_RUN or change_result.query_changed: + return + raise PlannerInputError( + f"model '{model.name}' sets full_refresh false, but the plan would fully rebuild it " + f"({full_rebuild_cause})", + code=FULL_REFRESH_DISABLED_REBUILD_CODE, + help=( + "set replay_on_change to forward or bounded- on this model to keep its " + "existing history, or remove full_refresh false to allow the rebuild" + ), + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/cascade.py b/src/sqlbuild/compiler/planner/_helpers/pruning/cascade.py deleted file mode 100644 index 7934622901..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/cascade.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Backfill cascade propagation through the dependency graph.""" - -from __future__ import annotations - -from datetime import timedelta - -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.changes.policy import resolve_replay_on_change -from sqlbuild.compiler.planner.models import ( - BackfillResult, - CascadeCause, - CascadeResult, - ChangeDetectionResult, - Duration, - FunctionChangeResult, - PlannerChangeResults, - PlannerResolvedActions, - PlannerScope, - ResolvedModelAction, -) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind, PlanReason -from sqlbuild.spec.contracts.main.get_config_str import get_config_str - -_APPROX_SECONDS_PER_MONTH: int = 2_629_746 - -_ACTION_RANK: dict[BackfillAction, int] = { - BackfillAction.FORWARD_ONLY: 0, - BackfillAction.BOUNDED: 1, - BackfillAction.FULL: 2, -} - - -def resolve_cascades( - *, - scope: PlannerScope, - changes: PlannerChangeResults, -) -> PlannerResolvedActions: - """Resolve effective model actions after upstream cascade propagation.""" - - effective_cascades: dict[str, CascadeResult] = {} - function_name: str - function_change: FunctionChangeResult - for function_name, function_change in changes.functions.items(): - if function_change.backfill.action == BackfillAction.FORWARD_ONLY: - continue - effective_cascades[function_name] = build_self_cascade( - backfill=function_change.backfill, - root_cause=function_name, - root_reason=( - PlanReason.FUNCTION_CHANGED - if function_change.reason == PlanReason.QUERY_CHANGED - else function_change.reason - ), - ) - - model_cursor_types: dict[str, str | None] = {} - resolved: dict[str, ResolvedModelAction] = {} - key: CompiledObjectKey - for key in scope.execution_order: - if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModel | None = scope.models_by_name.get(key.name) - change: ChangeDetectionResult | None = changes.models.get(key.name) - if model is None or change is None: - continue - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - model_cursor_types[model.name] = cursor_type - local_policy: str | None = get_config_str( - values=model.config.values, key="replay_on_change" - ) - cascade: CascadeResult | None = resolve_cascade( - model_name=model.name, - own_backfill=change.backfill, - local_backfill=resolve_replay_on_change(replay_on_change=local_policy), - own_cursor_type=cursor_type, - upstream_keys=scope.upstream_deps.get(key, ()), - effective_cascades=effective_cascades, - model_cursor_types=model_cursor_types, - ) - if cascade is None: - resolved[model.name] = ResolvedModelAction( - change=change, - backfill=change.backfill, - ) - effective_cascades[model.name] = build_self_cascade( - backfill=change.backfill, - root_cause=model.name, - root_reason=_reason_for_change(change), - ) - continue - backfill: BackfillResult = BackfillResult( - action=cascade.effective_action, - duration=cascade.effective_duration, - ) - resolved[model.name] = ResolvedModelAction( - change=change, - backfill=backfill, - cascade=cascade, - ) - effective_cascades[model.name] = cascade - - return PlannerResolvedActions(models=resolved) - - -def resolve_cascade( - *, - model_name: str, - own_backfill: BackfillResult, - local_backfill: BackfillResult, - own_cursor_type: str | None, - upstream_keys: tuple[CompiledObjectKey, ...], - effective_cascades: dict[str, CascadeResult], - model_cursor_types: dict[str, str | None], -) -> CascadeResult | None: - """Resolve the effective backfill for a model after upstream cascade propagation.""" - - candidates: list[CascadeCause] = _gather_cascade_candidates( - own_cursor_type=own_cursor_type, - upstream_keys=upstream_keys, - effective_cascades=effective_cascades, - model_cursor_types=model_cursor_types, - ) - - if not candidates: - return None - - winning: CascadeCause | None = _pick_winner(candidates=candidates) - if winning is None: - return None - - resolved_backfill: BackfillResult = _resolve_effective_backfill( - own_backfill=own_backfill, - local_backfill=local_backfill, - incoming_cascade=winning, - ) - if _backfills_match(a=resolved_backfill, b=own_backfill): - return None - - return CascadeResult( - effective_action=resolved_backfill.action, - effective_duration=resolved_backfill.duration, - root_cause=winning.root_cause or winning.model_name, - root_reason=winning.root_reason, - causes=tuple(candidates), - ) - - -def build_self_cascade( - *, - backfill: BackfillResult, - root_cause: str | None = None, - root_reason: PlanReason | None = None, -) -> CascadeResult: - """Build a CascadeResult representing a model's own backfill for the accumulator.""" - - return CascadeResult( - effective_action=backfill.action, - effective_duration=backfill.duration, - root_cause=root_cause, - root_reason=root_reason, - causes=(), - ) - - -def _gather_cascade_candidates( - *, - own_cursor_type: str | None, - upstream_keys: tuple[CompiledObjectKey, ...], - effective_cascades: dict[str, CascadeResult], - model_cursor_types: dict[str, str | None], -) -> list[CascadeCause]: - """Gather upstream effective backfills that can cascade to this model.""" - - candidates: list[CascadeCause] = [] - key: CompiledObjectKey - for key in upstream_keys: - if key.resource_type not in ( - CompiledResourceType.MODEL, - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - ): - continue - upstream_cascade: CascadeResult | None = effective_cascades.get(key.name) - if upstream_cascade is None: - continue - if upstream_cascade.effective_action == BackfillAction.FORWARD_ONLY: - continue - - if key.resource_type in {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN}: - candidates.append( - CascadeCause( - model_name=key.name, - effective_action=upstream_cascade.effective_action, - effective_duration=upstream_cascade.effective_duration, - root_cause=upstream_cascade.root_cause or key.name, - root_reason=upstream_cascade.root_reason, - ) - ) - continue - - upstream_cursor_type: str | None = model_cursor_types.get(key.name) - same_cursor_type: bool = ( - own_cursor_type is not None - and upstream_cursor_type is not None - and own_cursor_type == upstream_cursor_type - ) - - if upstream_cascade.effective_action == BackfillAction.FULL: - candidates.append( - CascadeCause( - model_name=key.name, - effective_action=upstream_cascade.effective_action, - effective_duration=upstream_cascade.effective_duration, - root_cause=upstream_cascade.root_cause or key.name, - root_reason=upstream_cascade.root_reason, - ) - ) - elif same_cursor_type: - candidates.append( - CascadeCause( - model_name=key.name, - effective_action=upstream_cascade.effective_action, - effective_duration=upstream_cascade.effective_duration, - root_cause=upstream_cascade.root_cause or key.name, - root_reason=upstream_cascade.root_reason, - ) - ) - - return candidates - - -def _pick_winner( - *, - candidates: list[CascadeCause], -) -> CascadeCause | None: - """Pick the strongest incoming candidate.""" - - best: CascadeCause | None = None - best_rank: tuple[int, int] | None = None - - candidate: CascadeCause - for candidate in candidates: - candidate_rank: tuple[int, int] = _backfill_rank( - action=candidate.effective_action, duration=candidate.effective_duration - ) - if best_rank is None or candidate_rank > best_rank: - best = candidate - best_rank = candidate_rank - elif candidate_rank == best_rank and best is not None: - if candidate.model_name < best.model_name: - best = candidate - - return best - - -def _resolve_effective_backfill( - *, - own_backfill: BackfillResult, - local_backfill: BackfillResult, - incoming_cascade: CascadeCause, -) -> BackfillResult: - """Resolve the model's outgoing pressure from local and incoming state.""" - - if own_backfill.action != BackfillAction.FORWARD_ONLY: - return own_backfill - if local_backfill.action != BackfillAction.FORWARD_ONLY: - return local_backfill - return BackfillResult( - action=incoming_cascade.effective_action, - duration=incoming_cascade.effective_duration, - ) - - -def _backfills_match(*, a: BackfillResult, b: BackfillResult) -> bool: - """Return True when two backfill results resolve identically.""" - - return a.action == b.action and a.duration == b.duration - - -def _backfill_rank(*, action: BackfillAction, duration: str | None) -> tuple[int, int]: - """Produce a comparable rank tuple for a backfill action.""" - - action_rank: int = _ACTION_RANK[action] - duration_seconds: int = 0 - if action == BackfillAction.BOUNDED and duration is not None: - td: timedelta | None = _parse_duration(duration) - if td is not None: - duration_seconds = int(td.total_seconds()) - return (action_rank, duration_seconds) - - -def _reason_for_change(change: ChangeDetectionResult) -> PlanReason: - if change.change_kind == ChangeKind.QUERY_CHANGED: - return PlanReason.QUERY_CHANGED - if change.change_kind == ChangeKind.CONFIG_CHANGED: - return PlanReason.CONFIG_CHANGED - if change.change_kind == ChangeKind.SCHEMA_CHANGED: - return PlanReason.SCHEMA_CHANGED - if change.change_kind == ChangeKind.FIRST_RUN: - return PlanReason.FIRST_RUN - return PlanReason.NO_CHANGE - - -def _parse_duration(duration: str) -> timedelta | None: - """Return a comparable magnitude for ranking a bounded backfill window.""" - - parsed: Duration | None = Duration.parse(duration) - if parsed is None: - return None - return timedelta(seconds=parsed.fixed_seconds + parsed.total_months * _APPROX_SECONDS_PER_MONTH) diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py index c0816766ee..cc86a108fa 100644 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py +++ b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py @@ -24,6 +24,16 @@ from sqlbuild.compiler.planner.types import ChangeKind, WarningSeverity from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult +_OWN_CHANGE_KINDS: frozenset[ChangeKind] = frozenset( + { + ChangeKind.FIRST_RUN, + ChangeKind.QUERY_CHANGED, + ChangeKind.FUNCTION_CHANGED, + ChangeKind.CONFIG_CHANGED, + ChangeKind.SCHEMA_CHANGED, + } +) + def build_stale_out_of_selection_warnings( *, @@ -136,13 +146,7 @@ def _changed_model_names( changed: set[str] = { model_name for model_name, change in changes.models.items() - if change.change_kind - in { - ChangeKind.FIRST_RUN, - ChangeKind.QUERY_CHANGED, - ChangeKind.CONFIG_CHANGED, - ChangeKind.SCHEMA_CHANGED, - } + if change.change_kind in _OWN_CHANGE_KINDS } changed.update( model_name @@ -194,21 +198,9 @@ def _model_own_identity_changed( ) -> bool: selected_change: ChangeDetectionResult | None = changes.models.get(model_name) if selected_change is not None and ( - model_changes_complete - or selected_change.change_kind - in { - ChangeKind.FIRST_RUN, - ChangeKind.QUERY_CHANGED, - ChangeKind.CONFIG_CHANGED, - ChangeKind.SCHEMA_CHANGED, - } + model_changes_complete or selected_change.change_kind in _OWN_CHANGE_KINDS ): - return selected_change.change_kind in { - ChangeKind.FIRST_RUN, - ChangeKind.QUERY_CHANGED, - ChangeKind.CONFIG_CHANGED, - ChangeKind.SCHEMA_CHANGED, - } + return selected_change.change_kind in _OWN_CHANGE_KINDS model: CompiledModel | None = original_scope.models_by_name.get(model_name) if model is None: return False @@ -221,12 +213,7 @@ def _model_own_identity_changed( expected_version_hash=version_identities.model_version_hashes.get(model_name), expected_metadata_json=version_identities.model_metadata_jsons.get(model_name), ) - return change.change_kind in { - ChangeKind.FIRST_RUN, - ChangeKind.QUERY_CHANGED, - ChangeKind.CONFIG_CHANGED, - ChangeKind.SCHEMA_CHANGED, - } + return change.change_kind in _OWN_CHANGE_KINDS def _seed_identity_changed( diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index 39d5f40763..0fe0422d33 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -77,9 +77,9 @@ def resolve_model_sql( backfill: BackfillResult, full_refresh: bool, cursor_overrides: CursorOverridePair, - suppress_runtime_cursor_bounds: bool = False, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, replaces_relation: bool = False, + full_rebuild_cause: str | None = None, ) -> str: """Resolve all references in a model's query SQL to produce executable SQL.""" @@ -100,7 +100,6 @@ def resolve_model_sql( start_cursor_override=cursor_overrides.start_cursor_override, end_cursor_override=cursor_overrides.end_cursor_override, runtime_cursor_producer_names=context.runtime_cursor_producer_names, - suppress_runtime_cursor_bounds=suppress_runtime_cursor_bounds, context=context, replaces_relation=replaces_relation, ) @@ -148,14 +147,10 @@ def resolve_model_sql( _, has_intrinsics = resolve_cursor_intrinsics(sql=query_sql) if has_intrinsics: if cursor_bounds is None: - if full_refresh: - raise PlannerInputError( - f"Model '{model.name}' uses cursor intrinsics, but non-microbatch full " - "refresh has no cursor interval" - ) - raise PlannerInputError( - f"Model '{model.name}' uses cursor intrinsics, but cursor bounds could not be " - "resolved" + raise _unresolved_cursor_bounds_error( + model_name=model.name, + full_rebuild_cause=full_rebuild_cause + or ("--full-refresh" if full_refresh else None), ) query_sql = render_cursor_intrinsic_bounds( sql=query_sql, bounds=cursor_bounds, cursor_type=cursor_type, adapter=adapter @@ -165,6 +160,24 @@ def resolve_model_sql( return query_sql +def _unresolved_cursor_bounds_error( + *, model_name: str, full_rebuild_cause: str | None +) -> PlannerInputError: + if full_rebuild_cause is None: + return PlannerInputError( + f"Model '{model_name}' uses cursor intrinsics, but cursor bounds could not be resolved" + ) + return PlannerInputError( + f"Model '{model_name}' was given a FULL backfill ({full_rebuild_cause}), but it uses " + "cursor intrinsics and a non-microbatch full rebuild has no cursor interval", + help=( + "rebuild it with an explicit cursor interval (--start-cursor-ts and " + "--end-cursor-ts, or the -int forms), or make it a microbatch model so a full " + "rebuild can run batch by batch" + ), + ) + + def resolve_function_sql( *, adapter: BaseAdapter, @@ -204,7 +217,6 @@ def _compute_model_cursor_bounds( start_cursor_override: str | None, end_cursor_override: str | None, runtime_cursor_producer_names: frozenset[str], - suppress_runtime_cursor_bounds: bool, context: ModelPlanContext, replaces_relation: bool = False, ) -> CursorBounds | None: @@ -246,7 +258,6 @@ def _compute_model_cursor_bounds( ) if ( not full_refresh - and not suppress_runtime_cursor_bounds and not runtime_owned and cursor_snapshot is not None and not cursor_snapshot.watermarks_available @@ -258,7 +269,7 @@ def _compute_model_cursor_bounds( code="S302", ) - if full_refresh or suppress_runtime_cursor_bounds: + if full_refresh: return None if runtime_owned: diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 59b474cb02..e5849f7f12 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -61,6 +61,8 @@ QUALIFIED_RELATION_MAX_PARTS: int = 3 MIGRATION_MODEL_NAME_METADATA_KEY: str = "model_name" MIGRATION_FINGERPRINT_METADATA_KEY: str = "migration_fingerprint" +LOCAL_FUNCTION_HASHES_METADATA_KEY: str = "local_function_hashes" +FULL_REFRESH_DISABLED_REBUILD_CODE: str = "S203" MIGRATION_REF_PLACEHOLDER_PREFIX: str = "__sqb_migration_ref__" MIGRATION_LOCAL_NAME_PREFIX: str = "__sqb_local_" MIGRATION_FINGERPRINT_ALGORITHM: str = "migration-fingerprint-v1" diff --git a/src/sqlbuild/compiler/planner/main/execution/execution.py b/src/sqlbuild/compiler/planner/main/execution/execution.py index 3348cf14a0..7419711535 100644 --- a/src/sqlbuild/compiler/planner/main/execution/execution.py +++ b/src/sqlbuild/compiler/planner/main/execution/execution.py @@ -8,6 +8,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.planner._helpers.changes.actions import resolve_model_actions from sqlbuild.compiler.planner._helpers.changes.detect import detect_changes from sqlbuild.compiler.planner._helpers.planning.buildability import ( check_selected_scope_buildability, @@ -29,7 +30,6 @@ from sqlbuild.compiler.planner._helpers.planning.warehouse_state import ( gather_planner_warehouse_state, ) -from sqlbuild.compiler.planner._helpers.pruning.cascade import resolve_cascades from sqlbuild.compiler.planner._helpers.warehouse.source_freshness import ( build_planner_source_freshness_result, ) @@ -118,7 +118,7 @@ def build_execution_plan( overrides=overrides, policies=policies, ) - resolved_actions: PlannerResolvedActions = resolve_cascades( + resolved_actions: PlannerResolvedActions = resolve_model_actions( scope=scopes.inspection_scope, changes=changes, ) diff --git a/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py b/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py index ed273117d0..d5628746d8 100644 --- a/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py +++ b/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py @@ -4,6 +4,7 @@ import json +from sqlbuild.compiler.planner.constants import LOCAL_FUNCTION_HASHES_METADATA_KEY from sqlbuild.compiler.planner.main.identity._version_identity_config import ( build_version_identity_config, ) @@ -22,7 +23,7 @@ def build_version_identity_metadata_json( { "config": build_version_identity_config(config_values), "execution_signature": execution_signature or {}, - "local_function_hashes": local_function_hashes or {}, + LOCAL_FUNCTION_HASHES_METADATA_KEY: local_function_hashes or {}, "model_name": model_name, }, sort_keys=True, diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 8afa5db8ca..78643b2f4b 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -688,28 +688,6 @@ class BackfillResult: duration: str | None = None -@dataclass(frozen=True) -class CascadeCause: - """One upstream model that contributed to a backfill cascade.""" - - model_name: str - effective_action: BackfillAction - effective_duration: str | None - root_cause: str | None = None - root_reason: PlanReason | None = None - - -@dataclass(frozen=True) -class CascadeResult: - """Effective backfill after upstream cascade propagation.""" - - effective_action: BackfillAction - effective_duration: str | None - root_cause: str | None - root_reason: PlanReason | None = None - causes: tuple[CascadeCause, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class ChangeDetectionResult: """Per-model output from change detection and policy resolution.""" @@ -727,6 +705,7 @@ class ChangeDetectionResult: default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY) ) recorded_build_relation_missing: bool = False + changed_functions: tuple[str, ...] = field(default_factory=tuple) @dataclass(frozen=True) @@ -805,9 +784,6 @@ class FunctionChangeResult: fingerprint_sql: str reason: PlanReason = PlanReason.NO_CHANGE - backfill: BackfillResult = field( - default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY) - ) @dataclass(frozen=True) @@ -820,16 +796,15 @@ class PlannerChangeResults: @dataclass(frozen=True) class ResolvedModelAction: - """Effective model change and backfill after cascade resolution.""" + """Effective model change and backfill decided from the model's own change.""" change: ChangeDetectionResult backfill: BackfillResult - cascade: CascadeResult | None = None @dataclass(frozen=True) class PlannerResolvedActions: - """Cascade-resolved planning decisions keyed by model name.""" + """Per-model planning decisions keyed by model name.""" models: dict[str, ResolvedModelAction] @@ -1192,7 +1167,7 @@ class ModelPlanEntry: backfill: BackfillResult = field( default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY) ) - cascade: CascadeResult | None = None + changed_functions: tuple[str, ...] = field(default_factory=tuple) custom_materialization_name: str | None = None custom_config: dict[str, object] = field(default_factory=dict) custom_placeholders: dict[str, str] = field(default_factory=dict) @@ -1262,9 +1237,6 @@ class FunctionPlanEntry: packages: tuple[str, ...] = field(default_factory=tuple) previous_query_sql: str | None = None reason: PlanReason = PlanReason.NO_CHANGE - backfill: BackfillResult = field( - default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY) - ) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index 201122727e..1e00c3c70a 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -57,6 +57,7 @@ class ChangeKind(StrEnum): FIRST_RUN = "first_run" RENAMED = "renamed" QUERY_CHANGED = "query_changed" + FUNCTION_CHANGED = "function_changed" CONFIG_CHANGED = "config_changed" SCHEMA_CHANGED = "schema_changed" NO_CHANGE = "no_change" @@ -126,7 +127,6 @@ class PlanReason(StrEnum): FUNCTION_CHANGED = "function_changed" CONFIG_CHANGED = "config_changed" SCHEMA_CHANGED = "schema_changed" - UPSTREAM_CHANGED = "upstream_changed" NORMAL_INCREMENTAL = "normal_incremental" NO_CHANGE = "no_change" DISABLED = "disabled" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 859ec396d7..9ca3a1bad5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -208,8 +208,7 @@ def test_given_direct_function_dependency_when_building_then_persists_function_h 'database = "warehouse.duckdb"\n' ), "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, " - "replay_on_change full);\n\namount > 100\n" + "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 100\n" ), "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT __udf("is_large_order")(150) AS is_large\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py index 9b28706871..f803825f7b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py @@ -150,7 +150,7 @@ ), ), QueryPropagationBuildE2ETestCase( - description="changed SQL UDF propagates full rebuild to downstream incremental model", + description="changed SQL UDF applies the direct caller's own full replay", repo_files={ "sqlbuild_project.toml": dedent( """ @@ -189,7 +189,6 @@ FUNCTION ( arguments (amount_cents INTEGER), returns BOOLEAN, - replay_on_change full, ); amount_cents > 100 @@ -206,7 +205,8 @@ cursor_inputs ( raw_orders id, ), - unique_key order_id + unique_key order_id, + replay_on_change full ); SELECT @@ -224,12 +224,12 @@ before_text="amount_cents > 100", after_text="amount_cents >= 100", expected_exit_code=0, - expected_reasons={"fact_orders": "upstream_changed"}, + expected_reasons={"fact_orders": "function_changed"}, expected_actions={"fact_orders": "create_table"}, expected_fingerprint_models=("fact_orders", "is_high_value_order"), ), QueryPropagationBuildE2ETestCase( - description="bounded query change propagates downstream", + description="bounded query change does not propagate downstream", repo_files={ "sqlbuild_project.toml": dedent( """ @@ -347,7 +347,7 @@ expected_exit_code=0, expected_reasons={ "hourly_order_activity": "query_changed", - "daily_activity_rollup": "upstream_changed", + "daily_activity_rollup": "normal_incremental", }, expected_fingerprint_models=( "daily_activity_rollup", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index 76027ea038..56203d1729 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -40,6 +40,7 @@ class DiamondPlanE2ETestCase: description: str expected_fragments: tuple[str, ...] expected_max_lines_per_model: int + unexpected_fragments: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index 927dc90025..53f419c3d2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -86,7 +86,7 @@ def prepare_incremental_diamond_project(*, tmp_path: Path) -> Path: def change_incremental_diamond_root(*, project_dir: Path) -> None: - """Change the root query so every downstream model inherits a full replay.""" + """Change the root query, which replays fully under its own replay_on_change.""" (project_dir / _DIAMOND_ROOT_PATH).write_text( _diamond_root_sql(amount_offset=1), encoding="utf-8" @@ -94,7 +94,7 @@ def change_incremental_diamond_root(*, project_dir: Path) -> None: def prepare_changed_incremental_diamond_project(*, tmp_path: Path) -> Path: - """Build the incremental diamonds, then change the root so the plan cascades.""" + """Build the incremental diamonds, then change the root query.""" project_dir: Path = prepare_incremental_diamond_project(tmp_path=tmp_path) build: subprocess.CompletedProcess[str] = run_sqb( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py index b479d6fa54..42845dc31e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_diamond_plan.py @@ -29,13 +29,13 @@ "test_case", [ DiamondPlanE2ETestCase( - description="plan tree lists each diamond model once with its cascade cause", + description="plan tree lists each diamond model once without inherited replay", expected_fragments=( "Query changed (1)", - f"Upstream changed ({len(diamond_model_names()) - 1})", + f"Models ({len(diamond_model_names()) - 1})", f"└── orders_{DIAMOND_LAYERS} ", - "cause orders_0 (query changed)", ), + unexpected_fragments=("Upstream changed", "cause"), expected_max_lines_per_model=4, ) ], @@ -54,6 +54,8 @@ def test_given_changed_diamond_root_when_planning_then_tree_lists_each_model_onc assert result.returncode == 0, result.stdout + result.stderr for fragment in test_case.expected_fragments: assert fragment in result.stdout, result.stdout + for fragment in test_case.unexpected_fragments: + assert fragment not in result.stdout, result.stdout plan_output: str = result.stdout[result.stdout.index("Plan ready") :] assert sorted(_PLAN_ENTRY.findall(plan_output)) == sorted(diamond_model_names()) assert len(plan_output.splitlines()) <= test_case.expected_max_lines_per_model * len( @@ -65,10 +67,10 @@ def test_given_changed_diamond_root_when_planning_then_tree_lists_each_model_onc "test_case", [ DiamondPlanJsonE2ETestCase( - description="plan json lists each diamond model once with its cascade reason", + description="plan json lists each diamond model once with its own reason", expected_reasons={ "orders_0": "query_changed", - **{name: "upstream_changed" for name in diamond_model_names()[1:]}, + **{name: "normal_incremental" for name in diamond_model_names()[1:]}, }, ) ], diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py index 46b51b2328..6b1af280d5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py @@ -130,9 +130,6 @@ def test_given_renamed_chain_with_full_replay_when_building_then_migrated_histor assert model_entry(plan=first_plan, name="customer_orders_enriched")["reason"] == ( test_case.expected_first_view_reason ) - assert model_entry(plan=first_plan, name="customer_daily_order_totals")["reason"] != ( - "upstream_changed" - ) assert model_entry(plan=retry_plan, name="customer_daily_order_totals")["reason"] == ( test_case.expected_retry_totals_reason ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index fa0f96534c..a961f090bf 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -42,7 +42,6 @@ class BuildExecutionPlanTestCase: function_locations: dict[str, str] = field(default_factory=dict) function_bodies: dict[str, str] = field(default_factory=dict) previous_function_bodies: dict[str, str] = field(default_factory=dict) - function_replay_on_changes: dict[str, str] = field(default_factory=dict) function_languages: dict[str, FunctionLanguage] = field(default_factory=dict) function_deps: dict[str, tuple[str, ...]] = field(default_factory=dict) select: tuple[str, ...] = () @@ -50,9 +49,7 @@ class BuildExecutionPlanTestCase: expected_model_count: int | None = None effective_connection: dict[str, object] = field(default_factory=dict) model_deps: dict[str, tuple[str, ...]] = field(default_factory=dict) - expected_cascade_action: dict[str, BackfillAction] = field(default_factory=dict) - expected_cascade_duration: dict[str, str | None] = field(default_factory=dict) - expected_cascade_root_cause: dict[str, str] = field(default_factory=dict) + expected_backfill_action: dict[str, BackfillAction] = field(default_factory=dict) expected_progress_fragments: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index ca25110464..d9c28180e0 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -161,7 +161,6 @@ def build_project_from_test_case( ), language=language, entry_point=(None, "main")[language == FunctionLanguage.PYTHON], - replay_on_change=test_case.function_replay_on_changes.get(function_name), ) ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 3b8bc5bbb1..8e38febc00 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -11,12 +11,10 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.output.main.plan import format_plan from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.planner._helpers.changes import detect as change_detection from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( - CascadeResult, CursorBounds, ModelPlanEntry, PlanOutput, @@ -702,7 +700,7 @@ def test_given_cursor_type_mismatch_when_building_plan_then_produces_warning( "test_case", [ BuildExecutionPlanTestCase( - description="upstream first run full cascades to existing downstream", + description="upstream first run does not replay existing downstream table", setup_sql=("CREATE TABLE staging.fact_orders AS SELECT 1 AS id",), model_locations={ "stg_orders": "staging", @@ -724,13 +722,12 @@ def test_given_cursor_type_mismatch_when_building_plan_then_produces_warning( }, expected_reason={ "stg_orders": PlanReason.FIRST_RUN, - "fact_orders": PlanReason.FULL_REFRESH, + "fact_orders": PlanReason.NO_CHANGE, }, - expected_cascade_action={"fact_orders": BackfillAction.FULL}, - expected_cascade_root_cause={"fact_orders": "stg_orders"}, + expected_backfill_action={"fact_orders": BackfillAction.FORWARD_ONLY}, ), BuildExecutionPlanTestCase( - description="upstream full cascade forces existing incremental downstream rebuild", + description="upstream first run leaves existing incremental downstream incremental", setup_sql=("CREATE TABLE staging.fact_orders AS SELECT 1 AS id",), model_locations={ "stg_orders": "staging", @@ -754,83 +751,16 @@ def test_given_cursor_type_mismatch_when_building_plan_then_produces_warning( full_refresh=False, expected_action={ "stg_orders": PlanAction.CREATE_TABLE, - "fact_orders": PlanAction.CREATE_TABLE, + "fact_orders": PlanAction.INCREMENTAL_DELETE_INSERT, }, expected_reason={ "stg_orders": PlanReason.FIRST_RUN, - "fact_orders": PlanReason.FULL_REFRESH, - }, - expected_ddl_fragments={ - "fact_orders": "CREATE OR REPLACE TABLE staging.fact_orders AS", - }, - expected_cascade_action={"fact_orders": BackfillAction.FULL}, - expected_cascade_root_cause={"fact_orders": "stg_orders"}, - ), - BuildExecutionPlanTestCase( - description="changed SQL UDF cascades full rebuild to incremental downstream", - setup_sql=("CREATE TABLE staging.fact_orders AS SELECT 1 AS id",), - model_locations={"fact_orders": "staging"}, - function_locations={"is_priority_order": "staging"}, - function_bodies={"is_priority_order": "value = 2"}, - previous_function_bodies={"is_priority_order": "value = 1"}, - function_replay_on_changes={"is_priority_order": "full"}, - function_deps={"fact_orders": ("is_priority_order",)}, - model_configs={ - "fact_orders": { - "materialized": "incremental", - "incremental_strategy": "delete_insert", - "cursor": "id", - "cursor_type": "integer", - "unique_key": "id", - }, - }, - model_queries={"fact_orders": "SELECT 1 AS id"}, - full_refresh=False, - expected_action={"fact_orders": PlanAction.CREATE_TABLE}, - expected_reason={"fact_orders": PlanReason.FULL_REFRESH}, - expected_ddl_fragments={ - "fact_orders": "CREATE OR REPLACE TABLE staging.fact_orders AS", - }, - expected_cascade_action={"fact_orders": BackfillAction.FULL}, - expected_cascade_root_cause={"fact_orders": "is_priority_order"}, - ), - BuildExecutionPlanTestCase( - description="changed Python UDF cascades full rebuild to incremental downstream", - setup_sql=("CREATE TABLE staging.fact_orders AS SELECT 1 AS id",), - model_locations={"fact_orders": "staging"}, - function_locations={"is_priority_order_py": "staging"}, - function_languages={"is_priority_order_py": FunctionLanguage.PYTHON}, - function_bodies={ - "is_priority_order_py": "def main(value: int) -> int:\n return value + 2", - }, - previous_function_bodies={ - "is_priority_order_py": "def main(value: int) -> int:\n return value + 1", - }, - function_replay_on_changes={"is_priority_order_py": "full"}, - function_deps={"fact_orders": ("is_priority_order_py",)}, - model_configs={ - "fact_orders": { - "materialized": "incremental", - "incremental_strategy": "delete_insert", - "cursor": "id", - "cursor_type": "integer", - "unique_key": "id", - }, - }, - model_queries={"fact_orders": "SELECT 1 AS id"}, - full_refresh=False, - expected_action={"fact_orders": PlanAction.CREATE_TABLE}, - expected_reason={"fact_orders": PlanReason.FULL_REFRESH}, - expected_ddl_fragments={ - "fact_orders": "CREATE OR REPLACE TABLE staging.fact_orders AS", + "fact_orders": PlanReason.NORMAL_INCREMENTAL, }, - expected_cascade_action={"fact_orders": BackfillAction.FULL}, - expected_cascade_root_cause={"fact_orders": "is_priority_order_py"}, + expected_backfill_action={"fact_orders": BackfillAction.FORWARD_ONLY}, ), BuildExecutionPlanTestCase( - description=( - "downstream local bounded policy replaces upstream full and propagates downstream" - ), + description="downstream replay policy applies only to its own change", setup_sql=( "CREATE TABLE staging.fact_orders AS SELECT 1 AS id", "CREATE TABLE staging.order_metrics AS SELECT 1 AS id", @@ -878,23 +808,15 @@ def test_given_cursor_type_mismatch_when_building_plan_then_produces_warning( "fact_orders": PlanReason.NORMAL_INCREMENTAL, "order_metrics": PlanReason.NORMAL_INCREMENTAL, }, - expected_cascade_action={ - "fact_orders": BackfillAction.BOUNDED, - "order_metrics": BackfillAction.BOUNDED, - }, - expected_cascade_duration={ - "fact_orders": "1d", - "order_metrics": "1d", - }, - expected_cascade_root_cause={ - "fact_orders": "stg_orders", - "order_metrics": "stg_orders", + expected_backfill_action={ + "fact_orders": BackfillAction.FORWARD_ONLY, + "order_metrics": BackfillAction.FORWARD_ONLY, }, ), ], ids=lambda case: case.description, ) -def test_given_upstream_first_run_when_building_plan_then_cascades_to_downstream( +def test_given_upstream_first_run_when_building_plan_then_downstream_keeps_own_backfill( test_case: BuildExecutionPlanTestCase, adapter: DuckDbAdapter, connection: Any, @@ -932,23 +854,9 @@ def test_given_upstream_first_run_when_building_plan_then_cascades_to_downstream for model_name, expected_fragment in test_case.expected_ddl_fragments.items(): assert expected_fragment in entry_map[model_name].logical_ddl - expected_cascade_action: BackfillAction - for model_name, expected_cascade_action in test_case.expected_cascade_action.items(): - cascade: CascadeResult | None = entry_map[model_name].cascade - assert cascade is not None - assert cascade.effective_action == expected_cascade_action - - expected_cascade_duration: str | None - for model_name, expected_cascade_duration in test_case.expected_cascade_duration.items(): - cascade_for_duration: CascadeResult | None = entry_map[model_name].cascade - assert cascade_for_duration is not None - assert cascade_for_duration.effective_duration == expected_cascade_duration - - expected_root: str - for model_name, expected_root in test_case.expected_cascade_root_cause.items(): - cascade_for_root: CascadeResult | None = entry_map[model_name].cascade - assert cascade_for_root is not None - assert cascade_for_root.root_cause == expected_root + expected_backfill: BackfillAction + for model_name, expected_backfill in test_case.expected_backfill_action.items(): + assert entry_map[model_name].backfill.action == expected_backfill @pytest.mark.parametrize( @@ -984,7 +892,7 @@ def test_given_upstream_first_run_when_building_plan_then_cascades_to_downstream unexpected_format_fragments=("Normal", "Query changed", "Upstream changed"), ), FormatPlanIntegrationTestCase( - description="cascade formats with upstream changed group and cause line", + description="upstream first run formats downstream table as routine work", setup_sql=("CREATE TABLE staging.fact_orders AS SELECT 1 AS id",), model_locations={ "stg_orders": "staging", @@ -1004,11 +912,10 @@ def test_given_upstream_first_run_when_building_plan_then_cascades_to_downstream "Plan ready", "\033[1mFirst run\033[0m \033[2m(1)\033[0m", "stg_orders", - "\033[1mUpstream changed\033[0m \033[2m(1)\033[0m", + "\033[1mModels\033[0m \033[2m(1)\033[0m", "fact_orders", - "full rebuild", - "cause stg_orders", ), + unexpected_format_fragments=("Upstream changed", "full rebuild", "cause"), ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index e12ddccf95..3c99950166 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -12,7 +12,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage from sqlbuild.compiler.planner.models import ( BackfillResult, - CascadeResult, ColumnRenameHint, CursorBounds, CursorInputRelation, @@ -67,7 +66,7 @@ def build_model_entry( observed_at_column: str | None = None, historical_input: str | None = None, schema_findings: tuple[SchemaFinding, ...] = (), - cascade: CascadeResult | None = None, + changed_functions: tuple[str, ...] = (), custom_materialization_name: str | None = None, query_changed: bool = False, config_changed: bool = False, @@ -113,7 +112,7 @@ def build_model_entry( previous_version_hash=previous_version_hash, schema_findings=schema_findings, backfill=BackfillResult(action=backfill_action, duration=backfill_duration), - cascade=cascade, + changed_functions=changed_functions, custom_materialization_name=custom_materialization_name, query_changed=query_changed, config_changed=config_changed, @@ -254,8 +253,6 @@ def build_function_entry( name: str, language: FunctionLanguage = FunctionLanguage.SQL, reason: PlanReason = PlanReason.NO_CHANGE, - backfill_action: BackfillAction = BackfillAction.FORWARD_ONLY, - backfill_duration: str | None = None, previous_query_sql: str | None = None, ) -> FunctionPlanEntry: """Build a minimal FunctionPlanEntry for formatter tests.""" @@ -277,7 +274,6 @@ def build_function_entry( language=language, previous_query_sql=previous_query_sql, reason=reason, - backfill=BackfillResult(action=backfill_action, duration=backfill_duration), ) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index c49569f7e1..b692c775ff 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -10,8 +10,6 @@ from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( - CascadeCause, - CascadeResult, ColumnRenameHint, CursorBounds, CursorInputRelation, @@ -553,90 +551,27 @@ ), ), FormatPlanTestCase( - description="upstream changed shows cascade cause", + description="direct caller of a changed function shows function changed cause", plan_output=build_plan_output( model_entries=( build_model_entry( - name="fact_daily_revenue", + name="hourly_order_activity", action=PlanAction.INCREMENTAL_DELETE_INSERT, - reason=PlanReason.NORMAL_INCREMENTAL, + reason=PlanReason.FUNCTION_CHANGED, materialization_type=MaterializationType.INCREMENTAL, incremental_strategy="delete_insert", - cursor_column="event_time", - cursor_type="timestamp", backfill_action=BackfillAction.FORWARD_ONLY, - cascade=CascadeResult( - effective_action=BackfillAction.BOUNDED, - effective_duration="90d", - root_cause="fact_orders", - causes=( - CascadeCause( - model_name="fact_orders", - effective_action=BackfillAction.BOUNDED, - effective_duration="90d", - ), - ), - ), - ), - ), - ), - expected_fragments=( - "Upstream changed (1)", - "fact_daily_revenue", - "rebuild last 90d", - "cause fact_orders (90d)", - ), - unexpected_fragments=("Normal",), - ), - FormatPlanTestCase( - description="upstream changed with full shows full in cause", - plan_output=build_plan_output( - model_entries=( - build_model_entry( - name="dim_summary", - action=PlanAction.CREATE_TABLE, - reason=PlanReason.NO_CHANGE, - cascade=CascadeResult( - effective_action=BackfillAction.FULL, - effective_duration=None, - root_cause="fact_orders", - causes=( - CascadeCause( - model_name="fact_orders", - effective_action=BackfillAction.FULL, - effective_duration=None, - ), - ), - ), + changed_functions=("is_completed_order",), ), ), ), expected_fragments=( - "Upstream changed (1)", - "dim_summary", - "full rebuild", - "cause fact_orders (full)", - ), - ), - FormatPlanTestCase( - description="upstream changed view shows recreate action", - plan_output=build_plan_output( - model_entries=( - build_model_entry( - name="stg_orders", - action=PlanAction.CREATE_VIEW, - reason=PlanReason.NO_CHANGE, - materialization_type=MaterializationType.VIEW, - cascade=CascadeResult( - effective_action=BackfillAction.FULL, - effective_duration=None, - root_cause="raw_orders", - ), - ), - ), + "Function changed (1)", + "hourly_order_activity", + "continue forward", + "cause function is_completed_order changed", ), - expected_fragments=("stg_orders", "recreate view"), - unexpected_fragments=("full rebuild",), + unexpected_fragments=("Upstream changed", "Normal"), ), FormatPlanTestCase( description="seeds section shows seed names", @@ -848,7 +783,7 @@ ), ), FormatPlanTestCase( - description="function query change shows diff and policy", + description="function query change shows diff without replay policy", plan_output=build_plan_output( model_entries=(build_model_entry(name="orders", action=PlanAction.CREATE_TABLE),), function_entries=( @@ -856,7 +791,6 @@ name="is_completed_order", language=FunctionLanguage.SQL, reason=PlanReason.QUERY_CHANGED, - backfill_action=BackfillAction.FULL, previous_query_sql="returns=BOOLEAN\nbody=\norder_status = 'completed'", ), ), @@ -865,47 +799,11 @@ "Changed functions (1)", "is_completed_order", "sql udf", - "policy replay_on_change=full", "query diff:", "--- previous", "+++ current", ), ), - FormatPlanTestCase( - description="upstream changed prefers root function changed cause", - plan_output=build_plan_output( - model_entries=( - build_model_entry( - name="daily_activity_rollup", - action=PlanAction.CREATE_TABLE, - reason=PlanReason.NO_CHANGE, - cascade=CascadeResult( - effective_action=BackfillAction.FULL, - effective_duration=None, - root_cause="is_completed_order", - root_reason=PlanReason.FUNCTION_CHANGED, - causes=( - CascadeCause( - model_name="hourly_order_activity", - effective_action=BackfillAction.FULL, - effective_duration=None, - ), - ), - ), - ), - ), - ), - expected_fragments=( - "Upstream changed (1)", - "daily_activity_rollup", - "full rebuild", - "cause is_completed_order (function changed)", - ), - unexpected_fragments=( - "cause hourly_order_activity", - "cause is_completed_order (full)", - ), - ), FormatPlanTestCase( description="warnings section shows warning messages", plan_output=build_plan_output( @@ -1132,7 +1030,6 @@ build_function_entry( name="normalize_email", reason=PlanReason.QUERY_CHANGED, - backfill_action=BackfillAction.FULL, previous_query_sql="returns=TEXT\nbody=old_email", ), ), diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index e6fe42c491..3cc8cf25ca 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -10,8 +10,6 @@ from sqlbuild.cli.output.main._plan_json import format_plan_json from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( - CascadeCause, - CascadeResult, CursorBounds, CursorInputEvidence, CursorInputRelation, @@ -435,34 +433,21 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence ), ), JsonOutputTestCase( - description="plan json includes cascade when present", + description="plan json lists changed functions on the direct caller", plan_output=build_plan_output( model_entries=( build_model_entry( - name="fact_daily", - action=PlanAction.CREATE_TABLE, - reason=PlanReason.NO_CHANGE, - cascade=CascadeResult( - effective_action=BackfillAction.BOUNDED, - effective_duration="90d", - root_cause="fact_orders", - causes=( - CascadeCause( - model_name="fact_orders", - effective_action=BackfillAction.BOUNDED, - effective_duration="90d", - ), - ), - ), + name="hourly_order_activity", + action=PlanAction.INCREMENTAL_DELETE_INSERT, + reason=PlanReason.FUNCTION_CHANGED, + changed_functions=("is_completed_order",), ), ), ), expected_keys=("selected_count", "models"), expected_fragments=( - '"reason": "upstream_changed"', - '"cascade"', - '"root_cause": "fact_orders"', - '"effective_duration": "90d"', + '"reason": "function_changed"', + '"changed_functions": [\n "is_completed_order"\n ]', ), ), JsonOutputTestCase( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_test_types.py index 6372dc5b17..c527c52cd5 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_test_types.py @@ -72,7 +72,6 @@ class BuildCompileInputsTestCase: expected_sql_function_runtime_versions: tuple[str | None, ...] = field(default_factory=tuple) expected_sql_function_entry_points: tuple[str | None, ...] = field(default_factory=tuple) expected_sql_function_packages: tuple[tuple[str, ...], ...] = field(default_factory=tuple) - expected_sql_function_replay_on_changes: tuple[str | None, ...] = field(default_factory=tuple) expected_model_references: tuple[tuple[tuple[str, str], ...], ...] = field( default_factory=tuple ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index f26922c593..820e0a99a4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -1707,7 +1707,6 @@ def source_columns() -> str: expected_sql_function_runtime_versions=(None,), expected_sql_function_entry_points=(None,), expected_sql_function_packages=((),), - expected_sql_function_replay_on_changes=(None,), expected_effective_target_name=None, expected_effective_connection={}, expected_effective_vars={ @@ -2019,7 +2018,6 @@ def status_match(column_name: str, status: str) -> str: schema ${udf_schema}, arguments (order_status ${status_type}), returns ${return_type}, - replay_on_change bounded-${backfill_days}d ); @status_match("order_status", "completed") AND order_status <> @@cancelled_status @@ -2050,7 +2048,6 @@ def status_match(column_name: str, status: str) -> str: expected_sql_function_runtime_versions=(None,), expected_sql_function_entry_points=(None,), expected_sql_function_packages=((),), - expected_sql_function_replay_on_changes=("bounded-30d",), expected_effective_target_name="dev", expected_effective_connection={}, expected_effective_vars={ @@ -2120,7 +2117,6 @@ def status_match(column_name: str, status: str) -> str: expected_sql_function_runtime_versions=(None,), expected_sql_function_entry_points=(None,), expected_sql_function_packages=((),), - expected_sql_function_replay_on_changes=(None,), expected_effective_target_name=None, expected_effective_connection={}, expected_effective_vars={}, @@ -2177,7 +2173,6 @@ def status_match(column_name: str, status: str) -> str: expected_sql_function_runtime_versions=(None,), expected_sql_function_entry_points=(None,), expected_sql_function_packages=((),), - expected_sql_function_replay_on_changes=(None,), expected_effective_target_name=None, expected_effective_connection={}, expected_effective_vars={}, @@ -2352,7 +2347,6 @@ def main(order_status): expected_sql_function_runtime_versions=("3.11",), expected_sql_function_entry_points=("main",), expected_sql_function_packages=(("faker",),), - expected_sql_function_replay_on_changes=(None,), expected_effective_target_name="dev", expected_effective_connection={}, expected_effective_vars={ @@ -2974,12 +2968,6 @@ def test_given_discovered_inputs_when_building_compile_inputs_then_it_attaches_m tuple(function_input.packages for function_input in compile_inputs.sql_function_inputs) == test_case.expected_sql_function_packages ) - assert ( - tuple( - function_input.replay_on_change for function_input in compile_inputs.sql_function_inputs - ) - == test_case.expected_sql_function_replay_on_changes - ) actual_model_references: list[tuple[tuple[str, str], ...]] = [] for model_input in compile_inputs.model_inputs: references: list[tuple[str, str]] = [] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 6f2f61ede5..4428fb9bb9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -841,39 +841,12 @@ class CursorTypeCheckTestCase: expected_message_fragment: str | None = None -@dataclass(frozen=True) -class ResolveCascadeTestCase: - description: str - own_action: BackfillAction - own_duration: str | None - own_cursor_type: str | None - upstream_entries: tuple[tuple[str, BackfillAction, str | None, str | None], ...] - expected_cascade: bool - local_policy: str | None = None - expected_action: BackfillAction | None = None - expected_duration: str | None = None - expected_root_cause: str | None = None - expected_cause_count: int = 0 - - -@dataclass(frozen=True) -class ResolveCascadeRootCauseTestCase: - description: str - expected_action: BackfillAction - expected_root_cause: str - expected_root_reason: PlanReason - expected_immediate_cause: str - - @dataclass(frozen=True) class DetectFunctionChangeTestCase: description: str body_sql: str existing_function_fingerprints: dict[str, Fingerprint] - replay_on_change: str | None expected_reason: PlanReason - expected_action: BackfillAction - expected_duration: str | None = None target_schema: str = "main" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index 4ddd126509..292943e305 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -24,6 +24,7 @@ build_version_identity_metadata_json, ) from sqlbuild.compiler.planner.models import PlannerScope, WarehouseFingerprints, WarehouseSnapshot +from sqlbuild.compiler.planner.types import ChangeKind from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectModelChangesTestCase, @@ -267,3 +268,27 @@ def _build_fingerprints(test_case: DetectModelChangesTestCase) -> dict[str, Fing ) } return ({}, fingerprints)[test_case.fingerprint_query_hash is not None] + + +def build_function_caller_metadata_case( + *, + config_values: dict[str, object], + function_local_hashes: dict[str, str], + previous_function_hashes: dict[str, str], + previous_config_values: dict[str, object], +) -> DetectModelMetadataTestCase: + """Build metadata detection inputs for a model calling the given functions.""" + + return DetectModelMetadataTestCase( + description="caller metadata", + config_values=config_values, + schema_columns=(), + deps=tuple(function_local_hashes), + function_local_hashes=function_local_hashes, + previous_metadata_json=build_version_identity_metadata_json( + model_name="orders", + config_values=previous_config_values, + local_function_hashes=previous_function_hashes, + ), + expected_change_kind=ChangeKind.FUNCTION_CHANGED, + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 69d8a56770..9be6626992 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -49,6 +49,16 @@ class DetectModelMetadataTestCase: expected_metadata_fragments: tuple[str, ...] = () +@dataclass(frozen=True) +class DetectFunctionCallerChangeTestCase: + description: str + metadata_case: DetectModelMetadataTestCase + query_change_tracking: bool + expected_changed_functions: tuple[str, ...] + expected_config_changed: bool + expected_backfill: BackfillResult + + @dataclass(frozen=True) class DetectQueryChangeTestCase: description: str @@ -75,3 +85,4 @@ class ResolveBackfillTestCase: description: str raw_value: str | None expected_result: BackfillResult + diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 831348ae7e..0c3031764b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -19,6 +19,7 @@ build_version_identity_metadata_json, ) from sqlbuild.compiler.planner.models import ( + BackfillResult, ChangeDetectionResult, PlannerChangeResults, PlannerScope, @@ -34,6 +35,7 @@ WarningSeverity, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_helpers import ( + build_function_caller_metadata_case, build_metadata_json_with_audit_gate, build_model_from_metadata_test_case, build_model_from_test_case, @@ -43,6 +45,7 @@ build_snapshot_from_test_case, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( + DetectFunctionCallerChangeTestCase, DetectModelChangesTestCase, DetectModelMetadataTestCase, DroppedRelationPlanActionTestCase, @@ -379,7 +382,7 @@ def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_fir "test_case", [ DetectModelMetadataTestCase( - description="detects config change when dependent function hash changes", + description="detects function change when dependent function hash changes", config_values={}, schema_columns=(), deps=("is_large_order",), @@ -389,7 +392,7 @@ def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_fir config_values={}, local_function_hashes={"is_large_order": "old"}, ), - expected_change_kind=ChangeKind.CONFIG_CHANGED, + expected_change_kind=ChangeKind.FUNCTION_CHANGED, expected_metadata_fragments=('"local_function_hashes":{"is_large_order":"new"}',), ), DetectModelMetadataTestCase( @@ -470,6 +473,96 @@ def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_ assert fragment in (result.fingerprint_metadata_json or "") +@pytest.mark.parametrize( + "test_case", + [ + DetectFunctionCallerChangeTestCase( + description="changed function continues forward by default", + metadata_case=build_function_caller_metadata_case( + config_values={}, + function_local_hashes={"is_large_order": "new"}, + previous_function_hashes={"is_large_order": "old"}, + previous_config_values={}, + ), + query_change_tracking=True, + expected_changed_functions=("is_large_order",), + expected_config_changed=False, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), + DetectFunctionCallerChangeTestCase( + description="changed function applies the caller's own bounded replay", + metadata_case=build_function_caller_metadata_case( + config_values={"replay_on_change": "bounded-7d"}, + function_local_hashes={"is_large_order": "new", "normalize_status": "same"}, + previous_function_hashes={"is_large_order": "old", "normalize_status": "same"}, + previous_config_values={"replay_on_change": "bounded-7d"}, + ), + query_change_tracking=True, + expected_changed_functions=("is_large_order",), + expected_config_changed=False, + expected_backfill=BackfillResult(action=BackfillAction.BOUNDED, duration="7d"), + ), + DetectFunctionCallerChangeTestCase( + description="function change alongside a config change reports both", + metadata_case=build_function_caller_metadata_case( + config_values={"replay_on_change": "full", "lookback": "1d"}, + function_local_hashes={"is_large_order": "new"}, + previous_function_hashes={"is_large_order": "old"}, + previous_config_values={"replay_on_change": "full"}, + ), + query_change_tracking=True, + expected_changed_functions=("is_large_order",), + expected_config_changed=True, + expected_backfill=BackfillResult(action=BackfillAction.FULL), + ), + DetectFunctionCallerChangeTestCase( + description="newly called function is not a function change", + metadata_case=build_function_caller_metadata_case( + config_values={"replay_on_change": "full"}, + function_local_hashes={"is_large_order": "new"}, + previous_function_hashes={}, + previous_config_values={"replay_on_change": "full"}, + ), + query_change_tracking=True, + expected_changed_functions=(), + expected_config_changed=True, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), + DetectFunctionCallerChangeTestCase( + description="disabled query change tracking keeps a function change as config only", + metadata_case=build_function_caller_metadata_case( + config_values={"replay_on_change": "full"}, + function_local_hashes={"is_large_order": "new"}, + previous_function_hashes={"is_large_order": "old"}, + previous_config_values={"replay_on_change": "full"}, + ), + query_change_tracking=False, + expected_changed_functions=(), + expected_config_changed=True, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), + ], + ids=lambda case: case.description, +) +def test_given_called_function_change_when_detecting_caller_changes_then_caller_policy_decides( + test_case: DetectFunctionCallerChangeTestCase, +) -> None: + result: ChangeDetectionResult = detect_model_changes( + model=build_model_from_metadata_test_case(test_case.metadata_case), + snapshot=build_snapshot_for_metadata_test_case(test_case.metadata_case), + sql_analysis_enabled=False, + query_change_tracking=test_case.query_change_tracking, + full_refresh=False, + function_local_hashes=test_case.metadata_case.function_local_hashes, + ) + + assert (result.changed_functions, result.config_changed, result.backfill) == ( + test_case.expected_changed_functions, + test_case.expected_config_changed, + test_case.expected_backfill, + ) + + @pytest.mark.parametrize( "test_case", [ @@ -480,7 +573,7 @@ def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_ deps=("is_large_order",), function_local_hashes={"is_large_order": "old"}, previous_metadata_json="{}", - expected_change_kind=ChangeKind.CONFIG_CHANGED, + expected_change_kind=ChangeKind.FUNCTION_CHANGED, ) ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 75d88ebc68..de3896c33e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -59,7 +59,6 @@ from sqlbuild.compiler.planner._helpers.scenario.artifacts import build_scenario_relation_map from sqlbuild.compiler.planner.models import ( BackfillResult, - CascadeResult, ChangeDetectionResult, GraphIdentityNode, GraphNodeKey, @@ -1271,40 +1270,7 @@ def build_microbatch_lookback_model( ) -def build_cascade_upstream_state( - entries: tuple[tuple[str, BackfillAction, str | None, str | None], ...], -) -> tuple[ - tuple[CompiledObjectKey, ...], - dict[str, CascadeResult], - dict[str, str | None], -]: - """Build upstream keys, effective cascades, and cursor types from test tuples. - - Each tuple is (model_name, effective_action, effective_duration, cursor_type). - """ - - keys: list[CompiledObjectKey] = [] - cascades: dict[str, CascadeResult] = {} - cursor_types: dict[str, str | None] = {} - name: str - action: BackfillAction - duration: str | None - cursor_type: str | None - for name, action, duration, cursor_type in entries: - keys.append(model_key(name)) - cascades[name] = CascadeResult( - effective_action=action, - effective_duration=duration, - root_cause=None, - causes=(), - ) - cursor_types[name] = cursor_type - return tuple(keys), cascades, cursor_types - - -def build_compiled_function( - *, body_sql: str, replay_on_change: str | None = None, target_schema: str = "main" -) -> CompiledFunction: +def build_compiled_function(*, body_sql: str, target_schema: str = "main") -> CompiledFunction: """Build a minimal compiled function for planner tests.""" return CompiledFunction( @@ -1330,7 +1296,6 @@ def build_compiled_function( name="is_completed_order", qualified_name=f"{target_schema}.is_completed_order", ), - replay_on_change=replay_on_change, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py index cea5f69ec4..28651b6839 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from sqlbuild.adapter.contract.models import RetentionState +from sqlbuild.compiler.planner.models import ChangeDetectionResult from sqlbuild.compiler.planner.types import RetentionDirection, RetentionPlanPhase from sqlbuild.spec.contracts.types import TableType, TableTypeDowngradePolicy @@ -74,3 +75,11 @@ class PlannerScopesTestCase: select: tuple[str, ...] expected_selected_names: frozenset[str] expected_stale_warning_names: tuple[str, ...] + + +@dataclass(frozen=True) +class FullRebuildCheckTestCase: + description: str + model_config: dict[str, object] + change: ChangeDetectionResult + expected_cause: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index 4696e862dc..4fd899ae11 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -144,3 +144,21 @@ def _retention_model( qualified_name=f"warehouse.analytics.{name}", ), ) + + +def build_full_refresh_model(*, config_values: dict[str, object]) -> CompiledModel: + """Build an incremental model with the given header configuration.""" + + return CompiledModel( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="order_history"), + deps=(), + name="order_history", + relative_path=Path("models/order_history.sql"), + query_sql="SELECT 1 AS id", + config=CompileModelConfig( + values={"materialized": "incremental", "incremental_strategy": "merge", **config_values} + ), + destination=CompiledRelationLocation( + database=None, schema="main", name="order_history", qualified_name=None + ), + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py new file mode 100644 index 0000000000..5d599a3301 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py @@ -0,0 +1,121 @@ +"""Tests for full-rebuild protection of full_refresh false models.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( + check_full_rebuild_allowed, + describe_full_rebuild_cause, +) +from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.planner.models import BackfillResult, ChangeDetectionResult +from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind +from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( + FullRebuildCheckTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.planning.helpers import ( + build_full_refresh_model, +) + +_FULL: BackfillResult = BackfillResult(action=BackfillAction.FULL) +_PROTECTED: dict[str, object] = {"full_refresh": False} + + +@pytest.mark.parametrize( + "test_case", + [ + FullRebuildCheckTestCase( + description="first run of a protected model", + model_config=_PROTECTED, + change=ChangeDetectionResult( + model_name="order_history", change_kind=ChangeKind.FIRST_RUN, backfill=_FULL + ), + expected_cause="first run", + ), + FullRebuildCheckTestCase( + description="own query change with replay full on a protected model", + model_config=_PROTECTED, + change=ChangeDetectionResult( + model_name="order_history", + change_kind=ChangeKind.QUERY_CHANGED, + query_changed=True, + backfill=_FULL, + ), + expected_cause="query changed with replay_on_change full", + ), + FullRebuildCheckTestCase( + description="called function change on an unprotected model", + model_config={}, + change=ChangeDetectionResult( + model_name="order_history", + change_kind=ChangeKind.FUNCTION_CHANGED, + changed_functions=("normalize_status",), + backfill=_FULL, + ), + expected_cause="function normalize_status changed with replay_on_change full", + ), + ], + ids=lambda case: case.description, +) +def test_given_own_full_rebuild_cause_when_checking_protection_then_allows_rebuild( + test_case: FullRebuildCheckTestCase, +) -> None: + cause: str = describe_full_rebuild_cause(change_result=test_case.change, full_refresh=False) + + check_full_rebuild_allowed( + model=build_full_refresh_model(config_values=test_case.model_config), + change_result=test_case.change, + full_rebuild_cause=cause, + ) + + assert cause == test_case.expected_cause + + +@pytest.mark.parametrize( + "test_case", + [ + FullRebuildCheckTestCase( + description="called function change with replay full", + model_config=_PROTECTED, + change=ChangeDetectionResult( + model_name="order_history", + change_kind=ChangeKind.FUNCTION_CHANGED, + changed_functions=("normalize_status",), + backfill=_FULL, + ), + expected_cause="function normalize_status changed with replay_on_change full", + ), + FullRebuildCheckTestCase( + description="full backfill without an own query change", + model_config=_PROTECTED, + change=ChangeDetectionResult( + model_name="order_history", change_kind=ChangeKind.SCHEMA_CHANGED, backfill=_FULL + ), + expected_cause="full backfill", + ), + ], + ids=lambda case: case.description, +) +def test_given_foreign_full_rebuild_cause_when_checking_protection_then_refuses_with_cause( + test_case: FullRebuildCheckTestCase, +) -> None: + cause: str = describe_full_rebuild_cause(change_result=test_case.change, full_refresh=False) + + with pytest.raises(PlannerInputError) as raised: + check_full_rebuild_allowed( + model=build_full_refresh_model(config_values=test_case.model_config), + change_result=test_case.change, + full_rebuild_cause=cause, + ) + + assert cause == test_case.expected_cause + assert raised.value.code == "S203" + assert raised.value.message == ( + "model 'order_history' sets full_refresh false, but the plan would fully rebuild it " + f"({test_case.expected_cause})" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 9b4ac29faa..e49767fc62 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -94,7 +94,9 @@ def test_given_cursor_interval_when_resolving_then_renders_expected_sql( }, full_refresh=True, cursor_snapshots={}, - expected_error_fragment="full refresh has no cursor interval", + expected_error_fragment=( + r"given a FULL backfill \(--full-refresh\).*full rebuild has no cursor interval" + ), ) ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cascade.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cascade.py deleted file mode 100644 index 2d93b58e74..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cascade.py +++ /dev/null @@ -1,317 +0,0 @@ -"""Tests for backfill cascade propagation.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.changes.policy import resolve_replay_on_change -from sqlbuild.compiler.planner._helpers.pruning.cascade import ( - build_self_cascade, - resolve_cascade, -) -from sqlbuild.compiler.planner.models import ( - BackfillResult, - CascadeResult, -) -from sqlbuild.compiler.planner.types import BackfillAction, PlanReason -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( - ResolveCascadeRootCauseTestCase, - ResolveCascadeTestCase, -) -from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( - build_cascade_upstream_state, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveCascadeTestCase( - description="upstream full cascades to same cursor type downstream", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=(("orders", BackfillAction.FULL, None, "timestamp"),), - expected_cascade=True, - expected_action=BackfillAction.FULL, - expected_root_cause="orders", - expected_cause_count=1, - ), - ResolveCascadeTestCase( - description="upstream bounded cascades to same cursor type downstream", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=(("orders", BackfillAction.BOUNDED, "90d", "timestamp"),), - expected_cascade=True, - expected_action=BackfillAction.BOUNDED, - expected_duration="90d", - expected_root_cause="orders", - expected_cause_count=1, - ), - ResolveCascadeTestCase( - description="upstream full cascades across cursor types", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="integer", - upstream_entries=(("orders", BackfillAction.FULL, None, "timestamp"),), - expected_cascade=True, - expected_action=BackfillAction.FULL, - expected_root_cause="orders", - expected_cause_count=1, - ), - ResolveCascadeTestCase( - description="picks most aggressive upstream when multiple cascade", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=( - ("orders", BackfillAction.BOUNDED, "30d", "timestamp"), - ("payments", BackfillAction.BOUNDED, "90d", "timestamp"), - ), - expected_cascade=True, - expected_action=BackfillAction.BOUNDED, - expected_duration="90d", - expected_root_cause="payments", - expected_cause_count=2, - ), - ResolveCascadeTestCase( - description="alphabetical tiebreak among equal upstream windows", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=( - ("orders", BackfillAction.BOUNDED, "90d", "timestamp"), - ("customers", BackfillAction.BOUNDED, "90d", "timestamp"), - ), - expected_cascade=True, - expected_action=BackfillAction.BOUNDED, - expected_duration="90d", - expected_root_cause="customers", - expected_cause_count=2, - ), - ResolveCascadeTestCase( - description="full beats bounded in upstream comparison", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=( - ("orders", BackfillAction.BOUNDED, "90d", "timestamp"), - ("payments", BackfillAction.FULL, None, "timestamp"), - ), - expected_cascade=True, - expected_action=BackfillAction.FULL, - expected_root_cause="payments", - expected_cause_count=2, - ), - ResolveCascadeTestCase( - description="local bounded policy replaces stronger upstream full", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=( - ("orders", BackfillAction.FULL, None, "timestamp"), - ("customers", BackfillAction.BOUNDED, "30d", "timestamp"), - ), - local_policy="bounded-1d", - expected_cascade=True, - expected_action=BackfillAction.BOUNDED, - expected_duration="1d", - expected_root_cause="orders", - expected_cause_count=2, - ), - ResolveCascadeTestCase( - description="local full policy replaces weaker upstream bounded", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=(("orders", BackfillAction.BOUNDED, "30d", "timestamp"),), - local_policy="full", - expected_cascade=True, - expected_action=BackfillAction.FULL, - expected_root_cause="orders", - expected_cause_count=1, - ), - ], - ids=lambda case: case.description, -) -def test_given_upstream_cascade_when_resolving_then_returns_cascade_result( - test_case: ResolveCascadeTestCase, -) -> None: - upstream_keys: tuple[CompiledObjectKey, ...] - effective_cascades: dict[str, CascadeResult] - model_cursor_types: dict[str, str | None] - upstream_keys, effective_cascades, model_cursor_types = build_cascade_upstream_state( - test_case.upstream_entries - ) - - result: CascadeResult | None = resolve_cascade( - model_name="test_model", - own_backfill=BackfillResult(action=test_case.own_action, duration=test_case.own_duration), - local_backfill=resolve_replay_on_change(replay_on_change=test_case.local_policy), - own_cursor_type=test_case.own_cursor_type, - upstream_keys=upstream_keys, - effective_cascades=effective_cascades, - model_cursor_types=model_cursor_types, - ) - - assert result is not None - assert result.effective_action == test_case.expected_action - assert result.effective_duration == test_case.expected_duration - assert result.root_cause == test_case.expected_root_cause - assert len(result.causes) == test_case.expected_cause_count - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveCascadeTestCase( - description="no upstream produces no cascade", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=(), - expected_cascade=False, - ), - ResolveCascadeTestCase( - description="upstream forward produces no cascade", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=(("orders", BackfillAction.FORWARD_ONLY, None, "timestamp"),), - expected_cascade=False, - ), - ResolveCascadeTestCase( - description="upstream bounded does not cascade across cursor types", - own_action=BackfillAction.FORWARD_ONLY, - own_duration=None, - own_cursor_type="integer", - upstream_entries=(("orders", BackfillAction.BOUNDED, "90d", "timestamp"),), - expected_cascade=False, - ), - ResolveCascadeTestCase( - description="upstream weaker than own backfill produces no cascade", - own_action=BackfillAction.BOUNDED, - own_duration="90d", - own_cursor_type="timestamp", - upstream_entries=(("orders", BackfillAction.BOUNDED, "30d", "timestamp"),), - expected_cascade=False, - ), - ResolveCascadeTestCase( - description="own bounded backfill replaces stronger upstream bounded", - own_action=BackfillAction.BOUNDED, - own_duration="30d", - own_cursor_type="timestamp", - upstream_entries=(("orders", BackfillAction.BOUNDED, "90d", "timestamp"),), - expected_cascade=False, - ), - ResolveCascadeTestCase( - description="own full is not exceeded by upstream bounded", - own_action=BackfillAction.FULL, - own_duration=None, - own_cursor_type="timestamp", - upstream_entries=(("orders", BackfillAction.BOUNDED, "90d", "timestamp"),), - expected_cascade=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_no_stronger_upstream_when_resolving_then_returns_none( - test_case: ResolveCascadeTestCase, -) -> None: - upstream_keys: tuple[CompiledObjectKey, ...] - effective_cascades: dict[str, CascadeResult] - model_cursor_types: dict[str, str | None] - upstream_keys, effective_cascades, model_cursor_types = build_cascade_upstream_state( - test_case.upstream_entries - ) - - result: CascadeResult | None = resolve_cascade( - model_name="test_model", - own_backfill=BackfillResult(action=test_case.own_action, duration=test_case.own_duration), - local_backfill=resolve_replay_on_change(replay_on_change=test_case.local_policy), - own_cursor_type=test_case.own_cursor_type, - upstream_keys=upstream_keys, - effective_cascades=effective_cascades, - model_cursor_types=model_cursor_types, - ) - - assert result is None - assert not test_case.expected_cascade - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveCascadeTestCase( - description="build_self_cascade preserves own backfill in accumulator", - own_action=BackfillAction.BOUNDED, - own_duration="30d", - own_cursor_type="timestamp", - upstream_entries=(), - expected_cascade=False, - expected_action=BackfillAction.BOUNDED, - expected_duration="30d", - ), - ], - ids=lambda case: case.description, -) -def test_given_own_backfill_when_building_self_cascade_then_preserves_values( - test_case: ResolveCascadeTestCase, -) -> None: - result: CascadeResult = build_self_cascade( - backfill=BackfillResult(action=test_case.own_action, duration=test_case.own_duration) - ) - - assert result.effective_action == test_case.expected_action - assert result.effective_duration == test_case.expected_duration - assert result.root_cause is None - assert result.causes == () - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveCascadeRootCauseTestCase( - description="multihop cascade preserves function root cause", - expected_action=BackfillAction.FULL, - expected_root_cause="is_completed_order", - expected_root_reason=PlanReason.QUERY_CHANGED, - expected_immediate_cause="hourly_order_activity", - ) - ], - ids=lambda case: case.description, -) -def test_given_multihop_cascade_when_resolving_then_preserves_root_cause( - test_case: ResolveCascadeRootCauseTestCase, -) -> None: - hourly_key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.MODEL, name="hourly_order_activity" - ) - upstream_keys: tuple[CompiledObjectKey, ...] = (hourly_key,) - effective_cascades: dict[str, CascadeResult] = { - "hourly_order_activity": build_self_cascade( - backfill=BackfillResult(action=BackfillAction.FULL), - root_cause="is_completed_order", - root_reason=PlanReason.QUERY_CHANGED, - ) - } - model_cursor_types: dict[str, str | None] = {"hourly_order_activity": "timestamp"} - - result: CascadeResult | None = resolve_cascade( - model_name="daily_activity_rollup", - own_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), - local_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), - own_cursor_type="timestamp", - upstream_keys=upstream_keys, - effective_cascades=effective_cascades, - model_cursor_types=model_cursor_types, - ) - - assert result is not None - assert result.effective_action == test_case.expected_action - assert result.root_cause == test_case.expected_root_cause - assert result.root_reason == test_case.expected_root_reason - assert result.causes[0].model_name == test_case.expected_immediate_cause diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py index e9b5406bc4..ab5d2d167b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py @@ -9,12 +9,7 @@ build_compiled_function_fingerprint_sql, detect_function_change, ) -from sqlbuild.compiler.planner.models import ( - BackfillResult, - WarehouseFingerprints, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction, PlanReason +from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( DetectFunctionChangeTestCase, ) @@ -28,46 +23,20 @@ "test_case", [ DetectFunctionChangeTestCase( - description="first run function without policy returns first run reason and warn only", + description="first run function returns first run reason", body_sql="order_status = 'completed'", existing_function_fingerprints={}, - replay_on_change=None, expected_reason=PlanReason.FIRST_RUN, - expected_action=BackfillAction.FORWARD_ONLY, - ), - DetectFunctionChangeTestCase( - description="first run function with policy returns first run reason and bounded backfill", - body_sql="order_status = 'completed'", - existing_function_fingerprints={}, - replay_on_change="bounded-30d", - expected_reason=PlanReason.FIRST_RUN, - expected_action=BackfillAction.BOUNDED, - expected_duration="30d", - ), - DetectFunctionChangeTestCase( - description="changed function with policy returns query reason and bounded backfill", - body_sql="order_status = 'completed'", - existing_function_fingerprints={ - "is_completed_order": build_fingerprint( - query_sql="name=is_completed_order\nbody=\norder_status = 'complete'" - ) - }, - replay_on_change="bounded-30d", - expected_reason=PlanReason.QUERY_CHANGED, - expected_action=BackfillAction.BOUNDED, - expected_duration="30d", ), DetectFunctionChangeTestCase( - description="changed function without policy returns query reason and warn only", + description="changed function returns query reason", body_sql="order_status = 'completed'", existing_function_fingerprints={ "is_completed_order": build_fingerprint( query_sql="name=is_completed_order\nbody=\norder_status = 'complete'" ) }, - replay_on_change=None, expected_reason=PlanReason.QUERY_CHANGED, - expected_action=BackfillAction.FORWARD_ONLY, ), DetectFunctionChangeTestCase( description="target schema case change does not cause function query change", @@ -87,38 +56,24 @@ ) ) }, - replay_on_change=None, expected_reason=PlanReason.NO_CHANGE, - expected_action=BackfillAction.FORWARD_ONLY, target_schema="DEV", ), ], ids=lambda case: case.description, ) -def test_given_function_fingerprint_when_detecting_change_then_returns_reason_and_backfill( +def test_given_function_fingerprint_when_detecting_change_then_returns_reason( test_case: DetectFunctionChangeTestCase, ) -> None: function: CompiledFunction = build_compiled_function( body_sql=test_case.body_sql, - replay_on_change=test_case.replay_on_change, target_schema=test_case.target_schema, ) - snapshot: WarehouseSnapshot = WarehouseSnapshot( - fingerprints=WarehouseFingerprints(functions=test_case.existing_function_fingerprints) - ) - - reason: PlanReason - backfill: BackfillResult - reason, backfill = detect_function_change( - function=function, + reason: PlanReason = detect_function_change( fingerprint_sql=build_compiled_function_fingerprint_sql(function), - snapshot=snapshot, + fingerprint=test_case.existing_function_fingerprints.get(function.name), query_change_tracking=True, full_refresh=False, ) assert reason == test_case.expected_reason - assert backfill == BackfillResult( - action=test_case.expected_action, - duration=test_case.expected_duration, - ) From 70015e7d92da1053d503d136558ecd4bed6d7211 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:59:05 +0100 Subject: [PATCH 02/12] fix(migrations): plan renames with changed SQL as renamed query changes A recognised migrate_from rename of a table or view whose handed-over definition differs from its new query now keeps the handed-over identity and plans as renamed with a query change, instead of a first run. Its backfill comes from the model's own replay_on_change (forward-only by default). Once the renamed model has built after the recorded rename, its own fingerprint is used, so later plans no longer report the query change. --- src/sqlbuild/cli/output/_helpers/plan_json.py | 4 +- .../planner/_helpers/changes/detect.py | 20 +++++- .../planner/_helpers/migrations/planning.py | 44 +++++------- .../output/main/plan_json/test_plan_json.py | 15 ++++ .../planner/_helpers/changes/_test_types.py | 8 +++ .../planner/_helpers/changes/test_detect.py | 68 +++++++++++++++++++ 6 files changed, 129 insertions(+), 30 deletions(-) diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 68b99a06d4..05f6b8fc97 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -25,7 +25,7 @@ SeedPlanEntry, SourceLoadPlanEntry, ) -from sqlbuild.compiler.planner.types import IncrementalMode, MaterializationType +from sqlbuild.compiler.planner.types import IncrementalMode, MaterializationType, PlanReason from sqlbuild.compiler.python_nodes.types import PythonIdentityStatus from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token @@ -187,6 +187,8 @@ def _serialize_model_entry(entry: ModelPlanEntry) -> dict[str, object]: "duration": entry.backfill.duration, } + if entry.reason == PlanReason.RENAMED and entry.query_changed: + model["query_changed"] = True if entry.changed_functions: model["changed_functions"] = list(entry.changed_functions) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 550a8dd3c2..43de45020e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -174,12 +174,30 @@ def detect_model_changes( fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model_name) if not relation_exists and model_name in snapshot.renamed_models: + renamed_query_changed: bool = ( + query_change_tracking + and fingerprint is not None + and detect_query_change( + compiled_query_hash=compute_query_hash(model.query_sql), fingerprint=fingerprint + ) + ) return ChangeDetectionResult( model_name=model_name, change_kind=ChangeKind.RENAMED, - backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + query_changed=renamed_query_changed, + backfill=( + resolve_replay_on_change( + replay_on_change=get_config_str( + values=model.config.values, key="replay_on_change" + ) + ) + if renamed_query_changed + else BackfillResult(action=BackfillAction.FORWARD_ONLY) + ), fingerprint_metadata_json=metadata_json, + previous_metadata_json=fingerprint.metadata_json if fingerprint is not None else None, fingerprint_version_hash=expected_version_hash, + previous_version_hash=fingerprint.version_hash if fingerprint is not None else None, ) if not relation_exists: return ChangeDetectionResult( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index b6af40b215..8ac44b3272 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -208,9 +208,7 @@ def plan_model_migrations( request.model.name, () ), ) - renamed: frozenset[str] = _renamed_models( - runtime=runtime, entries=tuple(entries), handovers=handovers - ) + renamed: frozenset[str] = _renamed_models(entries=tuple(entries), handovers=handovers) return ModelMigrationPlanning( snapshot=replace( _overlay_snapshot( @@ -218,9 +216,7 @@ def plan_model_migrations( scope=scope, snapshot=snapshot, entries=tuple(entries), - handovers=_effective_handovers( - entries=tuple(entries), handovers=handovers, renamed=renamed - ), + handovers=handovers, state=state, overrides=overrides, deferral=deferral, @@ -237,38 +233,19 @@ def plan_model_migrations( def _renamed_models( *, - runtime: PlannerRuntime, entries: tuple[ModelMigrationPlanEntry, ...], handovers: dict[str, Fingerprint | None], ) -> frozenset[str]: - """Return renamed tables and views whose handed-over definition matches their own.""" + """Return renamed tables and views that inherit their predecessor's identity.""" - query_sql: dict[str, str] = {model.name: model.query_sql for model in runtime.project.models} return frozenset( entry.model_name for entry in entries if entry.decision == MigrationDecision.RENAMED - and (handover := handovers.get(entry.model_name)) is not None - and handover.definition == query_sql.get(entry.model_name) + and handovers.get(entry.model_name) is not None ) -def _effective_handovers( - *, - entries: tuple[ModelMigrationPlanEntry, ...], - handovers: dict[str, Fingerprint | None], - renamed: frozenset[str], -) -> dict[str, Fingerprint | None]: - """Drop the handover of any rename whose definition no longer matches.""" - - unmatched: frozenset[str] = frozenset( - entry.model_name - for entry in entries - if entry.decision == MigrationDecision.RENAMED and entry.model_name not in renamed - ) - return {name: None if name in unmatched else handover for name, handover in handovers.items()} - - def _equivalent_definition( *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprints: tuple[str, ...] ) -> Fingerprint | None: @@ -448,6 +425,7 @@ def _decide( origin_fingerprint=origin_fingerprint, origin_exists=origin_relation is not None, manual=request.discovery == MigrationDiscovery.MANUAL, + snapshot=snapshot, ) if ( newest is not None @@ -519,6 +497,7 @@ def _decide_rename( origin_fingerprint: Fingerprint | None, origin_exists: bool, manual: bool, + snapshot: WarehouseSnapshot, ) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: """Hand a renamed table or view's identity to its successor; no data moves.""" @@ -530,13 +509,22 @@ def _decide_rename( ) if manual and not recorded and not origin_exists: return replace(base, decision=MigrationDecision.ORIGIN_MISSING), None, False + built_since_rename: bool = ( + recorded + and newest is not None + and not _done_handover_applies(model=model, event=newest, snapshot=snapshot) + ) return ( replace( base, decision=MigrationDecision.RENAMED, completed_at=newest.created_at if recorded and newest is not None else None, ), - _handover_fingerprint(model=model, fingerprint=origin_fingerprint, version_hash=""), + ( + snapshot.fingerprints.models.get(model.name) + if built_since_rename + else _handover_fingerprint(model=model, fingerprint=origin_fingerprint, version_hash="") + ), True, ) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index 3cc8cf25ca..2e3644d217 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -450,6 +450,21 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence '"changed_functions": [\n "is_completed_order"\n ]', ), ), + JsonOutputTestCase( + description="plan json marks a renamed model whose query changed", + plan_output=build_plan_output( + model_entries=( + build_model_entry( + name="order_lines", + action=PlanAction.CREATE_TABLE, + reason=PlanReason.RENAMED, + query_changed=True, + ), + ), + ), + expected_keys=("models",), + expected_fragments=('"reason": "renamed"', '"query_changed": true'), + ), JsonOutputTestCase( description="plan json includes seeds and warnings", plan_output=build_plan_output( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 9be6626992..e4e002a9ee 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -86,3 +86,11 @@ class ResolveBackfillTestCase: raw_value: str | None expected_result: BackfillResult + +@dataclass(frozen=True) +class DetectRenamedModelTestCase: + description: str + previous_definition: str + config_values: dict[str, object] + expected_query_changed: bool + expected_backfill: BackfillResult diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 0c3031764b..929dfea053 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -1,5 +1,6 @@ from __future__ import annotations +from dataclasses import replace from datetime import datetime import pytest @@ -48,6 +49,7 @@ DetectFunctionCallerChangeTestCase, DetectModelChangesTestCase, DetectModelMetadataTestCase, + DetectRenamedModelTestCase, DroppedRelationPlanActionTestCase, ) @@ -628,3 +630,69 @@ def test_given_direct_project_function_hash_change_when_detecting_changes_then_m ) assert result.models["orders"].change_kind == test_case.expected_change_kind + + +@pytest.mark.parametrize( + "test_case", + [ + DetectRenamedModelTestCase( + description="rename with the handed-over definition continues forward", + previous_definition="SELECT 1 AS order_id", + config_values={"replay_on_change": "full"}, + expected_query_changed=False, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), + DetectRenamedModelTestCase( + description="rename with changed SQL follows the model's own replay policy", + previous_definition="SELECT 2 AS order_id", + config_values={"replay_on_change": "bounded-3d"}, + expected_query_changed=True, + expected_backfill=BackfillResult(action=BackfillAction.BOUNDED, duration="3d"), + ), + DetectRenamedModelTestCase( + description="rename with changed SQL defaults to forward only", + previous_definition="SELECT 2 AS order_id", + config_values={}, + expected_query_changed=True, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_model_when_detecting_changes_then_reports_rename_and_own_query_change( + test_case: DetectRenamedModelTestCase, +) -> None: + metadata_case: DetectModelMetadataTestCase = DetectModelMetadataTestCase( + description="renamed model", + config_values=test_case.config_values, + schema_columns=(), + deps=(), + function_local_hashes={}, + previous_metadata_json="{}", + expected_change_kind=ChangeKind.RENAMED, + ) + snapshot: WarehouseSnapshot = build_snapshot_for_metadata_test_case(metadata_case) + handover: Fingerprint = replace( + snapshot.fingerprints.models["orders"], + definition=test_case.previous_definition, + definition_hash=compute_query_hash(test_case.previous_definition), + ) + + result: ChangeDetectionResult = detect_model_changes( + model=build_model_from_metadata_test_case(metadata_case), + snapshot=replace( + snapshot, + existing_relations={}, + fingerprints=WarehouseFingerprints(models={"orders": handover}), + renamed_models=frozenset({"orders"}), + ), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + ) + + assert (result.change_kind, result.query_changed, result.backfill) == ( + ChangeKind.RENAMED, + test_case.expected_query_changed, + test_case.expected_backfill, + ) From b5e09b5fb03d86577d4bc0df71266afee0cb164f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:03:04 +0100 Subject: [PATCH 03/12] test(e2e): cover own-change replay for new, renamed and function-changed upstreams --- Makefile | 1 + .../main/build/own_change_replay/__init__.py | 0 .../build/own_change_replay/_test_types.py | 41 ++ .../main/build/own_change_replay/helpers.py | 309 +++++++++++++++ .../test_own_change_replay.py | 361 ++++++++++++++++++ 5 files changed, 712 insertions(+) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/__init__.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py diff --git a/Makefile b/Makefile index 3f16e3e5ad..1a4f206025 100644 --- a/Makefile +++ b/Makefile @@ -94,6 +94,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ + tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/_test_types.py new file mode 100644 index 0000000000..9d3209de27 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/_test_types.py @@ -0,0 +1,41 @@ +"""Test case types for own-change replay e2e tests.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class OwnChangePlanBuildTestCase: + description: str + expected_plan_fragments: tuple[str, ...] + unexpected_plan_fragments: tuple[str, ...] + expected_reasons: dict[str, str] + expected_rows: dict[str, list[tuple[Any, ...]]] + + +@dataclass(frozen=True) +class RenamePlanBuildTestCase: + description: str + extra_sql: str + expected_plan_fragments: tuple[str, ...] + unexpected_plan_fragments: tuple[str, ...] + expected_query_changed: bool + expected_rows: dict[str, list[tuple[Any, ...]]] + expected_relation_types: dict[str, list[str]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class FunctionCallerReplayTestCase: + description: str + caller_config: str + expected_plan_fragments: tuple[str, ...] + expected_caller_action: str + expected_rows: dict[str, list[tuple[Any, ...]]] + + +@dataclass(frozen=True) +class RefusedPlanTestCase: + description: str + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py new file mode 100644 index 0000000000..877a1dc845 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py @@ -0,0 +1,309 @@ +"""Project builders for own-change replay e2e tests.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +DATABASE_FILE: str = "warehouse.duckdb" +FIRST_DAY: str = "2026-09-01" +_PROJECT_TOML: str = ( + 'name = "own_change_replay"\nadapter = "duckdb"\n\n' + f'[connection]\ndatabase = "{DATABASE_FILE}"\n' +) +_EVENTS_SEED_YML: str = ( + "seeds:\n" + "- name: events\n" + " columns:\n" + " - name: id\n" + " type: INTEGER\n" + " - name: event_date\n" + " type: DATE\n" +) +_EVENTS_CSV: str = "id,event_date\n1,2026-09-01\n2,2026-09-02\n3,2026-09-03\n" +_UPSTREAM_SQL: str = ( + "MODEL (\n materialized table,\n);\n\n" + "SELECT\n id,\n CAST(event_date AS DATE) AS event_date\n" + 'FROM __seed("events")\n' +) +_DOWNSTREAM_MB_SQL: str = ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor event_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (\n" + " upstream_v (\n" + " column event_date,\n" + " roles [filter, watermark],\n" + " ),\n" + " ),\n" + " incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n" + " batch_size 1d,\n" + " cursor_watermark_mode all,\n" + " lookback 1d,\n" + " full_refresh false,\n" + ' cursor_start "2026-09-01",\n' + ");\n\n" + 'SELECT\n id,\n event_date\nFROM __ref("upstream_v")\n' +) +_DOWNSTREAM_MERGE_SQL: str = ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [id],\n" + " cursor event_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " lookback 1d,\n" + " cursor_inputs (\n" + " upstream_v event_date,\n" + " ),\n" + "{extra_config}" + ");\n\n" + 'SELECT\n id,\n event_date\nFROM __ref("upstream_v")\n' + "WHERE\n event_date >= __cursor_start()::DATE\n" +) +_ADJUST_FUNCTION_PATH: str = "functions/sql/udf__adjust_amount.sql" + + +def _upstream_view_sql(*, upstream: str) -> str: + return ( + "MODEL (\n materialized view,\n);\n\n" + f'SELECT\n id,\n event_date\nFROM __ref("{upstream}")\n' + ) + + +def build_event_project(*, tmp_path: Path, extra_files: dict[str, str]) -> Path: + """Write and build the event chain of table, view and microbatch plus any extra files.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_TOML, + "seeds/seeds.yml": _EVENTS_SEED_YML, + "seeds/events.csv": _EVENTS_CSV, + "models/upstream.sql": _UPSTREAM_SQL, + "models/upstream_v.sql": _upstream_view_sql(upstream="upstream"), + "models/downstream_mb.sql": _DOWNSTREAM_MB_SQL, + **extra_files, + } + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="own_change_replay", repo_files=files + ) + _build_ok(project_dir=project_dir) + return project_dir + + +def merge_downstream_files(*, extra_config: str) -> dict[str, str]: + """Return a merge model whose SQL reads its interval through cursor intrinsics.""" + + return {"models/downstream.sql": _DOWNSTREAM_MERGE_SQL.format(extra_config=extra_config)} + + +def mark_first_day(*, project_dir: Path, relations: tuple[str, ...]) -> None: + """Offset ids of the first, out-of-window day so a replay of that day is detectable.""" + + relation: str + for relation in relations: + execute_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=f"UPDATE main.{relation} SET id = id + 100 WHERE event_date = DATE '{FIRST_DAY}'", + ) + + +def insert_table_between_upstream_and_view(*, project_dir: Path) -> None: + """Add a new table between the upstream table and its view.""" + + (project_dir / "models/upstream_new.sql").write_text( + "MODEL (\n materialized table,\n);\n\n" + 'SELECT\n id,\n event_date\nFROM __ref("upstream")\n', + encoding="utf-8", + ) + _write_upstream_view(project_dir=project_dir, upstream="upstream_new") + + +def rename_upstream_with_migrate_from(*, project_dir: Path, extra_sql: str) -> None: + """Rename the upstream table by hand, declaring migrate_from, optionally changing its SQL.""" + + (project_dir / "models/upstream.sql").unlink() + (project_dir / "models/upstream_renamed.sql").write_text( + _UPSTREAM_SQL.replace( + " materialized table,\n", " materialized table,\n migrate_from upstream,\n" + ) + + extra_sql, + encoding="utf-8", + ) + _write_upstream_view(project_dir=project_dir, upstream="upstream_renamed") + + +def rename_upstream_with_cli(*, project_dir: Path, extra_sql: str) -> None: + """Rename the upstream table with sqb rename, then append SQL to the renamed model.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "rename", "model:upstream", "upstream_renamed"), + project_dir=project_dir, + ) + assert result.returncode == 0, result.stdout + result.stderr + renamed_path: Path = project_dir / "models/upstream_renamed.sql" + renamed_path.write_text(renamed_path.read_text(encoding="utf-8") + extra_sql, encoding="utf-8") + + +def _write_upstream_view(*, project_dir: Path, upstream: str) -> None: + (project_dir / "models/upstream_v.sql").write_text( + _upstream_view_sql(upstream=upstream), encoding="utf-8" + ) + + +def build_function_project(*, tmp_path: Path, caller_config: str) -> Path: + """Write and build a UDF called by an incremental model that feeds another incremental.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_TOML, + "seeds/seeds.yml": ( + "seeds:\n" + "- name: orders\n" + " columns:\n" + " - name: id\n" + " type: INTEGER\n" + " - name: ordered_at\n" + " type: DATE\n" + " - name: amount\n" + " type: INTEGER\n" + ), + "seeds/orders.csv": ( + "id,ordered_at,amount\n1,2026-09-01,10\n2,2026-09-02,20\n3,2026-09-03,30\n" + ), + _ADJUST_FUNCTION_PATH: ( + "FUNCTION (\n arguments (amount INTEGER),\n returns INTEGER,\n);\n\namount + 1\n" + ), + "models/order_amounts.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [id],\n" + " cursor ordered_at,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (\n orders ordered_at,\n ),\n" + f"{caller_config}" + ");\n\n" + "SELECT\n id,\n CAST(ordered_at AS DATE) AS ordered_at,\n" + ' __udf("udf__adjust_amount")(amount) AS adjusted_amount\n' + 'FROM __seed("orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [id],\n" + " cursor ordered_at,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (\n order_amounts ordered_at,\n ),\n" + ");\n\n" + "SELECT\n id,\n ordered_at,\n adjusted_amount * 2 AS doubled_amount\n" + 'FROM __ref("order_amounts")\n' + ), + } + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="function_caller_replay", repo_files=files + ) + _build_ok(project_dir=project_dir) + (project_dir / _ADJUST_FUNCTION_PATH).write_text( + (project_dir / _ADJUST_FUNCTION_PATH) + .read_text(encoding="utf-8") + .replace("amount + 1", "amount + 100"), + encoding="utf-8", + ) + return project_dir + + +def plan(*, project_dir: Path, args: tuple[str, ...] = ()) -> subprocess.CompletedProcess[str]: + """Run sqb plan without colour.""" + + return run_sqb(command=("--no-color", "plan", *args), project_dir=project_dir) + + +def plan_models(*, project_dir: Path) -> dict[str, dict[str, Any]]: + """Return plan JSON model entries keyed by model name.""" + + result: subprocess.CompletedProcess[str] = plan(project_dir=project_dir, args=("--json",)) + assert result.returncode == 0, result.stdout + result.stderr + return {str(model["name"]): model for model in json.loads(result.stdout)["models"]} + + +def build(*, project_dir: Path) -> subprocess.CompletedProcess[str]: + """Run sqb build without colour.""" + + return run_sqb(command=("--no-color", "build"), project_dir=project_dir) + + +def _build_ok(*, project_dir: Path) -> None: + result: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + assert result.returncode == 0, result.stdout + result.stderr + + +def relation_rows(*, project_dir: Path, relation: str) -> list[tuple[Any, ...]]: + """Return every row of a relation ordered by its first column.""" + + return query_duckdb( + db_path=project_dir / DATABASE_FILE, sql=f"SELECT * FROM main.{relation} ORDER BY 1" + ) + + +def relation_types(*, project_dir: Path, name: str) -> list[str]: + """Return the information-schema table types of a main-schema relation name.""" + + rows: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / DATABASE_FILE, + sql=( + "SELECT table_type FROM information_schema.tables " + f"WHERE table_schema = 'main' AND table_name = '{name}'" + ), + ) + return [str(row[0]) for row in rows] + + +def build_and_assert_rows( + *, project_dir: Path, expected_rows: dict[str, list[tuple[Any, ...]]] +) -> None: + """Build the project and compare every listed relation's rows.""" + + result: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + assert result.returncode == 0, result.stdout + result.stderr + assert { + relation: relation_rows(project_dir=project_dir, relation=relation) + for relation in expected_rows + } == expected_rows + + +def assert_plan_text( + *, project_dir: Path, expected: tuple[str, ...], unexpected: tuple[str, ...] +) -> None: + """Plan the project and check the text output for expected and unexpected fragments.""" + + result: subprocess.CompletedProcess[str] = plan(project_dir=project_dir) + assert result.returncode == 0, result.stdout + result.stderr + assert all(fragment in result.stdout for fragment in expected), result.stdout + assert not any(fragment in result.stdout for fragment in unexpected), result.stdout + + +def assert_plan_fails( + *, project_dir: Path, args: tuple[str, ...], expected: tuple[str, ...] +) -> None: + """Plan the project, expect failure, and check the output for expected fragments.""" + + result: subprocess.CompletedProcess[str] = plan(project_dir=project_dir, args=args) + output: str = result.stdout + result.stderr + assert result.returncode != 0, output + assert all(fragment in output for fragment in expected), output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py new file mode 100644 index 0000000000..5158328aa6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py @@ -0,0 +1,361 @@ +"""E2E coverage: every model's replay comes only from its own change and replay_on_change.""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build.own_change_replay._test_types import ( + FunctionCallerReplayTestCase, + OwnChangePlanBuildTestCase, + RefusedPlanTestCase, + RenamePlanBuildTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.own_change_replay.helpers import ( + assert_plan_fails, + assert_plan_text, + build_and_assert_rows, + build_event_project, + build_function_project, + insert_table_between_upstream_and_view, + mark_first_day, + merge_downstream_files, + plan_models, + relation_types, + rename_upstream_with_cli, + rename_upstream_with_migrate_from, +) + +_REPLAYED_TAIL: list[tuple[Any, ...]] = [ + (2, date(2026, 9, 2)), + (3, date(2026, 9, 3)), + (101, date(2026, 9, 1)), +] +_CHANGED_UPSTREAM_SQL: str = "WHERE\n NOT id IS NULL\n" +_RENAMED_ROWS: dict[str, list[tuple[Any, ...]]] = { + "upstream_renamed": [ + (1, date(2026, 9, 1)), + (2, date(2026, 9, 2)), + (3, date(2026, 9, 3)), + ], + "downstream_mb": _REPLAYED_TAIL, +} + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="new upstream table is a first run and the microbatch continues forward", + expected_plan_fragments=( + "First run (1)", + "upstream_new", + "Query changed (1)", + "upstream_v recreate view", + "downstream_mb delete_insert (timestamp, microbatch)", + ), + unexpected_plan_fragments=("Upstream changed", "full rebuild", "cause"), + expected_reasons={ + "upstream_new": "first_run", + "upstream_v": "query_changed", + "downstream_mb": "normal_incremental", + }, + expected_rows={"downstream_mb": _REPLAYED_TAIL}, + ) + ], + ids=lambda case: case.description, +) +def test_given_new_upstream_table_when_planning_and_building_then_downstream_continues_forward( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project(tmp_path=tmp_path, extra_files={}) + mark_first_day(project_dir=project_dir, relations=("downstream_mb",)) + insert_table_between_upstream_and_view(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + assert all("cascade" not in model for model in models.values()) + assert models["downstream_mb"]["backfill"] == {"action": "forward", "duration": None} + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="merge model with cursor intrinsics plans and continues forward", + expected_plan_fragments=( + "First run (1)", + "downstream merge (timestamp)", + "downstream_mb delete_insert (timestamp, microbatch)", + ), + unexpected_plan_fragments=("Upstream changed", "full rebuild", "error"), + expected_reasons={ + "downstream": "normal_incremental", + "downstream_mb": "normal_incremental", + }, + expected_rows={"downstream": _REPLAYED_TAIL, "downstream_mb": _REPLAYED_TAIL}, + ) + ], + ids=lambda case: case.description, +) +def test_given_cursor_intrinsic_merge_downstream_when_upstream_is_new_then_plan_succeeds( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project( + tmp_path=tmp_path, + extra_files=merge_downstream_files(extra_config=" full_refresh false,\n"), + ) + mark_first_day(project_dir=project_dir, relations=("downstream", "downstream_mb")) + insert_table_between_upstream_and_view(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + + +@pytest.mark.parametrize( + "test_case", + [ + RenamePlanBuildTestCase( + description="rename with identical SQL continues forward", + extra_sql="", + expected_plan_fragments=( + "upstream_renamed migrate main.upstream -> main.upstream_renamed", + "Renamed (1)", + "upstream_renamed continue forward", + ), + unexpected_plan_fragments=("First run", "Upstream changed", "full rebuild"), + expected_query_changed=False, + expected_rows=_RENAMED_ROWS, + expected_relation_types={"upstream": ["VIEW"], "upstream_renamed": ["BASE TABLE"]}, + ), + RenamePlanBuildTestCase( + description="rename with changed SQL is a renamed query change", + extra_sql=_CHANGED_UPSTREAM_SQL, + expected_plan_fragments=( + "upstream_renamed migrate main.upstream -> main.upstream_renamed", + "Renamed (1)", + "upstream_renamed continue forward", + "+ NOT id IS NULL", + ), + unexpected_plan_fragments=("First run", "Upstream changed", "full rebuild"), + expected_query_changed=True, + expected_rows=_RENAMED_ROWS, + expected_relation_types={"upstream": ["VIEW"], "upstream_renamed": ["BASE TABLE"]}, + ), + ], + ids=lambda case: case.description, +) +def test_given_migrate_from_rename_when_planning_and_building_then_plans_rename_not_first_run( + test_case: RenamePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project(tmp_path=tmp_path, extra_files={}) + mark_first_day(project_dir=project_dir, relations=("downstream_mb",)) + rename_upstream_with_migrate_from(project_dir=project_dir, extra_sql=test_case.extra_sql) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert models["upstream_renamed"]["reason"] == "renamed" + assert models["upstream_renamed"].get("query_changed", False) is ( + test_case.expected_query_changed + ) + assert models["downstream_mb"]["reason"] == "normal_incremental" + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + assert { + name: relation_types(project_dir=project_dir, name=name) + for name in test_case.expected_relation_types + } == test_case.expected_relation_types + assert plan_models(project_dir=project_dir)["upstream_renamed"]["reason"] == "no_change" + + +@pytest.mark.parametrize( + "test_case", + [ + RenamePlanBuildTestCase( + description="sqb rename followed by a SQL edit plans a renamed query change", + extra_sql=_CHANGED_UPSTREAM_SQL, + expected_plan_fragments=("Renamed (1)", "upstream_renamed", "+ NOT id IS NULL"), + unexpected_plan_fragments=("First run", "Upstream changed", "full rebuild"), + expected_query_changed=True, + expected_rows=_RENAMED_ROWS, + ) + ], + ids=lambda case: case.description, +) +def test_given_sqb_rename_when_planning_then_model_is_renamed_not_first_run( + test_case: RenamePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project(tmp_path=tmp_path, extra_files={}) + mark_first_day(project_dir=project_dir, relations=("downstream_mb",)) + rename_upstream_with_cli(project_dir=project_dir, extra_sql=test_case.extra_sql) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert (models["upstream_renamed"]["reason"], models["downstream_mb"]["reason"]) == ( + "renamed", + "normal_incremental", + ) + assert models["upstream_renamed"].get("query_changed", False) is ( + test_case.expected_query_changed + ) + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + + +@pytest.mark.parametrize( + "test_case", + [ + FunctionCallerReplayTestCase( + description="forward-only caller and its downstream both continue forward", + caller_config="", + expected_plan_fragments=( + "Function changed (1)", + "order_amounts continue forward", + "cause function udf__adjust_amount changed", + "order_totals merge (timestamp)", + ), + expected_caller_action="incremental_merge", + expected_rows={ + "order_amounts": [ + (1, date(2026, 9, 1), 11), + (2, date(2026, 9, 2), 21), + (3, date(2026, 9, 3), 130), + ], + "order_totals": [ + (1, date(2026, 9, 1), 22), + (2, date(2026, 9, 2), 42), + (3, date(2026, 9, 3), 260), + ], + }, + ), + FunctionCallerReplayTestCase( + description="full replay caller rebuilds while its downstream continues forward", + caller_config=" replay_on_change full,\n", + expected_plan_fragments=( + "Function changed (1)", + "order_amounts full rebuild", + "cause function udf__adjust_amount changed", + "order_totals merge (timestamp)", + ), + expected_caller_action="create_table", + expected_rows={ + "order_amounts": [ + (1, date(2026, 9, 1), 110), + (2, date(2026, 9, 2), 120), + (3, date(2026, 9, 3), 130), + ], + "order_totals": [ + (1, date(2026, 9, 1), 22), + (2, date(2026, 9, 2), 42), + (3, date(2026, 9, 3), 260), + ], + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_changed_function_when_planning_and_building_then_only_direct_caller_replays( + test_case: FunctionCallerReplayTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_function_project( + tmp_path=tmp_path, caller_config=test_case.caller_config + ) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=("Upstream changed",), + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert models["order_amounts"]["reason"] == "function_changed" + assert models["order_amounts"]["changed_functions"] == ["udf__adjust_amount"] + assert models["order_amounts"]["action"] == test_case.expected_caller_action + assert models["order_totals"]["reason"] == "normal_incremental" + assert "changed_functions" not in models["order_totals"] + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + + +@pytest.mark.parametrize( + "test_case", + [ + RefusedPlanTestCase( + description="full_refresh false caller refuses a function-driven full rebuild", + expected_fragments=( + "error[S203]", + "model 'order_amounts' sets full_refresh false", + "function udf__adjust_amount changed with replay_on_change full", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_full_refresh_false_caller_when_function_forces_full_rebuild_then_plan_fails( + test_case: RefusedPlanTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_function_project( + tmp_path=tmp_path, caller_config=" replay_on_change full,\n full_refresh false,\n" + ) + + assert_plan_fails(project_dir=project_dir, args=(), expected=test_case.expected_fragments) + + +@pytest.mark.parametrize( + "test_case", + [ + RefusedPlanTestCase( + description="explicit full refresh of a cursor-intrinsic merge names the FULL cause", + expected_fragments=( + "Model 'downstream' was given a FULL backfill (--full-refresh)", + "non-microbatch full rebuild has no cursor interval", + "--start-cursor-ts", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_full_refresh_flag_on_cursor_intrinsic_merge_when_planning_then_names_full_cause( + test_case: RefusedPlanTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project( + tmp_path=tmp_path, extra_files=merge_downstream_files(extra_config="") + ) + + assert_plan_fails( + project_dir=project_dir, + args=("--full-refresh", "--select", "downstream"), + expected=test_case.expected_fragments, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 24134cc9e8abe073c49f505c97c1210396d04a7e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:05:19 +0100 Subject: [PATCH 04/12] docs: describe per-model replay decisions, function callers and full_refresh protection --- .../sqlbuild/references/docs/CONTENTS.md | 2 +- .../sqlbuild/references/docs/cli/plan.md | 4 +- .../references/docs/concepts/functions.md | 6 ++- .../references/docs/concepts/incremental.md | 18 +++++-- .../docs/concepts/models/migrations.md | 4 +- .../references/docs/concepts/planning.md | 25 +++++----- .../concepts/planning/cascade-propagation.md | 41 ---------------- .../concepts/planning/replay-decisions.md | 49 +++++++++++++++++++ .../references/docs/feature-comparison.md | 2 +- .../sqlbuild/references/incremental-models.md | 12 +++-- website/src/content/docs/docs/cli/plan.mdx | 4 +- .../content/docs/docs/concepts/functions.mdx | 6 ++- .../docs/docs/concepts/incremental.mdx | 18 +++++-- .../docs/docs/concepts/models/migrations.mdx | 4 +- .../content/docs/docs/concepts/planning.mdx | 23 +++++---- .../concepts/planning/cascade-propagation.mdx | 38 -------------- .../concepts/planning/replay-decisions.mdx | 46 +++++++++++++++++ .../content/docs/docs/feature-comparison.mdx | 2 +- website/src/navigation.mjs | 3 +- website/src/sidebar.json | 2 +- 20 files changed, 178 insertions(+), 131 deletions(-) delete mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/cascade-propagation.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md delete mode 100644 website/src/content/docs/docs/concepts/planning/cascade-propagation.mdx create mode 100644 website/src/content/docs/docs/concepts/planning/replay-decisions.mdx diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index db76c4d019..d34e58b97b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -55,7 +55,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Functions](concepts/functions.md) (`concepts/functions`) - User-defined functions and table functions managed as part of your project. - [Incremental Models](concepts/incremental.md) (`concepts/incremental`) - Cursor-based incremental strategies, microbatch execution, and backfill policies. - [Planning and change detection](concepts/planning.md) (`concepts/planning`) - How SQLBuild explains build work, detects changes, and chooses safe model actions. -- [Cascade propagation](concepts/planning/cascade-propagation.md) (`concepts/planning/cascade-propagation`) - How a change signal propagates downstream through the DAG, and how each materialization type responds. +- [Replay decisions](concepts/planning/replay-decisions.md) (`concepts/planning/replay-decisions`) - How much history each model reprocesses: decided by its own change and its own replay_on_change, never inherited from upstream. - [Source freshness](concepts/planning/source-freshness.md) (`concepts/planning/source-freshness`) - Observe external source changes and propagate them through planning. - [Selection and staleness](concepts/planning/selection-and-staleness.md) (`concepts/planning/selection-and-staleness`) - How selection interacts with change detection, and the stale warnings that stop silent partial rebuilds. - [Snapshots (SCD Type 2)](concepts/snapshots.md) (`concepts/snapshots`) - Preserve row history over time using SCD Type 2 semantics with timestamp or check-based change detection. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 06a440fc17..ba85022841 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -85,7 +85,9 @@ Seeds (1) ✓ Plan complete 13 selected ``` -When query or schema changes are detected, the plan shows the affected models with backfill actions and cascade information. +When query, function, or schema changes are detected, the plan lists each affected model under its own change with its backfill action. A model that calls a changed function directly appears under `Function changed` with `cause: function changed`; models further downstream continue forward. See [Replay decisions](../concepts/planning/replay-decisions.md). + +In `sqb plan --json`, each model's `reason` is its own change. A direct caller of a changed function has `"reason": "function_changed"` and a `changed_functions` list, and a renamed model whose query also changed has `"reason": "renamed"` and `"query_changed": true`. Warnings and maintenance notices, such as `migrate_from can be removed`, are listed in a `Warnings` section before the final `Plan complete` line. `sqb plan --json` output is unchanged. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md index 6421a8ad5f..fae251cf7d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md @@ -16,7 +16,7 @@ Online: https://sqlbuild.com/docs/concepts/functions/ - Python UDF options - Adapter support -Functions are SQL or Python definitions under `functions/` that SQLBuild compiles and deploys to the warehouse alongside your models. They participate in the DAG - if a function definition changes, every model that uses it is rebuilt. +Functions are SQL or Python definitions under `functions/` that SQLBuild compiles and deploys to the warehouse alongside your models. They participate in the DAG: if a function definition changes, SQLBuild redeploys it, and every model that calls it directly treats the change as a change to its own query. ## Scalar UDFs @@ -144,7 +144,9 @@ function folder or to that folder and its children. ## Change propagation -Functions participate in fingerprint-based change detection. Their identity includes dependencies and declared return contracts in addition to the function body and runtime metadata. If a function changes, SQLBuild redeploys it and marks dependent models as changed. +Functions participate in fingerprint-based change detection. Their identity includes dependencies and declared return contracts in addition to the function body and runtime metadata. If a function changes, SQLBuild redeploys it. + +A model that calls a changed function directly plans as `Function changed`, with `cause: function changed`. The function change counts as a change to that model's own query, so the model's own `replay_on_change` decides how much history it reprocesses (forward-only by default). Models further downstream are not affected by the function change and continue forward. Functions have no replay setting of their own; to reprocess history when a function changes, set `replay_on_change` on the models that call it. See [Replay decisions](planning/replay-decisions.md). ## Project layout diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index 04eebb38e0..e13338a8e9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -114,7 +114,15 @@ MODEL ( ); ``` -The override is evaluated per model, so one selection can contain incrementally executed opt-outs and full-refreshed models. It controls execution mode rather than acting as a safety rejection: an opted-out model does not abort the rest of the build. +The override is evaluated per model, so one selection can contain incrementally executed opt-outs and full-refreshed models. With `--full-refresh`, an opted-out model simply runs incrementally; it does not abort the rest of the build. + +`full_refresh false` also protects the model from every other full rebuild. Only the model's own explicit configuration can fully rebuild it: its own first run, or its own query change with `replay_on_change full`. If anything else would plan a full rebuild, for example a changed function that the model calls while it sets `replay_on_change full`, the plan fails with `S203`, naming the model and the cause, instead of silently replaying its history: + +``` +error[S203]: model 'order_history' sets full_refresh false, but the plan would fully rebuild it (function normalize_status changed with replay_on_change full) +``` + +Set `replay_on_change` to `forward` or `bounded-` to keep the existing history, or remove `full_refresh false` to allow the rebuild. Upstream changes never cause a full rebuild of a downstream model; see [Replay decisions](planning/replay-decisions.md). This does not skip initial loading. If the destination relation does not exist, an incremental or microbatch model still builds the history required by its cursor policy. Cloning an existing destination before the build can provide a current watermark and avoid a first-run historical replay. @@ -173,7 +181,7 @@ WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() ``` -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails with a message naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. @@ -324,7 +332,7 @@ The legacy scalar `max_microbatches` model field remains supported as a fail/war ### Mixed-grain chains -When a downstream microbatch model reads from an upstream model with a coarser time grain, SQLBuild aligns the replay to the coarsest participating grain (the model's own grain and its cursor-input grains). This happens on every run that resolves cursor bounds from upstream models, not only when something changes. It is independent of the `replay_on_change` cascade behavior described below. +When a downstream microbatch model reads from an upstream model with a coarser time grain, SQLBuild aligns the replay to the coarsest participating grain (the model's own grain and its cursor-input grains). This happens on every run that resolves cursor bounds from upstream models, not only when something changes. It is independent of the `replay_on_change` policy described below. Alignment does two things: it floors the replay window edges to the coarsest grain, and it coarsens the batch size to that grain. For example, an hourly model downstream of a daily model processes in day-sized batches, so each batch lines up with a unit of upstream data that actually advances instead of producing empty or boundary-straddling windows: @@ -360,7 +368,7 @@ MODEL ( ## Replay on change -When a model's version identity changes (query, config, upstream cascade, or any other change reason), `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large downstream tables. You choose the cost per model: +When a model's own query changes, a function it calls directly changes, or its schema changes, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: | Value | Effect | |-------|--------| @@ -381,7 +389,7 @@ MODEL ( ); ``` -See [Cascade propagation](planning/cascade-propagation.md) for how replay policies propagate through the DAG and how downstream models can override inherited replay behavior. +See [Replay decisions](planning/replay-decisions.md) for what counts as a model's own change and how materialization types respond to upstream changes. ### on_schema_change diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index 6bce224583..b65efc1bd3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -40,6 +40,8 @@ Use `schema.name` when the old relation is in a different schema of the same dat On the next build, SQLBuild copies the old relation's data to the new name through a staging table (see [How a move runs](#how-a-move-runs)), records the move, and builds the model incrementally from the migrated state. There is no first-run rebuild, and no `replay_on_change` replay caused by the rename. After the model builds, the old relation is archived and a compatibility view takes its name (see [Old names](#old-names)). +A rename is usually made together with other edits, for example a changed filter or a renamed upstream reference. If the renamed model's query also differs from the old model's, the plan still recognises the rename: the model is listed as `Renamed` with its query diff, never as a first run. The query change follows the model's own `replay_on_change`, exactly as it would without the rename (forward-only by default). Models downstream of the renamed model are not replayed because of the rename; see [Replay decisions](../planning/replay-decisions.md). + After the move is recorded, `migrate_from` has no further effect, and SQLBuild tells you it can be removed. Keep it until every target that needs the move has built. ## Decisions @@ -66,7 +68,7 @@ Table and view models only ever see `migrate`, `done`, and `origin missing`: not If a selected model has never been built, SQLBuild compares it with models that were removed from the project but whose relations still exist. When exactly one removed model has equivalent logic, it is migrated automatically. Equivalent means the same query and configuration, ignoring the model's own name, CTE and table alias names, comments, formatting, and storage-only settings. Renamed upstream models are matched first, so a renamed chain of models is migrated together. Removed models whose data was already moved on by a recorded migration are not candidates, so a model renamed several times matches its latest table. -Renamed tables and views are matched the same way. They are rebuilt under the new name, since they hold no history, but they keep their identity for change detection, so incremental models downstream of a renamed view are not replayed or rebuilt. These renames are recorded as `renamed` events. +Renamed tables and views are matched the same way. They are rebuilt under the new name, since they hold no history, but they keep their identity for change detection. These renames are recorded as `renamed` events. Automatic discovery never guesses. If a match is ambiguous, SQLBuild warns (`M107`) and builds from scratch; declare `migrate_from` to choose. An explicit `migrate_from` always wins. Automatic discovery covers renames within the project's schemas; use `migrate_from` to move a model to another schema. Relations last built before this feature carry no stored fingerprint and aren't matched automatically. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md index 86291e7b00..1c69d187a6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning.md @@ -11,7 +11,7 @@ Online: https://sqlbuild.com/docs/concepts/planning/ - Reading a plan - What SQLBuild compares - Change reasons -- Cascades and incremental actions +- Replay and incremental actions - Selection and coherence - Recorded state - Related topics @@ -47,8 +47,8 @@ A plan answers four questions: 3. Why does each selected model need work? 4. What materialization or incremental action will the build take? -Plans may include query diffs, schema changes, backfill ranges, source freshness signals, cascade -reasons, and warnings about stale dependencies. Planning does not modify model relations. +Plans may include query diffs, schema changes, backfill ranges, source freshness signals, change +causes, and warnings about stale dependencies. Planning does not modify model relations. ## What SQLBuild compares @@ -94,26 +94,25 @@ The plan assigns reasons to explain detected work: | Query changed / checksum changed | Compiled SQL or seed content differs from the recorded identity | | Config changed | Version-relevant configuration differs | | Schema changed | A relevant warehouse or upstream schema changed | -| Function changed | A referenced function has a different identity | -| Upstream changed | A dependency change propagated to this model | +| Function changed | A function the model calls directly has a different definition | | Run despite unchanged | A table's explicit policy requires another run despite an unchanged identity | These reasons explain the plan; they are not a promise that a build will skip entries labelled current. Builds execute the selected scope. -## Cascades and incremental actions +## Replay and incremental actions -Changes propagate through the DAG in dependency order. The resulting action depends on the +Each model's action comes from its own change. The resulting action depends on the materialization: - views are recreated when selected; - tables rebuild when selected; -- incremental models use their cursor state, backfill policy, and `replay_on_change` setting to - determine the affected range; and -- a changed upstream can alter the action of downstream models even when their own SQL is unchanged. +- incremental models use their cursor state and their own `replay_on_change` setting to determine + the affected range; and +- a changed, new, renamed, or rebuilt upstream never replays a downstream incremental model, which + continues forward from its cursor. -See [Cascade propagation](planning/cascade-propagation.md) for the detailed materialization -rules. +See [Replay decisions](planning/replay-decisions.md) for the detailed rules. ## Selection and coherence @@ -136,6 +135,6 @@ The planner reads the latest applicable facts and appends new facts after succes ## Related topics -- [Cascade propagation](planning/cascade-propagation.md) +- [Replay decisions](planning/replay-decisions.md) - [Source freshness](planning/source-freshness.md) - [Selection and staleness](planning/selection-and-staleness.md) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/cascade-propagation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/cascade-propagation.md deleted file mode 100644 index 7ced95ea76..0000000000 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/cascade-propagation.md +++ /dev/null @@ -1,41 +0,0 @@ - - -# Cascade propagation - -> How a change signal propagates downstream through the DAG, and how each materialization type responds. - -Online: https://sqlbuild.com/docs/concepts/planning/cascade-propagation/ - -When a model or function changes, the change signal propagates downstream through the DAG. Every model downstream of a changed node is marked `Upstream changed` in the plan, even if its own SQL and config are identical. - -The cascade walk is topological: it processes models in dependency order, so each model sees the resolved state of all its upstreams before deciding its own effective action. - -## What cascades - -- A query, config, or schema change on any model cascades to all its downstream dependents. -- A function change cascades to every model that calls it (directly or transitively). -- Source freshness changes propagate downstream the same way. See [Source freshness](source-freshness.md). - -## How materialization types respond - -- **Views** are recreated on every build regardless, so a cascade has no extra cost. -- **Tables** are fully rebuilt, same as if they had changed themselves. -- **Incremental models** receive a replay window from the cascade. A `full` replay always cascades. A `bounded` replay only cascades when the upstream and downstream models share the same `cursor_type` (e.g. both use `timestamp`), so unrelated cursor types don't inherit bounded windows that don't apply. The downstream model's own `replay_on_change` policy takes precedence over any cascaded signal. - -## Resolution when multiple upstreams are stale - -If a model has multiple stale upstreams with different replay actions, the most aggressive action wins. `full` beats `bounded`, and among bounded actions, the longer duration wins. Ties are broken alphabetically by model name for determinism. - -## Overriding cascaded replay - -Downstream incremental models can set their own `replay_on_change` policy to override the cascaded signal. If a downstream model declares its own policy, that policy applies instead of the upstream's. If it has no policy, it inherits the upstream's replay scope. - -## Replay on change - -When a change is detected on an incremental model, `replay_on_change` controls how much data to reprocess: - -- **`forward`** (default) - run the normal incremental delta from the cursor. No reprocessing. -- **`bounded-`** (e.g. `bounded-14d`) - replay the specified window of data. -- **`full`** - drop and rebuild the entire table. - -See [Incremental Models: Replay on change](../incremental.md#replay-on-change) for configuration details. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md new file mode 100644 index 0000000000..e1a5fc65da --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md @@ -0,0 +1,49 @@ + + +# Replay decisions + +> How much history each model reprocesses: decided by its own change and its own replay_on_change, never inherited from upstream. + +Online: https://sqlbuild.com/docs/concepts/planning/replay-decisions/ + +Every model decides how much of its history to reprocess from two things only: its own change and its own `replay_on_change` setting. A change, rebuild, rename, or replay of an upstream model never forces a downstream model to replay. + +## What a model reacts to + +A model's own change is any of: + +- **Query changed**: its compiled SQL differs from the recorded definition. +- **Function changed**: a UDF or table function that the model calls directly has a new definition. The model's SQL text is unchanged, but its results can differ, so this counts as a query change of the model. Models further downstream do not see it. +- **Schema changed**: its declared or inferred columns differ from the warehouse. +- **Config changed**: version-relevant configuration differs. A config change alone does not replay history. +- **First run**: the model's relation does not exist yet. Only that model is built from scratch. + +For a query, function, or schema change, the model's own `replay_on_change` decides the replay: + +| `replay_on_change` | Effect on an incremental model | +|--------------------|-------------------------------| +| `forward` (default) | Continue forward from the cursor; existing history keeps its old results | +| `bounded-` | Reprocess that window, for example `bounded-14d` | +| `full` | Rebuild the whole table | + +Functions have no replay setting of their own. Set `replay_on_change` on the models that call the function. + +## How materialization types respond + +- **Views** are recreated on every build. A view whose upstream was added, renamed, or rebuilt is recreated as usual. +- **Tables** are rebuilt on every build from their current upstreams. +- **Incremental models** run their normal incremental step unless their own change asks for more. When an upstream is rebuilt or replayed, a downstream incremental model picks up whatever its cursor and `lookback` reach on its next run; older history is not reprocessed. + +For example, adding a new table between an existing table and a view plans the new table as a first run, recreates the view because its query changed, and continues every incremental model downstream of the view forward. + +## Replaying downstream history on purpose + +When an upstream change should also be applied to the existing history of downstream models, request it explicitly for those models: build them with an explicit cursor interval (`--start-cursor-ts` and `--end-cursor-ts`), or select them with `--full-refresh`. + +## Protected models + +A model that sets `full_refresh false` is never fully rebuilt by anything but its own explicit configuration: its own first run, or its own query change with `replay_on_change full`. If anything else would fully rebuild it, for example a called function's change with `replay_on_change full`, the plan fails with `S203`, naming the model and the cause. See [Incremental models](../incremental.md#full-refresh-overrides). + +## Plan output + +`sqb plan` lists each model under its own change: `Query changed`, `Function changed`, `Schema changed`, `Config changed`, `First run`, or `Renamed`. A direct caller of a changed function shows `cause: function changed`. In `sqb plan --json`, the same model has `"reason": "function_changed"` and a `changed_functions` list. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md index 8b2bcb405f..15ca62a78a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md @@ -70,7 +70,7 @@ SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common gr |---------|----------|-----|---------| | Warehouse-native state | Append-only tables in the warehouse; no external state database | manifest.json artifacts | Requires external state store (SQLite/PostgreSQL) | | Source freshness | `sqb freshness` with adapter/column/sql strategies, lag tolerance, and CI gating | `dbt source freshness` | No dedicated freshness command; `signals` gate model evaluation until external data is ready | -| Cascade propagation | Topological walk with `replay_on_change` policy inheritance and override | No cascade control | Cascades through version hashes | +| Replay on upstream change | Each model replays only for its own change, per its own `replay_on_change`; upstream changes never replay downstream history | No cascade control | Cascades through version hashes | ### Environments diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md index dc322397d2..bc9786ede5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md @@ -129,10 +129,14 @@ plan or build is a one-run hard ceiling and explicit authorization for an intent ## Replay, full refresh and schema changes -- `replay_on_change` decides how much to reprocess when the model's identity changes: - `forward` (default), `full`, or `bounded-` such as `bounded-14d`. +- `replay_on_change` decides how much to reprocess when the model's own query changes, a + function it calls directly changes, or its schema changes: `forward` (default), `full`, or + `bounded-` such as `bounded-14d`. Upstream changes, first runs and renames never + replay a downstream model; it continues forward. Functions have no replay setting of their own. - `--full-refresh` rebuilds selected models unless a model sets `full_refresh false`; - `full_refresh true` always rebuilds. + `full_refresh true` always rebuilds. A `full_refresh false` model is only fully rebuilt by its + own first run or its own query change with `replay_on_change full`; anything else fails the plan + with `S203`. - `on_schema_change`: `append_new_columns` (default), `sync_all_columns`, `ignore`, `fail`. - Cursor overrides for one run: `--start-cursor-ts/--end-cursor-ts` (ISO) or `--start-cursor-int/--end-cursor-int`. @@ -145,5 +149,5 @@ plan or build is a one-run hard ceiling and explicit authorization for an intent 4. Verify with `sqb diff prod:dev --bounded 14d --select ` or a query diff over the window. Full reference: [docs/concepts/incremental.md](docs/concepts/incremental.md), -[docs/concepts/planning/cascade-propagation.md](docs/concepts/planning/cascade-propagation.md), +[docs/concepts/planning/replay-decisions.md](docs/concepts/planning/replay-decisions.md), [docs/concepts/snapshots.md](docs/concepts/snapshots.md). diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 2049f0df59..b11e5ec5f0 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -77,7 +77,9 @@ Seeds (1) ✓ Plan complete 13 selected ``` -When query or schema changes are detected, the plan shows the affected models with backfill actions and cascade information. +When query, function, or schema changes are detected, the plan lists each affected model under its own change with its backfill action. A model that calls a changed function directly appears under `Function changed` with `cause: function changed`; models further downstream continue forward. See [Replay decisions](/docs/concepts/planning/replay-decisions). + +In `sqb plan --json`, each model's `reason` is its own change. A direct caller of a changed function has `"reason": "function_changed"` and a `changed_functions` list, and a renamed model whose query also changed has `"reason": "renamed"` and `"query_changed": true`. Warnings and maintenance notices, such as `migrate_from can be removed`, are listed in a `Warnings` section before the final `Plan complete` line. `sqb plan --json` output is unchanged. diff --git a/website/src/content/docs/docs/concepts/functions.mdx b/website/src/content/docs/docs/concepts/functions.mdx index 68233e8632..b39990add4 100644 --- a/website/src/content/docs/docs/concepts/functions.mdx +++ b/website/src/content/docs/docs/concepts/functions.mdx @@ -3,7 +3,7 @@ title: "Functions" description: "User-defined functions and table functions managed as part of your project." --- -Functions are SQL or Python definitions under `functions/` that SQLBuild compiles and deploys to the warehouse alongside your models. They participate in the DAG - if a function definition changes, every model that uses it is rebuilt. +Functions are SQL or Python definitions under `functions/` that SQLBuild compiles and deploys to the warehouse alongside your models. They participate in the DAG: if a function definition changes, SQLBuild redeploys it, and every model that calls it directly treats the change as a change to its own query. ## Scalar UDFs @@ -131,7 +131,9 @@ function folder or to that folder and its children. ## Change propagation -Functions participate in fingerprint-based change detection. Their identity includes dependencies and declared return contracts in addition to the function body and runtime metadata. If a function changes, SQLBuild redeploys it and marks dependent models as changed. +Functions participate in fingerprint-based change detection. Their identity includes dependencies and declared return contracts in addition to the function body and runtime metadata. If a function changes, SQLBuild redeploys it. + +A model that calls a changed function directly plans as `Function changed`, with `cause: function changed`. The function change counts as a change to that model's own query, so the model's own `replay_on_change` decides how much history it reprocesses (forward-only by default). Models further downstream are not affected by the function change and continue forward. Functions have no replay setting of their own; to reprocess history when a function changes, set `replay_on_change` on the models that call it. See [Replay decisions](/docs/concepts/planning/replay-decisions). ## Project layout diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index a953c7e067..f4e052855b 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -103,7 +103,15 @@ MODEL ( ); ``` -The override is evaluated per model, so one selection can contain incrementally executed opt-outs and full-refreshed models. It controls execution mode rather than acting as a safety rejection: an opted-out model does not abort the rest of the build. +The override is evaluated per model, so one selection can contain incrementally executed opt-outs and full-refreshed models. With `--full-refresh`, an opted-out model simply runs incrementally; it does not abort the rest of the build. + +`full_refresh false` also protects the model from every other full rebuild. Only the model's own explicit configuration can fully rebuild it: its own first run, or its own query change with `replay_on_change full`. If anything else would plan a full rebuild, for example a changed function that the model calls while it sets `replay_on_change full`, the plan fails with `S203`, naming the model and the cause, instead of silently replaying its history: + +``` +error[S203]: model 'order_history' sets full_refresh false, but the plan would fully rebuild it (function normalize_status changed with replay_on_change full) +``` + +Set `replay_on_change` to `forward` or `bounded-` to keep the existing history, or remove `full_refresh false` to allow the rebuild. Upstream changes never cause a full rebuild of a downstream model; see [Replay decisions](/docs/concepts/planning/replay-decisions). This does not skip initial loading. If the destination relation does not exist, an incremental or microbatch model still builds the history required by its cursor policy. Cloning an existing destination before the build can provide a current watermark and avoid a first-run historical replay. @@ -162,7 +170,7 @@ WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() ``` -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails with a message naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. @@ -313,7 +321,7 @@ The legacy scalar `max_microbatches` model field remains supported as a fail/war ### Mixed-grain chains -When a downstream microbatch model reads from an upstream model with a coarser time grain, SQLBuild aligns the replay to the coarsest participating grain (the model's own grain and its cursor-input grains). This happens on every run that resolves cursor bounds from upstream models, not only when something changes. It is independent of the `replay_on_change` cascade behavior described below. +When a downstream microbatch model reads from an upstream model with a coarser time grain, SQLBuild aligns the replay to the coarsest participating grain (the model's own grain and its cursor-input grains). This happens on every run that resolves cursor bounds from upstream models, not only when something changes. It is independent of the `replay_on_change` policy described below. Alignment does two things: it floors the replay window edges to the coarsest grain, and it coarsens the batch size to that grain. For example, an hourly model downstream of a daily model processes in day-sized batches, so each batch lines up with a unit of upstream data that actually advances instead of producing empty or boundary-straddling windows: @@ -349,7 +357,7 @@ MODEL ( ## Replay on change -When a model's version identity changes (query, config, upstream cascade, or any other change reason), `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large downstream tables. You choose the cost per model: +When a model's own query changes, a function it calls directly changes, or its schema changes, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: | Value | Effect | |-------|--------| @@ -370,7 +378,7 @@ MODEL ( ); ``` -See [Cascade propagation](/docs/concepts/planning/cascade-propagation) for how replay policies propagate through the DAG and how downstream models can override inherited replay behavior. +See [Replay decisions](/docs/concepts/planning/replay-decisions) for what counts as a model's own change and how materialization types respond to upstream changes. ### on_schema_change diff --git a/website/src/content/docs/docs/concepts/models/migrations.mdx b/website/src/content/docs/docs/concepts/models/migrations.mdx index fde311af7c..b142788471 100644 --- a/website/src/content/docs/docs/concepts/models/migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/migrations.mdx @@ -26,6 +26,8 @@ Use `schema.name` when the old relation is in a different schema of the same dat On the next build, SQLBuild copies the old relation's data to the new name through a staging table (see [How a move runs](#how-a-move-runs)), records the move, and builds the model incrementally from the migrated state. There is no first-run rebuild, and no `replay_on_change` replay caused by the rename. After the model builds, the old relation is archived and a compatibility view takes its name (see [Old names](#old-names)). +A rename is usually made together with other edits, for example a changed filter or a renamed upstream reference. If the renamed model's query also differs from the old model's, the plan still recognises the rename: the model is listed as `Renamed` with its query diff, never as a first run. The query change follows the model's own `replay_on_change`, exactly as it would without the rename (forward-only by default). Models downstream of the renamed model are not replayed because of the rename; see [Replay decisions](/docs/concepts/planning/replay-decisions). + After the move is recorded, `migrate_from` has no further effect, and SQLBuild tells you it can be removed. Keep it until every target that needs the move has built. ## Decisions @@ -52,7 +54,7 @@ Table and view models only ever see `migrate`, `done`, and `origin missing`: not If a selected model has never been built, SQLBuild compares it with models that were removed from the project but whose relations still exist. When exactly one removed model has equivalent logic, it is migrated automatically. Equivalent means the same query and configuration, ignoring the model's own name, CTE and table alias names, comments, formatting, and storage-only settings. Renamed upstream models are matched first, so a renamed chain of models is migrated together. Removed models whose data was already moved on by a recorded migration are not candidates, so a model renamed several times matches its latest table. -Renamed tables and views are matched the same way. They are rebuilt under the new name, since they hold no history, but they keep their identity for change detection, so incremental models downstream of a renamed view are not replayed or rebuilt. These renames are recorded as `renamed` events. +Renamed tables and views are matched the same way. They are rebuilt under the new name, since they hold no history, but they keep their identity for change detection. These renames are recorded as `renamed` events. Automatic discovery never guesses. If a match is ambiguous, SQLBuild warns (`M107`) and builds from scratch; declare `migrate_from` to choose. An explicit `migrate_from` always wins. Automatic discovery covers renames within the project's schemas; use `migrate_from` to move a model to another schema. Relations last built before this feature carry no stored fingerprint and aren't matched automatically. diff --git a/website/src/content/docs/docs/concepts/planning.mdx b/website/src/content/docs/docs/concepts/planning.mdx index 5904a668a2..db7f857471 100644 --- a/website/src/content/docs/docs/concepts/planning.mdx +++ b/website/src/content/docs/docs/concepts/planning.mdx @@ -36,8 +36,8 @@ A plan answers four questions: 3. Why does each selected model need work? 4. What materialization or incremental action will the build take? -Plans may include query diffs, schema changes, backfill ranges, source freshness signals, cascade -reasons, and warnings about stale dependencies. Planning does not modify model relations. +Plans may include query diffs, schema changes, backfill ranges, source freshness signals, change +causes, and warnings about stale dependencies. Planning does not modify model relations. ## What SQLBuild compares @@ -83,26 +83,25 @@ The plan assigns reasons to explain detected work: | Query changed / checksum changed | Compiled SQL or seed content differs from the recorded identity | | Config changed | Version-relevant configuration differs | | Schema changed | A relevant warehouse or upstream schema changed | -| Function changed | A referenced function has a different identity | -| Upstream changed | A dependency change propagated to this model | +| Function changed | A function the model calls directly has a different definition | | Run despite unchanged | A table's explicit policy requires another run despite an unchanged identity | These reasons explain the plan; they are not a promise that a build will skip entries labelled current. Builds execute the selected scope. -## Cascades and incremental actions +## Replay and incremental actions -Changes propagate through the DAG in dependency order. The resulting action depends on the +Each model's action comes from its own change. The resulting action depends on the materialization: - views are recreated when selected; - tables rebuild when selected; -- incremental models use their cursor state, backfill policy, and `replay_on_change` setting to - determine the affected range; and -- a changed upstream can alter the action of downstream models even when their own SQL is unchanged. +- incremental models use their cursor state and their own `replay_on_change` setting to determine + the affected range; and +- a changed, new, renamed, or rebuilt upstream never replays a downstream incremental model, which + continues forward from its cursor. -See [Cascade propagation](/docs/concepts/planning/cascade-propagation) for the detailed materialization -rules. +See [Replay decisions](/docs/concepts/planning/replay-decisions) for the detailed rules. ## Selection and coherence @@ -125,6 +124,6 @@ The planner reads the latest applicable facts and appends new facts after succes ## Related topics -- [Cascade propagation](/docs/concepts/planning/cascade-propagation) +- [Replay decisions](/docs/concepts/planning/replay-decisions) - [Source freshness](/docs/concepts/planning/source-freshness) - [Selection and staleness](/docs/concepts/planning/selection-and-staleness) diff --git a/website/src/content/docs/docs/concepts/planning/cascade-propagation.mdx b/website/src/content/docs/docs/concepts/planning/cascade-propagation.mdx deleted file mode 100644 index e77461cba7..0000000000 --- a/website/src/content/docs/docs/concepts/planning/cascade-propagation.mdx +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: "Cascade propagation" -description: "How a change signal propagates downstream through the DAG, and how each materialization type responds." ---- - -When a model or function changes, the change signal propagates downstream through the DAG. Every model downstream of a changed node is marked `Upstream changed` in the plan, even if its own SQL and config are identical. - -The cascade walk is topological: it processes models in dependency order, so each model sees the resolved state of all its upstreams before deciding its own effective action. - -## What cascades - -- A query, config, or schema change on any model cascades to all its downstream dependents. -- A function change cascades to every model that calls it (directly or transitively). -- Source freshness changes propagate downstream the same way. See [Source freshness](/docs/concepts/planning/source-freshness). - -## How materialization types respond - -- **Views** are recreated on every build regardless, so a cascade has no extra cost. -- **Tables** are fully rebuilt, same as if they had changed themselves. -- **Incremental models** receive a replay window from the cascade. A `full` replay always cascades. A `bounded` replay only cascades when the upstream and downstream models share the same `cursor_type` (e.g. both use `timestamp`), so unrelated cursor types don't inherit bounded windows that don't apply. The downstream model's own `replay_on_change` policy takes precedence over any cascaded signal. - -## Resolution when multiple upstreams are stale - -If a model has multiple stale upstreams with different replay actions, the most aggressive action wins. `full` beats `bounded`, and among bounded actions, the longer duration wins. Ties are broken alphabetically by model name for determinism. - -## Overriding cascaded replay - -Downstream incremental models can set their own `replay_on_change` policy to override the cascaded signal. If a downstream model declares its own policy, that policy applies instead of the upstream's. If it has no policy, it inherits the upstream's replay scope. - -## Replay on change - -When a change is detected on an incremental model, `replay_on_change` controls how much data to reprocess: - -- **`forward`** (default) - run the normal incremental delta from the cursor. No reprocessing. -- **`bounded-`** (e.g. `bounded-14d`) - replay the specified window of data. -- **`full`** - drop and rebuild the entire table. - -See [Incremental Models: Replay on change](/docs/concepts/incremental#replay-on-change) for configuration details. diff --git a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx new file mode 100644 index 0000000000..cd0fa0513e --- /dev/null +++ b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx @@ -0,0 +1,46 @@ +--- +title: "Replay decisions" +description: "How much history each model reprocesses: decided by its own change and its own replay_on_change, never inherited from upstream." +--- + +Every model decides how much of its history to reprocess from two things only: its own change and its own `replay_on_change` setting. A change, rebuild, rename, or replay of an upstream model never forces a downstream model to replay. + +## What a model reacts to + +A model's own change is any of: + +- **Query changed**: its compiled SQL differs from the recorded definition. +- **Function changed**: a UDF or table function that the model calls directly has a new definition. The model's SQL text is unchanged, but its results can differ, so this counts as a query change of the model. Models further downstream do not see it. +- **Schema changed**: its declared or inferred columns differ from the warehouse. +- **Config changed**: version-relevant configuration differs. A config change alone does not replay history. +- **First run**: the model's relation does not exist yet. Only that model is built from scratch. + +For a query, function, or schema change, the model's own `replay_on_change` decides the replay: + +| `replay_on_change` | Effect on an incremental model | +|--------------------|-------------------------------| +| `forward` (default) | Continue forward from the cursor; existing history keeps its old results | +| `bounded-` | Reprocess that window, for example `bounded-14d` | +| `full` | Rebuild the whole table | + +Functions have no replay setting of their own. Set `replay_on_change` on the models that call the function. + +## How materialization types respond + +- **Views** are recreated on every build. A view whose upstream was added, renamed, or rebuilt is recreated as usual. +- **Tables** are rebuilt on every build from their current upstreams. +- **Incremental models** run their normal incremental step unless their own change asks for more. When an upstream is rebuilt or replayed, a downstream incremental model picks up whatever its cursor and `lookback` reach on its next run; older history is not reprocessed. + +For example, adding a new table between an existing table and a view plans the new table as a first run, recreates the view because its query changed, and continues every incremental model downstream of the view forward. + +## Replaying downstream history on purpose + +When an upstream change should also be applied to the existing history of downstream models, request it explicitly for those models: build them with an explicit cursor interval (`--start-cursor-ts` and `--end-cursor-ts`), or select them with `--full-refresh`. + +## Protected models + +A model that sets `full_refresh false` is never fully rebuilt by anything but its own explicit configuration: its own first run, or its own query change with `replay_on_change full`. If anything else would fully rebuild it, for example a called function's change with `replay_on_change full`, the plan fails with `S203`, naming the model and the cause. See [Incremental models](/docs/concepts/incremental#full-refresh-overrides). + +## Plan output + +`sqb plan` lists each model under its own change: `Query changed`, `Function changed`, `Schema changed`, `Config changed`, `First run`, or `Renamed`. A direct caller of a changed function shows `cause: function changed`. In `sqb plan --json`, the same model has `"reason": "function_changed"` and a `changed_functions` list. diff --git a/website/src/content/docs/docs/feature-comparison.mdx b/website/src/content/docs/docs/feature-comparison.mdx index ad13aee605..01ad2944c0 100644 --- a/website/src/content/docs/docs/feature-comparison.mdx +++ b/website/src/content/docs/docs/feature-comparison.mdx @@ -61,7 +61,7 @@ SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common gr |---------|----------|-----|---------| | Warehouse-native state | Append-only tables in the warehouse; no external state database | manifest.json artifacts | Requires external state store (SQLite/PostgreSQL) | | Source freshness | `sqb freshness` with adapter/column/sql strategies, lag tolerance, and CI gating | `dbt source freshness` | No dedicated freshness command; `signals` gate model evaluation until external data is ready | -| Cascade propagation | Topological walk with `replay_on_change` policy inheritance and override | No cascade control | Cascades through version hashes | +| Replay on upstream change | Each model replays only for its own change, per its own `replay_on_change`; upstream changes never replay downstream history | No cascade control | Cascades through version hashes | ### Environments diff --git a/website/src/navigation.mjs b/website/src/navigation.mjs index f1084d9dd8..a06feae12d 100644 --- a/website/src/navigation.mjs +++ b/website/src/navigation.mjs @@ -4,5 +4,6 @@ export const sidebar = sidebarData; export const redirects = { "/docs/concepts/rules/overview": "/docs/concepts/rules", - "/docs/concepts/rules/custom-rules": "/docs/concepts/rules/custom-rules/overview" + "/docs/concepts/rules/custom-rules": "/docs/concepts/rules/custom-rules/overview", + "/docs/concepts/planning/cascade-propagation": "/docs/concepts/planning/replay-decisions" }; diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 98bdf37bb5..410841f956 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -92,7 +92,7 @@ "collapsed": true, "items": [ "docs/concepts/planning", - "docs/concepts/planning/cascade-propagation", + "docs/concepts/planning/replay-decisions", "docs/concepts/planning/source-freshness", "docs/concepts/planning/selection-and-staleness" ] From 248c263ea93accd837792541a5e9a357ea34dcfc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:33:26 +0100 Subject: [PATCH 05/12] fix(functions): reject unknown FUNCTION header and @udf keys SQL FUNCTION() headers and Python @udf(...) decorators now fail discovery with D002 when they use a key that functions do not support, naming the key and its file and line, the same way HOOK() and AUDIT() headers reject unsupported keys. Previously such keys were silently ignored. --- .../references/docs/concepts/functions.md | 4 +- .../discovery/_helpers/python/functions.py | 9 ++ .../discovery/_helpers/sql/functions.py | 24 +++- src/sqlbuild/compiler/discovery/constants.py | 15 +++ .../cli/commands/main/compile/_test_types.py | 7 ++ .../main/compile/test_function_header_keys.py | 57 +++++++++ .../sqlbuild/cli/commands/main/test_format.py | 1 - .../sqlbuild/cli/commands/main/test_rules.py | 1 - .../discovery/_helpers/_test_types.py | 9 ++ .../_helpers/test_function_header_keys.py | 118 ++++++++++++++++++ .../content/docs/docs/concepts/functions.mdx | 4 +- 11 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py create mode 100644 tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md index fae251cf7d..4043248d36 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md @@ -24,7 +24,7 @@ Scalar UDFs return a single value per row. They can be written in SQL or Python. ### SQL UDFs -Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body: +Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line: ```sql -- functions/sql/udf__is_completed_order.sql @@ -171,6 +171,8 @@ The `@udf` decorator accepts these keyword arguments: | `entry_point` | No | Function name to use as the entry point (defaults to the decorated function name) | | `packages` | No | List of Python packages required at runtime | +`database`, `schema`, and `tags` are also accepted. Any other keyword fails compilation with `D002`, naming the keyword and its file and line. + ## Adapter support All four supported adapters implement SQL UDFs, Python UDFs, and table functions: diff --git a/src/sqlbuild/compiler/discovery/_helpers/python/functions.py b/src/sqlbuild/compiler/discovery/_helpers/python/functions.py index cbab5a042f..7de63870b9 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/python/functions.py +++ b/src/sqlbuild/compiler/discovery/_helpers/python/functions.py @@ -8,6 +8,7 @@ from sqlbuild.compiler.discovery.constants import ( PYTHON_UDF_DECORATOR_NAME, PYTHON_UDF_IMPORT_MODULES, + PYTHON_UDF_KEYS, ) from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError @@ -61,6 +62,14 @@ def parse_python_function(*, contents: str, file_path: Path) -> tuple[dict[str, raise ModelSqlParseError( f"Python function '{file_path}' @udf(...) must use keyword arguments only" ) + unsupported: tuple[ast.keyword, ...] = tuple( + keyword for keyword in decorator_call.keywords if keyword.arg not in PYTHON_UDF_KEYS + ) + if unsupported: + raise ModelSqlParseError( + f"@udf(...) in '{file_path}:{unsupported[0].lineno}' has unsupported keys: " + f"{', '.join(str(keyword.arg) for keyword in unsupported)}" + ) entry_point: object | None = values.get("entry_point") if entry_point is None: diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py b/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py index 8e1fc1298c..243fcb99b9 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py @@ -6,7 +6,10 @@ from pathlib import Path from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values -from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN +from sqlbuild.compiler.discovery.constants import ( + SQL_FUNCTION_HEADER_KEYS, + STATEMENT_HEADER_BODY_PATTERN, +) from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError _FUNCTION_HEADER_PATTERN: re.Pattern[str] = re.compile( @@ -30,7 +33,26 @@ def parse_function_sql(*, contents: str, file_path: Path) -> tuple[dict[str, obj file_path=file_path, statement_name="FUNCTION", ) + unsupported_keys: tuple[str, ...] = tuple( + str(key) for key in header_values if key not in SQL_FUNCTION_HEADER_KEYS + ) + if unsupported_keys: + line: int = contents.count( + "\n", + 0, + header_match.start("header") + + _header_key_offset(header=header_match.group("header"), key=unsupported_keys[0]), + ) + raise ModelSqlParseError( + f"FUNCTION() in '{file_path}:{line + 1}' has unsupported keys: " + f"{', '.join(unsupported_keys)}" + ) body_sql: str = header_match.group("sql").strip() if not body_sql: raise ModelSqlParseError(f"SQL function '{file_path}' must contain SQL after FUNCTION(...)") return header_values, body_sql + + +def _header_key_offset(*, header: str, key: str) -> int: + match: re.Match[str] | None = re.search(rf"(?:^|[(,])\s*({re.escape(key)})\b", header) + return match.start(1) if match is not None else 0 diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 2de8bc4062..e0f95636b0 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -17,6 +17,21 @@ LANGUAGE_PYTHON_ROOT_PART_COUNT: int = 2 PYTHON_NODE_ROOT: str = "python" PYTHON_UDF_DECORATOR_NAME: str = "udf" +SQL_FUNCTION_HEADER_KEYS: frozenset[str] = frozenset( + {"arguments", "returns", "database", "schema", "tags"} +) +PYTHON_UDF_KEYS: frozenset[str] = frozenset( + { + "arguments", + "returns", + "runtime_version", + "entry_point", + "packages", + "database", + "schema", + "tags", + } +) PYTHON_UDF_IMPORT_MODULES: frozenset[str] = frozenset({"sqlbuild", "sqlbuild.functions"}) DLT_LOADER_KIND: str = "dlt" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index bd05de011b..7107a35c96 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -371,3 +371,10 @@ class CompileCacheDisabledTestCase: edit: Callable[[Path], None] = lambda _root: None expected_fact_cache_counts: tuple[int, int] = (0, 0) expected_fact_databases: tuple[Path, ...] = () + + +@dataclass(frozen=True) +class FunctionHeaderKeyCompileCase: + description: str + repo_files: dict[str, str] + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py new file mode 100644 index 0000000000..c3a809b4ae --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py @@ -0,0 +1,57 @@ +"""E2E coverage for rejecting unknown function header keys at compile time.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + FunctionHeaderKeyCompileCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + FunctionHeaderKeyCompileCase( + description="sql udf header with a removed replay setting", + repo_files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "functions/sql/normalize_status.sql": ( + "FUNCTION (\n arguments (status VARCHAR),\n returns VARCHAR,\n" + " replay_on_change full,\n);\n\nlower(status)\n" + ), + "models/order_statuses.sql": ( + "MODEL (materialized table);\n\nSELECT __udf(\"normalize_status\")('OPEN') " + "AS status\n" + ), + }, + expected_fragments=( + "error[D002]", + "functions/sql/normalize_status.sql:4' has unsupported keys: replay_on_change", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_unknown_function_header_key_when_compiling_then_fails_naming_key_and_location( + test_case: FunctionHeaderKeyCompileCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="orders", repo_files=test_case.repo_files + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode != 0, output + assert all(fragment in output for fragment in test_case.expected_fragments), output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py index 649bbe4dac..62836e041d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py @@ -259,7 +259,6 @@ def test_given_table_function_argument_when_formatting_then_intrinsic_restores_a function.parent.mkdir(parents=True) function.write_text( """FUNCTION ( - description "Expand one order", arguments (order_id INTEGER), returns table (order_id INTEGER) ); diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 6a18174026..c4773e46ec 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -1210,7 +1210,6 @@ def test_given_table_function_argument_when_running_alias_rule_then_left_alias_i function.parent.mkdir(parents=True) function.write_text( """FUNCTION ( - description "Expand one order", arguments (order_id INTEGER), returns table (order_id INTEGER) ); diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 7e9608c845..206f0c041b 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -903,3 +903,12 @@ class MissingMigrationOriginConfigTestCase: description: str target_lines: tuple[str, ...] expected_policy: MissingMigrationOriginPolicy + + +@dataclass(frozen=True) +class FunctionHeaderKeysTestCase: + description: str + file_name: str + contents: str + expected_error: str = "" + expected_keys: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py new file mode 100644 index 0000000000..2c9d80068c --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py @@ -0,0 +1,118 @@ +"""Tests for supported keys in SQL FUNCTION headers and Python @udf decorators.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function +from sqlbuild.compiler.discovery._helpers.sql.functions import parse_function_sql +from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError +from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( + FunctionHeaderKeysTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + FunctionHeaderKeysTestCase( + description="sql udf with every supported key", + file_name="normalize_status.sql", + contents=( + "FUNCTION (\n arguments (status VARCHAR),\n returns VARCHAR,\n" + " database analytics,\n schema udfs,\n tags [orders],\n);\n\nlower(status)\n" + ), + expected_keys=("arguments", "database", "returns", "schema", "tags"), + ), + FunctionHeaderKeysTestCase( + description="table function with a returns table declaration", + file_name="expand_order.sql", + contents=( + "FUNCTION (\n arguments (order_id INTEGER),\n" + " returns table (order_id INTEGER),\n);\n\nSELECT order_id\n" + ), + expected_keys=("arguments", "returns"), + ), + ], + ids=lambda case: case.description, +) +def test_given_supported_function_header_keys_when_parsing_then_accepts_header( + test_case: FunctionHeaderKeysTestCase, +) -> None: + header_values: dict[str, object] + header_values, _ = parse_function_sql( + contents=test_case.contents, file_path=Path(test_case.file_name) + ) + + assert tuple(sorted(header_values)) == test_case.expected_keys + + +@pytest.mark.parametrize( + "test_case", + [ + FunctionHeaderKeysTestCase( + description="sql udf with a replay setting", + file_name="normalize_status.sql", + contents=( + "FUNCTION (\n arguments (status VARCHAR),\n returns VARCHAR,\n" + " replay_on_change full,\n);\n\nlower(status)\n" + ), + expected_error=( + "FUNCTION() in 'normalize_status.sql:4' has unsupported keys: replay_on_change" + ), + ), + FunctionHeaderKeysTestCase( + description="table function with two unknown keys", + file_name="expand_order.sql", + contents=( + 'FUNCTION (\n description "Expand one order",\n arguments (order_id INTEGER),\n' + " returns table (order_id INTEGER),\n owner orders_team,\n);\n\nSELECT order_id\n" + ), + expected_error=( + "FUNCTION() in 'expand_order.sql:2' has unsupported keys: description, owner" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_sql_function_header_key_when_parsing_then_rejects_key_with_location( + test_case: FunctionHeaderKeysTestCase, +) -> None: + with pytest.raises(ModelSqlParseError) as raised: + parse_function_sql(contents=test_case.contents, file_path=Path(test_case.file_name)) + + assert raised.value.message == test_case.expected_error + assert raised.value.code == "D002" + + +@pytest.mark.parametrize( + "test_case", + [ + FunctionHeaderKeysTestCase( + description="python udf with a replay setting", + file_name="normalize_status_py.py", + contents=( + "from sqlbuild.functions import udf\n\n\n@udf(\n" + ' arguments={"status": "VARCHAR"},\n returns="VARCHAR",\n' + ' replay_on_change="full",\n)\ndef main(status):\n return status\n' + ), + expected_error=( + "@udf(...) in 'normalize_status_py.py:7' has unsupported keys: replay_on_change" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_unknown_python_udf_key_when_parsing_then_rejects_key_with_location( + test_case: FunctionHeaderKeysTestCase, +) -> None: + with pytest.raises(ModelSqlParseError) as raised: + parse_python_function(contents=test_case.contents, file_path=Path(test_case.file_name)) + + assert raised.value.message == test_case.expected_error + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/functions.mdx b/website/src/content/docs/docs/concepts/functions.mdx index b39990add4..56f9f86665 100644 --- a/website/src/content/docs/docs/concepts/functions.mdx +++ b/website/src/content/docs/docs/concepts/functions.mdx @@ -11,7 +11,7 @@ Scalar UDFs return a single value per row. They can be written in SQL or Python. ### SQL UDFs -Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body: +Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line: ```sql -- functions/sql/udf__is_completed_order.sql @@ -158,6 +158,8 @@ The `@udf` decorator accepts these keyword arguments: | `entry_point` | No | Function name to use as the entry point (defaults to the decorated function name) | | `packages` | No | List of Python packages required at runtime | +`database`, `schema`, and `tags` are also accepted. Any other keyword fails compilation with `D002`, naming the keyword and its file and line. + ## Adapter support All four supported adapters implement SQL UDFs, Python UDFs, and table functions: From d1ad4117688e1bb1493a3552e37db4952c7a116b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:32:45 +0100 Subject: [PATCH 06/12] fix(executor): report called-function changes to custom materializations Custom materializations now receive query_changed for function_changed models as well as query_changed ones, since a changed directly called function is the model's own query change. --- .../run/_helpers/materializations/custom.py | 8 +++- .../executor/run/_helpers/_test_types.py | 8 ++++ .../run/_helpers/test_custom_query_change.py | 45 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py diff --git a/src/sqlbuild/executor/run/_helpers/materializations/custom.py b/src/sqlbuild/executor/run/_helpers/materializations/custom.py index a5dc8b2c4d..e3e6fbaa28 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/custom.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/custom.py @@ -218,7 +218,7 @@ def _run_custom_materialization( declared_columns=declared_columns, is_first_run=existing_relation is None, is_full_refresh=entry.reason == PlanReason.FULL_REFRESH, - query_changed=entry.reason == PlanReason.QUERY_CHANGED, + query_changed=reports_query_change(reason=entry.reason), schema_findings=entry.schema_findings, run_audits=run_audits_fn, on_progress=on_progress, @@ -467,3 +467,9 @@ def _cleanup_relations( ) except Exception: pass + + +def reports_query_change(*, reason: PlanReason) -> bool: + """Return whether a custom materialization sees its model's query as changed.""" + + return reason in {PlanReason.QUERY_CHANGED, PlanReason.FUNCTION_CHANGED} diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index 00f2284dce..0f2eb18658 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent +from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.run.types import AuditGateReuseReason, AuditGateStatus, ExecutionPhase from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -470,3 +471,10 @@ class PythonHookHardCodedRelationTestCase: description: str enforce_explicit_references: bool expected_warning_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class CustomQueryChangeTestCase: + description: str + reason: PlanReason + expected_query_changed: bool diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py new file mode 100644 index 0000000000..2942843573 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py @@ -0,0 +1,45 @@ +"""Tests for the query-change flag custom materializations receive.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.planner.types import PlanReason +from sqlbuild.executor.run._helpers.materializations.custom import reports_query_change +from tests.unit.src.sqlbuild.executor.run._helpers._test_types import CustomQueryChangeTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + CustomQueryChangeTestCase( + description="own query change", + reason=PlanReason.QUERY_CHANGED, + expected_query_changed=True, + ), + CustomQueryChangeTestCase( + description="called function change", + reason=PlanReason.FUNCTION_CHANGED, + expected_query_changed=True, + ), + CustomQueryChangeTestCase( + description="config change", + reason=PlanReason.CONFIG_CHANGED, + expected_query_changed=False, + ), + CustomQueryChangeTestCase( + description="no change", + reason=PlanReason.NO_CHANGE, + expected_query_changed=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_plan_reason_when_building_custom_context_then_reports_query_change( + test_case: CustomQueryChangeTestCase, +) -> None: + assert reports_query_change(reason=test_case.reason) is test_case.expected_query_changed + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From a57ecea48cb2e4ce1ae11b384f447556efede2ea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:32:45 +0100 Subject: [PATCH 07/12] fix(planner): check full_refresh protection only for models that execute S203 is no longer evaluated for models the plan skips because a source freshness check or an external upstream blocked them. --- .../planner/_helpers/output/plan_entry.py | 5 +- .../compiler/planner/main/_test_types.py | 7 ++ .../sqlbuild/compiler/planner/main/helpers.py | 38 +++++++++++ .../compiler/planner/main/test_execution.py | 64 +++++++++++++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index ad6a54aea9..b0796256c6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -458,6 +458,8 @@ def build_plan_entries( ), change_result=resolved.change, external_sql_reference_resolver=project.external_sql_reference_resolver, + protect_full_refresh=key.name not in source_freshness_blocked_model_names + and key.name not in external_blocked_model_names, ) if entry.name in source_freshness_blocked_model_names: entry = replace( @@ -676,6 +678,7 @@ def plan_model_from_change( cursor_overrides: CursorOverridePair, change_result: ChangeDetectionResult, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + protect_full_refresh: bool = True, ) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: """Build a model plan entry from a resolved change result.""" @@ -710,7 +713,7 @@ def plan_model_from_change( if replaces_relation else None ) - if full_rebuild_cause is not None and not full_refresh: + if full_rebuild_cause is not None and protect_full_refresh and not full_refresh: check_full_rebuild_allowed( model=model, change_result=change_result, full_rebuild_cause=full_rebuild_cause ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index c439e4c0e4..621c272656 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -174,3 +174,10 @@ class FutureCursorSafetyTestCase: expected_end: str expected_has_safety: bool expected_error_fragment: str | None = None + + +@dataclass(frozen=True) +class ProtectedRebuildBlockedTestCase: + description: str + external_blocked_model_names: tuple[str, ...] + expected_action: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 7ef403a14d..a1743b8d61 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -21,6 +21,7 @@ PlannerSelection, PlanOutput, ) +from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry def model_definition_hash(project: CompiledProject, name: str) -> str: @@ -92,3 +93,40 @@ def grouped(model: type) -> dict[str, Any]: policies=policies, **kwargs, ) + + +def build_protected_schema_replay_project() -> CompiledProject: + """Build a full_refresh false incremental whose declared column forces a full replay.""" + + model: CompiledModel = CompiledModel( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="order_history"), + deps=(), + name="order_history", + relative_path=Path("models/order_history.sql"), + query_sql="SELECT 1 AS id, 2 AS amount", + config=CompileModelConfig( + values={ + "materialized": "incremental", + "incremental_strategy": "append", + "replay_on_change": "full", + "full_refresh": False, + } + ), + destination=CompiledRelationLocation( + database=None, schema="main", name="order_history", qualified_name="main.order_history" + ), + schema_entry=SchemaModelEntry( + name="order_history", + columns=( + SchemaColumn(name="id", type="INTEGER"), + SchemaColumn(name="amount", type="INTEGER"), + ), + ), + ) + return CompiledProject( + run_id="protected-rebuild-run", + effective_target_name=None, + effective_connection={}, + effective_vars={}, + models=(model,), + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index 3acb11cdfa..c4fdb2cfcd 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -9,6 +9,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.discovery.models import DiscoveredHookFunction +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( ModelPlanEntry, PlanOutput, @@ -21,9 +22,11 @@ DirectSourceFreshnessPlanOutputTestCase, ExternalBlockedPlanOutputTestCase, HookFunctionPlanOutputTestCase, + ProtectedRebuildBlockedTestCase, ) from tests.unit.src.sqlbuild.compiler.planner.main.helpers import ( build_execution_plan_from_kwargs, + build_protected_schema_replay_project, ) from tests.unit.src.sqlbuild.integrations.dbt.helpers import build_compiled_project_with_models @@ -146,3 +149,64 @@ def test_given_external_blocked_model_when_building_execution_plan_then_only_tha assert entries_by_name["blocked"].action == PlanAction.SKIP assert entries_by_name["blocked"].reason == PlanReason.EXTERNAL_UPSTREAM_FAILED assert entries_by_name["unrelated"].action != PlanAction.SKIP + + +@pytest.mark.parametrize( + "test_case", + [ + ProtectedRebuildBlockedTestCase( + description="externally blocked protected model is skipped without S203", + external_blocked_model_names=("order_history",), + expected_action="skip", + ) + ], + ids=lambda case: case.description, +) +def test_given_blocked_protected_model_when_planning_then_full_rebuild_protection_is_not_evaluated( + test_case: ProtectedRebuildBlockedTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + try: + adapter.execute(connection=connection, sql="CREATE TABLE main.order_history (id INTEGER)") + plan_output: PlanOutput = build_execution_plan_from_kwargs( + project=build_protected_schema_replay_project(), + adapter=adapter, + connection=connection, + external_blocked_model_names=test_case.external_blocked_model_names, + ) + finally: + adapter.close(connection) + + assert plan_output.model_entries[0].action.value == test_case.expected_action + + +@pytest.mark.parametrize( + "test_case", + [ + ProtectedRebuildBlockedTestCase( + description="unblocked protected model refuses the declared-column full replay", + external_blocked_model_names=(), + expected_action="S203", + ) + ], + ids=lambda case: case.description, +) +def test_given_unblocked_protected_model_when_planning_then_full_rebuild_is_refused( + test_case: ProtectedRebuildBlockedTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + try: + adapter.execute(connection=connection, sql="CREATE TABLE main.order_history (id INTEGER)") + with pytest.raises(PlannerInputError) as raised: + build_execution_plan_from_kwargs( + project=build_protected_schema_replay_project(), + adapter=adapter, + connection=connection, + external_blocked_model_names=test_case.external_blocked_model_names, + ) + finally: + adapter.close(connection) + + assert raised.value.code == test_case.expected_action From 22c27f932bbe184ad5062cf1f4ede6436fa55f30 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:32:46 +0100 Subject: [PATCH 08/12] fix(planner): ignore rename-rewritten references and inferred upstream columns for replay A downstream model whose only difference is __ref and cursor_inputs names rewritten for an upstream renamed with migrate_from in this plan is no longer a query or config change: its migration fingerprint with the renamed names mapped back to their origins must equal its stored one. Views are still recreated and incremental models continue forward; a real edit alongside the rewrite is a normal query change. Schema findings inferred from a model's unchanged SQL, such as a column an upstream added through SELECT *, no longer trigger replay_on_change; they evolve forward through on_schema_change. Changes to the model's own declared columns still follow its replay_on_change. --- .../references/docs/concepts/incremental.md | 2 +- .../concepts/planning/replay-decisions.md | 6 +- .../sqlbuild/references/incremental-models.md | 8 +- .../planner/_helpers/changes/detect.py | 14 +- .../planner/_helpers/changes/metadata.py | 29 ++++ .../planner/_helpers/migrations/discovery.py | 6 +- .../planner/_helpers/migrations/planning.py | 58 +++++++ src/sqlbuild/compiler/planner/models.py | 2 + .../main/build/own_change_replay/helpers.py | 85 ++++++++++ .../test_own_change_replay.py | 155 ++++++++++++++++++ .../planner/_helpers/changes/test_detect.py | 36 ++++ .../docs/docs/concepts/incremental.mdx | 2 +- .../concepts/planning/replay-decisions.mdx | 6 +- 13 files changed, 394 insertions(+), 15 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index e13338a8e9..597065914e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -368,7 +368,7 @@ MODEL ( ## Replay on change -When a model's own query changes, a function it calls directly changes, or its schema changes, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: +When a model's own query changes, a function it calls directly changes, or its own declared columns change, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: | Value | Effect | |-------|--------| diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md index e1a5fc65da..3c7ea800ae 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md @@ -14,11 +14,13 @@ A model's own change is any of: - **Query changed**: its compiled SQL differs from the recorded definition. - **Function changed**: a UDF or table function that the model calls directly has a new definition. The model's SQL text is unchanged, but its results can differ, so this counts as a query change of the model. Models further downstream do not see it. -- **Schema changed**: its declared or inferred columns differ from the warehouse. +- **Schema changed**: its declared or inferred columns differ from the warehouse. Only a change to the model's own declared columns (its `columns` block, contract, or schema YAML) follows `replay_on_change`. Columns inferred from its unchanged SQL, for example a column an upstream added that `SELECT *` passes through, are applied forward through [`on_schema_change`](../incremental.md#on_schema_change) and never replay history. - **Config changed**: version-relevant configuration differs. A config change alone does not replay history. - **First run**: the model's relation does not exist yet. Only that model is built from scratch. -For a query, function, or schema change, the model's own `replay_on_change` decides the replay: +References rewritten by a rename are not a change of their own. When an upstream model is renamed with `migrate_from` (for example by [`sqb rename`](../../cli/rename.md)), a downstream model whose only difference is the rewritten `__ref` and `cursor_inputs` name keeps its identity: views are recreated as usual, and incremental models continue forward. A downstream model that also has a real edit is a normal query change. + +For a query, function, or own declared schema change, the model's own `replay_on_change` decides the replay: | `replay_on_change` | Effect on an incremental model | |--------------------|-------------------------------| diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md index bc9786ede5..1d7f0702e8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md @@ -130,9 +130,11 @@ plan or build is a one-run hard ceiling and explicit authorization for an intent ## Replay, full refresh and schema changes - `replay_on_change` decides how much to reprocess when the model's own query changes, a - function it calls directly changes, or its schema changes: `forward` (default), `full`, or - `bounded-` such as `bounded-14d`. Upstream changes, first runs and renames never - replay a downstream model; it continues forward. Functions have no replay setting of their own. + function it calls directly changes, or its own declared columns change: `forward` (default), + `full`, or `bounded-` such as `bounded-14d`. Upstream changes, first runs and renames + never replay a downstream model; it continues forward. Columns inferred from upstream (for + example through `SELECT *`) evolve via `on_schema_change`, and `__ref`/`cursor_inputs` names + rewritten by a `migrate_from` rename are not a change. Functions have no replay setting. - `--full-refresh` rebuilds selected models unless a model sets `full_refresh false`; `full_refresh true` always rebuilds. A `full_refresh false` model is only fully rebuilt by its own first run or its own query change with `replay_on_change full`; anything else fails the plan diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 43de45020e..188a81a9d0 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -20,6 +20,7 @@ changed_local_function_names, non_function_identity_metadata_payload, version_identity_metadata_payload, + with_origin_cursor_input_names, ) from sqlbuild.compiler.planner._helpers.changes.policy import ( pick_more_aggressive, @@ -47,7 +48,7 @@ SchemaFinding, WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind +from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind, SchemaColumnSource from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql @@ -227,14 +228,19 @@ def detect_model_changes( if changed_functions else version_identity_metadata_payload ) + references_only: bool = model_name in snapshot.reference_only_changes config_changed: bool = recorded_metadata_json is not None and identity_payload( - metadata_json + with_origin_cursor_input_names( + metadata_json=metadata_json, renamed_refs=snapshot.renamed_refs + ) + if references_only + else metadata_json ) != identity_payload(recorded_metadata_json) replay_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) if query_change_tracking and fingerprint is not None: debug_logger: logging.Logger = logging.getLogger("sqlbuild.planner.changes") compiled_query_hash: str = compute_query_hash(model.query_sql) - query_changed = detect_query_change( + query_changed = not references_only and detect_query_change( compiled_query_hash=compiled_query_hash, fingerprint=fingerprint, ) @@ -280,7 +286,7 @@ def detect_model_changes( ), dialect=snapshot.column_dialect, ) - if schema_findings: + if any(finding.source == SchemaColumnSource.YML for finding in schema_findings): schema_backfill = resolve_replay_on_change( replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py index f76b1c2d02..a82bb1665e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py @@ -4,6 +4,7 @@ import json +from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY from sqlbuild.compiler.planner.constants import ( LOCAL_FUNCTION_HASHES_METADATA_KEY, @@ -64,3 +65,31 @@ def _local_function_hashes(metadata_json: str) -> dict[str, object]: return {} hashes: object = payload.get(LOCAL_FUNCTION_HASHES_METADATA_KEY) return {str(name): value for name, value in hashes.items()} if isinstance(hashes, dict) else {} + + +def with_origin_cursor_input_names(*, metadata_json: str, renamed_refs: dict[str, str]) -> str: + """Return metadata JSON whose cursor_inputs name renamed models by their origin names.""" + + try: + payload: object = json.loads(metadata_json) + except json.JSONDecodeError: + return metadata_json + config: object = payload.get("config") if isinstance(payload, dict) else None + cursor_inputs: object = ( + config.get(CURSOR_INPUTS_CONFIG_KEY) if isinstance(config, dict) else None + ) + if ( + not isinstance(payload, dict) + or not isinstance(config, dict) + or not isinstance(cursor_inputs, dict) + ): + return metadata_json + origin_inputs: dict[str, object] = { + renamed_refs.get(str(name), str(name)): value for name, value in cursor_inputs.items() + } + return json.dumps( + {**payload, "config": {**config, CURSOR_INPUTS_CONFIG_KEY: origin_inputs}}, + sort_keys=True, + separators=(",", ":"), + default=str, + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index d0a0c7daf2..ca3d5ff281 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -75,7 +75,7 @@ def discover_model_migrations( ) if not triggered and not manual_requests: return ModelMigrationDiscovery() - metadata_jsons: dict[str, str] = _metadata_jsons(runtime=runtime, models=models) + metadata_jsons: dict[str, str] = migration_metadata_jsons(runtime=runtime, models=models) dialect: str | None = runtime.adapter.sql_analysis_dialect() state.inspect_schemas(schemas=project_schemas) declared_renames: dict[str, str] = _declared_renames(declarations=declarations, state=state) @@ -353,9 +353,11 @@ def _new_model_names( ) -def _metadata_jsons( +def migration_metadata_jsons( *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] ) -> dict[str, str]: + """Return each model's current version identity metadata JSON.""" + function_hashes: dict[str, str] = build_function_local_hashes( functions=runtime.project.functions ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 8ac44b3272..efc042938a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -28,8 +28,12 @@ ) from sqlbuild.compiler.planner._helpers.migrations.discovery import ( discover_model_migrations, + migration_metadata_jsons, stored_migration_fingerprint, ) +from sqlbuild.compiler.planner._helpers.migrations.fingerprint import ( + migration_fingerprint_ref_names, +) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) @@ -209,6 +213,11 @@ def plan_model_migrations( ), ) renamed: frozenset[str] = _renamed_models(entries=tuple(entries), handovers=handovers) + renamed_refs: dict[str, str] = { + entry.model_name: entry.origin_model + for entry in entries + if entry.model_name in handovers and entry.origin_model is not None + } return ModelMigrationPlanning( snapshot=replace( _overlay_snapshot( @@ -222,6 +231,14 @@ def plan_model_migrations( deferral=deferral, ), renamed_models=renamed, + renamed_refs=renamed_refs, + reference_only_changes=_reference_only_changes( + runtime=runtime, + scope=scope, + snapshot=snapshot, + renamed_refs=renamed_refs, + fingerprints=fingerprints, + ), ), entries=tuple(entries), warnings=( @@ -246,6 +263,47 @@ def _renamed_models( ) +def _reference_only_changes( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + renamed_refs: dict[str, str], + fingerprints: MigrationFingerprintCache, +) -> frozenset[str]: + """Return selected models whose query differs only by references to renamed models.""" + + candidates: tuple[CompiledModel, ...] = tuple( + model + for key in scope.execution_order + if key in scope.selected_keys + and (model := scope.models_by_name.get(key.name)) is not None + and model.name not in renamed_refs + and (recorded := snapshot.fingerprints.models.get(model.name)) is not None + and compute_query_hash(model.query_sql) != recorded.definition_hash + and stored_migration_fingerprint(recorded) is not None + ) + if not candidates: + return frozenset() + metadata_jsons: dict[str, str] = migration_metadata_jsons(runtime=runtime, models=candidates) + dialect: str | None = runtime.adapter.sql_analysis_dialect() + return frozenset( + model.name + for model in candidates + if migration_fingerprint_ref_names( + query_sql=model.query_sql, metadata_json=metadata_jsons[model.name] + ) + & frozenset(renamed_refs) + and fingerprints.fingerprint( + query_sql=model.query_sql, + metadata_json=metadata_jsons[model.name], + ref_identities=renamed_refs, + dialect=dialect, + ) + == stored_migration_fingerprint(snapshot.fingerprints.models[model.name]) + ) + + def _equivalent_definition( *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprints: tuple[str, ...] ) -> Fingerprint | None: diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 78643b2f4b..7f3433d2e1 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -635,6 +635,8 @@ class WarehouseSnapshot: source_freshness_state_schemas: frozenset[str] = field(default_factory=frozenset) column_dialect: str | None = None renamed_models: frozenset[str] = field(default_factory=frozenset) + renamed_refs: dict[str, str] = field(default_factory=dict) + reference_only_changes: frozenset[str] = field(default_factory=frozenset) old_name_view_state_schemas: frozenset[str] = field(default_factory=frozenset) listed_state_schemas: frozenset[str] | None = field(default_factory=frozenset) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py index 877a1dc845..b06eb0b334 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py @@ -228,6 +228,91 @@ def build_function_project(*, tmp_path: Path, caller_config: str) -> Path: return project_dir +def direct_reference_merge_files() -> dict[str, str]: + """Return a protected full-replay merge model that reads the upstream table directly.""" + + return { + "models/down_merge.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [id],\n" + " cursor event_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (\n upstream event_date,\n ),\n" + " replay_on_change full,\n" + " full_refresh false,\n" + ");\n\n" + 'SELECT\n id,\n event_date\nFROM __ref("upstream")\n' + ) + } + + +def edit_direct_reference_merge(*, project_dir: Path) -> None: + """Make a real query edit to the direct-reference merge model.""" + + path: Path = project_dir / "models/down_merge.sql" + path.write_text( + path.read_text(encoding="utf-8").replace( + " event_date\nFROM", " event_date,\n id * 2 AS doubled_id\nFROM" + ), + encoding="utf-8", + ) + + +def build_star_project(*, tmp_path: Path, downstream_config: str) -> Path: + """Write and build a table and an append incremental that selects all of its columns.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="star_projection", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/up.sql": ( + "MODEL (\n materialized table,\n);\n\n" + "SELECT 1 AS id, DATE '2026-09-01' AS event_date\n" + ), + "models/down.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy append,\n" + " replay_on_change full,\n" + f"{downstream_config}" + ");\n\n" + 'SELECT * FROM __ref("up")\n' + ), + }, + ) + _build_ok(project_dir=project_dir) + return project_dir + + +def add_upstream_column(*, project_dir: Path) -> None: + """Add a column to the star project's upstream table.""" + + path: Path = project_dir / "models/up.sql" + path.write_text( + path.read_text(encoding="utf-8").replace( + "AS event_date", "AS event_date, 'web' AS channel" + ), + encoding="utf-8", + ) + + +def declare_downstream_column_type(*, project_dir: Path) -> None: + """Declare a new type for the star downstream's id column in its own header.""" + + path: Path = project_dir / "models/down.sql" + path.write_text( + path.read_text(encoding="utf-8").replace( + " replay_on_change full,\n", + " replay_on_change full,\n columns (\n id (type BIGINT),\n ),\n", + ), + encoding="utf-8", + ) + + def plan(*, project_dir: Path, args: tuple[str, ...] = ()) -> subprocess.CompletedProcess[str]: """Run sqb plan without colour.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py index 5158328aa6..b08b7a7bfb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py @@ -15,11 +15,16 @@ RenamePlanBuildTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.own_change_replay.helpers import ( + add_upstream_column, assert_plan_fails, assert_plan_text, build_and_assert_rows, build_event_project, build_function_project, + build_star_project, + declare_downstream_column_type, + direct_reference_merge_files, + edit_direct_reference_merge, insert_table_between_upstream_and_view, mark_first_day, merge_downstream_files, @@ -357,5 +362,155 @@ def test_given_full_refresh_flag_on_cursor_intrinsic_merge_when_planning_then_na ) +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="downstream references rewritten by sqb rename are not a query change", + expected_plan_fragments=("Renamed (1)", "down_merge merge (timestamp)"), + unexpected_plan_fragments=("Query changed", "full rebuild", "S203"), + expected_reasons={ + "upstream_renamed": "renamed", + "upstream_v": "no_change", + "down_merge": "normal_incremental", + "downstream_mb": "normal_incremental", + }, + expected_rows={"down_merge": _REPLAYED_TAIL, "downstream_mb": _REPLAYED_TAIL}, + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_upstream_when_downstream_refs_are_rewritten_then_downstream_continues_forward( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project( + tmp_path=tmp_path, extra_files=direct_reference_merge_files() + ) + mark_first_day(project_dir=project_dir, relations=("down_merge", "downstream_mb")) + rename_upstream_with_cli(project_dir=project_dir, extra_sql="") + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + assert models["upstream_v"]["action"] == "create_view" + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + rebuilt: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + assert (rebuilt["down_merge"]["reason"], rebuilt["upstream_v"]["reason"]) == ( + "normal_incremental", + "no_change", + ) + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="a real edit alongside rewritten references is a query change", + expected_plan_fragments=("Query changed (1)", "down_merge full rebuild"), + unexpected_plan_fragments=("S203",), + expected_reasons={"down_merge": "query_changed"}, + expected_rows={}, + ) + ], + ids=lambda case: case.description, +) +def test_given_renamed_upstream_and_real_downstream_edit_when_planning_then_query_change_applies( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project( + tmp_path=tmp_path, extra_files=direct_reference_merge_files() + ) + rename_upstream_with_cli(project_dir=project_dir, extra_sql="") + edit_direct_reference_merge(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + assert models["down_merge"]["action"] == "create_table" + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="upstream column added to a star downstream evolves it forward", + expected_plan_fragments=("Schema changed (1)", "down continue forward"), + unexpected_plan_fragments=("full rebuild", "S203"), + expected_reasons={"down": "schema_changed"}, + expected_rows={ + "down": [(1, date(2026, 9, 1), None), (1, date(2026, 9, 1), "web")], + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_upstream_column_added_when_downstream_selects_star_then_no_replay( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_star_project( + tmp_path=tmp_path, downstream_config=" full_refresh false,\n" + ) + add_upstream_column(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + assert models["down"]["backfill"] == {"action": "forward", "duration": None} + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="own declared column type change follows replay_on_change full", + expected_plan_fragments=("down full rebuild",), + unexpected_plan_fragments=("S203",), + expected_reasons={"down": "schema_changed"}, + expected_rows={}, + ) + ], + ids=lambda case: case.description, +) +def test_given_own_declared_column_change_when_planning_then_own_replay_policy_applies( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_star_project(tmp_path=tmp_path, downstream_config="") + declare_downstream_column_type(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + assert models["down"]["backfill"] == {"action": "full", "duration": None} + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 929dfea053..c65f2c3361 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -248,6 +248,42 @@ expected_change_kind=ChangeKind.NO_CHANGE, expected_backfill_action=BackfillAction.FORWARD_ONLY, ), + DetectModelChangesTestCase( + description="inferred upstream column never replays the model", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={"materialized": "incremental", "replay_on_change": "full"}, + fingerprint_config_values={"materialized": "incremental", "replay_on_change": "full"}, + schema_columns=(), + relation_exists=True, + fingerprint_query_hash=_MATCHING_HASH, + warehouse_column_names=(("id", "INTEGER"),), + sql_analysis_enabled=True, + query_change_tracking=True, + full_refresh=False, + inferred_columns=( + InferredColumn(name="id", type="INTEGER"), + InferredColumn(name="channel", type="VARCHAR"), + ), + expected_change_kind=ChangeKind.SCHEMA_CHANGED, + expected_backfill_action=BackfillAction.FORWARD_ONLY, + ), + DetectModelChangesTestCase( + description="own declared column follows the model's replay policy", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={"materialized": "incremental", "replay_on_change": "full"}, + fingerprint_config_values={"materialized": "incremental", "replay_on_change": "full"}, + schema_columns=(("id", "INTEGER"), ("channel", "VARCHAR")), + relation_exists=True, + fingerprint_query_hash=_MATCHING_HASH, + warehouse_column_names=(("id", "INTEGER"),), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + expected_change_kind=ChangeKind.SCHEMA_CHANGED, + expected_backfill_action=BackfillAction.FULL, + ), DetectModelChangesTestCase( description="does not report passthrough columns removed for unresolved star", model_name="orders", diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index f4e052855b..6aebcb410e 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -357,7 +357,7 @@ MODEL ( ## Replay on change -When a model's own query changes, a function it calls directly changes, or its schema changes, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: +When a model's own query changes, a function it calls directly changes, or its own declared columns change, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: | Value | Effect | |-------|--------| diff --git a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx index cd0fa0513e..1ba71c7330 100644 --- a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx +++ b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx @@ -11,11 +11,13 @@ A model's own change is any of: - **Query changed**: its compiled SQL differs from the recorded definition. - **Function changed**: a UDF or table function that the model calls directly has a new definition. The model's SQL text is unchanged, but its results can differ, so this counts as a query change of the model. Models further downstream do not see it. -- **Schema changed**: its declared or inferred columns differ from the warehouse. +- **Schema changed**: its declared or inferred columns differ from the warehouse. Only a change to the model's own declared columns (its `columns` block, contract, or schema YAML) follows `replay_on_change`. Columns inferred from its unchanged SQL, for example a column an upstream added that `SELECT *` passes through, are applied forward through [`on_schema_change`](/docs/concepts/incremental#on_schema_change) and never replay history. - **Config changed**: version-relevant configuration differs. A config change alone does not replay history. - **First run**: the model's relation does not exist yet. Only that model is built from scratch. -For a query, function, or schema change, the model's own `replay_on_change` decides the replay: +References rewritten by a rename are not a change of their own. When an upstream model is renamed with `migrate_from` (for example by [`sqb rename`](/docs/cli/rename)), a downstream model whose only difference is the rewritten `__ref` and `cursor_inputs` name keeps its identity: views are recreated as usual, and incremental models continue forward. A downstream model that also has a real edit is a normal query change. + +For a query, function, or own declared schema change, the model's own `replay_on_change` decides the replay: | `replay_on_change` | Effect on an incremental model | |--------------------|-------------------------------| From 8051dd924d3302721b3c24eb267e23cde365775a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:59:28 +0100 Subject: [PATCH 09/12] fix(planner): match rename-rewritten references by query hash and recorded renames A downstream model is no longer a query change when its compiled SQL, with __ref names of renamed models substituted back to their origin names, hashes to its stored definition hash. Substitution touches only reference calls outside comments and string literals, so it works for any SQL, including models the migration fingerprint parser cannot normalize. The rename map combines this plan's handovers with renames recorded in migration history after the downstream's last build, so renames built in an earlier run or whose migrate_from was removed are recognized too. Migration events are read only when a selected model's query hash changed and the migration table was already listed by the warehouse snapshot. A column removed from the warehouse is attributed to the model's declared columns only when no inferred columns exist, so an upstream dropping a column a partial columns block does not declare evolves forward instead of following replay_on_change. --- .../concepts/planning/replay-decisions.md | 2 +- .../planner/_helpers/changes/detect.py | 23 ++- .../_helpers/changes/recorded_renames.py | 72 +++++++ .../_helpers/changes/reference_renames.py | 105 ++++++++++ .../planner/_helpers/changes/schema.py | 2 +- .../planner/_helpers/migrations/discovery.py | 6 +- .../planner/_helpers/migrations/planning.py | 78 +++----- .../planner/_helpers/warehouse/snapshot.py | 9 +- .../classes/migration_state_inspection.py | 29 ++- src/sqlbuild/compiler/planner/models.py | 13 +- .../main/build/own_change_replay/helpers.py | 74 +++++++ .../test_recorded_renames.py | 184 ++++++++++++++++++ .../planner/_helpers/changes/_test_helpers.py | 1 + .../planner/_helpers/changes/_test_types.py | 12 +- .../planner/_helpers/changes/test_detect.py | 34 ++++ .../changes/test_reference_renames.py | 111 +++++++++++ .../concepts/planning/replay-decisions.mdx | 2 +- 17 files changed, 676 insertions(+), 81 deletions(-) create mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_recorded_renames.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md index 3c7ea800ae..097ce49bd0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md @@ -18,7 +18,7 @@ A model's own change is any of: - **Config changed**: version-relevant configuration differs. A config change alone does not replay history. - **First run**: the model's relation does not exist yet. Only that model is built from scratch. -References rewritten by a rename are not a change of their own. When an upstream model is renamed with `migrate_from` (for example by [`sqb rename`](../../cli/rename.md)), a downstream model whose only difference is the rewritten `__ref` and `cursor_inputs` name keeps its identity: views are recreated as usual, and incremental models continue forward. A downstream model that also has a real edit is a normal query change. +References rewritten by a rename are not a change of their own. When an upstream model is renamed with `migrate_from` (for example by [`sqb rename`](../../cli/rename.md)), a downstream model whose only difference is the rewritten `__ref` and `cursor_inputs` name keeps its identity: views are recreated as usual, and incremental models continue forward. This also holds when the renamed model was built in an earlier run, for example by a selective build, or when its `migrate_from` was removed afterwards, because recorded renames are read from migration history. A downstream model that also has a real edit is a normal query change. For a query, function, or own declared schema change, the model's own `replay_on_change` decides the replay: diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 188a81a9d0..60c4fc2652 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -27,6 +27,7 @@ resolve_replay_on_change, ) from sqlbuild.compiler.planner._helpers.changes.query import detect_query_change +from sqlbuild.compiler.planner._helpers.changes.reference_renames import origin_reference_names from sqlbuild.compiler.planner._helpers.changes.schema import detect_schema_changes from sqlbuild.compiler.planner._helpers.identity.functions import ( build_compiled_function_fingerprint_sql, @@ -228,19 +229,29 @@ def detect_model_changes( if changed_functions else version_identity_metadata_payload ) - references_only: bool = model_name in snapshot.reference_only_changes - config_changed: bool = recorded_metadata_json is not None and identity_payload( - with_origin_cursor_input_names( - metadata_json=metadata_json, renamed_refs=snapshot.renamed_refs + origin_names: dict[str, str] | None = ( + origin_reference_names( + query_sql=model.query_sql, + recorded=fingerprint, + renames=snapshot.reference_renames, + dialect=snapshot.column_dialect, ) - if references_only + if query_change_tracking + and fingerprint is not None + and snapshot.reference_renames + and compute_query_hash(model.query_sql) != fingerprint.definition_hash + else None + ) + config_changed: bool = recorded_metadata_json is not None and identity_payload( + with_origin_cursor_input_names(metadata_json=metadata_json, renamed_refs=origin_names) + if origin_names is not None else metadata_json ) != identity_payload(recorded_metadata_json) replay_backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) if query_change_tracking and fingerprint is not None: debug_logger: logging.Logger = logging.getLogger("sqlbuild.planner.changes") compiled_query_hash: str = compute_query_hash(model.query_sql) - query_changed = not references_only and detect_query_change( + query_changed = origin_names is None and detect_query_change( compiled_query_hash=compiled_query_hash, fingerprint=fingerprint, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py new file mode 100644 index 0000000000..1c9634cba5 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py @@ -0,0 +1,72 @@ +"""Collect the model renames that downstream references may have been rewritten for.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.planner.classes.migration_state_inspection import ( + MigrationStateInspection, +) +from sqlbuild.compiler.planner.models import ( + PlannerRuntime, + PlannerScope, + ReferenceRename, + WarehouseSnapshot, +) +from sqlbuild.compiler.references.types import SqlReferenceKind + + +def plan_reference_renames( + *, + runtime: PlannerRuntime, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + state: MigrationStateInspection, + schemas: set[str], + planned: dict[str, str], +) -> tuple[ReferenceRename, ...]: + """Return this plan's renames plus recorded renames, read only when a query changed.""" + + if not _has_changed_referencing_query(scope=scope, snapshot=snapshot): + return () + state.inspect_migration_events( + schemas={schema for schema in schemas if schema.lower() in snapshot.migration_state_schemas} + ) + target_name: str | None = runtime.project.effective_target_name + return ( + *( + ReferenceRename(new_name=new_name, origin_name=origin_name) + for new_name, origin_name in planned.items() + if new_name != origin_name + ), + *( + ReferenceRename( + new_name=event.destination_model, + origin_name=event.origin_model, + recorded_at=event.created_at, + ) + for event in state.events + if event.target_name in (None, target_name) + and event.origin_model is not None + and event.origin_model != event.destination_model + ), + ) + + +def _has_changed_referencing_query(*, scope: PlannerScope, snapshot: WarehouseSnapshot) -> bool: + selected: tuple[CompiledModel, ...] = tuple( + model + for key in scope.selected_keys + if (model := scope.models_by_name.get(key.name)) is not None + ) + return any(_referencing_query_changed(model=model, snapshot=snapshot) for model in selected) + + +def _referencing_query_changed(*, model: CompiledModel, snapshot: WarehouseSnapshot) -> bool: + recorded: Fingerprint | None = snapshot.fingerprints.models.get(model.name) + return ( + recorded is not None + and any(reference.ref_kind == SqlReferenceKind.REF for reference in model.references) + and compute_query_hash(model.query_sql) != recorded.definition_hash + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py new file mode 100644 index 0000000000..c5d439c829 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py @@ -0,0 +1,105 @@ +"""Recognize model queries whose only change is references rewritten for renamed models.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime +from itertools import islice, product + +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.planner.models import ReferenceRename +from sqlbuild.lint.main.scan_interpolation_sites import scan_interpolation_sites +from sqlbuild.lint.models import InterpolationSite + +_REF_CALL: re.Pattern[str] = re.compile( + r"^__ref\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" +) +_GENERIC_DIALECT: str = "generic" +_MAX_CANDIDATE_MAPPINGS: int = 64 + +type _NameSpan = tuple[int, int, str] + + +def origin_reference_names( + *, + query_sql: str, + recorded: Fingerprint, + renames: tuple[ReferenceRename, ...], + dialect: str | None, +) -> dict[str, str] | None: + """Return the rename mapping under which the query hashes to its recorded definition.""" + + origins: dict[str, list[str]] = {} + rename: ReferenceRename + for rename in renames: + if rename.recorded_at is None or _utc(recorded.ts) < _utc(rename.recorded_at): + origins.setdefault(rename.new_name, []).append(rename.origin_name) + if not origins: + return None + spans: tuple[_NameSpan, ...] = _ref_name_spans( + query_sql=query_sql, dialect=dialect or _GENERIC_DIALECT, names=frozenset(origins) + ) + names: tuple[str, ...] = tuple(sorted({name for _, _, name in spans})) + choices: tuple[tuple[str, ...], ...] = tuple( + _lineage(name=name, origins=origins) for name in names + ) + combination: tuple[str, ...] + for combination in islice(product(*choices), _MAX_CANDIDATE_MAPPINGS): + mapping: dict[str, str] = { + name: origin for name, origin in zip(names, combination, strict=True) if origin != name + } + if mapping and compute_query_hash( + _substitute(query_sql=query_sql, spans=spans, mapping=mapping) + ) == (recorded.definition_hash): + return mapping + return None + + +def _ref_name_spans( + *, query_sql: str, dialect: str, names: frozenset[str] +) -> tuple[_NameSpan, ...]: + spans: list[_NameSpan] = [] + site: InterpolationSite + for site in scan_interpolation_sites(body=query_sql, dialect=dialect): + match: re.Match[str] | None = _REF_CALL.match(site.original_text) + if match is not None and match.group("name") in names: + spans.append( + ( + site.original_start + match.start("name"), + site.original_start + match.end("name"), + match.group("name"), + ) + ) + return tuple(spans) + + +def _lineage(*, name: str, origins: dict[str, list[str]]) -> tuple[str, ...]: + seen: list[str] = [name] + index: int = 0 + while index < len(seen): + origin: str + for origin in origins.get(seen[index], ()): + if origin not in seen: + seen.append(origin) + index += 1 + return tuple(seen) + + +def _substitute(*, query_sql: str, spans: tuple[_NameSpan, ...], mapping: dict[str, str]) -> str: + pieces: list[str] = [] + copied_to: int = 0 + start: int + end: int + name: str + for start, end, name in spans: + pieces.extend((query_sql[copied_to:start], mapping.get(name, name))) + copied_to = end + pieces.append(query_sql[copied_to:]) + return "".join(pieces) + + +def _utc(value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py index 6da7f7e8f8..a2841c2f90 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py @@ -120,7 +120,7 @@ def detect_schema_changes( column_name=col_name, source=( SchemaColumnSource.YML - if yml_columns + if yml_columns and inferred_columns is None else SchemaColumnSource.SQL_ANALYSIS ), actual_type=col_type, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index ca3d5ff281..d0a0c7daf2 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -75,7 +75,7 @@ def discover_model_migrations( ) if not triggered and not manual_requests: return ModelMigrationDiscovery() - metadata_jsons: dict[str, str] = migration_metadata_jsons(runtime=runtime, models=models) + metadata_jsons: dict[str, str] = _metadata_jsons(runtime=runtime, models=models) dialect: str | None = runtime.adapter.sql_analysis_dialect() state.inspect_schemas(schemas=project_schemas) declared_renames: dict[str, str] = _declared_renames(declarations=declarations, state=state) @@ -353,11 +353,9 @@ def _new_model_names( ) -def migration_metadata_jsons( +def _metadata_jsons( *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] ) -> dict[str, str]: - """Return each model's current version identity metadata JSON.""" - function_hashes: dict[str, str] = build_function_local_hashes( functions=runtime.project.functions ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index efc042938a..f04432235f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -22,18 +22,15 @@ MigrationDiscovery, MigrationPromotion, ) +from sqlbuild.compiler.planner._helpers.changes.recorded_renames import plan_reference_renames from sqlbuild.compiler.planner._helpers.migrations.compatibility import ( MigrationCompatibilityResult, check_migration_compatibility, ) from sqlbuild.compiler.planner._helpers.migrations.discovery import ( discover_model_migrations, - migration_metadata_jsons, stored_migration_fingerprint, ) -from sqlbuild.compiler.planner._helpers.migrations.fingerprint import ( - migration_fingerprint_ref_names, -) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) @@ -167,7 +164,20 @@ def plan_model_migrations( fingerprints=fingerprints, ) if not discovery.requests: - return ModelMigrationPlanning(snapshot=snapshot, warnings=discovery.warnings) + return ModelMigrationPlanning( + snapshot=replace( + snapshot, + reference_renames=plan_reference_renames( + runtime=runtime, + scope=scope, + snapshot=snapshot, + state=state, + schemas=schemas, + planned={}, + ), + ), + warnings=discovery.warnings, + ) resolved: list[tuple[ModelMigrationRequest, CompiledRelationLocation, str | None]] = [ (request, *_resolve_origin(request=request, runtime=runtime, state=state)) for request in discovery.requests @@ -213,11 +223,6 @@ def plan_model_migrations( ), ) renamed: frozenset[str] = _renamed_models(entries=tuple(entries), handovers=handovers) - renamed_refs: dict[str, str] = { - entry.model_name: entry.origin_model - for entry in entries - if entry.model_name in handovers and entry.origin_model is not None - } return ModelMigrationPlanning( snapshot=replace( _overlay_snapshot( @@ -231,13 +236,17 @@ def plan_model_migrations( deferral=deferral, ), renamed_models=renamed, - renamed_refs=renamed_refs, - reference_only_changes=_reference_only_changes( + reference_renames=plan_reference_renames( runtime=runtime, scope=scope, snapshot=snapshot, - renamed_refs=renamed_refs, - fingerprints=fingerprints, + state=state, + schemas=schemas, + planned={ + entry.model_name: entry.origin_model + for entry in entries + if entry.model_name in handovers and entry.origin_model is not None + }, ), ), entries=tuple(entries), @@ -263,47 +272,6 @@ def _renamed_models( ) -def _reference_only_changes( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - renamed_refs: dict[str, str], - fingerprints: MigrationFingerprintCache, -) -> frozenset[str]: - """Return selected models whose query differs only by references to renamed models.""" - - candidates: tuple[CompiledModel, ...] = tuple( - model - for key in scope.execution_order - if key in scope.selected_keys - and (model := scope.models_by_name.get(key.name)) is not None - and model.name not in renamed_refs - and (recorded := snapshot.fingerprints.models.get(model.name)) is not None - and compute_query_hash(model.query_sql) != recorded.definition_hash - and stored_migration_fingerprint(recorded) is not None - ) - if not candidates: - return frozenset() - metadata_jsons: dict[str, str] = migration_metadata_jsons(runtime=runtime, models=candidates) - dialect: str | None = runtime.adapter.sql_analysis_dialect() - return frozenset( - model.name - for model in candidates - if migration_fingerprint_ref_names( - query_sql=model.query_sql, metadata_json=metadata_jsons[model.name] - ) - & frozenset(renamed_refs) - and fingerprints.fingerprint( - query_sql=model.query_sql, - metadata_json=metadata_jsons[model.name], - ref_identities=renamed_refs, - dialect=dialect, - ) - == stored_migration_fingerprint(snapshot.fingerprints.models[model.name]) - ) - - def _equivalent_definition( *, model: CompiledModel, handover: Fingerprint | None, destination_fingerprints: tuple[str, ...] ) -> Fingerprint | None: diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index c1935082a9..86d63659f7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -33,7 +33,7 @@ from sqlbuild.compiler.fingerprints.main.read import read_latest_fingerprints from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many -from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_TABLE_NAME +from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME from sqlbuild.compiler.planner._helpers.graph.buildability import ( check_buildability, missing_upstream_message, @@ -167,6 +167,7 @@ class _StateTableSchemas: fingerprints: frozenset[str] source_freshness: frozenset[str] old_name_views: frozenset[str] + migrations: frozenset[str] def build_warehouse_snapshot( @@ -307,6 +308,7 @@ def gather_warehouse_snapshot( source_freshness_state_schemas=freshness_state_schemas, column_dialect=adapter.sql_analysis_dialect(), old_name_view_state_schemas=state_schemas.old_name_views, + migration_state_schemas=state_schemas.migrations, listed_state_schemas=( frozenset(schema.lower() for schema in query_schemas) if query_schemas is not None @@ -489,6 +491,7 @@ def _build_metadata_name_filter( names.add(FINGERPRINT_TABLE_NAME) names.add(SOURCE_FRESHNESS_TABLE_NAME) names.add(OLD_NAME_VIEW_TABLE_NAME) + names.add(MIGRATION_TABLE_NAME) return tuple(sorted(names)) @@ -552,8 +555,11 @@ def _gather_relations( fingerprint_schemas: set[str] = set() freshness_schemas: set[str] = set() old_name_schemas: set[str] = set() + migration_schemas: set[str] = set() relation: RelationInfo for relation in relations: + if relation.name == MIGRATION_TABLE_NAME and relation.schema is not None: + migration_schemas.add(relation.schema.lower()) if relation.name == OLD_NAME_VIEW_TABLE_NAME: if relation.schema is not None: old_name_schemas.add(relation.schema.lower()) @@ -571,6 +577,7 @@ def _gather_relations( fingerprints=frozenset(fingerprint_schemas), source_freshness=frozenset(freshness_schemas), old_name_views=frozenset(old_name_schemas), + migrations=frozenset(migration_schemas), ) diff --git a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py index 3824981075..635e9d6b60 100644 --- a/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py +++ b/src/sqlbuild/compiler/planner/classes/migration_state_inspection.py @@ -43,6 +43,7 @@ def __init__(self, *, adapter: BaseAdapter, connection: Any, database: str | Non self._read_schemas: set[str] = set() self._fingerprints: dict[str, tuple[Fingerprint, ...]] = {} self._events: list[MigrationEvent] = [] + self._event_schemas: set[str] = set() self._relations: dict[tuple[str, str], RelationInfo] = {} self._columns: dict[tuple[str, str], tuple[ColumnInfo, ...]] = {} self._column_event_schemas: set[str] = set() @@ -158,6 +159,24 @@ def inspect_old_name_views(self, *, schemas: set[str]) -> None: ) ) + def inspect_migration_events(self, *, schemas: set[str]) -> None: + """Read migration events once from schemas known to hold the migration table.""" + + schema: str + for schema in sorted(schemas): + if schema.lower() in self._event_schemas: + continue + self._event_schemas.add(schema.lower()) + self._events.extend( + read_migration_events( + connection=self._connection, + execute=self._adapter.execute, + database=self._database, + schema=schema, + render_qualified_name=self._adapter.render_qualified_name, + ) + ) + def inspect_schemas(self, *, schemas: set[str]) -> None: """Read state tables once for every schema not inspected yet.""" @@ -179,15 +198,7 @@ def inspect_schemas(self, *, schemas: set[str]) -> None: schema: str for schema in pending: if (schema.lower(), MIGRATION_TABLE_NAME) in state_tables: - self._events.extend( - read_migration_events( - connection=self._connection, - execute=self._adapter.execute, - database=self._database, - schema=schema, - render_qualified_name=self._adapter.render_qualified_name, - ) - ) + self.inspect_migration_events(schemas={schema}) if (schema.lower(), FINGERPRINT_TABLE_NAME) in state_tables: self._fingerprints[schema.lower()] = self._read_model_fingerprints(schema=schema) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 7f3433d2e1..2c2c510c49 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -624,6 +624,15 @@ class MaximumStartPolicyInputs: incremental_mode: str | None = None +@dataclass(frozen=True) +class ReferenceRename: + """One model rename that downstream references may have been rewritten for.""" + + new_name: str + origin_name: str + recorded_at: datetime | None = None + + @dataclass(frozen=True) class WarehouseSnapshot: """Frozen point-in-time picture of warehouse state for planning.""" @@ -635,8 +644,8 @@ class WarehouseSnapshot: source_freshness_state_schemas: frozenset[str] = field(default_factory=frozenset) column_dialect: str | None = None renamed_models: frozenset[str] = field(default_factory=frozenset) - renamed_refs: dict[str, str] = field(default_factory=dict) - reference_only_changes: frozenset[str] = field(default_factory=frozenset) + reference_renames: tuple[ReferenceRename, ...] = () + migration_state_schemas: frozenset[str] = field(default_factory=frozenset) old_name_view_state_schemas: frozenset[str] = field(default_factory=frozenset) listed_state_schemas: frozenset[str] | None = field(default_factory=frozenset) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py index b06eb0b334..e7fba5fa27 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py @@ -313,6 +313,80 @@ def declare_downstream_column_type(*, project_dir: Path) -> None: ) +def udf_reference_merge_files() -> dict[str, str]: + """Return a protected full-replay merge model that reads the upstream through a UDF.""" + + return { + "functions/sql/udf__plus_one.sql": ( + "FUNCTION (\n arguments (value INTEGER),\n returns INTEGER,\n);\n\nvalue + 1\n" + ), + "models/down_merge.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [id],\n" + " cursor event_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (\n upstream event_date,\n ),\n" + " replay_on_change full,\n" + " full_refresh false,\n" + ");\n\n" + '-- reads __ref("upstream")\n' + "SELECT\n u.id,\n u.event_date,\n" + ' __udf("udf__plus_one")(u.id) AS next_id\n' + 'FROM __ref("upstream") AS u\n' + ), + } + + +def incremental_upstream_files() -> dict[str, str]: + """Return an incremental upstream so a rename moves its data instead of renaming it.""" + + return { + "models/upstream.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [id],\n" + " cursor event_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (\n events event_date,\n ),\n" + ");\n\n" + "SELECT\n id,\n CAST(event_date AS DATE) AS event_date\n" + 'FROM __seed("events")\n' + ), + } + + +def build_selected(*, project_dir: Path, selector: str) -> None: + """Build only the selected nodes.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", selector), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def remove_migrate_from(*, project_dir: Path) -> None: + """Drop the migrate_from declaration from the renamed upstream model.""" + + path: Path = project_dir / "models/upstream_renamed.sql" + path.write_text( + path.read_text(encoding="utf-8").replace(" migrate_from upstream,\n", ""), + encoding="utf-8", + ) + + +def drop_upstream_column(*, project_dir: Path) -> None: + """Drop the event_date column from the star project's upstream table.""" + + (project_dir / "models/up.sql").write_text( + "MODEL (\n materialized table,\n);\n\nSELECT 1 AS id\n", encoding="utf-8" + ) + + def plan(*, project_dir: Path, args: tuple[str, ...] = ()) -> subprocess.CompletedProcess[str]: """Run sqb plan without colour.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_recorded_renames.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_recorded_renames.py new file mode 100644 index 0000000000..1c8659c6bc --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_recorded_renames.py @@ -0,0 +1,184 @@ +"""E2E coverage: rename-rewritten references and inferred upstream columns never replay a model.""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build.own_change_replay._test_types import ( + OwnChangePlanBuildTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.own_change_replay.helpers import ( + assert_plan_text, + build_and_assert_rows, + build_event_project, + build_selected, + build_star_project, + direct_reference_merge_files, + drop_upstream_column, + incremental_upstream_files, + plan_models, + remove_migrate_from, + rename_upstream_with_cli, + udf_reference_merge_files, +) + +_MERGE_ROWS: list[tuple[Any, ...]] = [ + (1, date(2026, 9, 1)), + (2, date(2026, 9, 2)), + (3, date(2026, 9, 3)), +] +_UDF_MERGE_ROWS: list[tuple[Any, ...]] = [ + (1, date(2026, 9, 1), 2), + (2, date(2026, 9, 2), 3), + (3, date(2026, 9, 3), 4), +] + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="a UDF-calling downstream of a renamed upstream continues forward", + expected_plan_fragments=("down_merge merge (timestamp)",), + unexpected_plan_fragments=("Query changed", "full rebuild", "S203"), + expected_reasons={"down_merge": "normal_incremental"}, + expected_rows={"down_merge": _UDF_MERGE_ROWS}, + ) + ], + ids=lambda case: case.description, +) +def test_given_udf_downstream_when_upstream_is_renamed_then_downstream_continues_forward( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project( + tmp_path=tmp_path, extra_files=udf_reference_merge_files() + ) + rename_upstream_with_cli(project_dir=project_dir, extra_sql="") + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + assert plan_models(project_dir=project_dir)["down_merge"]["reason"] == "normal_incremental" + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="incremental upstream renamed and built alone in an earlier run", + expected_plan_fragments=("down_merge merge (timestamp)",), + unexpected_plan_fragments=("Query changed", "full rebuild", "S203"), + expected_reasons={"down_merge": "normal_incremental"}, + expected_rows={"down_merge": _MERGE_ROWS}, + ) + ], + ids=lambda case: case.description, +) +def test_given_upstream_renamed_in_earlier_run_when_planning_then_downstream_continues_forward( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project( + tmp_path=tmp_path, + extra_files={**incremental_upstream_files(), **direct_reference_merge_files()}, + ) + rename_upstream_with_cli(project_dir=project_dir, extra_sql="") + build_selected(project_dir=project_dir, selector="upstream_renamed") + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="table renamed earlier and migrate_from removed afterwards", + expected_plan_fragments=("down_merge merge (timestamp)",), + unexpected_plan_fragments=("Query changed", "full rebuild", "S203"), + expected_reasons={"upstream_renamed": "no_change", "down_merge": "normal_incremental"}, + expected_rows={"down_merge": _MERGE_ROWS}, + ) + ], + ids=lambda case: case.description, +) +def test_given_migrate_from_removed_after_rename_when_planning_then_downstream_continues_forward( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_event_project( + tmp_path=tmp_path, extra_files=direct_reference_merge_files() + ) + rename_upstream_with_cli(project_dir=project_dir, extra_sql="") + build_selected(project_dir=project_dir, selector="upstream_renamed") + remove_migrate_from(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + build_and_assert_rows(project_dir=project_dir, expected_rows=test_case.expected_rows) + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="upstream drops a column a partial columns block does not declare", + expected_plan_fragments=("Schema changed (1)", "down continue forward"), + unexpected_plan_fragments=("full rebuild", "S203"), + expected_reasons={"down": "schema_changed"}, + expected_rows={}, + ) + ], + ids=lambda case: case.description, +) +def test_given_partial_columns_when_upstream_drops_undeclared_column_then_no_replay( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_star_project( + tmp_path=tmp_path, + downstream_config=" full_refresh false,\n columns (\n id (type INTEGER),\n ),\n", + ) + drop_upstream_column(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + assert models["down"]["backfill"] == {"action": "forward", "duration": None} + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index 292943e305..8e5fcfc25e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -214,6 +214,7 @@ def _build_schema_entry(test_case: DetectModelChangesTestCase) -> SchemaModelEnt schema_entry: SchemaModelEntry = SchemaModelEntry( name=test_case.model_name, columns=tuple(SchemaColumn(name=c[0], type=c[1]) for c in schema_cols), + type_enforcement=test_case.schema_type_enforcement, ) return (None, schema_entry)[bool(schema_cols)] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index e4e002a9ee..05ff7cbe7a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -2,7 +2,7 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner.models import BackfillResult, SchemaFinding +from sqlbuild.compiler.planner.models import BackfillResult, ReferenceRename, SchemaFinding from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind, PlanAction, PlanReason from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -27,6 +27,7 @@ class DetectModelChangesTestCase: fast_lineage_has_star: bool = False expected_recorded_build_relation_missing: bool = False renamed_by_migration: bool = False + schema_type_enforcement: bool | None = None @dataclass(frozen=True) @@ -94,3 +95,12 @@ class DetectRenamedModelTestCase: config_values: dict[str, object] expected_query_changed: bool expected_backfill: BackfillResult + + +@dataclass(frozen=True) +class OriginReferenceNamesTestCase: + description: str + query_sql: str + recorded_query_sql: str + renames: tuple[ReferenceRename, ...] + expected_mapping: dict[str, str] | None diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index c65f2c3361..f48b6e7282 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -284,6 +284,40 @@ expected_change_kind=ChangeKind.SCHEMA_CHANGED, expected_backfill_action=BackfillAction.FULL, ), + DetectModelChangesTestCase( + description="undeclared column dropped from inferred columns never replays", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={"materialized": "incremental", "replay_on_change": "full"}, + fingerprint_config_values={"materialized": "incremental", "replay_on_change": "full"}, + schema_columns=(("id", "INTEGER"),), + relation_exists=True, + fingerprint_query_hash=_MATCHING_HASH, + warehouse_column_names=(("id", "INTEGER"), ("event_date", "DATE")), + sql_analysis_enabled=True, + query_change_tracking=True, + full_refresh=False, + inferred_columns=(InferredColumn(name="id", type="INTEGER"),), + expected_change_kind=ChangeKind.SCHEMA_CHANGED, + expected_backfill_action=BackfillAction.FORWARD_ONLY, + ), + DetectModelChangesTestCase( + description="column removed from an enforced declared contract follows replay policy", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={"materialized": "incremental", "replay_on_change": "full"}, + fingerprint_config_values={"materialized": "incremental", "replay_on_change": "full"}, + schema_columns=(("id", "INTEGER"),), + schema_type_enforcement=True, + relation_exists=True, + fingerprint_query_hash=_MATCHING_HASH, + warehouse_column_names=(("id", "INTEGER"), ("event_date", "DATE")), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + expected_change_kind=ChangeKind.SCHEMA_CHANGED, + expected_backfill_action=BackfillAction.FULL, + ), DetectModelChangesTestCase( description="does not report passthrough columns removed for unresolved star", model_name="orders", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py new file mode 100644 index 0000000000..0325b7da41 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py @@ -0,0 +1,111 @@ +"""Tests for recognizing queries changed only by references rewritten for renamed models.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.planner._helpers.changes.reference_renames import origin_reference_names +from sqlbuild.compiler.planner.models import ReferenceRename +from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( + OriginReferenceNamesTestCase, +) + +_BUILT_AT: datetime = datetime(2026, 9, 1, tzinfo=UTC) +_RENAMED_AFTER_BUILD: datetime = datetime(2026, 9, 2, tzinfo=UTC) +_RENAMED_BEFORE_BUILD: datetime = datetime(2026, 8, 31, tzinfo=UTC) + + +@pytest.mark.parametrize( + "test_case", + [ + OriginReferenceNamesTestCase( + description="planned rename maps the reference back outside comments", + query_sql='-- __ref("orders")\nSELECT * FROM __ref("orders_v2")', + recorded_query_sql='-- __ref("orders")\nSELECT * FROM __ref("orders")', + renames=(ReferenceRename(new_name="orders_v2", origin_name="orders"),), + expected_mapping={"orders_v2": "orders"}, + ), + OriginReferenceNamesTestCase( + description="names inside string literals are never substituted", + query_sql='SELECT \'__ref("orders_v2")\' AS note, * FROM __ref("orders_v2")', + recorded_query_sql='SELECT \'__ref("orders")\' AS note, * FROM __ref("orders")', + renames=(ReferenceRename(new_name="orders_v2", origin_name="orders"),), + expected_mapping=None, + ), + OriginReferenceNamesTestCase( + description="recorded rename after the downstream build maps back", + query_sql='SELECT * FROM __ref("orders_v2")', + recorded_query_sql='SELECT * FROM __ref("orders")', + renames=( + ReferenceRename( + new_name="orders_v2", origin_name="orders", recorded_at=_RENAMED_AFTER_BUILD + ), + ), + expected_mapping={"orders_v2": "orders"}, + ), + OriginReferenceNamesTestCase( + description="recorded rename before the downstream build is ignored", + query_sql='SELECT * FROM __ref("orders_v2")', + recorded_query_sql='SELECT * FROM __ref("orders")', + renames=( + ReferenceRename( + new_name="orders_v2", origin_name="orders", recorded_at=_RENAMED_BEFORE_BUILD + ), + ), + expected_mapping=None, + ), + OriginReferenceNamesTestCase( + description="chained renames map back to the oldest recorded name", + query_sql='SELECT * FROM __ref("orders_v3")', + recorded_query_sql='SELECT * FROM __ref("orders")', + renames=( + ReferenceRename( + new_name="orders_v2", origin_name="orders", recorded_at=_RENAMED_AFTER_BUILD + ), + ReferenceRename(new_name="orders_v3", origin_name="orders_v2"), + ), + expected_mapping={"orders_v3": "orders"}, + ), + OriginReferenceNamesTestCase( + description="a real edit alongside the rewritten reference is not reference-only", + query_sql='SELECT id, 1 AS flag FROM __ref("orders_v2")', + recorded_query_sql='SELECT id FROM __ref("orders")', + renames=(ReferenceRename(new_name="orders_v2", origin_name="orders"),), + expected_mapping=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_renamed_references_when_matching_recorded_query_then_returns_mapping( + test_case: OriginReferenceNamesTestCase, +) -> None: + recorded: Fingerprint = Fingerprint( + node_type="model", + node_name="order_summary", + target_database=None, + target_schema="main", + target_name="order_summary", + run_id="run", + definition_hash=compute_query_hash(test_case.recorded_query_sql), + schema_fingerprint="", + definition=test_case.recorded_query_sql, + ts=_BUILT_AT, + ) + + assert ( + origin_reference_names( + query_sql=test_case.query_sql, + recorded=recorded, + renames=test_case.renames, + dialect="duckdb", + ) + == test_case.expected_mapping + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx index 1ba71c7330..db7a9436f7 100644 --- a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx +++ b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx @@ -15,7 +15,7 @@ A model's own change is any of: - **Config changed**: version-relevant configuration differs. A config change alone does not replay history. - **First run**: the model's relation does not exist yet. Only that model is built from scratch. -References rewritten by a rename are not a change of their own. When an upstream model is renamed with `migrate_from` (for example by [`sqb rename`](/docs/cli/rename)), a downstream model whose only difference is the rewritten `__ref` and `cursor_inputs` name keeps its identity: views are recreated as usual, and incremental models continue forward. A downstream model that also has a real edit is a normal query change. +References rewritten by a rename are not a change of their own. When an upstream model is renamed with `migrate_from` (for example by [`sqb rename`](/docs/cli/rename)), a downstream model whose only difference is the rewritten `__ref` and `cursor_inputs` name keeps its identity: views are recreated as usual, and incremental models continue forward. This also holds when the renamed model was built in an earlier run, for example by a selective build, or when its `migrate_from` was removed afterwards, because recorded renames are read from migration history. A downstream model that also has a real edit is a normal query change. For a query, function, or own declared schema change, the model's own `replay_on_change` decides the replay: From 3eea8d64c4a07398de059c33f9cc81f07954f0c9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:08:59 +0100 Subject: [PATCH 10/12] fix(planner): replay schema differences only when the model's own definition changed A schema finding follows replay_on_change only when the model's own SQL, header config, contract, or called functions changed in this plan, with rename-rewritten references not counting as a change. When the model is unchanged, every schema difference comes from upstream, whether added, removed or retyped, declared or inferred; it evolves through on_schema_change and contract enforcement as before, so an enforced contract whose declared column disappeared upstream still fails the build instead of replaying. The interim removed-column source heuristic is reverted. --- .../references/docs/concepts/incremental.md | 2 +- .../concepts/planning/replay-decisions.md | 4 +- .../sqlbuild/references/incremental-models.md | 11 +-- .../planner/_helpers/changes/detect.py | 4 +- .../planner/_helpers/changes/schema.py | 2 +- .../build/own_change_replay/_test_types.py | 7 ++ .../main/build/own_change_replay/helpers.py | 24 ++++-- .../test_own_change_replay.py | 9 ++- .../test_recorded_renames.py | 74 +++++++++++++++++++ .../planner/_helpers/changes/test_detect.py | 16 ++-- .../sqlbuild/compiler/planner/main/helpers.py | 43 +++++++++++ .../compiler/planner/main/test_execution.py | 7 +- .../docs/docs/concepts/incremental.mdx | 2 +- .../concepts/planning/replay-decisions.mdx | 4 +- 14 files changed, 178 insertions(+), 31 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index 597065914e..23251aa6b0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -368,7 +368,7 @@ MODEL ( ## Replay on change -When a model's own query changes, a function it calls directly changes, or its own declared columns change, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: +When a model's own query changes, a function it calls directly changes, or its own header or contract changes the columns it produces, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: | Value | Effect | |-------|--------| diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md index 097ce49bd0..4de8df0ca5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md @@ -14,13 +14,13 @@ A model's own change is any of: - **Query changed**: its compiled SQL differs from the recorded definition. - **Function changed**: a UDF or table function that the model calls directly has a new definition. The model's SQL text is unchanged, but its results can differ, so this counts as a query change of the model. Models further downstream do not see it. -- **Schema changed**: its declared or inferred columns differ from the warehouse. Only a change to the model's own declared columns (its `columns` block, contract, or schema YAML) follows `replay_on_change`. Columns inferred from its unchanged SQL, for example a column an upstream added that `SELECT *` passes through, are applied forward through [`on_schema_change`](../incremental.md#on_schema_change) and never replay history. +- **Schema changed**: its declared or inferred columns differ from the warehouse. A schema difference follows `replay_on_change` only when the model's own definition also changed in this plan: its SQL, its header config, or its contract. When the model itself is unchanged, every schema difference comes from upstream (an added, removed, or retyped column, declared or inferred) and never replays history. It is applied forward through [`on_schema_change`](../incremental.md#on_schema_change), and an enforced contract whose declared column disappeared upstream still fails the build. - **Config changed**: version-relevant configuration differs. A config change alone does not replay history. - **First run**: the model's relation does not exist yet. Only that model is built from scratch. References rewritten by a rename are not a change of their own. When an upstream model is renamed with `migrate_from` (for example by [`sqb rename`](../../cli/rename.md)), a downstream model whose only difference is the rewritten `__ref` and `cursor_inputs` name keeps its identity: views are recreated as usual, and incremental models continue forward. This also holds when the renamed model was built in an earlier run, for example by a selective build, or when its `migrate_from` was removed afterwards, because recorded renames are read from migration history. A downstream model that also has a real edit is a normal query change. -For a query, function, or own declared schema change, the model's own `replay_on_change` decides the replay: +For a query, function, or own-definition schema change, the model's own `replay_on_change` decides the replay: | `replay_on_change` | Effect on an incremental model | |--------------------|-------------------------------| diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md index 1d7f0702e8..fa443414cf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md @@ -130,11 +130,12 @@ plan or build is a one-run hard ceiling and explicit authorization for an intent ## Replay, full refresh and schema changes - `replay_on_change` decides how much to reprocess when the model's own query changes, a - function it calls directly changes, or its own declared columns change: `forward` (default), - `full`, or `bounded-` such as `bounded-14d`. Upstream changes, first runs and renames - never replay a downstream model; it continues forward. Columns inferred from upstream (for - example through `SELECT *`) evolve via `on_schema_change`, and `__ref`/`cursor_inputs` names - rewritten by a `migrate_from` rename are not a change. Functions have no replay setting. + function it calls directly changes, or its own header or contract changes its columns: + `forward` (default), `full`, or `bounded-` such as `bounded-14d`. Upstream changes, + first runs and renames never replay a downstream model; it continues forward. Schema + differences of an unchanged model come from upstream and evolve via `on_schema_change` (an + enforced contract still fails loudly), and `__ref`/`cursor_inputs` names rewritten by a + `migrate_from` rename are not a change. Functions have no replay setting. - `--full-refresh` rebuilds selected models unless a model sets `full_refresh false`; `full_refresh true` always rebuilds. A `full_refresh false` model is only fully rebuilt by its own first run or its own query change with `replay_on_change full`; anything else fails the plan diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 60c4fc2652..da3de9d5a9 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -49,7 +49,7 @@ SchemaFinding, WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind, SchemaColumnSource +from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql @@ -297,7 +297,7 @@ def detect_model_changes( ), dialect=snapshot.column_dialect, ) - if any(finding.source == SchemaColumnSource.YML for finding in schema_findings): + if schema_findings and (query_changed or changed_functions or config_changed): schema_backfill = resolve_replay_on_change( replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py index a2841c2f90..6da7f7e8f8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py @@ -120,7 +120,7 @@ def detect_schema_changes( column_name=col_name, source=( SchemaColumnSource.YML - if yml_columns and inferred_columns is None + if yml_columns else SchemaColumnSource.SQL_ANALYSIS ), actual_type=col_type, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/_test_types.py index 9d3209de27..5fd99769ce 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/_test_types.py @@ -39,3 +39,10 @@ class FunctionCallerReplayTestCase: class RefusedPlanTestCase: description: str expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class UpstreamContractFailureTestCase: + description: str + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py index e7fba5fa27..ff7086fcfd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py @@ -300,15 +300,29 @@ def add_upstream_column(*, project_dir: Path) -> None: ) +ENFORCED_CONTRACT_CONFIG: str = ( + " contract enforced,\n columns (\n id (type INTEGER),\n event_date (type DATE),\n ),\n" +) + + def declare_downstream_column_type(*, project_dir: Path) -> None: - """Declare a new type for the star downstream's id column in its own header.""" + """Change the declared type of the star downstream's id column in its own contract.""" path: Path = project_dir / "models/down.sql" path.write_text( - path.read_text(encoding="utf-8").replace( - " replay_on_change full,\n", - " replay_on_change full,\n columns (\n id (type BIGINT),\n ),\n", - ), + path.read_text(encoding="utf-8").replace("id (type INTEGER)", "id (type BIGINT)"), + encoding="utf-8", + ) + + +def remove_downstream_declared_column(*, project_dir: Path) -> None: + """Remove event_date from the star downstream's own contract and select list.""" + + path: Path = project_dir / "models/down.sql" + path.write_text( + path.read_text(encoding="utf-8") + .replace(" event_date (type DATE),\n", "") + .replace("SELECT *", "SELECT id"), encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py index b08b7a7bfb..a6f720053c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py @@ -15,6 +15,7 @@ RenamePlanBuildTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.own_change_replay.helpers import ( + ENFORCED_CONTRACT_CONFIG, add_upstream_column, assert_plan_fails, assert_plan_text, @@ -484,10 +485,10 @@ def test_given_upstream_column_added_when_downstream_selects_star_then_no_replay "test_case", [ OwnChangePlanBuildTestCase( - description="own declared column type change follows replay_on_change full", + description="own contract column type change follows replay_on_change full", expected_plan_fragments=("down full rebuild",), unexpected_plan_fragments=("S203",), - expected_reasons={"down": "schema_changed"}, + expected_reasons={"down": "config_changed"}, expected_rows={}, ) ], @@ -496,7 +497,9 @@ def test_given_upstream_column_added_when_downstream_selects_star_then_no_replay def test_given_own_declared_column_change_when_planning_then_own_replay_policy_applies( test_case: OwnChangePlanBuildTestCase, tmp_path: Path ) -> None: - project_dir: Path = build_star_project(tmp_path=tmp_path, downstream_config="") + project_dir: Path = build_star_project( + tmp_path=tmp_path, downstream_config=ENFORCED_CONTRACT_CONFIG + ) declare_downstream_column_type(project_dir=project_dir) assert_plan_text( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_recorded_renames.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_recorded_renames.py index 1c8659c6bc..a8d4d937ef 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_recorded_renames.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_recorded_renames.py @@ -2,6 +2,7 @@ from __future__ import annotations +import subprocess from datetime import date from pathlib import Path from typing import Any @@ -10,9 +11,12 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build.own_change_replay._test_types import ( OwnChangePlanBuildTestCase, + UpstreamContractFailureTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.own_change_replay.helpers import ( + ENFORCED_CONTRACT_CONFIG, assert_plan_text, + build, build_and_assert_rows, build_event_project, build_selected, @@ -21,6 +25,7 @@ drop_upstream_column, incremental_upstream_files, plan_models, + remove_downstream_declared_column, remove_migrate_from, rename_upstream_with_cli, udf_reference_merge_files, @@ -180,5 +185,74 @@ def test_given_partial_columns_when_upstream_drops_undeclared_column_then_no_rep assert models["down"]["backfill"] == {"action": "forward", "duration": None} +@pytest.mark.parametrize( + "test_case", + [ + UpstreamContractFailureTestCase( + description="upstream drops a contract column of an unchanged model", + expected_fragments=("K008", "runtime contract missing columns: event_date"), + unexpected_fragments=("S203", "full rebuild"), + ) + ], + ids=lambda case: case.description, +) +def test_given_unchanged_contract_model_when_upstream_drops_declared_column_then_build_fails( + test_case: UpstreamContractFailureTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_star_project( + tmp_path=tmp_path, + downstream_config=f" full_refresh false,\n{ENFORCED_CONTRACT_CONFIG}", + ) + drop_upstream_column(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, expected=(), unexpected=test_case.unexpected_fragments + ) + assert plan_models(project_dir=project_dir)["down"]["backfill"] == { + "action": "forward", + "duration": None, + } + result: subprocess.CompletedProcess[str] = build(project_dir=project_dir) + output: str = result.stdout + result.stderr + + assert result.returncode != 0 + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert not any(fragment in output for fragment in test_case.unexpected_fragments), output + + +@pytest.mark.parametrize( + "test_case", + [ + OwnChangePlanBuildTestCase( + description="own removal of a declared column follows replay_on_change full", + expected_plan_fragments=("Query changed (1)", "down full rebuild"), + unexpected_plan_fragments=("S203",), + expected_reasons={"down": "query_changed"}, + expected_rows={}, + ) + ], + ids=lambda case: case.description, +) +def test_given_own_declared_column_removed_when_planning_then_own_replay_policy_applies( + test_case: OwnChangePlanBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = build_star_project( + tmp_path=tmp_path, downstream_config=ENFORCED_CONTRACT_CONFIG + ) + remove_downstream_declared_column(project_dir=project_dir) + + assert_plan_text( + project_dir=project_dir, + expected=test_case.expected_plan_fragments, + unexpected=test_case.unexpected_plan_fragments, + ) + models: dict[str, dict[str, Any]] = plan_models(project_dir=project_dir) + + assert {name: models[name]["reason"] for name in test_case.expected_reasons} == ( + test_case.expected_reasons + ) + assert models["down"]["backfill"] == {"action": "full", "duration": None} + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index f48b6e7282..89170cb630 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -172,7 +172,7 @@ expected_backfill_action=BackfillAction.BOUNDED, ), DetectModelChangesTestCase( - description="detects schema change when expected column is missing from warehouse", + description="schema finding of an unchanged model never replays", model_name="orders", query_sql=_QUERY_SQL, config_values={"replay_on_change": "full"}, @@ -184,7 +184,7 @@ query_change_tracking=True, full_refresh=False, expected_change_kind=ChangeKind.SCHEMA_CHANGED, - expected_backfill_action=BackfillAction.FULL, + expected_backfill_action=BackfillAction.FORWARD_ONLY, ), DetectModelChangesTestCase( description="returns full backfill when full refresh is requested", @@ -269,11 +269,15 @@ expected_backfill_action=BackfillAction.FORWARD_ONLY, ), DetectModelChangesTestCase( - description="own declared column follows the model's replay policy", + description="schema finding with an own header change follows the replay policy", model_name="orders", query_sql=_QUERY_SQL, config_values={"materialized": "incremental", "replay_on_change": "full"}, - fingerprint_config_values={"materialized": "incremental", "replay_on_change": "full"}, + fingerprint_config_values={ + "materialized": "incremental", + "replay_on_change": "full", + "on_schema_change": "fail", + }, schema_columns=(("id", "INTEGER"), ("channel", "VARCHAR")), relation_exists=True, fingerprint_query_hash=_MATCHING_HASH, @@ -302,7 +306,7 @@ expected_backfill_action=BackfillAction.FORWARD_ONLY, ), DetectModelChangesTestCase( - description="column removed from an enforced declared contract follows replay policy", + description="declared column removed upstream of an unchanged model never replays", model_name="orders", query_sql=_QUERY_SQL, config_values={"materialized": "incremental", "replay_on_change": "full"}, @@ -316,7 +320,7 @@ query_change_tracking=True, full_refresh=False, expected_change_kind=ChangeKind.SCHEMA_CHANGED, - expected_backfill_action=BackfillAction.FULL, + expected_backfill_action=BackfillAction.FORWARD_ONLY, ), DetectModelChangesTestCase( description="does not report passthrough columns removed for unresolved star", diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index a1743b8d61..f2eb067281 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -1,9 +1,11 @@ from __future__ import annotations from dataclasses import fields +from datetime import UTC, datetime from pathlib import Path from typing import Any +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -13,7 +15,12 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.fingerprints.main.write import write_fingerprint +from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan +from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( + build_version_identity_metadata_json, +) from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, @@ -130,3 +137,39 @@ def build_protected_schema_replay_project() -> CompiledProject: effective_vars={}, models=(model,), ) + + +def create_protected_model_state(*, adapter: DuckDbAdapter, connection: Any) -> None: + """Create the protected model's table and a fingerprint from an older header.""" + + adapter.execute(connection=connection, sql="CREATE TABLE main.order_history (id INTEGER)") + query_sql: str = build_protected_schema_replay_project().models[0].query_sql + write_fingerprint( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprint=Fingerprint( + node_type="model", + node_name="order_history", + target_database=None, + target_schema="main", + target_name="order_history", + run_id="previous-run", + definition_hash=compute_query_hash(query_sql), + schema_fingerprint="", + definition=query_sql, + metadata_json=build_version_identity_metadata_json( + model_name="order_history", + config_values={ + "materialized": "incremental", + "incremental_strategy": "append", + "full_refresh": False, + "on_schema_change": "fail", + }, + ), + ts=datetime(2026, 9, 1, tzinfo=UTC), + ), + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index c4fdb2cfcd..7f077e6953 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -27,6 +27,7 @@ from tests.unit.src.sqlbuild.compiler.planner.main.helpers import ( build_execution_plan_from_kwargs, build_protected_schema_replay_project, + create_protected_model_state, ) from tests.unit.src.sqlbuild.integrations.dbt.helpers import build_compiled_project_with_models @@ -168,7 +169,7 @@ def test_given_blocked_protected_model_when_planning_then_full_rebuild_protectio adapter: DuckDbAdapter = DuckDbAdapter() connection: Any = adapter.connect({"database": ":memory:"}) try: - adapter.execute(connection=connection, sql="CREATE TABLE main.order_history (id INTEGER)") + create_protected_model_state(adapter=adapter, connection=connection) plan_output: PlanOutput = build_execution_plan_from_kwargs( project=build_protected_schema_replay_project(), adapter=adapter, @@ -185,7 +186,7 @@ def test_given_blocked_protected_model_when_planning_then_full_rebuild_protectio "test_case", [ ProtectedRebuildBlockedTestCase( - description="unblocked protected model refuses the declared-column full replay", + description="unblocked protected model refuses its own header-change full replay", external_blocked_model_names=(), expected_action="S203", ) @@ -198,7 +199,7 @@ def test_given_unblocked_protected_model_when_planning_then_full_rebuild_is_refu adapter: DuckDbAdapter = DuckDbAdapter() connection: Any = adapter.connect({"database": ":memory:"}) try: - adapter.execute(connection=connection, sql="CREATE TABLE main.order_history (id INTEGER)") + create_protected_model_state(adapter=adapter, connection=connection) with pytest.raises(PlannerInputError) as raised: build_execution_plan_from_kwargs( project=build_protected_schema_replay_project(), diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index 6aebcb410e..588bcfe353 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -357,7 +357,7 @@ MODEL ( ## Replay on change -When a model's own query changes, a function it calls directly changes, or its own declared columns change, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: +When a model's own query changes, a function it calls directly changes, or its own header or contract changes the columns it produces, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: | Value | Effect | |-------|--------| diff --git a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx index db7a9436f7..96f6472b49 100644 --- a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx +++ b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx @@ -11,13 +11,13 @@ A model's own change is any of: - **Query changed**: its compiled SQL differs from the recorded definition. - **Function changed**: a UDF or table function that the model calls directly has a new definition. The model's SQL text is unchanged, but its results can differ, so this counts as a query change of the model. Models further downstream do not see it. -- **Schema changed**: its declared or inferred columns differ from the warehouse. Only a change to the model's own declared columns (its `columns` block, contract, or schema YAML) follows `replay_on_change`. Columns inferred from its unchanged SQL, for example a column an upstream added that `SELECT *` passes through, are applied forward through [`on_schema_change`](/docs/concepts/incremental#on_schema_change) and never replay history. +- **Schema changed**: its declared or inferred columns differ from the warehouse. A schema difference follows `replay_on_change` only when the model's own definition also changed in this plan: its SQL, its header config, or its contract. When the model itself is unchanged, every schema difference comes from upstream (an added, removed, or retyped column, declared or inferred) and never replays history. It is applied forward through [`on_schema_change`](/docs/concepts/incremental#on_schema_change), and an enforced contract whose declared column disappeared upstream still fails the build. - **Config changed**: version-relevant configuration differs. A config change alone does not replay history. - **First run**: the model's relation does not exist yet. Only that model is built from scratch. References rewritten by a rename are not a change of their own. When an upstream model is renamed with `migrate_from` (for example by [`sqb rename`](/docs/cli/rename)), a downstream model whose only difference is the rewritten `__ref` and `cursor_inputs` name keeps its identity: views are recreated as usual, and incremental models continue forward. This also holds when the renamed model was built in an earlier run, for example by a selective build, or when its `migrate_from` was removed afterwards, because recorded renames are read from migration history. A downstream model that also has a real edit is a normal query change. -For a query, function, or own declared schema change, the model's own `replay_on_change` decides the replay: +For a query, function, or own-definition schema change, the model's own `replay_on_change` decides the replay: | `replay_on_change` | Effect on an incremental model | |--------------------|-------------------------------| From c8fa5e822e6a9004e375300c420b46cc381f598a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:46:53 +0100 Subject: [PATCH 11/12] fix(planner): record declared columns so their own edits replay schema changes Model fingerprint metadata now carries a declared_columns_hash key covering the model's columns block or schema YAML columns (name, type, nullable), dynamic column families, type enforcement and contract setting. The key is additive, sits outside version identity, and is ignored by config-change comparison. A schema finding follows replay_on_change when this hash differs from the recorded one, alongside SQL, config, contract and function changes. Rows written before the key existed have no hash, so their declared columns count as unchanged and the model continues forward once. --- .../references/docs/concepts/incremental.md | 2 +- .../concepts/planning/replay-decisions.md | 2 +- .../sqlbuild/references/incremental-models.md | 2 +- .../planner/_helpers/changes/detect.py | 24 +++++++-- .../planner/_helpers/changes/metadata.py | 34 +++++++++++++ .../_helpers/identity/model_metadata.py | 31 ++++++++++++ src/sqlbuild/compiler/planner/constants.py | 1 + .../planner/_helpers/changes/_test_helpers.py | 15 ++++-- .../planner/_helpers/changes/_test_types.py | 10 +++- .../changes/test_declared_columns_metadata.py | 49 +++++++++++++++++++ .../planner/_helpers/changes/test_detect.py | 16 ++++++ .../docs/docs/concepts/incremental.mdx | 2 +- .../concepts/planning/replay-decisions.mdx | 2 +- 13 files changed, 176 insertions(+), 14 deletions(-) create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index 23251aa6b0..3bc4b8ee01 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -368,7 +368,7 @@ MODEL ( ## Replay on change -When a model's own query changes, a function it calls directly changes, or its own header or contract changes the columns it produces, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: +When a model's own query changes, a function it calls directly changes, or its own header, contract, or declared columns change, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: | Value | Effect | |-------|--------| diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md index 4de8df0ca5..d74154dcca 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/planning/replay-decisions.md @@ -14,7 +14,7 @@ A model's own change is any of: - **Query changed**: its compiled SQL differs from the recorded definition. - **Function changed**: a UDF or table function that the model calls directly has a new definition. The model's SQL text is unchanged, but its results can differ, so this counts as a query change of the model. Models further downstream do not see it. -- **Schema changed**: its declared or inferred columns differ from the warehouse. A schema difference follows `replay_on_change` only when the model's own definition also changed in this plan: its SQL, its header config, or its contract. When the model itself is unchanged, every schema difference comes from upstream (an added, removed, or retyped column, declared or inferred) and never replays history. It is applied forward through [`on_schema_change`](../incremental.md#on_schema_change), and an enforced contract whose declared column disappeared upstream still fails the build. +- **Schema changed**: its declared or inferred columns differ from the warehouse. A schema difference follows `replay_on_change` only when the model's own definition also changed in this plan: its SQL, its header config, its contract, or its declared columns (`columns` block or schema YAML). When the model itself is unchanged, every schema difference comes from upstream (an added, removed, or retyped column, declared or inferred) and never replays history. It is applied forward through [`on_schema_change`](../incremental.md#on_schema_change), and an enforced contract whose declared column disappeared upstream still fails the build. - **Config changed**: version-relevant configuration differs. A config change alone does not replay history. - **First run**: the model's relation does not exist yet. Only that model is built from scratch. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md index fa443414cf..c66516a74e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md @@ -130,7 +130,7 @@ plan or build is a one-run hard ceiling and explicit authorization for an intent ## Replay, full refresh and schema changes - `replay_on_change` decides how much to reprocess when the model's own query changes, a - function it calls directly changes, or its own header or contract changes its columns: + function it calls directly changes, or its own header, contract or declared columns change: `forward` (default), `full`, or `bounded-` such as `bounded-14d`. Upstream changes, first runs and renames never replay a downstream model; it continues forward. Schema differences of an unchanged model come from upstream and evolve via `on_schema_change` (an diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index da3de9d5a9..4912df0155 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -19,7 +19,9 @@ from sqlbuild.compiler.planner._helpers.changes.metadata import ( changed_local_function_names, non_function_identity_metadata_payload, + recorded_declared_columns_hash, version_identity_metadata_payload, + with_declared_columns_hash, with_origin_cursor_input_names, ) from sqlbuild.compiler.planner._helpers.changes.policy import ( @@ -33,6 +35,7 @@ build_compiled_function_fingerprint_sql, detect_function_change, ) +from sqlbuild.compiler.planner._helpers.identity.model_metadata import declared_columns_hash from sqlbuild.compiler.planner.constants import EMPTY_FINGERPRINT_METADATA_JSON from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( build_function_local_hashes, @@ -157,10 +160,15 @@ def detect_model_changes( """Detect changes for one model and resolve backfill policy.""" model_name: str = model.name - metadata_json: str = expected_metadata_json or build_model_version_identity_metadata_json( - model=model, - function_local_hashes=function_local_hashes, - hook_version_hashes=hook_version_hashes, + current_columns_hash: str = declared_columns_hash(model=model) + metadata_json: str = with_declared_columns_hash( + metadata_json=expected_metadata_json + or build_model_version_identity_metadata_json( + model=model, + function_local_hashes=function_local_hashes, + hook_version_hashes=hook_version_hashes, + ), + declared_columns_hash=current_columns_hash, ) if full_refresh: @@ -297,7 +305,13 @@ def detect_model_changes( ), dialect=snapshot.column_dialect, ) - if schema_findings and (query_changed or changed_functions or config_changed): + recorded_columns_hash: str | None = recorded_declared_columns_hash(recorded_metadata_json) + declared_columns_changed: bool = ( + recorded_columns_hash is not None and recorded_columns_hash != current_columns_hash + ) + if schema_findings and ( + query_changed or changed_functions or config_changed or declared_columns_changed + ): schema_backfill = resolve_replay_on_change( replay_on_change=get_config_str(values=model.config.values, key="replay_on_change") ) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py index a82bb1665e..e75cde0e69 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY from sqlbuild.compiler.planner.constants import ( + DECLARED_COLUMNS_METADATA_KEY, LOCAL_FUNCTION_HASHES_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, ) @@ -26,6 +27,7 @@ def version_identity_metadata_payload(metadata_json: str | None) -> object: identity_payload: dict[str, object] = dict(payload) identity_payload.pop(AUDIT_GATE_METADATA_KEY, None) identity_payload.pop(MIGRATION_FINGERPRINT_METADATA_KEY, None) + identity_payload.pop(DECLARED_COLUMNS_METADATA_KEY, None) return identity_payload @@ -93,3 +95,35 @@ def with_origin_cursor_input_names(*, metadata_json: str, renamed_refs: dict[str separators=(",", ":"), default=str, ) + + +def recorded_declared_columns_hash(metadata_json: str | None) -> str | None: + """Return the declared-columns hash a fingerprint row recorded, if it has one.""" + + if metadata_json is None: + return None + try: + payload: object = json.loads(metadata_json) + except json.JSONDecodeError: + return None + value: object = ( + payload.get(DECLARED_COLUMNS_METADATA_KEY) if isinstance(payload, dict) else None + ) + return value if isinstance(value, str) else None + + +def with_declared_columns_hash(*, metadata_json: str, declared_columns_hash: str) -> str: + """Return fingerprint metadata JSON that also records the model's declared-columns hash.""" + + try: + payload: object = json.loads(metadata_json) + except json.JSONDecodeError: + return metadata_json + if not isinstance(payload, dict): + return metadata_json + return json.dumps( + {**payload, DECLARED_COLUMNS_METADATA_KEY: declared_columns_hash}, + sort_keys=True, + separators=(",", ":"), + default=str, + ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py b/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py index 3df39762f1..119438a09d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py @@ -2,6 +2,9 @@ from __future__ import annotations +import hashlib +import json + from sqlbuild.compiler.compile.models import CompiledModel from sqlbuild.compiler.discovery.models import EnumDeclaration from sqlbuild.compiler.planner.types import ContractPolicy @@ -53,3 +56,31 @@ def _column_output_signature(*, model: CompiledModel, column: SchemaColumn) -> d ], } return signature + + +def declared_columns_hash(*, model: CompiledModel) -> str: + """Hash the model's own declared columns, column families, and contract setting.""" + + schema_entry: SchemaModelEntry | None = model.schema_entry + payload: dict[str, object] = { + "contract": model.config.values.get("contract"), + "columns": [ + {"name": column.name, "type": column.type, "nullable": column.nullable} + for column in (schema_entry.columns if schema_entry is not None else ()) + ], + "dynamic_columns": [ + { + "name": family.name, + "pivot_column": family.pivot_column, + "value_column": family.value_column, + "aggregate": family.aggregate, + "type": family.type, + "name_pattern": family.name_pattern, + } + for family in (schema_entry.dynamic_columns if schema_entry is not None else ()) + ], + "type_enforcement": schema_entry.type_enforcement if schema_entry is not None else None, + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") + ).hexdigest() diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index e5849f7f12..59f1ed55e5 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -61,6 +61,7 @@ QUALIFIED_RELATION_MAX_PARTS: int = 3 MIGRATION_MODEL_NAME_METADATA_KEY: str = "model_name" MIGRATION_FINGERPRINT_METADATA_KEY: str = "migration_fingerprint" +DECLARED_COLUMNS_METADATA_KEY: str = "declared_columns_hash" LOCAL_FUNCTION_HASHES_METADATA_KEY: str = "local_function_hashes" FULL_REFRESH_DISABLED_REBUILD_CODE: str = "S203" MIGRATION_REF_PLACEHOLDER_PREFIX: str = "__sqb_migration_ref__" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index 8e5fcfc25e..c9af14752d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -260,9 +260,18 @@ def _build_fingerprints(test_case: DetectModelChangesTestCase) -> dict[str, Fing definition_hash=test_case.fingerprint_query_hash or "", schema_fingerprint="schema_a", definition="SELECT 1", - metadata_json=build_version_identity_metadata_json( - model_name=test_case.model_name, - config_values=fingerprint_config_values, + metadata_json=json.dumps( + { + **json.loads( + build_version_identity_metadata_json( + model_name=test_case.model_name, + config_values=fingerprint_config_values, + ) + ), + **test_case.fingerprint_extra_metadata, + }, + sort_keys=True, + separators=(",", ":"), ), ts=_STUB_TS, version_hash="recorded_version", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 05ff7cbe7a..099e4f5d7c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn @@ -28,6 +28,7 @@ class DetectModelChangesTestCase: expected_recorded_build_relation_missing: bool = False renamed_by_migration: bool = False schema_type_enforcement: bool | None = None + fingerprint_extra_metadata: dict[str, object] = field(default_factory=dict) @dataclass(frozen=True) @@ -104,3 +105,10 @@ class OriginReferenceNamesTestCase: recorded_query_sql: str renames: tuple[ReferenceRename, ...] expected_mapping: dict[str, str] | None + + +@dataclass(frozen=True) +class DeclaredColumnsMetadataTestCase: + description: str + metadata_json: str + expected_recorded_hash: str | None diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py new file mode 100644 index 0000000000..6b7797669c --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py @@ -0,0 +1,49 @@ +"""Tests for the declared-columns hash recorded in model fingerprint metadata.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.planner._helpers.changes.metadata import ( + recorded_declared_columns_hash, + version_identity_metadata_payload, + with_declared_columns_hash, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( + DeclaredColumnsMetadataTestCase, +) + +_METADATA_JSON: str = '{"config":{"materialized":"table"}}' + + +@pytest.mark.parametrize( + "test_case", + [ + DeclaredColumnsMetadataTestCase( + description="written hash is read back", + metadata_json=with_declared_columns_hash( + metadata_json=_METADATA_JSON, declared_columns_hash="abc123" + ), + expected_recorded_hash="abc123", + ), + DeclaredColumnsMetadataTestCase( + description="row written before the key existed has no recorded hash", + metadata_json=_METADATA_JSON, + expected_recorded_hash=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_fingerprint_metadata_when_reading_declared_columns_hash_then_tolerates_absence( + test_case: DeclaredColumnsMetadataTestCase, +) -> None: + assert recorded_declared_columns_hash(test_case.metadata_json) == ( + test_case.expected_recorded_hash + ) + assert version_identity_metadata_payload(test_case.metadata_json) == ( + version_identity_metadata_payload(_METADATA_JSON) + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 89170cb630..6399426ed4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -322,6 +322,22 @@ expected_change_kind=ChangeKind.SCHEMA_CHANGED, expected_backfill_action=BackfillAction.FORWARD_ONLY, ), + DetectModelChangesTestCase( + description="schema finding after an own declared-column edit follows replay policy", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={"replay_on_change": "full"}, + schema_columns=(("id", "INTEGER"), ("status", "VARCHAR")), + relation_exists=True, + fingerprint_query_hash=_MATCHING_HASH, + fingerprint_extra_metadata={"declared_columns_hash": "previous-declared-columns"}, + warehouse_column_names=(("id", "INTEGER"),), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + expected_change_kind=ChangeKind.SCHEMA_CHANGED, + expected_backfill_action=BackfillAction.FULL, + ), DetectModelChangesTestCase( description="does not report passthrough columns removed for unresolved star", model_name="orders", diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index 588bcfe353..8858022557 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -357,7 +357,7 @@ MODEL ( ## Replay on change -When a model's own query changes, a function it calls directly changes, or its own header or contract changes the columns it produces, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: +When a model's own query changes, a function it calls directly changes, or its own header, contract, or declared columns change, `replay_on_change` is the explicit, per-model policy for how much data to reprocess. Reprocessing is a policy you set, not an automatic forced rebuild, so a definition change does not silently trigger a full rebuild of large tables. Upstream changes never replay a downstream model. You choose the cost per model: | Value | Effect | |-------|--------| diff --git a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx index 96f6472b49..69b67841c5 100644 --- a/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx +++ b/website/src/content/docs/docs/concepts/planning/replay-decisions.mdx @@ -11,7 +11,7 @@ A model's own change is any of: - **Query changed**: its compiled SQL differs from the recorded definition. - **Function changed**: a UDF or table function that the model calls directly has a new definition. The model's SQL text is unchanged, but its results can differ, so this counts as a query change of the model. Models further downstream do not see it. -- **Schema changed**: its declared or inferred columns differ from the warehouse. A schema difference follows `replay_on_change` only when the model's own definition also changed in this plan: its SQL, its header config, or its contract. When the model itself is unchanged, every schema difference comes from upstream (an added, removed, or retyped column, declared or inferred) and never replays history. It is applied forward through [`on_schema_change`](/docs/concepts/incremental#on_schema_change), and an enforced contract whose declared column disappeared upstream still fails the build. +- **Schema changed**: its declared or inferred columns differ from the warehouse. A schema difference follows `replay_on_change` only when the model's own definition also changed in this plan: its SQL, its header config, its contract, or its declared columns (`columns` block or schema YAML). When the model itself is unchanged, every schema difference comes from upstream (an added, removed, or retyped column, declared or inferred) and never replays history. It is applied forward through [`on_schema_change`](/docs/concepts/incremental#on_schema_change), and an enforced contract whose declared column disappeared upstream still fails the build. - **Config changed**: version-relevant configuration differs. A config change alone does not replay history. - **First run**: the model's relation does not exist yet. Only that model is built from scratch. From e66af49df886678a2a2a4270dcd56572a65f6723 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:10:00 +0100 Subject: [PATCH 12/12] fix(planner): keep migration fingerprint work at one computation per model The migration fingerprint cache keyed entries on the raw fingerprint metadata JSON, so the declared_columns_hash key made build-time entries miss the value computed during planning. The cache now strips that key before keying, as the key never contributes to the fingerprint. --- .../planner/_helpers/changes/metadata.py | 17 +++++++++++++++++ .../classes/migration_fingerprint_cache.py | 2 ++ .../changes/test_declared_columns_metadata.py | 2 ++ 3 files changed, 21 insertions(+) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py index e75cde0e69..bbc84be95a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/metadata.py @@ -127,3 +127,20 @@ def with_declared_columns_hash(*, metadata_json: str, declared_columns_hash: str separators=(",", ":"), default=str, ) + + +def without_declared_columns_hash(metadata_json: str) -> str: + """Return fingerprint metadata JSON without the declared-columns hash key, if present.""" + + try: + payload: object = json.loads(metadata_json) + except json.JSONDecodeError: + return metadata_json + if not isinstance(payload, dict) or DECLARED_COLUMNS_METADATA_KEY not in payload: + return metadata_json + return json.dumps( + {key: value for key, value in payload.items() if key != DECLARED_COLUMNS_METADATA_KEY}, + sort_keys=True, + separators=(",", ":"), + default=str, + ) diff --git a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py index a8b7454141..ddc30ab436 100644 --- a/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py +++ b/src/sqlbuild/compiler/planner/classes/migration_fingerprint_cache.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, cast +from sqlbuild.compiler.planner._helpers.changes.metadata import without_declared_columns_hash from sqlbuild.compiler.planner._helpers.migrations.fingerprint import ( build_migration_fingerprint, migration_fingerprint_ref_names, @@ -57,6 +58,7 @@ def fingerprint( ) -> str | None: """Return the migration fingerprint, keyed only by the renames this model can see.""" + metadata_json = without_declared_columns_hash(metadata_json) definition: tuple[str, str] = (query_sql, metadata_json) names: frozenset[str] | None = self._ref_names.get(definition) if names is None: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py index 6b7797669c..ea10347407 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_declared_columns_metadata.py @@ -8,6 +8,7 @@ recorded_declared_columns_hash, version_identity_metadata_payload, with_declared_columns_hash, + without_declared_columns_hash, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DeclaredColumnsMetadataTestCase, @@ -43,6 +44,7 @@ def test_given_fingerprint_metadata_when_reading_declared_columns_hash_then_tole assert version_identity_metadata_payload(test_case.metadata_json) == ( version_identity_metadata_payload(_METADATA_JSON) ) + assert without_declared_columns_hash(test_case.metadata_json) == _METADATA_JSON if __name__ == "__main__":