From 53e376215885c5a0605466ab3170171cf3c36f33 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:18:04 +0100 Subject: [PATCH 1/3] fix: plan relations missing from the warehouse as first runs A model whose relation was dropped outside SQLBuild was still planned as built because a stored fingerprint existed: incremental models kept their cursor bounds and failed on the missing table, and views and tables reported no_change. Change detection now treats the warehouse snapshot as the source of truth for relation existence and consults stored fingerprints only when the relation exists. - Plan a first run for any model whose relation is absent, regardless of stored fingerprints, and emit a RECORDED_RELATION_MISSING plan warning when stored state recorded a previous build. - Report the built version as missing in plan JSON and expose warning codes. - Plan a seed whose relation is absent as a first run. - Let microbatch full-refresh builds create an absent live target instead of failing reconciliation. --- Makefile | 1 + src/sqlbuild/cli/output/_helpers/plan_json.py | 2 + .../planner/_helpers/changes/detect.py | 3 +- .../planner/_helpers/output/plan_output.py | 2 + .../planner/_helpers/output/strategy.py | 15 + src/sqlbuild/compiler/planner/constants.py | 1 + src/sqlbuild/compiler/planner/models.py | 1 + .../_helpers/materializations/microbatch.py | 6 - .../cli/commands/main/build/_test_types.py | 7 + .../cli/commands/main/build/helpers.py | 26 ++ .../build/test_dropped_relation_rebuild.py | 135 ++++++++ .../sqlbuild/cli/commands/main/_test_types.py | 46 +++ .../src/sqlbuild/cli/commands/main/helpers.py | 134 +++++++ .../main/test_dropped_relation_first_run.py | 326 ++++++++++++++++++ .../output/main/plan_json/test_plan_json.py | 26 ++ .../planner/_helpers/changes/_test_helpers.py | 1 + .../planner/_helpers/changes/_test_types.py | 11 +- .../planner/_helpers/changes/test_detect.py | 143 +++++++- 18 files changed, 877 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py diff --git a/Makefile b/Makefile index 1043a3ee74..8a7c9edbad 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py \ diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 23381f91b7..f845a467d0 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -305,6 +305,8 @@ def _serialize_warning(warning: PlanWarning) -> dict[str, object]: } if warning.model_name is not None: result["model_name"] = warning.model_name + if warning.code is not None: + result["code"] = warning.code return result diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 57cda287d5..b8600036f5 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -175,13 +175,14 @@ def detect_model_changes( relation_exists: bool = model_name in snapshot.existing_relations fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model_name) - if not relation_exists and fingerprint is None: + if not relation_exists: return ChangeDetectionResult( model_name=model_name, change_kind=ChangeKind.FIRST_RUN, backfill=BackfillResult(action=BackfillAction.FULL), fingerprint_metadata_json=metadata_json, fingerprint_version_hash=expected_version_hash, + recorded_build_relation_missing=fingerprint is not None, ) query_changed: bool = False diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index 60608ac5e6..b1f7d5d2d7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -181,6 +181,8 @@ def _resolve_seed_plan_reason( reason: PlanReason | None = (seed_plan_reasons or {}).get(seed_name) if reason is not None: return reason + if seed_name not in snapshot.existing_relations: + return PlanReason.FIRST_RUN fingerprint: Fingerprint | None = snapshot.fingerprints.seeds.get(seed_name) if fingerprint is None: return PlanReason.FIRST_RUN diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index d9c5ab26f3..cc1945fbb4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -7,6 +7,7 @@ CompiledModel, CompiledRelationLocation, ) +from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, @@ -184,6 +185,20 @@ def build_model_warnings( effective: OnSchemaChange = on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE warnings: list[PlanWarning] = [] + if change_result.recorded_build_relation_missing: + warnings.append( + PlanWarning( + model_name=model_name, + severity=WarningSeverity.WARNING, + code=RECORDED_RELATION_MISSING_WARNING_CODE, + message=( + f"{model_name}: SQLBuild state records a previous build, but the relation " + "no longer exists in the warehouse. It may have been dropped outside " + "SQLBuild; planning a first run to recreate it." + ), + ) + ) + if effective == OnSchemaChange.FAIL and change_result.schema_findings: warnings.append( PlanWarning( diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index dc7fa0bb6a..bd2aaf2da1 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -10,6 +10,7 @@ "'checks/', or 'loaders/'" ) EMPTY_FINGERPRINT_METADATA_JSON: str = "{}" +RECORDED_RELATION_MISSING_WARNING_CODE: str = "RECORDED_RELATION_MISSING" MODEL_SELECTOR_ROOT: str = "models" SQL_FILE_SELECTOR_SUFFIX: str = ".sql" MODEL_SELECTOR_ROOT_PREFIX: str = "models/" diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 58ff77e55e..e39830789a 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -749,6 +749,7 @@ class ChangeDetectionResult: backfill: BackfillResult = field( default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY) ) + recorded_build_relation_missing: bool = False @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index f5e2e88e0d..ef23d1a7bb 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -43,7 +43,6 @@ IncrementalStrategy, MicrobatchStrategy, OnSchemaChange, - PlanReason, ) from sqlbuild.cursor_algebra.main.cap_from_end import cap_from_end from sqlbuild.cursor_algebra.main.cap_from_start import cap_from_start @@ -3106,11 +3105,6 @@ def _prepare_full_refresh_rebuild( destination=relations.target_qualified, statement_recorder=state.statement_recorder, ) - elif not live_exists and context.entry.reason == PlanReason.FULL_REFRESH: - raise ExecutorInputError( - "full-refresh reconciliation found neither the live target nor its rebuild " - f"relation for '{context.entry.name}'" - ) elif rebuild_exists: context.adapter.drop( connection=context.connection, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 5ae694aeed..e4e491923d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1109,3 +1109,10 @@ class SqlHookIdentityBuildE2ETestCase: added_hook_path: str added_hook_contents: str expected_reason: str + + +@dataclass(frozen=True) +class DroppedIncrementalRelationE2ETestCase: + description: str + incremental_strategy: str + expected_rows: tuple[tuple[int, str], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 1dbd92dbcd..e0949c0c95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -862,3 +862,29 @@ def physical_orders_relation(*, state_path: Path) -> tuple[str, str]: ), )[0] return str(schema_name), str(relation_name) + + +def dropped_incremental_project_files(*, incremental_strategy: str) -> dict[str, str]: + """Return a DuckDB project with one timestamp-cursor incremental order model.""" + + return { + "sqlbuild_project.toml": ( + 'name = "dropped_orders"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "dropped_orders.duckdb"\n' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + ), + "models/orders.sql": ( + "MODEL (\n" + " materialized incremental,\n" + f" incremental_strategy {incremental_strategy},\n" + " unique_key id,\n" + " cursor ordered_at,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_start '2026-01-02',\n" + ");\n\n" + 'SELECT id, ordered_at FROM __source("raw_orders")\n' + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py new file mode 100644 index 0000000000..96aaf0040d --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py @@ -0,0 +1,135 @@ +"""E2E regression tests for incremental relations dropped outside SQLBuild.""" + +from __future__ import annotations + +import json +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + DroppedIncrementalRelationE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + dropped_incremental_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DroppedIncrementalRelationE2ETestCase( + description="delete insert", + incremental_strategy="delete_insert", + expected_rows=( + (2, "2026-01-02 01:00:00"), + (3, "2026-01-03 01:00:00"), + (4, "2026-01-04 01:00:00"), + ), + ), + DroppedIncrementalRelationE2ETestCase( + description="append", + incremental_strategy="append", + expected_rows=( + (2, "2026-01-02 01:00:00"), + (3, "2026-01-03 01:00:00"), + (4, "2026-01-04 01:00:00"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_incremental_table_dropped_outside_sqlbuild_when_planning_and_building_then_recreates( + tmp_path: Path, + test_case: DroppedIncrementalRelationE2ETestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="dropped_orders", + repo_files=dropped_incremental_project_files( + incremental_strategy=test_case.incremental_strategy + ), + ) + db_path: Path = project_dir / "dropped_orders.duckdb" + execute_duckdb( + db_path=db_path, + sql=( + "CREATE TABLE main.raw_orders (id INTEGER, ordered_at TIMESTAMP); " + "INSERT INTO main.raw_orders VALUES " + "(1, '2026-01-01 05:00:00'), (2, '2026-01-02 01:00:00'), " + "(3, '2026-01-03 01:00:00')" + ), + ) + first_build: CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first_build.returncode == 0, first_build.stdout + first_build.stderr + execute_duckdb( + db_path=db_path, + sql=( + "DROP TABLE main.orders; INSERT INTO main.raw_orders VALUES (4, '2026-01-04 01:00:00')" + ), + ) + + plan: CompletedProcess[str] = run_sqb(command=("plan", "--json"), project_dir=project_dir) + rebuild: CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + document: dict[str, object] = json.loads(plan.stdout) + models: object = document["models"] + assert isinstance(models, list) + model: dict[str, object] = models[0] + assert (model["name"], model["action"], model["reason"]) == ( + "orders", + "create_table", + "first_run", + ) + assert model["built_version_present"] is False + assert model["identity_status"] == "missing" + assert model["cursor_bounds"] == { + "start": "2026-01-02T00:00:00", + "end": "2026-01-05T00:00:00", + } + assert document["warnings"] == [ + { + "severity": "warning", + "message": ( + "orders: SQLBuild state records a previous build, but the relation no longer " + "exists in the warehouse. It may have been dropped outside SQLBuild; planning a " + "first run to recreate it." + ), + "model_name": "orders", + "code": "RECORDED_RELATION_MISSING", + } + ] + assert rebuild.returncode == 0, rebuild.stdout + rebuild.stderr + assert "RECORDED_RELATION_MISSING" not in rebuild.stdout + assert "no longer exists in the warehouse" in rebuild.stdout + rebuild.stderr + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT id, CAST(ordered_at AS VARCHAR) FROM main.orders ORDER BY id", + ) + ) + == test_case.expected_rows + ) + assert ( + query_duckdb( + db_path=db_path, + sql=( + "SELECT table_name FROM information_schema.tables " + "WHERE table_name LIKE 'orders\\_\\_%' ESCAPE '\\'" + ), + ) + == [] + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 33339a443e..b62ef7cc5f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -622,3 +622,49 @@ class LeadingCteCommentFormatIntegrationTestCase: description: str authored_sql: str expected_fragment: str + + +@dataclass(frozen=True) +class DroppedRelationRecoveryTestCase: + """One model whose relation is dropped outside SQLBuild after a successful build.""" + + description: str + model_sql: str + drop_sql: str + expected_action: str + expected_rows: tuple[tuple[object, ...], ...] + settings_toml: str = "" + stale_artifact_sql: str = "SELECT 1" + + +@dataclass(frozen=True) +class DroppedIncrementalFirstRunRangeTestCase: + """One incremental model rebuilt from its configured start after an external drop.""" + + description: str + model_sql: str + expected_start: str + expected_end: str + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class DroppedSeedPlanTestCase: + """One seed dropped outside SQLBuild after a successful build.""" + + description: str + model_sql: str + expected_steady_reasons: dict[str, object] + expected_dropped_reasons: dict[str, object] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class AbsentMicrobatchFullRefreshTestCase: + """One full-refresh build of a microbatch model whose live target is absent.""" + + description: str + batch_concurrency: int + setup_build_flags: tuple[str, ...] + setup_drop_sql: str + expected_rows: tuple[tuple[object, ...], ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index f8f8786c90..0c46702399 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -246,3 +246,137 @@ def run_build( exit_code: int = main(["--project-dir", str(project_dir), "--no-color", "build", *flags]) return exit_code, "".join(capsys.readouterr()) + + +def write_dropped_relation_project( + *, project_dir: Path, model_name: str, model_sql: str, settings_toml: str = "" +) -> Path: + """Write a DuckDB project with raw order sources, one seed, and one model.""" + + (project_dir / "models").mkdir(parents=True, exist_ok=True) + (project_dir / "sources").mkdir(parents=True, exist_ok=True) + (project_dir / "seeds").mkdir(parents=True, exist_ok=True) + _ = (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' + + settings_toml, + encoding="utf-8", + ) + _ = (project_dir / "sources" / "raw.yml").write_text( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " - name: raw_customers\n" + " schema: main\n" + " table: raw_customers\n", + encoding="utf-8", + ) + _ = (project_dir / "seeds" / "order_statuses.csv").write_text( + "status_code,status_name\n1,placed\n2,shipped\n", encoding="utf-8" + ) + _ = (project_dir / "seeds" / "lookups.yml").write_text( + "seeds:\n" + " - name: order_statuses\n" + " columns:\n" + " - name: status_code\n" + " type: INTEGER\n" + " - name: status_name\n" + " type: VARCHAR\n", + encoding="utf-8", + ) + _ = (project_dir / "models" / f"{model_name}.sql").write_text(model_sql, encoding="utf-8") + db_path: Path = project_dir / "orders.duckdb" + execute_duckdb_sql( + db_path=db_path, + sql=( + "CREATE TABLE main.raw_orders (id INTEGER, ordered_at TIMESTAMP); " + "INSERT INTO main.raw_orders VALUES " + "(1, '2026-01-01 05:00:00'), (2, '2026-01-02 01:00:00'), " + "(3, '2026-01-03 01:00:00'); " + "CREATE TABLE main.raw_customers AS " + "SELECT 1 AS customer_id, 'basic' AS plan, TIMESTAMP '2026-01-01' AS updated_at" + ), + ) + return db_path + + +def execute_duckdb_sql(*, db_path: Path, sql: str) -> None: + """Run mutating SQL directly against a DuckDB file, outside SQLBuild.""" + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(db_path)) + try: + _ = connection.execute(sql) + finally: + connection.close() + + +def query_duckdb_rows(*, db_path: Path, sql: str) -> tuple[tuple[object, ...], ...]: + """Read rows directly from a DuckDB file.""" + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(db_path), read_only=True) + try: + return tuple(tuple(row) for row in connection.execute(sql).fetchall()) + finally: + connection.close() + + +def run_plan_json( + *, project_dir: Path, flags: tuple[str, ...], capsys: pytest.CaptureFixture[str] +) -> dict[str, object]: + """Run one real CLI JSON plan and return the parsed document.""" + + _ = capsys.readouterr() + exit_code: int = main(["--project-dir", str(project_dir), "plan", "--json", *flags]) + out: str + err: str + out, err = capsys.readouterr() + assert exit_code == 0, out + err + return cast(dict[str, object], json.loads(out)) + + +def plan_model(*, plan: dict[str, object], name: str) -> dict[str, object]: + """Return one serialized model entry from a JSON plan.""" + + models: list[dict[str, object]] = cast(list[dict[str, object]], plan["models"]) + return {str(model["name"]): model for model in models}[name] + + +def plan_warning_identities(*, plan: dict[str, object]) -> tuple[tuple[object, object], ...]: + """Return the resource name and diagnostic code of every plan warning.""" + + warnings: list[dict[str, object]] = cast(list[dict[str, object]], plan["warnings"]) + return tuple((warning.get("model_name"), warning.get("code")) for warning in warnings) + + +def plan_cursor_bounds(*, plan: dict[str, object], name: str) -> dict[str, object]: + """Return the serialized cursor bounds of one planned model.""" + + return cast(dict[str, object], plan_model(plan=plan, name=name)["cursor_bounds"]) + + +def plan_seed_reasons(*, plan: dict[str, object]) -> dict[str, object]: + """Return plan reasons keyed by seed name.""" + + seeds: list[dict[str, object]] = cast(list[dict[str, object]], plan["seeds"]) + return {str(seed["name"]): seed["reason"] for seed in seeds} + + +def dropped_relation_microbatch_sql(*, batch_concurrency: int) -> str: + """Render an integer-cursor microbatch model over the raw order source.""" + + return ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n" + " cursor_watermark_mode all,\n" + " cursor id,\n" + " cursor_type integer,\n" + " cursor_inputs (raw_orders (column id, roles [filter, watermark]),),\n" + ' batch_size "1",\n' + f" batch_concurrency {batch_concurrency},\n" + ");\n\n" + 'SELECT id, ordered_at FROM __source("raw_orders")\n' + "WHERE id >= __cursor_start() AND id < __cursor_end()\n" + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py b/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py new file mode 100644 index 0000000000..4502280b6d --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py @@ -0,0 +1,326 @@ +"""Integration coverage for relations dropped outside SQLBuild through real CLI plans and builds.""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +import pytest + +from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + AbsentMicrobatchFullRefreshTestCase, + DroppedIncrementalFirstRunRangeTestCase, + DroppedRelationRecoveryTestCase, + DroppedSeedPlanTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + dropped_relation_microbatch_sql, + execute_duckdb_sql, + plan_cursor_bounds, + plan_model, + plan_seed_reasons, + plan_warning_identities, + query_duckdb_rows, + run_build, + run_plan_json, + write_dropped_relation_project, +) + +_INCREMENTAL_SQL: str = ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key id,\n" + " cursor ordered_at,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_start '2026-01-02',\n" + ");\n\n" + 'SELECT id, ordered_at FROM __source("raw_orders")\n' +) +_TABLE_SQL: str = 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' +_MICROBATCH_CONCURRENCY_SETTINGS: str = ( + "\n[settings]\nconcurrency = 3\nmicrobatch_concurrency = true\n" +) +_BUILD_ARTIFACTS_SQL: str = ( + "SELECT table_name FROM information_schema.tables " + "WHERE table_name LIKE '%orders\\_\\_%' ESCAPE '\\'" +) + + +@pytest.mark.parametrize( + "test_case", + [ + DroppedRelationRecoveryTestCase( + description="incremental table with stale delta relation", + model_sql=_INCREMENTAL_SQL, + drop_sql="DROP TABLE main.orders", + stale_artifact_sql="CREATE TABLE main.orders__delta AS SELECT 99 AS id", + expected_action="create_table", + expected_rows=((2,), (3,)), + ), + DroppedRelationRecoveryTestCase( + description="table", + model_sql=_TABLE_SQL, + drop_sql="DROP TABLE main.orders", + expected_action="create_table", + expected_rows=((1,), (2,), (3,)), + ), + DroppedRelationRecoveryTestCase( + description="view", + model_sql='MODEL (materialized view);\n\nSELECT id FROM __source("raw_orders")\n', + drop_sql="DROP VIEW main.orders", + expected_action="create_view", + expected_rows=((1,), (2,), (3,)), + ), + DroppedRelationRecoveryTestCase( + description="snapshot", + model_sql=( + "MODEL (\n" + " materialized snapshot,\n" + " unique_key [id],\n" + " snapshot_strategy timestamp,\n" + " updated_at updated_at\n" + ");\n\n" + 'SELECT customer_id AS id, plan, updated_at FROM __source("raw_customers")\n' + ), + drop_sql="DROP TABLE main.orders", + expected_action="snapshot", + expected_rows=((1,),), + ), + DroppedRelationRecoveryTestCase( + description="sequential microbatch incremental with stale delta relation", + model_sql=dropped_relation_microbatch_sql(batch_concurrency=1), + drop_sql="DROP TABLE main.orders", + stale_artifact_sql="CREATE TABLE main.orders__delta AS SELECT 99 AS id", + expected_action="create_table", + expected_rows=((1,), (2,), (3,)), + ), + DroppedRelationRecoveryTestCase( + description="concurrent microbatch incremental with recorded completions", + model_sql=dropped_relation_microbatch_sql(batch_concurrency=3), + drop_sql="DROP TABLE main.orders", + settings_toml=_MICROBATCH_CONCURRENCY_SETTINGS, + expected_action="create_table", + expected_rows=((1,), (2,), (3,)), + ), + ], + ids=lambda case: case.description, +) +def test_given_relation_dropped_outside_sqlbuild_when_planning_and_building_then_recreates_it( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + test_case: DroppedRelationRecoveryTestCase, +) -> None: + db_path: Path = write_dropped_relation_project( + project_dir=tmp_path, + model_name="orders", + model_sql=test_case.model_sql, + settings_toml=test_case.settings_toml, + ) + first_exit_code: int + first_output: str + first_exit_code, first_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + assert first_exit_code == 0, first_output + execute_duckdb_sql(db_path=db_path, sql=test_case.drop_sql) + execute_duckdb_sql(db_path=db_path, sql=test_case.stale_artifact_sql) + + plan: dict[str, object] = run_plan_json(project_dir=tmp_path, flags=(), capsys=capsys) + rebuild_exit_code: int + rebuild_output: str + rebuild_exit_code, rebuild_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + rebuilt_rows: tuple[tuple[object, ...], ...] = query_duckdb_rows( + db_path=db_path, sql="SELECT id FROM main.orders ORDER BY id" + ) + steady_exit_code: int + steady_output: str + steady_exit_code, steady_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + steady_plan: dict[str, object] = run_plan_json(project_dir=tmp_path, flags=(), capsys=capsys) + + model: dict[str, object] = plan_model(plan=plan, name="orders") + assert (model["action"], model["reason"]) == (test_case.expected_action, "first_run") + assert ( + model["built_version_hash"], + model["built_version_present"], + model["identity_status"], + ) == (None, False, "missing") + assert plan_warning_identities(plan=plan) == ( + ("orders", RECORDED_RELATION_MISSING_WARNING_CODE), + ) + assert rebuild_exit_code == 0, rebuild_output + assert rebuilt_rows == test_case.expected_rows + assert steady_exit_code == 0, steady_output + assert query_duckdb_rows(db_path=db_path, sql=_BUILD_ARTIFACTS_SQL) == () + assert ("orders", RECORDED_RELATION_MISSING_WARNING_CODE) not in plan_warning_identities( + plan=steady_plan + ) + assert plan_model(plan=steady_plan, name="orders")["identity_status"] == "current" + + +@pytest.mark.parametrize( + "test_case", + [ + DroppedIncrementalFirstRunRangeTestCase( + description="delete insert timestamp cursor restarts at cursor_start", + model_sql=_INCREMENTAL_SQL, + expected_start="2026-01-02T00:00:00", + expected_end="2026-01-07T00:00:00", + expected_rows=( + (2, datetime(2026, 1, 2, 1)), + (3, datetime(2026, 1, 3, 1)), + (4, datetime(2026, 1, 5, 1)), + (5, datetime(2026, 1, 6, 1)), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_incremental_relation_dropped_after_later_builds_when_rebuilding_then_uses_first_run_range( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + test_case: DroppedIncrementalFirstRunRangeTestCase, +) -> None: + db_path: Path = write_dropped_relation_project( + project_dir=tmp_path, model_name="orders", model_sql=test_case.model_sql + ) + first_plan: dict[str, object] = run_plan_json(project_dir=tmp_path, flags=(), capsys=capsys) + first_exit_code: int + first_output: str + first_exit_code, first_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + assert first_exit_code == 0, first_output + execute_duckdb_sql( + db_path=db_path, sql="INSERT INTO main.raw_orders VALUES (4, '2026-01-05 01:00:00')" + ) + second_exit_code: int + second_output: str + second_exit_code, second_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + assert second_exit_code == 0, second_output + execute_duckdb_sql( + db_path=db_path, + sql=( + "DROP TABLE main.orders; INSERT INTO main.raw_orders VALUES (5, '2026-01-06 01:00:00')" + ), + ) + + recreate_plan: dict[str, object] = run_plan_json(project_dir=tmp_path, flags=(), capsys=capsys) + rebuild_exit_code: int + rebuild_output: str + rebuild_exit_code, rebuild_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + + assert plan_model(plan=recreate_plan, name="orders")["reason"] == "first_run" + assert plan_cursor_bounds(plan=first_plan, name="orders")["start"] == test_case.expected_start + assert plan_cursor_bounds(plan=recreate_plan, name="orders") == { + "start": test_case.expected_start, + "end": test_case.expected_end, + } + assert rebuild_exit_code == 0, rebuild_output + assert ( + query_duckdb_rows(db_path=db_path, sql="SELECT id, ordered_at FROM main.orders ORDER BY id") + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DroppedSeedPlanTestCase( + description="seed with recorded fingerprint", + model_sql=_TABLE_SQL, + expected_steady_reasons={"order_statuses": "no_change"}, + expected_dropped_reasons={"order_statuses": "first_run"}, + expected_rows=((1,), (2,)), + ), + ], + ids=lambda case: case.description, +) +def test_given_seed_dropped_outside_sqlbuild_when_planning_then_reports_first_run( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + test_case: DroppedSeedPlanTestCase, +) -> None: + db_path: Path = write_dropped_relation_project( + project_dir=tmp_path, model_name="orders", model_sql=test_case.model_sql + ) + first_exit_code: int + first_output: str + first_exit_code, first_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + assert first_exit_code == 0, first_output + steady_plan: dict[str, object] = run_plan_json(project_dir=tmp_path, flags=(), capsys=capsys) + execute_duckdb_sql(db_path=db_path, sql="DROP TABLE main.order_statuses") + + plan: dict[str, object] = run_plan_json(project_dir=tmp_path, flags=(), capsys=capsys) + rebuild_exit_code: int + rebuild_output: str + rebuild_exit_code, rebuild_output = run_build(project_dir=tmp_path, flags=(), capsys=capsys) + + assert plan_seed_reasons(plan=steady_plan) == test_case.expected_steady_reasons + assert plan_seed_reasons(plan=plan) == test_case.expected_dropped_reasons + assert rebuild_exit_code == 0, rebuild_output + assert ( + query_duckdb_rows( + db_path=db_path, + sql="SELECT status_code FROM main.order_statuses ORDER BY status_code", + ) + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + AbsentMicrobatchFullRefreshTestCase( + description="sequential microbatch never built", + batch_concurrency=1, + setup_build_flags=("--select", "order_statuses"), + setup_drop_sql="SELECT 1", + expected_rows=((1,), (2,), (3,)), + ), + AbsentMicrobatchFullRefreshTestCase( + description="sequential microbatch dropped outside sqlbuild", + batch_concurrency=1, + setup_build_flags=(), + setup_drop_sql="DROP TABLE main.orders", + expected_rows=((1,), (2,), (3,)), + ), + AbsentMicrobatchFullRefreshTestCase( + description="concurrent microbatch dropped outside sqlbuild", + batch_concurrency=3, + setup_build_flags=(), + setup_drop_sql="DROP TABLE main.orders", + expected_rows=((1,), (2,), (3,)), + ), + ], + ids=lambda case: case.description, +) +def test_given_microbatch_target_absent_when_full_refresh_building_then_creates_target( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + test_case: AbsentMicrobatchFullRefreshTestCase, +) -> None: + db_path: Path = write_dropped_relation_project( + project_dir=tmp_path, + model_name="orders", + model_sql=dropped_relation_microbatch_sql(batch_concurrency=test_case.batch_concurrency), + settings_toml=_MICROBATCH_CONCURRENCY_SETTINGS, + ) + setup_exit_code: int + setup_output: str + setup_exit_code, setup_output = run_build( + project_dir=tmp_path, flags=test_case.setup_build_flags, capsys=capsys + ) + assert setup_exit_code == 0, setup_output + execute_duckdb_sql(db_path=db_path, sql=test_case.setup_drop_sql) + + exit_code: int + output: str + exit_code, output = run_build( + project_dir=tmp_path, flags=("--full-refresh", "--select", "orders"), capsys=capsys + ) + + assert exit_code == 0, output + assert ( + query_duckdb_rows(db_path=db_path, sql="SELECT id FROM main.orders ORDER BY id") + == test_case.expected_rows + ) + assert query_duckdb_rows(db_path=db_path, sql=_BUILD_ARTIFACTS_SQL) == () diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index 1cf523d5fd..e10780b8b5 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -486,6 +486,32 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence '"severity": "warning"', ), ), + JsonOutputTestCase( + description="plan json exposes warning diagnostic codes", + plan_output=build_plan_output( + model_entries=( + build_model_entry( + name="orders", + action=PlanAction.CREATE_TABLE, + reason=PlanReason.FIRST_RUN, + fingerprint_version_hash="expected_hash", + ), + ), + warnings=( + build_warning( + model_name="orders", + message="relation no longer exists", + code="RECORDED_RELATION_MISSING", + ), + ), + ), + expected_keys=("models", "warnings"), + expected_fragments=( + '"code": "RECORDED_RELATION_MISSING"', + '"built_version_present": false', + '"identity_status": "missing"', + ), + ), JsonOutputTestCase( description="plan json includes source load count", plan_output=build_plan_output( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index 2e44bb28f1..fc823518e3 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -260,6 +260,7 @@ def _build_fingerprints(test_case: DetectModelChangesTestCase) -> dict[str, Fing config_values=fingerprint_config_values, ), ts=_STUB_TS, + version_hash="recorded_version", ) } return ({}, fingerprints)[test_case.fingerprint_query_hash is not None] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index b38cfcd1f9..657dbe3237 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -3,7 +3,7 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.planner.models import BackfillResult, SchemaFinding -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind +from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind, PlanAction, PlanReason from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -25,6 +25,15 @@ class DetectModelChangesTestCase: fingerprint_config_values: dict[str, object] | None = None inferred_columns: tuple[InferredColumn, ...] | None = None fast_lineage_has_star: bool = False + expected_recorded_build_relation_missing: bool = False + + +@dataclass(frozen=True) +class DroppedRelationPlanActionTestCase: + description: str + config_values: dict[str, object] + expected_action: PlanAction + expected_reason: PlanReason @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index aac5def677..6b2293caf8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -9,6 +9,12 @@ from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.changes.detect import detect_changes, detect_model_changes +from sqlbuild.compiler.planner._helpers.output.strategy import ( + build_model_warnings, + get_materialization_type, + resolve_model_plan_action, +) +from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( build_version_identity_metadata_json, ) @@ -16,10 +22,17 @@ ChangeDetectionResult, PlannerChangeResults, PlannerScope, + PlanWarning, WarehouseFingerprints, WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind +from sqlbuild.compiler.planner.types import ( + BackfillAction, + ChangeKind, + PlanAction, + PlanReason, + WarningSeverity, +) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_helpers import ( build_metadata_json_with_audit_gate, build_model_from_metadata_test_case, @@ -32,6 +45,7 @@ from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectModelChangesTestCase, DetectModelMetadataTestCase, + DroppedRelationPlanActionTestCase, ) _QUERY_SQL: str = "SELECT id, name FROM orders" @@ -57,6 +71,39 @@ expected_change_kind=ChangeKind.FIRST_RUN, expected_backfill_action=BackfillAction.FULL, ), + DetectModelChangesTestCase( + description="detects first run when relation is missing despite matching fingerprint", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={}, + schema_columns=(), + relation_exists=False, + fingerprint_query_hash=_MATCHING_HASH, + warehouse_column_names=(), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + expected_change_kind=ChangeKind.FIRST_RUN, + expected_backfill_action=BackfillAction.FULL, + expected_recorded_build_relation_missing=True, + ), + DetectModelChangesTestCase( + description="detects first run when relation is missing despite changed fingerprint", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={"replay_on_change": "bounded-30d"}, + schema_columns=(("id", "INTEGER"), ("status", "VARCHAR")), + relation_exists=False, + fingerprint_query_hash=_DIFFERENT_HASH, + fingerprint_config_values={"materialized": "view"}, + warehouse_column_names=(), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + expected_change_kind=ChangeKind.FIRST_RUN, + expected_backfill_action=BackfillAction.FULL, + expected_recorded_build_relation_missing=True, + ), DetectModelChangesTestCase( description="detects no change when query hash matches and no schema columns declared", model_name="orders", @@ -200,6 +247,100 @@ def test_given_model_and_snapshot_when_detecting_changes_then_returns_expected( assert result.change_kind == test_case.expected_change_kind assert result.backfill.action == test_case.expected_backfill_action + assert ( + result.recorded_build_relation_missing == test_case.expected_recorded_build_relation_missing + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DroppedRelationPlanActionTestCase( + description="incremental plans create table", + config_values={"materialized": "incremental", "incremental_strategy": "delete_insert"}, + expected_action=PlanAction.CREATE_TABLE, + expected_reason=PlanReason.FIRST_RUN, + ), + DroppedRelationPlanActionTestCase( + description="table plans create table", + config_values={"materialized": "table"}, + expected_action=PlanAction.CREATE_TABLE, + expected_reason=PlanReason.FIRST_RUN, + ), + DroppedRelationPlanActionTestCase( + description="view plans create view", + config_values={"materialized": "view"}, + expected_action=PlanAction.CREATE_VIEW, + expected_reason=PlanReason.FIRST_RUN, + ), + DroppedRelationPlanActionTestCase( + description="snapshot plans first run", + config_values={"materialized": "snapshot"}, + expected_action=PlanAction.SNAPSHOT, + expected_reason=PlanReason.FIRST_RUN, + ), + DroppedRelationPlanActionTestCase( + description="custom materialization plans first run", + config_values={"materialized": "partition_tracked"}, + expected_action=PlanAction.CUSTOM, + expected_reason=PlanReason.FIRST_RUN, + ), + ], + ids=lambda case: case.description, +) +def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_first_run( + test_case: DroppedRelationPlanActionTestCase, +) -> None: + detection_case: DetectModelChangesTestCase = DetectModelChangesTestCase( + description=test_case.description, + model_name="orders", + query_sql=_QUERY_SQL, + config_values=test_case.config_values, + schema_columns=(), + relation_exists=False, + fingerprint_query_hash=_MATCHING_HASH, + warehouse_column_names=(), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + expected_change_kind=ChangeKind.FIRST_RUN, + expected_backfill_action=BackfillAction.FULL, + ) + model: CompiledModel = build_model_from_test_case(detection_case) + + change: ChangeDetectionResult = detect_model_changes( + model=model, + snapshot=build_snapshot_from_test_case(detection_case), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + ) + action: PlanAction + reason: PlanReason + action, reason = resolve_model_plan_action( + model=model, change_result=change, full_refresh=False + ) + warnings: tuple[PlanWarning, ...] = build_model_warnings( + model_name=model.name, + materialization_type=get_materialization_type(model), + change_result=change, + schema_actions=(), + on_schema_change=None, + type_enforcement=False, + ) + + assert (action, reason) == (test_case.expected_action, test_case.expected_reason) + assert change.backfill.action == BackfillAction.FULL + assert change.previous_version_hash is None + assert change.previous_metadata_json is None + assert (change.query_changed, change.config_changed, change.schema_findings) == ( + False, + False, + (), + ) + assert [warning.code for warning in warnings] == [RECORDED_RELATION_MISSING_WARNING_CODE] + assert warnings[0].severity == WarningSeverity.WARNING + assert warnings[0].model_name == "orders" @pytest.mark.parametrize( From 0441157bc47f79ba2b36fde2325c72392405456a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:18:05 +0100 Subject: [PATCH 2/3] docs: document state versus live warehouse source of truth --- AGENTS.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 6bb5192c5e..0c114aa6bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,6 +69,20 @@ - State the expected verification scope in subagent prompts and explicitly prohibit unnecessary full-suite runs. - Do not delay committing and pushing a focused fix solely to repeat checks already completed successfully by another agent or CI. +## State and Source of Truth + +- Do not store what can be calculated. If a value can be derived from the project or the live + warehouse at the point of use, derive it there and then instead of persisting it and reading it + back later. +- Stored state is for history, auditing, logging, and facts that genuinely cannot be recalculated: + what was built, when, at which version, and whether a migration or batch completed. +- The warehouse catalog and planner snapshot are authoritative for live facts such as relation + existence, relation type, columns and types, table type, and retention. Never infer a live fact + from fingerprints, events, bindings, or a plan reason when the warehouse can answer; built-up + state drifts through external drops and edits and then needs reconciliation. +- Consult stored state only after the live facts it describes are confirmed. For example, a model + whose relation is missing plans as a first run regardless of any recorded fingerprint. + ## Direct-Mode Warehouse State - NEVER implement a mutable lifecycle state machine in raw warehouse state tables in direct mode. Do not represent progress by repeatedly updating one row through statuses such as `PLANNED`, `RUNNING`, and `COMPLETE`. From 7cff19bc93a661afbe3ae1aedf9fc307d61f3e26 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:23:23 +0100 Subject: [PATCH 3/3] fix: keep verified rename handover for pending migration destinations After integrating model migrations, the unconditional missing-relation first run preceded the RENAMED branch, so a verified rename whose destination is intentionally not created yet planned as a first run, emitted a spurious RECORDED_RELATION_MISSING warning, and replayed downstream history. Check the migration-verified rename before treating a missing relation as a first run. --- .../planner/_helpers/changes/detect.py | 14 ++++---- .../planner/_helpers/changes/_test_helpers.py | 3 ++ .../planner/_helpers/changes/_test_types.py | 1 + .../planner/_helpers/changes/test_detect.py | 32 +++++++++++++++++++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 462d1eced8..f6fd3f3d1e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -175,22 +175,22 @@ def detect_model_changes( relation_exists: bool = model_name in snapshot.existing_relations fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model_name) - if not relation_exists: + if not relation_exists and model_name in snapshot.renamed_models: return ChangeDetectionResult( model_name=model_name, - change_kind=ChangeKind.FIRST_RUN, - backfill=BackfillResult(action=BackfillAction.FULL), + change_kind=ChangeKind.RENAMED, + backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), fingerprint_metadata_json=metadata_json, fingerprint_version_hash=expected_version_hash, - recorded_build_relation_missing=fingerprint is not None, ) - if not relation_exists and model_name in snapshot.renamed_models: + if not relation_exists: return ChangeDetectionResult( model_name=model_name, - change_kind=ChangeKind.RENAMED, - backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + change_kind=ChangeKind.FIRST_RUN, + backfill=BackfillResult(action=BackfillAction.FULL), fingerprint_metadata_json=metadata_json, fingerprint_version_hash=expected_version_hash, + recorded_build_relation_missing=fingerprint is not None, ) query_changed: bool = False diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index fc823518e3..4ddd126509 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -202,6 +202,9 @@ def build_snapshot_from_test_case(test_case: DetectModelChangesTestCase) -> Ware existing_relations=relations, existing_columns=columns, fingerprints=WarehouseFingerprints(models=fingerprints), + renamed_models=(frozenset(), frozenset({test_case.model_name}))[ + test_case.renamed_by_migration + ], ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index c590f01462..69d8a56770 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -26,6 +26,7 @@ class DetectModelChangesTestCase: inferred_columns: tuple[InferredColumn, ...] | None = None fast_lineage_has_star: bool = False expected_recorded_build_relation_missing: bool = False + renamed_by_migration: bool = False @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 6b2293caf8..831348ae7e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -104,6 +104,38 @@ expected_backfill_action=BackfillAction.FULL, expected_recorded_build_relation_missing=True, ), + DetectModelChangesTestCase( + description="keeps verified rename handover when migration destination is pending", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={}, + schema_columns=(), + relation_exists=False, + fingerprint_query_hash=_MATCHING_HASH, + warehouse_column_names=(), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + expected_change_kind=ChangeKind.RENAMED, + expected_backfill_action=BackfillAction.FORWARD_ONLY, + renamed_by_migration=True, + ), + DetectModelChangesTestCase( + description="uses stored state once renamed destination exists", + model_name="orders", + query_sql=_QUERY_SQL, + config_values={}, + schema_columns=(), + relation_exists=True, + fingerprint_query_hash=_MATCHING_HASH, + warehouse_column_names=(), + sql_analysis_enabled=False, + query_change_tracking=True, + full_refresh=False, + expected_change_kind=ChangeKind.NO_CHANGE, + expected_backfill_action=BackfillAction.FORWARD_ONLY, + renamed_by_migration=True, + ), DetectModelChangesTestCase( description="detects no change when query hash matches and no schema columns declared", model_name="orders",