fix: recreate relations missing from the warehouse snapshot (#818)

* fix: plan relations missing from the warehouse as first runs

A model whose relation was dropped outside SQLBuild was still planned as
built because a stored fingerprint existed: incremental models kept their
cursor bounds and failed on the missing table, and views and tables reported
no_change. Change detection now treats the warehouse snapshot as the source of
truth for relation existence and consults stored fingerprints only when the
relation exists.

- Plan a first run for any model whose relation is absent, regardless of
  stored fingerprints, and emit a RECORDED_RELATION_MISSING plan warning when
  stored state recorded a previous build.
- Report the built version as missing in plan JSON and expose warning codes.
- Plan a seed whose relation is absent as a first run.
- Let microbatch full-refresh builds create an absent live target instead of
  failing reconciliation.

* docs: document state versus live warehouse source of truth

* fix: keep verified rename handover for pending migration destinations

After integrating model migrations, the unconditional missing-relation first
run preceded the RENAMED branch, so a verified rename whose destination is
intentionally not created yet planned as a first run, emitted a spurious
RECORDED_RELATION_MISSING warning, and replayed downstream history. Check the
migration-verified rename before treating a missing relation as a first run.

Author: kvlonge

AGENTS.md

@@ -69,6 +69,20 @@
 - State the expected verification scope in subagent prompts and explicitly prohibit unnecessary full-suite runs.
 - Do not delay committing and pushing a focused fix solely to repeat checks already completed successfully by another agent or CI.
 
+## State and Source of Truth
+
+- Do not store what can be calculated. If a value can be derived from the project or the live
+  warehouse at the point of use, derive it there and then instead of persisting it and reading it
+  back later.
+- Stored state is for history, auditing, logging, and facts that genuinely cannot be recalculated:
+  what was built, when, at which version, and whether a migration or batch completed.
+- The warehouse catalog and planner snapshot are authoritative for live facts such as relation
+  existence, relation type, columns and types, table type, and retention. Never infer a live fact
+  from fingerprints, events, bindings, or a plan reason when the warehouse can answer; built-up
+  state drifts through external drops and edits and then needs reconciliation.
+- Consult stored state only after the live facts it describes are confirmed. For example, a model
+  whose relation is missing plans as a first run regardless of any recorded fingerprint.
+
 ## Direct-Mode Warehouse State
 
 - NEVER implement a mutable lifecycle state machine in raw warehouse state tables in direct mode. Do not represent progress by repeatedly updating one row through statuses such as `PLANNED`, `RUNNING`, and `COMPLETE`.

Makefile

@@ -91,6 +91,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py \

src/sqlbuild/cli/output/_helpers/plan_json.py

@@ -306,6 +306,8 @@ def _serialize_warning(warning: PlanWarning) -> dict[str, object]:
     }
     if warning.model_name is not None:
         result["model_name"] = warning.model_name
+    if warning.code is not None:
+        result["code"] = warning.code
     return result
 
 

src/sqlbuild/compiler/planner/_helpers/changes/detect.py

@@ -175,21 +175,22 @@ def detect_model_changes(
     relation_exists: bool = model_name in snapshot.existing_relations
     fingerprint: Fingerprint | None = snapshot.fingerprints.models.get(model_name)
 
-    if not relation_exists and fingerprint is None:
+    if not relation_exists and model_name in snapshot.renamed_models:
         return ChangeDetectionResult(
             model_name=model_name,
-            change_kind=ChangeKind.FIRST_RUN,
-            backfill=BackfillResult(action=BackfillAction.FULL),
+            change_kind=ChangeKind.RENAMED,
+            backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY),
             fingerprint_metadata_json=metadata_json,
             fingerprint_version_hash=expected_version_hash,
         )
-    if not relation_exists and model_name in snapshot.renamed_models:
+    if not relation_exists:
         return ChangeDetectionResult(
             model_name=model_name,
-            change_kind=ChangeKind.RENAMED,
-            backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY),
+            change_kind=ChangeKind.FIRST_RUN,
+            backfill=BackfillResult(action=BackfillAction.FULL),
             fingerprint_metadata_json=metadata_json,
             fingerprint_version_hash=expected_version_hash,
+            recorded_build_relation_missing=fingerprint is not None,
         )
 
     query_changed: bool = False

src/sqlbuild/compiler/planner/_helpers/output/plan_output.py

@@ -181,6 +181,8 @@ def _resolve_seed_plan_reason(
     reason: PlanReason | None = (seed_plan_reasons or {}).get(seed_name)
     if reason is not None:
         return reason
+    if seed_name not in snapshot.existing_relations:
+        return PlanReason.FIRST_RUN
     fingerprint: Fingerprint | None = snapshot.fingerprints.seeds.get(seed_name)
     if fingerprint is None:
         return PlanReason.FIRST_RUN

src/sqlbuild/compiler/planner/_helpers/output/strategy.py

@@ -5,6 +5,7 @@
 from sqlbuild.compiler.compile.models import (
     CompiledModel,
 )
+from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE
 from sqlbuild.compiler.planner.exceptions import PlannerInputError
 from sqlbuild.compiler.planner.models import (
     ChangeDetectionResult,
@@ -142,6 +143,20 @@ def build_model_warnings(
     effective: OnSchemaChange = on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE
     warnings: list[PlanWarning] = []
 
+    if change_result.recorded_build_relation_missing:
+        warnings.append(
+            PlanWarning(
+                model_name=model_name,
+                severity=WarningSeverity.WARNING,
+                code=RECORDED_RELATION_MISSING_WARNING_CODE,
+                message=(
+                    f"{model_name}: SQLBuild state records a previous build, but the relation "
+                    "no longer exists in the warehouse. It may have been dropped outside "
+                    "SQLBuild; planning a first run to recreate it."
+                ),
+            )
+        )
+
     if effective == OnSchemaChange.FAIL and change_result.schema_findings:
         warnings.append(
             PlanWarning(

src/sqlbuild/compiler/planner/constants.py

@@ -10,6 +10,7 @@
     "'checks/', or 'loaders/'"
 )
 EMPTY_FINGERPRINT_METADATA_JSON: str = "{}"
+RECORDED_RELATION_MISSING_WARNING_CODE: str = "RECORDED_RELATION_MISSING"
 MODEL_SELECTOR_ROOT: str = "models"
 SQL_FILE_SELECTOR_SUFFIX: str = ".sql"
 MODEL_SELECTOR_ROOT_PREFIX: str = "models/"

src/sqlbuild/compiler/planner/models.py

@@ -736,6 +736,7 @@ class ChangeDetectionResult:
     backfill: BackfillResult = field(
         default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY)
     )
+    recorded_build_relation_missing: bool = False
 
 
 @dataclass(frozen=True)

src/sqlbuild/executor/run/_helpers/materializations/microbatch.py

@@ -43,7 +43,6 @@
     IncrementalStrategy,
     MicrobatchStrategy,
     OnSchemaChange,
-    PlanReason,
 )
 from sqlbuild.cursor_algebra.main.cap_from_end import cap_from_end
 from sqlbuild.cursor_algebra.main.cap_from_start import cap_from_start
@@ -3086,11 +3085,6 @@ def _prepare_full_refresh_rebuild(
                 destination=relations.target_qualified,
                 statement_recorder=state.statement_recorder,
             )
-        elif not live_exists and context.entry.reason == PlanReason.FULL_REFRESH:
-            raise ExecutorInputError(
-                "full-refresh reconciliation found neither the live target nor its rebuild "
-                f"relation for '{context.entry.name}'"
-            )
         elif rebuild_exists:
             context.adapter.drop(
                 connection=context.connection,

tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py

@@ -1113,3 +1113,10 @@ class PythonNodeIdentityBuildE2ETestCase:
     edited_text: str
     expected_asset_identity_status: str
     expected_loader_version_count: int
+
+
+@dataclass(frozen=True)
+class DroppedIncrementalRelationE2ETestCase:
+    description: str
+    incremental_strategy: str
+    expected_rows: tuple[tuple[int, str], ...]