From 630234c3eeef6d659d76507708a449af4a9bc1a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:49:51 +0100 Subject: [PATCH 1/2] fix(incremental): settle cursor windows before staging when inputs are missing or empty Breaking: a cursor-based incremental model that reads no __ref(), __source() or __seed() and lists no cursor_inputs (for example one that reads a hard-coded table) is now compile error P011, pointing at its cursor declaration. This applies to every incremental strategy (delete_insert, append and merge). Such models previously re-read everything on every run because no input could bound their window. When cursor inputs have no rows: - An incremental run succeeds without changing the destination, still runs the model's audits against it, and reports "no input rows (.)" in plan and build output. Build JSON lists empty_cursor_inputs; plan JSON reports resolution_status "no_input_rows" with empty_inputs. - When only some inputs are empty and others have new rows, the model makes no progress and plan and build show a warning such as "waiting on empty input stg_refunds.refunded_at; other inputs have new rows". It counts toward the build summary's WARN= total and the asset's JSON warnings, and JSON adds "waiting_on_empty_inputs": true. Exit code and status are unchanged. - A first build creates the destination empty. - A rebuild that would replace an existing destination with nothing (replay_on_change full after a query change, including aliased models, or a watermark microbatch --full-refresh in any watermark mode) fails with S302 and keeps the existing rows. Models whose inputs are built earlier in the same run follow the same rules; previously such a model could advance past an empty input and skip rows that input later delivered. Inputs are named by their project names, including inputs built earlier in the same run. The window is decided before any delta or staging relation is created, and other window failures name the model, its inputs and the --start-cursor/--end-cursor flags. --- Makefile | 1 + .../references/docs/concepts/incremental.md | 28 +- .../classes/build_progress_callbacks.py | 30 +- .../cli/output/_helpers/cursor_plan.py | 8 +- .../_helpers/execution_result_document.py | 2 + src/sqlbuild/cli/output/_helpers/plan_json.py | 2 + src/sqlbuild/cli/output/_helpers/plan_text.py | 12 + src/sqlbuild/cli/output/models.py | 2 + src/sqlbuild/cli/output/types.py | 1 + .../compile/_helpers/config/cursor_inputs.py | 103 ++ src/sqlbuild/compiler/compile/constants.py | 1 + .../compile/main/_build_compile_inputs.py | 4 + .../discovery/_helpers/sql/model_files.py | 27 + .../main/_model_header_entry_span.py | 13 + .../planner/_helpers/output/plan_entry.py | 45 + .../planner/_helpers/resolve/cursor.py | 110 +++ .../planner/_helpers/resolve/resolve.py | 72 +- .../planner/_helpers/warehouse/snapshot.py | 59 +- .../planner/main/cursor_window/__init__.py | 0 .../cursor_window_override_flags.py | 13 + .../empty_cursor_input_decision.py | 25 + .../main/cursor_window/empty_cursor_window.py | 14 + .../empty_input_rebuild_refusal.py | 17 + .../cursor_window/no_input_rows_display.py | 13 + .../waiting_on_empty_inputs_display.py | 13 + src/sqlbuild/compiler/planner/models.py | 14 + .../executor/build/_helpers/scheduler.py | 30 + .../run/_helpers/execution/results.py | 20 + .../_helpers/materializations/incremental.py | 150 ++- .../_helpers/materializations/microbatch.py | 55 +- .../_helpers/materializations/table_cursor.py | 58 +- .../run/_helpers/validation/cursor_bounds.py | 91 +- src/sqlbuild/executor/run/exceptions.py | 16 + src/sqlbuild/executor/run/main/_execute.py | 10 + src/sqlbuild/executor/run/models.py | 8 + .../cli/commands/main/build/_test_types.py | 88 ++ .../cli/commands/main/build/helpers.py | 173 ++++ .../main/build/test_empty_cursor_inputs.py | 880 ++++++++++++++++++ .../build/test_model_backed_cursor_build.py | 5 +- .../integrations/dagster/_test_types.py | 10 + .../dagster/test_sqlbuild_assets.py | 72 ++ .../planner/_helpers/resolve/_test_types.py | 15 +- .../planner/_helpers/resolve/helpers.py | 64 +- .../planner/_helpers/resolve/test_resolve.py | 110 ++- .../executor/build/test_build_execution.py | 6 +- .../executor/build/test_table_type_routing.py | 8 +- .../test_microbatch_reconciliation.py | 10 +- .../compiler/compile/_helpers/_test_types.py | 11 + .../compiler/compile/_helpers/helpers.py | 54 ++ .../_helpers/test_cursor_model_inputs.py | 113 +++ .../sqlbuild/compiler/compile/test_main.py | 15 + .../compiler/planner/_helpers/_test_types.py | 12 + .../_helpers/test_warehouse_cursor_queries.py | 102 +- .../test_warehouse_cursor_concurrency.py | 4 +- .../executor/run/_helpers/_test_types.py | 13 +- .../run/_helpers/test_cursor_bounds.py | 46 +- .../docs/docs/concepts/incremental.mdx | 28 +- 57 files changed, 2789 insertions(+), 117 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/_helpers/config/cursor_inputs.py create mode 100644 src/sqlbuild/compiler/discovery/main/_model_header_entry_span.py create mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/__init__.py create mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py create mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py create mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py create mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py create mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/no_input_rows_display.py create mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/waiting_on_empty_inputs_display.py create mode 100644 src/sqlbuild/executor/run/exceptions.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_empty_cursor_inputs.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_model_inputs.py diff --git a/Makefile b/Makefile index 0cc2ba70e3..38d0f4e140 100644 --- a/Makefile +++ b/Makefile @@ -102,6 +102,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_empty_cursor_inputs.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py \ diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index edc9f7eaab..ac86af1841 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -162,7 +162,33 @@ MODEL ( SQLBuild uses these to determine the replay window. With multiple listed inputs, the end is the conservative common watermark: the minimum of their maxima. This prevents a faster input from advancing the model beyond data available from a slower input. -On a first build, SQLBuild derives the interval from the declared cursor inputs and cursor policy. If it cannot establish a valid interval, the build fails before mutating the destination. +On a first build, SQLBuild derives the interval from the declared cursor inputs and cursor policy. SQLBuild decides the window while planning, before any staging or delta relation is created. When an input is built earlier in the same run, the window is decided just before the model runs, still before any staging work. + +#### Models with no inputs + +A cursor-based model must read at least one input that can bound its window, either listed in `cursor_inputs` or read through `__ref()`, `__source()` or `__seed()`. A model that reads none of them could build once but could never work out a window afterwards, so compile rejects it with `P011`: + +``` +error[P011]: incremental model 'daily_revenue' reads no inputs, so builds after the first cannot work out their cursor window + --> models/finance/daily_revenue.sql:5:3 + | + 5 | cursor revenue_date, + | ^^^^^^^^^^^^^^^^^^^ + = help: read the data through __ref(), __source() or __seed() so its cursor can bound the window, or use materialized table if the model has no upstream data +``` + +A `rolling_window` microbatch model takes its window from the invocation time, so it may read no inputs. + +#### Inputs with no rows + +Empty inputs are normal: a new development or CI environment, a source in the middle of a reload, or an upstream model that is legitimately empty. When a listed input has no rows, there is no new data to process: + +- On an incremental run, the model succeeds without changing its destination, and the plan and build output show the reason, for example `no input rows (stg_payments.paid_at)`. The model's audits still run against the unchanged destination, and downstream models continue. The `--json` build output lists the inputs as `empty_cursor_inputs`, and `sqb plan --json` reports `"resolution_status": "no_input_rows"` with the same inputs in `empty_inputs`. +- When only some listed inputs are empty, the window still waits for them, so the model makes no progress even though other inputs have new rows. The plan and build output show this as a warning, for example `waiting on empty input stg_refunds.refunded_at; other inputs have new rows`. The build counts it in the summary's `WARN=` total and lists it in the asset's `warnings` in `--json` output, and the JSON output adds `"waiting_on_empty_inputs": true`. The exit code and model status do not change. A watermark microbatch model with `cursor_watermark_mode any` instead progresses from the inputs that have rows. +- On a first build, the destination is created empty so that models that reference it can build. The next build derives its window from the inputs as usual. +- A rebuild that would replace an existing destination, such as a query change with `replay_on_change full` or a microbatch `--full-refresh`, fails instead of leaving the table empty. Load the inputs first, or set the window explicitly with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms). + +Use [source freshness](sources.md) checks and source audits to catch an input that should not be empty. If SQLBuild cannot read the cursor bounds of an input at all, the plan fails with `S302`, naming the model and its inputs. ### Cursor bounds in model SQL diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py index 6d0f3d9417..58c7d8adda 100644 --- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py +++ b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py @@ -22,6 +22,10 @@ from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.planner.main.cursor_window.no_input_rows_display import format_no_input_rows +from sqlbuild.compiler.planner.main.cursor_window.waiting_on_empty_inputs_display import ( + format_waiting_on_empty_inputs, +) from sqlbuild.compiler.planner.main.execution.cursor_bound_display import cursor_bound_display from sqlbuild.compiler.planner.main.execution.inclusive_cursor_end import inclusive_cursor_end from sqlbuild.compiler.planner.main.execution.materialization_type_display import ( @@ -597,10 +601,18 @@ def _write_model_result(self, *, ctr: str, model_result: ModelExecutionResult) - elif event.kind == LifeCycleEventKind.LOG: self._write_log_block(event.content) - batch_line: str | None = _format_streaming_batch_summary(model_result=model_result) + batch_line: str | None = _format_streaming_batch_summary( + model_result=model_result + ) or _format_empty_cursor_inputs(model_result=model_result) if batch_line is not None: self._stream.write(f"{self._style.muted(batch_line)}\n") self._stream.flush() + if model_result.waiting_on_empty_inputs and model_result.empty_cursor_inputs: + waiting_line: str = ( + f" {format_waiting_on_empty_inputs(model_result.empty_cursor_inputs)}" + ) + self._stream.write(f"{self._style.warning(waiting_line)}\n") + self._stream.flush() sub_pad: str = " " * (self._prefix_width + _SUB_INDENT) sub_nw: int = self._name_width - _SUB_INDENT @@ -1524,7 +1536,11 @@ def _format_streaming_batch_summary(*, model_result: ModelExecutionResult) -> st if model_result.batch_size is not None: count_text = f"{count_text} ({model_result.batch_size})" parts: list[str] = [count_text] - if model_result.cursor_range_start is not None and model_result.cursor_range_end is not None: + if ( + model_result.cursor_range_start is not None + and model_result.cursor_range_end is not None + and not model_result.empty_cursor_inputs + ): start: str = cursor_bound_display( value=model_result.cursor_range_start, cursor_type=model_result.cursor_type, @@ -1538,6 +1554,8 @@ def _format_streaming_batch_summary(*, model_result: ModelExecutionResult) -> st parts.append(f"range {start} \u2192 {end}") if model_result.rows_affected is not None: parts.append(_format_abbreviated_rows(count=model_result.rows_affected)) + if model_result.empty_cursor_inputs and not model_result.waiting_on_empty_inputs: + parts.append(format_no_input_rows(model_result.empty_cursor_inputs)) if model_result.microbatch_recovery_batch_count: parts.append(f"{model_result.microbatch_recovery_batch_count} recovery") if model_result.microbatch_synthetic_completion_count: @@ -1547,6 +1565,14 @@ def _format_streaming_batch_summary(*, model_result: ModelExecutionResult) -> st return f" {' '.join(parts)}" +def _format_empty_cursor_inputs(*, model_result: ModelExecutionResult) -> str | None: + """Return why a cursor window selected no rows, if its inputs were empty.""" + + if not model_result.empty_cursor_inputs or model_result.waiting_on_empty_inputs: + return None + return f" {format_no_input_rows(model_result.empty_cursor_inputs)}" + + def _format_abbreviated_rows(*, count: int) -> str: """Format a row count with K/M/B abbreviation above 10,000.""" diff --git a/src/sqlbuild/cli/output/_helpers/cursor_plan.py b/src/sqlbuild/cli/output/_helpers/cursor_plan.py index a3629ef093..a18d1a9b6a 100644 --- a/src/sqlbuild/cli/output/_helpers/cursor_plan.py +++ b/src/sqlbuild/cli/output/_helpers/cursor_plan.py @@ -60,8 +60,10 @@ def build_cursor_plan_details(*, entry: ModelPlanEntry) -> CursorPlanDetails | N batch_size=effective_batch_size, cursor_type=entry.cursor_type, ) - if resolved_bounds is not None: - resolution_status: CursorResolutionStatus = CursorResolutionStatus.RESOLVED + if entry.empty_cursor_inputs: + resolution_status: CursorResolutionStatus = CursorResolutionStatus.NO_INPUT_ROWS + elif resolved_bounds is not None: + resolution_status = CursorResolutionStatus.RESOLVED elif runtime_owned: resolution_status = CursorResolutionStatus.DEFERRED else: @@ -77,6 +79,8 @@ def build_cursor_plan_details(*, entry: ModelPlanEntry) -> CursorPlanDetails | N declared_batch_size=entry.batch_size, effective_batch_size=effective_batch_size, planned_batch_count=planned_batch_count, + empty_inputs=entry.empty_cursor_inputs, + waiting_on_empty_inputs=entry.waiting_on_empty_inputs, ) diff --git a/src/sqlbuild/cli/output/_helpers/execution_result_document.py b/src/sqlbuild/cli/output/_helpers/execution_result_document.py index 960550078a..fe6cfd50eb 100644 --- a/src/sqlbuild/cli/output/_helpers/execution_result_document.py +++ b/src/sqlbuild/cli/output/_helpers/execution_result_document.py @@ -623,6 +623,8 @@ def _format_model_assets( else None ), "warnings": result.warning_messages, + "empty_cursor_inputs": list(result.empty_cursor_inputs) or None, + "waiting_on_empty_inputs": result.waiting_on_empty_inputs or None, "future_cursor_safety": serialize_future_cursor_safety(result.future_cursor_safety), "maximum_start_safety": serialize_maximum_start_safety(result.maximum_start_safety), "microbatch": _format_microbatch_result(result), diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 05f6b8fc97..24389270f5 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -231,6 +231,8 @@ def _serialize_cursor_details(*, details: CursorPlanDetails) -> dict[str, object "declared_batch_size": details.declared_batch_size, "effective_batch_size": details.effective_batch_size, "planned_batch_count": details.planned_batch_count, + "empty_inputs": list(details.empty_inputs), + "waiting_on_empty_inputs": details.waiting_on_empty_inputs, } diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index 0b1fd548a5..329f94b679 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -19,6 +19,10 @@ from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.constants import SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE from sqlbuild.compiler.planner.main.changes.query_diff import format_query_diff +from sqlbuild.compiler.planner.main.cursor_window.no_input_rows_display import format_no_input_rows +from sqlbuild.compiler.planner.main.cursor_window.waiting_on_empty_inputs_display import ( + format_waiting_on_empty_inputs, +) from sqlbuild.compiler.planner.main.execution.cursor_bound_display import cursor_bound_display from sqlbuild.compiler.planner.main.execution.inclusive_cursor_end import inclusive_cursor_end from sqlbuild.compiler.planner.main.execution.model_materialization_label import ( @@ -942,6 +946,14 @@ def _append_cursor_detail( requested_start: str = details.requested_start or "earliest available" requested_end: str = details.requested_end or "latest available" lines.append(f" requested: {requested_start} -> {requested_end}") + if details.empty_inputs and details.waiting_on_empty_inputs: + lines.append( + CliStyle(use_color=True).warning( + f" window: {format_waiting_on_empty_inputs(details.empty_inputs)}" + ) + ) + elif details.empty_inputs: + lines.append(f" window: {format_no_input_rows(details.empty_inputs)}") if show_range and details.resolved_bounds is not None: lines.append( f" range: {_format_cursor_range(bounds=details.resolved_bounds, entry=entry)}" diff --git a/src/sqlbuild/cli/output/models.py b/src/sqlbuild/cli/output/models.py index c3f4f83752..0e0603c16c 100644 --- a/src/sqlbuild/cli/output/models.py +++ b/src/sqlbuild/cli/output/models.py @@ -91,6 +91,8 @@ class CursorPlanDetails: declared_batch_size: str | None effective_batch_size: str | None planned_batch_count: int | None + empty_inputs: tuple[str, ...] = () + waiting_on_empty_inputs: bool = False @dataclass(frozen=True) diff --git a/src/sqlbuild/cli/output/types.py b/src/sqlbuild/cli/output/types.py index 3a8ca3333f..f84f12bde5 100644 --- a/src/sqlbuild/cli/output/types.py +++ b/src/sqlbuild/cli/output/types.py @@ -25,6 +25,7 @@ class CursorResolutionStatus(StrEnum): DEFERRED = "deferred" RESOLVED = "resolved" UNAVAILABLE = "unavailable" + NO_INPUT_ROWS = "no_input_rows" class PlanRowKind(StrEnum): diff --git a/src/sqlbuild/compiler/compile/_helpers/config/cursor_inputs.py b/src/sqlbuild/compiler/compile/_helpers/config/cursor_inputs.py new file mode 100644 index 0000000000..3434f811d2 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/config/cursor_inputs.py @@ -0,0 +1,103 @@ +"""Reject cursor-based incremental models that have no input to bound their cursor window.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.compile.constants import ( + CURSOR_INPUTS_CONFIG_KEY, + CURSOR_MODEL_WITHOUT_INPUTS_CODE, +) +from sqlbuild.compiler.compile.models import CompileModelInput, CompilerDiagnostic +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.discovery.main._model_header_entry_span import get_model_header_entry_span +from sqlbuild.compiler.planner.types import MaterializationType, MicrobatchStrategy +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.models import SourceLocation + +_CURSOR_CONFIG_KEY: str = "cursor" +_RELATION_REFERENCE_KINDS: frozenset[SqlReferenceKind] = frozenset( + { + SqlReferenceKind.REF, + SqlReferenceKind.SOURCE, + SqlReferenceKind.SEED, + SqlReferenceKind.DBT_REF, + } +) +_HELP: str = ( + "read the data through __ref(), __source() or __seed() so its cursor can bound the window, " + "or use materialized table if the model has no upstream data" +) + + +def cursor_model_without_inputs_diagnostics( + *, model_inputs: tuple[CompileModelInput, ...] +) -> tuple[CompilerDiagnostic, ...]: + """Return one error per cursor-based incremental model with no cursor inputs.""" + + return tuple( + _diagnostic(model_input) + for model_input in model_inputs + if _lacks_cursor_inputs(model_input) + ) + + +def _lacks_cursor_inputs(model_input: CompileModelInput) -> bool: + values: dict[str, object] = model_input.config.values + if get_config_str(values=values, key="materialized") != MaterializationType.INCREMENTAL: + return False + if get_config_str(values=values, key=_CURSOR_CONFIG_KEY) is None: + return False + if ( + get_config_str(values=values, key="microbatch_strategy") + == MicrobatchStrategy.ROLLING_WINDOW + ): + return False + if values.get(CURSOR_INPUTS_CONFIG_KEY) is not None: + return False + return not any( + reference.ref_kind in _RELATION_REFERENCE_KINDS for reference in model_input.references + ) + + +def _diagnostic(model_input: CompileModelInput) -> CompilerDiagnostic: + name: str = model_input.model_file.file_path.stem + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=CURSOR_MODEL_WITHOUT_INPUTS_CODE, + message=( + f"incremental model '{name}' reads no inputs, so builds after the first " + "cannot work out their cursor window" + ), + resource_type=CompiledResourceType.MODEL, + resource_name=name, + location=_cursor_location( + path=model_input.model_file.relative_path, + contents=model_input.model_file.contents, + ), + help=_HELP, + ) + + +def _cursor_location(*, path: Path, contents: str) -> SourceLocation: + span: tuple[int, int] | None = get_model_header_entry_span( + contents=contents, key=_CURSOR_CONFIG_KEY + ) + start: int = 0 if span is None else span[0] + line: int = contents.count("\n", 0, start) + 1 + column: int = start - (contents.rfind("\n", 0, start) + 1) + 1 + if span is None: + return SourceLocation(path=path, line=line, column=column) + return SourceLocation( + path=path, + line=line, + column=column, + end_line=line, + end_column=column + span[1] - span[0], + ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 44905feecc..e137192e6f 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -20,6 +20,7 @@ HARD_CODED_PROJECT_RELATION_CODE: str = "P008" UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: str = "P009" MISSING_DESCRIPTION_CODE: str = "P010" +CURSOR_MODEL_WITHOUT_INPUTS_CODE: str = "P011" DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { "model_files": "model", "scenario_files": "scenario", diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index af15698a4f..4d49ec40b0 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -22,6 +22,9 @@ from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, ) +from sqlbuild.compiler.compile._helpers.config.cursor_inputs import ( + cursor_model_without_inputs_diagnostics, +) from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( with_collected_compile_diagnostics, ) @@ -197,6 +200,7 @@ def build_compile_inputs( source_inputs=source_inputs, function_inputs=sql_function_inputs, ), + *cursor_model_without_inputs_diagnostics(model_inputs=model_build.inputs), ) return CompileProjectInputs( project_config=discovered_inputs.project_config, diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index e9876628a6..d103451a2a 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -287,6 +287,33 @@ def model_header_columns_span( ) +def model_header_entry_span(*, contents: str, key: str) -> tuple[int, int] | None: + """Return authored byte offsets for one top-level `key value` MODEL header entry.""" + + header_match: re.Match[str] | None = _MODEL_HEADER_PATTERN.match(contents) + if header_match is None: + return None + header_start: int = header_match.start("header") + tokens: list[_ModelHeaderToken] = _tokenize_model_header_for_spans(header_match.group("header")) + depth: int = 0 + for index, token in enumerate(tokens): + if token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == _MODEL_HEADER_OPEN_PAREN: + depth += 1 + elif token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == _MODEL_HEADER_CLOSE_PAREN: + depth -= 1 + elif ( + depth == 0 + and token.kind == _MODEL_HEADER_WORD_TOKEN + and token.value == key + and index + 2 < len(tokens) + ): + entry_end: int = header_start + tokens[index + 2].position + while entry_end > header_start + token.position and contents[entry_end - 1].isspace(): + entry_end -= 1 + return header_start + token.position, entry_end + return None + + def model_header_body_span(*, contents: str) -> tuple[int, int] | None: """Return authored byte offsets for the MODEL header body.""" diff --git a/src/sqlbuild/compiler/discovery/main/_model_header_entry_span.py b/src/sqlbuild/compiler/discovery/main/_model_header_entry_span.py new file mode 100644 index 0000000000..70860d1a13 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/main/_model_header_entry_span.py @@ -0,0 +1,13 @@ +"""Public authored MODEL header entry span query.""" + +from __future__ import annotations + +from sqlbuild.compiler.discovery._helpers.sql.model_files import ( + model_header_entry_span as _model_header_entry_span, +) + + +def get_model_header_entry_span(*, contents: str, key: str) -> tuple[int, int] | None: + """Return authored byte offsets for one top-level `key value` MODEL header entry.""" + + return _model_header_entry_span(contents=contents, key=key) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index c59e7180ba..60df83ed6f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -53,6 +53,7 @@ ) from sqlbuild.compiler.planner._helpers.resolve.cursor import ( compute_cursor_bounds, + cursor_inputs_without_rows, normalize_cursor_snapshot_grain, resolve_effective_timestamp_grain, without_destination_cursor, @@ -106,6 +107,7 @@ CursorOverrides, DeferralInputs, Duration, + EmptyCursorInputDecision, MaximumStartPolicyInputs, ModelCursorSnapshot, ModelPlanContext, @@ -806,6 +808,16 @@ def plan_model_from_change( invocation_time=context.invocation_time, ) + empty_cursor_inputs: tuple[str, ...] + waiting_on_empty_inputs: bool + empty_cursor_inputs, waiting_on_empty_inputs = _resolve_empty_cursor_inputs( + model=model, + snapshot=snapshot, + full_refresh=full_refresh, + runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, + cursor_overrides=cursor_overrides, + ) + cursor_type_warning: PlanWarning | None = check_cursor_type_consistency( model_name=model.name, cursor_column=cursor_column, @@ -908,6 +920,8 @@ def plan_model_from_change( else get_config_str(values=model.config.values, key="lookback") ), cursor_bounds=cursor_bounds, + empty_cursor_inputs=empty_cursor_inputs, + waiting_on_empty_inputs=waiting_on_empty_inputs, cursor_input_relations=cursor_input_relations, batch_size=plan_config.batch_size, batch_concurrency=get_config_int(values=model.config.values, key="batch_concurrency") or 1, @@ -1381,6 +1395,37 @@ def _compute_plan_cursor_bounds( ) +def _resolve_empty_cursor_inputs( + *, + model: CompiledModel, + snapshot: WarehouseSnapshot, + full_refresh: bool, + runtime_owned_cursor_bounds: bool, + cursor_overrides: CursorOverridePair, +) -> tuple[tuple[str, ...], bool]: + """Return empty inputs that leave a planner-owned window empty, and whether others have rows.""" + + if ( + full_refresh + or runtime_owned_cursor_bounds + or get_config_str(values=model.config.values, key="materialized") + != MaterializationType.INCREMENTAL + or get_config_str(values=model.config.values, key="cursor") is None + or get_config_str(values=model.config.values, key="microbatch_strategy") + == MicrobatchStrategy.ROLLING_WINDOW + or ( + cursor_overrides.start_cursor_override is not None + and cursor_overrides.end_cursor_override is not None + ) + ): + return (), False + cursor_snapshot: ModelCursorSnapshot | None = snapshot.cursor_snapshots.get(model.name) + if cursor_snapshot is None: + return (), False + decision: EmptyCursorInputDecision = cursor_inputs_without_rows(cursor_snapshot=cursor_snapshot) + return decision.input_names, decision.waiting_on_empty_inputs + + def _apply_future_safety( *, bounds: CursorBounds | None, diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py index 03a0be528d..f202904a34 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py @@ -5,9 +5,11 @@ from dataclasses import replace from datetime import datetime +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorBounds, Duration, + EmptyCursorInputDecision, MaximumStartPolicyInputs, ModelCursorSnapshot, ) @@ -24,10 +26,14 @@ from sqlbuild.cursor_algebra.main.max_bound import max_bound from sqlbuild.cursor_algebra.main.min_bound import min_bound from sqlbuild.cursor_algebra.main.observed_partition import observed_partition +from sqlbuild.cursor_algebra.main.parse import parse from sqlbuild.cursor_algebra.main.try_parse import try_parse from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar +_EMPTY_INTEGER_CURSOR_BOUND: int = 0 +_EMPTY_TIMESTAMP_CURSOR_BOUND: datetime = datetime(1970, 1, 1) + def compute_cursor_bounds( *, @@ -101,6 +107,108 @@ def compute_cursor_bounds( ) +def cursor_inputs_without_rows(*, cursor_snapshot: ModelCursorSnapshot) -> EmptyCursorInputDecision: + """Return empty cursor inputs that leave no window; any unreadable input is a failure.""" + + if cursor_snapshot.unreadable_input_names: + return EmptyCursorInputDecision() + return decide_empty_cursor_inputs( + empty_input_names=cursor_snapshot.empty_input_names, + waits_for_every_input=cursor_snapshot.cursor_watermark_mode != CursorWatermarkMode.ANY, + inputs_with_rows=bool( + cursor_snapshot.upstream_maxes + or cursor_snapshot.upstream_terminal_ends + or cursor_snapshot.upstream_availability_ends + ), + has_window_start=bool( + cursor_snapshot.target_max is not None + or cursor_snapshot.upstream_mins + or cursor_snapshot.upstream_terminal_starts + ), + ) + + +def decide_empty_cursor_inputs( + *, + empty_input_names: tuple[str, ...], + waits_for_every_input: bool, + inputs_with_rows: bool, + has_window_start: bool, +) -> EmptyCursorInputDecision: + """Decide whether empty cursor inputs leave no window, shared by planning and runtime.""" + + if not empty_input_names: + return EmptyCursorInputDecision() + if not waits_for_every_input and inputs_with_rows and has_window_start: + return EmptyCursorInputDecision() + return EmptyCursorInputDecision( + input_names=empty_input_names, waiting_on_empty_inputs=inputs_with_rows + ) + + +def empty_cursor_window(*, cursor_type: str | None, cursor_start: str | None) -> CursorBounds: + """Return a zero-width cursor window, which selects no rows from any filtered input.""" + + bound: CursorScalar + if cursor_start is not None: + bound = parse(raw=cursor_start, cursor_type=cursor_type or CursorType.TIMESTAMP) + elif cursor_type == CursorType.INTEGER: + bound = IntegerValue(value=_EMPTY_INTEGER_CURSOR_BOUND) + else: + bound = TimestampValue(value=_EMPTY_TIMESTAMP_CURSOR_BOUND) + return CursorBounds(start=bound, end=bound) + + +def format_no_input_rows(input_names: tuple[str, ...]) -> str: + """Explain that a cursor window is empty because its inputs have no rows.""" + + return f"no input rows ({', '.join(input_names)})" + + +def format_waiting_on_empty_inputs(input_names: tuple[str, ...]) -> str: + """Explain that a cursor window waits on empty inputs while other inputs have rows.""" + + noun: str = "input" if len(input_names) == 1 else "inputs" + return f"waiting on empty {noun} {', '.join(input_names)}; other inputs have new rows" + + +def cursor_window_override_flags(*, cursor_type: str | None) -> str: + """Name the CLI flags that set a cursor window explicitly for this cursor type.""" + + suffix: str = "int" if cursor_type == CursorType.INTEGER else "ts" + return f"--start-cursor-{suffix} and --end-cursor-{suffix}" + + +def empty_input_rebuild_refusal( + *, model_name: str, input_names: tuple[str, ...], cursor_type: str | None +) -> tuple[str, str]: + """Return the message and help for refusing to rebuild an existing table from empty inputs.""" + + return ( + f"model '{model_name}' was not rebuilt: its cursor inputs have no rows " + f"({', '.join(input_names)}), so rebuilding it now would leave it empty", + "load the inputs first, or set the window explicitly with " + + cursor_window_override_flags(cursor_type=cursor_type), + ) + + +def unreadable_cursor_inputs_error( + *, model_name: str, cursor_type: str | None, input_names: tuple[str, ...] +) -> PlannerInputError: + """Report cursor inputs whose bounds could not be read, in terms of the model's inputs.""" + + return PlannerInputError( + f"model '{model_name}' cannot work out its cursor window: cursor bounds could not be " + f"read from {', '.join(input_names)}", + code="S302", + help=( + "check that each input exists and has its cursor column (`sqb debug` checks the " + "connection), or set the window explicitly with " + f"{cursor_window_override_flags(cursor_type=cursor_type)}" + ), + ) + + def without_destination_cursor(*, cursor_snapshot: ModelCursorSnapshot) -> ModelCursorSnapshot: """Return the snapshot as a first run sees it, ignoring the target's existing cursor.""" @@ -184,6 +292,8 @@ def normalize_cursor_snapshot_grain( for physical, terminal in cursor_snapshot.upstream_end_inputs ), upstream_availability_ends=cursor_snapshot.upstream_availability_ends, + empty_input_names=cursor_snapshot.empty_input_names, + unreadable_input_names=cursor_snapshot.unreadable_input_names, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index 0fe0422d33..0d747a75c8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -6,7 +6,7 @@ from datetime import datetime from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup from sqlbuild.compiler.compile.main._cursor_roles import resolve_cursor_input_roles from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.compile.models import ( @@ -24,6 +24,10 @@ ) from sqlbuild.compiler.planner._helpers.resolve.cursor import ( compute_cursor_bounds, + cursor_inputs_without_rows, + empty_cursor_window, + empty_input_rebuild_refusal, + unreadable_cursor_inputs_error, without_destination_cursor, ) from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( @@ -256,18 +260,40 @@ def _compute_model_cursor_bounds( set(resolve_cursor_input_roles(model=model).watermark_inputs) & runtime_cursor_producer_names ) - if ( - not full_refresh - and not runtime_owned - and cursor_snapshot is not None - and not cursor_snapshot.watermarks_available - ): - unavailable: str = ", ".join(cursor_snapshot.unavailable_watermark_tags) - raise PlannerInputError( - f"model '{model.name}': required cursor watermark bounds are unavailable: " - f"{unavailable}", - code="S302", - ) + if not full_refresh and not runtime_owned and cursor_snapshot is not None: + empty_inputs: tuple[str, ...] = cursor_inputs_without_rows( + cursor_snapshot=cursor_snapshot + ).input_names + if ( + empty_inputs + and replaces_relation + and ( + _destination_exists(model=model, snapshot=snapshot) + or _history_redirected(model=model, cursor_snapshot=cursor_snapshot) + ) + ): + message, help_text = empty_input_rebuild_refusal( + model_name=model.name, + input_names=empty_inputs, + cursor_type=get_config_str(values=model.config.values, key="cursor_type"), + ) + raise PlannerInputError(message, code="S302", help=help_text) + if empty_inputs: + return empty_cursor_window( + cursor_type=get_config_str(values=model.config.values, key="cursor_type"), + cursor_start=get_config_cursor_bound( + values=model.config.values, key="cursor_start" + ), + ) + if not cursor_snapshot.watermarks_available: + raise unreadable_cursor_inputs_error( + model_name=model.name, + cursor_type=get_config_str(values=model.config.values, key="cursor_type"), + input_names=( + cursor_snapshot.unreadable_input_names + or cursor_snapshot.unavailable_watermark_tags + ), + ) if full_refresh: return None @@ -321,6 +347,26 @@ def _compute_model_cursor_bounds( ) +def _destination_exists(*, model: CompiledModel, snapshot: WarehouseSnapshot) -> bool: + """Return whether the model's resolved destination, alias included, exists in full identity.""" + + relation: RelationInfo | None = snapshot.existing_relations.get(model.name) + return relation is not None and relation.identity == RelationLookup.key( + database=model.destination.database, + schema=model.destination.schema, + name=model.destination.name, + ) + + +def _history_redirected(*, model: CompiledModel, cursor_snapshot: ModelCursorSnapshot) -> bool: + """Return whether the model's cursor history lives in a relation other than its destination.""" + + return ( + cursor_snapshot.target_relation is not None + and cursor_snapshot.target_relation != model.destination.qualified_name + ) + + def resolve_model_hook_entries( *, model: CompiledModel, diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index aebb030fd9..c7e8126b1f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -129,6 +129,14 @@ class _CursorQueryOutcome: error: Exception | None = None +@dataclass(frozen=True) +class _CursorQueryResults: + """Merged cursor values plus the tags of physical reads that failed.""" + + values: dict[str, CursorScalar] + failed_tags: frozenset[str] = frozenset() + + @dataclass(frozen=True) class _UpstreamCursorInfo: """Pre-resolved upstream cursor metadata for one ref.""" @@ -137,6 +145,7 @@ class _UpstreamCursorInfo: tag_max: str relation: str cursor_column: str + input_name: str = "" cursor_grain: CursorGrain | None = None terminal_cursor_start: CursorScalar | None = None terminal_cursor_end: CursorScalar | None = None @@ -816,16 +825,16 @@ def _gather_cursor_snapshots( queries: list[_PhysicalCursorQuery] = _build_cursor_queries(cursor_models) cursor_start: float = time.monotonic() concurrency: int = _inspection_concurrency(adapter=adapter, connection=connection) - results: dict[str, CursorScalar] = _execute_cursor_queries( + query_results: _CursorQueryResults = _execute_cursor_queries( queries=queries, connection=connection, execute=execute, on_progress=on_progress, concurrency=concurrency, ) - results = _gather_eligible_target_maxes( + results: dict[str, CursorScalar] = _gather_eligible_target_maxes( cursor_models=cursor_models, - results=results, + results=query_results.values, adapter=adapter, connection=connection, execute=execute, @@ -839,7 +848,9 @@ def _gather_cursor_snapshots( f"{physical_total} physical relation reads). ({time.monotonic() - cursor_start:.2f}s)" ) - return _assemble_cursor_snapshots(cursor_models=cursor_models, results=results) + return _assemble_cursor_snapshots( + cursor_models=cursor_models, results=results, failed_tags=query_results.failed_tags + ) def _collect_cursor_models( @@ -974,6 +985,7 @@ def _collect_cursor_models( tag_max=f"{model.name}__{ref.ref_name}__max", relation=upstream_relation, cursor_column=upstream_cursor_col, + input_name=ref.ref_name, cursor_grain=( CursorGrain(input_grain) if (input_grain := _cursor_input_grain(ref=ref, model_map=model_map)) @@ -1142,12 +1154,12 @@ def _execute_cursor_queries( execute: AdapterExecute[Any, Any], on_progress: Callable[[str], None] | None, concurrency: int = 1, -) -> dict[str, CursorScalar]: +) -> _CursorQueryResults: """Run one unmerged MIN/MAX statement per relation, bounded-parallel, merged in query order.""" total: int = len(queries) if concurrency <= 1 or total <= 1: - results: dict[str, CursorScalar] = {} + sequential_outcomes: list[_CursorQueryOutcome] = [] query_index: int query: _PhysicalCursorQuery for query_index, query in enumerate(queries, start=1): @@ -1160,8 +1172,8 @@ def _execute_cursor_queries( _report_cursor_query( query=query, outcome=outcome, identity=identity, on_progress=on_progress ) - results.update(_fan_out_cursor_row(query=query, row=outcome.row)) - return results + sequential_outcomes.append(outcome) + return _merge_cursor_outcomes(queries=queries, outcomes=sequential_outcomes) if on_progress is not None: on_progress( f"Inspecting cursor bounds for {total} relations " @@ -1178,11 +1190,21 @@ def _execute_cursor_queries( concurrency=concurrency, on_complete=progress.report, ) + return _merge_cursor_outcomes(queries=queries, outcomes=outcomes) + + +def _merge_cursor_outcomes( + *, queries: list[_PhysicalCursorQuery], outcomes: list[_CursorQueryOutcome] +) -> _CursorQueryResults: merged: dict[str, CursorScalar] = {} - outcome_item: _CursorQueryOutcome - for query, outcome_item in zip(queries, outcomes, strict=True): - merged.update(_fan_out_cursor_row(query=query, row=outcome_item.row)) - return merged + failed_tags: set[str] = set() + query: _PhysicalCursorQuery + outcome: _CursorQueryOutcome + for query, outcome in zip(queries, outcomes, strict=True): + if outcome.error is not None: + failed_tags.update((*query.min_tags, *query.max_tags)) + merged.update(_fan_out_cursor_row(query=query, row=outcome.row)) + return _CursorQueryResults(values=merged, failed_tags=frozenset(failed_tags)) class _CursorBoundsProgress: @@ -1442,6 +1464,7 @@ def _assemble_cursor_snapshots( *, cursor_models: list[_CursorModelInfo], results: dict[str, CursorScalar], + failed_tags: frozenset[str] = frozenset(), ) -> dict[str, ModelCursorSnapshot]: """Fan batch query results back into per-model cursor snapshots.""" @@ -1458,10 +1481,20 @@ def _assemble_cursor_snapshots( terminal_ends: list[CursorScalar] = [] end_inputs: list[tuple[CursorScalar | None, CursorScalar | None]] = [] availability_ends: list[CursorScalar] = [] + empty_input_names: list[str] = [] + unreadable_input_names: list[str] = [] upstream: _UpstreamCursorInfo for upstream in info.upstreams: min_val: CursorScalar | None = results.get(upstream.tag_min) max_val: CursorScalar | None = results.get(upstream.tag_max) + if max_val is None and upstream.terminal_cursor_end is None: + input_label: str = ( + f"{upstream.input_name or upstream.relation}.{upstream.cursor_column}" + ) + if upstream.tag_max in failed_tags: + unreadable_input_names.append(input_label) + else: + empty_input_names.append(input_label) effective_max: CursorScalar | None = ( exclusive_to_inclusive( value=upstream.terminal_cursor_end, @@ -1539,6 +1572,8 @@ def _assemble_cursor_snapshots( upstream_terminal_ends=tuple(terminal_ends), upstream_end_inputs=tuple(end_inputs) if terminal_ends else (), upstream_availability_ends=tuple(availability_ends), + empty_input_names=tuple(empty_input_names), + unreadable_input_names=tuple(unreadable_input_names), ) return snapshots diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/__init__.py b/src/sqlbuild/compiler/planner/main/cursor_window/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py b/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py new file mode 100644 index 0000000000..132826c442 --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py @@ -0,0 +1,13 @@ +"""Public naming of the CLI flags that set a cursor window explicitly.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner._helpers.resolve.cursor import ( + cursor_window_override_flags as _cursor_window_override_flags, +) + + +def cursor_window_override_flags(*, cursor_type: str | None) -> str: + """Name the CLI flags that set a cursor window explicitly for this cursor type.""" + + return _cursor_window_override_flags(cursor_type=cursor_type) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py new file mode 100644 index 0000000000..543483c310 --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py @@ -0,0 +1,25 @@ +"""Public decision on whether empty cursor inputs leave a model with no window.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner._helpers.resolve.cursor import ( + decide_empty_cursor_inputs as _decide_empty_cursor_inputs, +) +from sqlbuild.compiler.planner.models import EmptyCursorInputDecision + + +def decide_empty_cursor_inputs( + *, + empty_input_names: tuple[str, ...], + waits_for_every_input: bool, + inputs_with_rows: bool, + has_window_start: bool, +) -> EmptyCursorInputDecision: + """Decide whether empty cursor inputs leave no window, shared by planning and runtime.""" + + return _decide_empty_cursor_inputs( + empty_input_names=empty_input_names, + waits_for_every_input=waits_for_every_input, + inputs_with_rows=inputs_with_rows, + has_window_start=has_window_start, + ) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py new file mode 100644 index 0000000000..4cc7058cf2 --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py @@ -0,0 +1,14 @@ +"""Public zero-width cursor window entrypoint.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner._helpers.resolve.cursor import ( + empty_cursor_window as _empty_cursor_window, +) +from sqlbuild.compiler.planner.models import CursorBounds + + +def empty_cursor_window(*, cursor_type: str | None, cursor_start: str | None) -> CursorBounds: + """Return a zero-width cursor window, which selects no rows from any filtered input.""" + + return _empty_cursor_window(cursor_type=cursor_type, cursor_start=cursor_start) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py new file mode 100644 index 0000000000..fae1a78827 --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py @@ -0,0 +1,17 @@ +"""Public wording for refusing to rebuild an existing table from empty cursor inputs.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner._helpers.resolve.cursor import ( + empty_input_rebuild_refusal as _empty_input_rebuild_refusal, +) + + +def empty_input_rebuild_refusal( + *, model_name: str, input_names: tuple[str, ...], cursor_type: str | None +) -> tuple[str, str]: + """Return the message and help for refusing to rebuild an existing table from empty inputs.""" + + return _empty_input_rebuild_refusal( + model_name=model_name, input_names=input_names, cursor_type=cursor_type + ) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/no_input_rows_display.py b/src/sqlbuild/compiler/planner/main/cursor_window/no_input_rows_display.py new file mode 100644 index 0000000000..2088eba3ce --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/cursor_window/no_input_rows_display.py @@ -0,0 +1,13 @@ +"""Public display of cursor inputs that currently have no rows.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner._helpers.resolve.cursor import ( + format_no_input_rows as _format_no_input_rows, +) + + +def format_no_input_rows(input_names: tuple[str, ...]) -> str: + """Explain that a cursor window is empty because its inputs have no rows.""" + + return _format_no_input_rows(input_names) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/waiting_on_empty_inputs_display.py b/src/sqlbuild/compiler/planner/main/cursor_window/waiting_on_empty_inputs_display.py new file mode 100644 index 0000000000..3779375e8b --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/cursor_window/waiting_on_empty_inputs_display.py @@ -0,0 +1,13 @@ +"""Public display of a cursor window that waits on empty inputs while others have rows.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner._helpers.resolve.cursor import ( + format_waiting_on_empty_inputs as _format_waiting_on_empty_inputs, +) + + +def format_waiting_on_empty_inputs(input_names: tuple[str, ...]) -> str: + """Explain that a cursor window waits on empty inputs while other inputs have rows.""" + + return _format_waiting_on_empty_inputs(input_names) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index e1e1411c6b..6c15d2058f 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -284,6 +284,14 @@ class PathSelector: downstream: bool = False +@dataclass(frozen=True) +class EmptyCursorInputDecision: + """Cursor inputs whose lack of rows leaves no window, and whether other inputs have rows.""" + + input_names: tuple[str, ...] = () + waiting_on_empty_inputs: bool = False + + @dataclass(frozen=True, kw_only=True, init=False) class ModelCursorSnapshot: """Cursor MIN/MAX values gathered from warehouse for one incremental model.""" @@ -303,6 +311,8 @@ class ModelCursorSnapshot: upstream_terminal_ends: tuple[CursorScalar, ...] = () upstream_end_inputs: tuple[tuple[CursorScalar | None, CursorScalar | None], ...] = () upstream_availability_ends: tuple[CursorScalar, ...] = () + empty_input_names: tuple[str, ...] = field(default=(), compare=False) + unreadable_input_names: tuple[str, ...] = field(default=(), compare=False) def __init__(self, **values: Any) -> None: defaults: dict[str, object] = { @@ -318,6 +328,8 @@ def __init__(self, **values: Any) -> None: "upstream_terminal_ends": (), "upstream_end_inputs": (), "upstream_availability_ends": (), + "empty_input_names": (), + "unreadable_input_names": (), } defaults.update(values) field_name: str @@ -1138,6 +1150,8 @@ class ModelPlanEntry: cursor_end: str | None = None lookback: str | None = None cursor_bounds: CursorBounds | None = None + empty_cursor_inputs: tuple[str, ...] = () + waiting_on_empty_inputs: bool = False cursor_input_relations: tuple[CursorInputRelation, ...] = field(default_factory=tuple) batch_size: str | None = None batch_concurrency: int = 1 diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 98a75846db..7bb314d305 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -43,6 +43,7 @@ execute_snapshot_entry, execute_table_entry, execute_view_entry, + with_waiting_on_empty_inputs_warning, ) from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -90,6 +91,35 @@ def _dispatch_model( effective_vars: dict[str, object] | None = None, warehouse_relations: dict[str, RelationInfo] | None = None, on_progress: Callable[[str], None] | None = None, +) -> ModelExecutionResult: + """Route a model to the correct executor, then report a stalled cursor window as a warning.""" + + return with_waiting_on_empty_inputs_warning( + _dispatch_model_entry( + context=context, + promotion_mode=promotion_mode, + snapshots=snapshots, + allow_snapshot_schema_change=allow_snapshot_schema_change, + custom_materializations=custom_materializations, + target=target, + effective_vars=effective_vars, + warehouse_relations=warehouse_relations, + on_progress=on_progress, + ) + ) + + +def _dispatch_model_entry( + *, + context: ModelMaterializationContext, + promotion_mode: TablePromotionMode, + snapshots: SnapshotsConfig, + allow_snapshot_schema_change: bool, + custom_materializations: Mapping[str, Callable[..., MaterializationResult]] | None = None, + target: str = "", + effective_vars: dict[str, object] | None = None, + warehouse_relations: dict[str, RelationInfo] | None = None, + on_progress: Callable[[str], None] | None = None, ) -> ModelExecutionResult: """Route a model to the correct executor based on action and mode.""" diff --git a/src/sqlbuild/executor/run/_helpers/execution/results.py b/src/sqlbuild/executor/run/_helpers/execution/results.py index 049eac1ecb..d574daa74e 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/results.py +++ b/src/sqlbuild/executor/run/_helpers/execution/results.py @@ -2,7 +2,12 @@ from __future__ import annotations +from dataclasses import replace + from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.compiler.planner.main.cursor_window.waiting_on_empty_inputs_display import ( + format_waiting_on_empty_inputs, +) from sqlbuild.compiler.planner.models import ( CursorBounds, FutureCursorSafetyEvidence, @@ -188,3 +193,18 @@ def _fallback_code_for_phase(phase: ExecutionPhase) -> str: if phase == ExecutionPhase.CUSTOM_MATERIALIZATION: return RUN_CUSTOM_MATERIALIZATION_FAILED_CODE return RUN_UNKNOWN_FAILED_CODE + + +def with_waiting_on_empty_inputs_warning(result: ModelExecutionResult) -> ModelExecutionResult: + """Count a cursor window stalled on empty inputs as a model warning.""" + + if ( + result.status != ExecutionStatus.SUCCESS + or not result.waiting_on_empty_inputs + or not result.empty_cursor_inputs + ): + return result + message: str = format_waiting_on_empty_inputs(result.empty_cursor_inputs) + if message in result.warning_messages: + return result + return replace(result, warning_messages=(*result.warning_messages, message)) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 53f7f01d5a..169ba3adf7 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -40,12 +40,14 @@ from sqlbuild.executor.run._helpers.validation.contracts import validate_runtime_contract from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, + cursor_window_unavailable_error, has_authoritative_cursor_override, has_runtime_owned_cursor_watermarks, resolve_runtime_cursor_bounds, substitute_cursor_sentinels, ) from sqlbuild.executor.run._helpers.validation.type_enforcement import enforce_types_staged +from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import ( BoundViewGuard, FinalAuditRun, @@ -76,6 +78,16 @@ class _DeltaPreparation: runtime_cursor_bounds: CursorBounds | None +@dataclass(frozen=True) +class _IncrementalTarget: + """Physical destination of one incremental model.""" + + database: str | None + schema: str | None + table: str + qualified: str + + @dataclass(frozen=True) class _DeltaTypeInputs: """Physical delta and target schema inputs for pre-DML type preparation.""" @@ -118,24 +130,19 @@ def execute_incremental_entry( hook_results: list[HookExecutionResult] = [] statement_recorder: StatementRecorder = StatementRecorder() - try: - preparation: _DeltaPreparation = _resolve_incremental_cursor_bounds( - context=context, - target_database=target_database, - target_schema=target_schema, - target_table=target_table, - target_qualified=target_qualified, - ) - except Exception as exc: - return build_failed_result( - entry=entry, - phase=ExecutionPhase.STAGING, - error=exc, - warnings=warnings, - audit_results=audit_results, - statement_recorder=statement_recorder, - hook_results=hook_results, - ) + prepared: _DeltaPreparation | ModelExecutionResult = _prepare_cursor_window( + context=context, + targets=_IncrementalTarget( + database=target_database, + schema=target_schema, + table=target_table, + qualified=target_qualified, + ), + statement_recorder=statement_recorder, + ) + if isinstance(prepared, ModelExecutionResult): + return prepared + preparation: _DeltaPreparation = prepared context = _context_with_runtime_cursor(context=context, preparation=preparation) entry = context.entry warnings.extend(_cursor_safety_warnings(entry.cursor_bounds)) @@ -414,6 +421,45 @@ def execute_incremental_entry( ) +def _no_input_rows_result( + *, + context: ModelMaterializationContext, + targets: _IncrementalTarget, + input_names: tuple[str, ...], + waiting_on_empty_inputs: bool, + statement_recorder: StatementRecorder, +) -> ModelExecutionResult: + """Leave the destination unchanged when the cursor inputs have no rows, then audit it.""" + + entry: ModelPlanEntry = context.entry + final_audit_run: FinalAuditRun = run_final_scope_audits(context=context) + if final_audit_run.has_error: + return replace( + build_failed_result( + entry=entry, + phase=ExecutionPhase.AUDIT, + error=f"final audit for '{entry.name}' failed with severity level: error", + promoted_relation=targets.qualified, + warnings=[], + audit_results=list(final_audit_run.results), + statement_recorder=statement_recorder, + ), + empty_cursor_inputs=input_names, + waiting_on_empty_inputs=waiting_on_empty_inputs, + ) + return ModelExecutionResult( + model_name=entry.name, + status=ExecutionStatus.SUCCESS, + promoted_relation=targets.qualified, + cursor_type=entry.cursor_type, + cursor_grain=entry.cursor_grain, + empty_cursor_inputs=input_names, + waiting_on_empty_inputs=waiting_on_empty_inputs, + audit_results=final_audit_run.results, + lifecycle_events=statement_recorder.snapshot(), + ) + + def _prepare_delta_relation( *, context: ModelMaterializationContext, @@ -487,7 +533,7 @@ def _resolve_incremental_cursor_bounds( watermark_resolver=context.watermark_resolver, ) if runtime_cursor_bounds is None: - raise ExecutorInputError(f"runtime cursor bounds could not be resolved for '{entry.name}'") + raise cursor_window_unavailable_error(entry=entry) return _DeltaPreparation( resolved_sql=substitute_cursor_sentinels( sql=entry.resolved_sql, bounds=runtime_cursor_bounds @@ -496,6 +542,66 @@ def _resolve_incremental_cursor_bounds( ) +def _prepare_cursor_window( + *, + context: ModelMaterializationContext, + targets: _IncrementalTarget, + statement_recorder: StatementRecorder, +) -> _DeltaPreparation | ModelExecutionResult: + """Settle the cursor window before hooks or staging, or return the run's final result.""" + + entry: ModelPlanEntry = context.entry + if entry.empty_cursor_inputs: + return _no_input_rows_result( + context=context, + targets=targets, + input_names=entry.empty_cursor_inputs, + waiting_on_empty_inputs=entry.waiting_on_empty_inputs, + statement_recorder=statement_recorder, + ) + try: + preparation: _DeltaPreparation = _resolve_incremental_cursor_bounds( + context=context, + target_database=targets.database, + target_schema=targets.schema, + target_table=targets.table, + target_qualified=targets.qualified, + ) + if _lacks_required_cursor_window( + entry=_context_with_runtime_cursor(context=context, preparation=preparation).entry + ): + raise cursor_window_unavailable_error(entry=entry) + except EmptyCursorInputsError as exc: + return _no_input_rows_result( + context=context, + targets=targets, + input_names=exc.input_names, + waiting_on_empty_inputs=exc.waiting_on_empty_inputs, + statement_recorder=statement_recorder, + ) + except Exception as exc: + return build_failed_result( + entry=entry, + phase=ExecutionPhase.STAGING, + error=exc, + warnings=[], + audit_results=[], + statement_recorder=statement_recorder, + hook_results=[], + ) + return preparation + + +def _lacks_required_cursor_window(*, entry: ModelPlanEntry) -> bool: + """Return whether a cursor delete would have no window, before any staging work.""" + + return ( + entry.incremental_strategy == IncrementalStrategy.DELETE_INSERT + and entry.cursor_column is not None + and entry.cursor_bounds is None + ) + + def _context_with_runtime_cursor( *, context: ModelMaterializationContext, preparation: _DeltaPreparation ) -> ModelMaterializationContext: @@ -772,11 +878,7 @@ def _execute_dml( cursor_column: str | None = entry.cursor_column if cursor_column is not None: if cursor_start is None or cursor_end is None: - raise ExecutorInputError( - f"cursor-based delete_insert for '{entry.name}' requires both " - f"cursor_start and cursor_end but got " - f"cursor_start={cursor_start}, cursor_end={cursor_end}" - ) + raise cursor_window_unavailable_error(entry=entry) return adapter.delete_insert_cursor( connection=connection, destination=target_qualified, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 1393bb9325..23c5db6c41 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -22,6 +22,7 @@ resolve_relation_location_qualified_name, ) from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( aligned_timestamp_bounds, ) @@ -91,6 +92,7 @@ from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, + empty_input_rebuild_error, has_authoritative_cursor_override, has_runtime_owned_cursor_watermarks, resolve_effective_timestamp_grain, @@ -98,6 +100,7 @@ substitute_cursor_sentinels, ) from sqlbuild.executor.run._helpers.validation.type_enforcement import enforce_types_staged +from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import ( BatchWindow, FinalAuditRun, @@ -165,8 +168,6 @@ _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") -_EMPTY_INTEGER_CURSOR_BOUND: str = "0" -_EMPTY_TIMESTAMP_CURSOR_BOUND: str = "1970-01-01T00:00:00" _UNACCOUNTED_COUNT_CHUNK_SIZE: int = 100 @@ -180,6 +181,8 @@ class _MicrobatchPlan: early_exit: ModelExecutionResult | None = None runtime_discovery: bool = False resolved_intervals: tuple[MicrobatchInterval, ...] = () + empty_cursor_inputs: tuple[str, ...] = () + waiting_on_empty_inputs: bool = False @dataclass(frozen=True) @@ -437,7 +440,7 @@ def execute_microbatch_entry( ) history_context = replace(history_context, resolution_reason=resolution_reason) state = _with_reconciliation_warnings(context=context, state=state, history=history_context) - if not batch_plan.batches: + if not batch_plan.batches and not batch_plan.empty_cursor_inputs: state = replace(state, warnings=[*state.warnings, "no batches to process"]) state, safety_failure = _enforce_microbatch_safety_limit( context=context, state=state, batch_plan=batch_plan @@ -628,6 +631,8 @@ def execute_microbatch_entry( ), cursor_type=context.entry.cursor_type, cursor_grain=context.entry.cursor_grain, + empty_cursor_inputs=batch_plan.empty_cursor_inputs, + waiting_on_empty_inputs=batch_plan.waiting_on_empty_inputs, future_cursor_safety=(resolved_range.future_safety if resolved_range is not None else None), maximum_start_safety=( resolved_range.maximum_start_safety if resolved_range is not None else None @@ -682,6 +687,8 @@ def _no_work_microbatch_result( ), cursor_type=context.entry.cursor_type, cursor_grain=context.entry.cursor_grain, + empty_cursor_inputs=batch_plan.empty_cursor_inputs, + waiting_on_empty_inputs=batch_plan.waiting_on_empty_inputs, warning_messages=tuple(state.warnings), lifecycle_events=state.statement_recorder.snapshot(), skip_reason=( @@ -3198,6 +3205,8 @@ def _plan_microbatch_windows( or (is_full_refresh and entry.microbatch_strategy != MicrobatchStrategy.ROLLING_WINDOW) ) microbatch_range: CursorBounds | None = entry.microbatch_range + empty_cursor_inputs: tuple[str, ...] = entry.empty_cursor_inputs + waiting_on_empty_inputs: bool = entry.waiting_on_empty_inputs if runtime_discovery: if entry.cursor_column is None: return _MicrobatchPlan( @@ -3225,6 +3234,31 @@ def _plan_microbatch_windows( on_progress=on_progress, watermark_resolver=context.watermark_resolver, ) + except EmptyCursorInputsError as exc: + if ( + is_full_refresh + and entry.microbatch_strategy == MicrobatchStrategy.WATERMARK + and cached_relation_exists( + adapter=adapter, + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + name=entry.destination.name, + ) + ): + return _MicrobatchPlan( + early_exit=build_failed_result( + entry=entry, + phase=ExecutionPhase.STAGING, + error=empty_input_rebuild_error(entry=entry, input_names=exc.input_names), + warnings=warnings, + audit_results=audit_results, + statement_recorder=statement_recorder, + ) + ) + microbatch_range = None + empty_cursor_inputs = exc.input_names + waiting_on_empty_inputs = exc.waiting_on_empty_inputs except Exception as exc: return _MicrobatchPlan( early_exit=build_failed_result( @@ -3305,6 +3339,8 @@ def _plan_microbatch_windows( effective_batch_size=effective_batch_size, resolved_range=resolved_range, runtime_discovery=runtime_discovery, + empty_cursor_inputs=empty_cursor_inputs, + waiting_on_empty_inputs=waiting_on_empty_inputs, ) @@ -3325,14 +3361,11 @@ def _batches_for_intervals( def _empty_microbatch_bound(*, entry: ModelPlanEntry) -> CursorScalar: - if entry.cursor_start is not None: - return parse( - raw=entry.cursor_start, - cursor_type=entry.cursor_type or CursorType.TIMESTAMP, - ) - if entry.cursor_type == CursorType.INTEGER: - return IntegerValue(value=int(_EMPTY_INTEGER_CURSOR_BOUND)) - return TimestampValue(value=datetime.fromisoformat(_EMPTY_TIMESTAMP_CURSOR_BOUND)) + return _concrete_bound( + bound=empty_cursor_window( + cursor_type=entry.cursor_type, cursor_start=entry.cursor_start + ).start + ) def _concrete_bound(*, bound: Bound) -> CursorScalar: diff --git a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py index 89ea1fbabe..70b435d7e3 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py @@ -4,9 +4,11 @@ from dataclasses import replace +from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.compiler.planner.main.changes.relation_replacement import ( replaces_incremental_relation, ) +from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window from sqlbuild.compiler.planner.main.execution.future_cursor_warning import ( future_cursor_cap_warning, ) @@ -14,11 +16,14 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, + cursor_window_unavailable_error, + empty_input_rebuild_error, has_authoritative_cursor_override, has_runtime_owned_cursor_watermarks, resolve_runtime_cursor_bounds, substitute_cursor_sentinels, ) +from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import ( ModelExecutionResult, ModelMaterializationContext, @@ -50,23 +55,44 @@ def resolve_table_cursor( ) if entry.cursor_column is None: raise ExecutorInputError("runtime-owned cursor resolution requires cursor_column") - bounds: CursorBounds | None = resolve_runtime_cursor_bounds( - adapter=context.adapter, - connection=context.connection, - target_relation=targets.target_qualified, - target_database=targets.target_database, - target_schema=targets.target_schema, - target_name=targets.target_table, - spec=build_runtime_cursor_spec( - entry=entry, - read_destination_cursor=not replaces_incremental_relation( - materialization_type=entry.materialization_type, action=entry.action + try: + bounds: CursorBounds | None = resolve_runtime_cursor_bounds( + adapter=context.adapter, + connection=context.connection, + target_relation=targets.target_qualified, + target_database=targets.target_database, + target_schema=targets.target_schema, + target_name=targets.target_table, + spec=build_runtime_cursor_spec( + entry=entry, + read_destination_cursor=not replaces_incremental_relation( + materialization_type=entry.materialization_type, action=entry.action + ), ), - ), - watermark_resolver=context.watermark_resolver, - ) + watermark_resolver=context.watermark_resolver, + ) + except EmptyCursorInputsError as exc: + if cached_relation_exists( + adapter=context.adapter, + connection=context.connection, + database=targets.target_database, + schema=targets.target_schema, + name=targets.target_table, + ): + raise empty_input_rebuild_error(entry=entry, input_names=exc.input_names) from exc + return TableCursorResolution( + resolved_sql=substitute_cursor_sentinels( + sql=entry.resolved_sql, + bounds=empty_cursor_window( + cursor_type=entry.cursor_type, cursor_start=entry.cursor_start + ), + ), + bounds=None, + empty_cursor_inputs=exc.input_names, + waiting_on_empty_inputs=exc.waiting_on_empty_inputs, + ) if bounds is None: - raise ExecutorInputError(f"runtime cursor bounds could not be resolved for '{entry.name}'") + raise cursor_window_unavailable_error(entry=entry) return TableCursorResolution( resolved_sql=substitute_cursor_sentinels(sql=entry.resolved_sql, bounds=bounds), bounds=bounds, @@ -82,6 +108,8 @@ def result_with_cursor_safety( bounds: CursorBounds | None = entry.microbatch_range or entry.cursor_bounds return replace( result, + empty_cursor_inputs=entry.empty_cursor_inputs, + waiting_on_empty_inputs=entry.waiting_on_empty_inputs, future_cursor_safety=bounds.future_safety if bounds is not None else None, maximum_start_safety=(bounds.maximum_start_safety if bounds is not None else None), ) diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index c07bbbf56e..ee9a4512c1 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -12,6 +12,15 @@ from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL +from sqlbuild.compiler.planner.main.cursor_window.cursor_window_override_flags import ( + cursor_window_override_flags, +) +from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_input_decision import ( + decide_empty_cursor_inputs, +) +from sqlbuild.compiler.planner.main.cursor_window.empty_input_rebuild_refusal import ( + empty_input_rebuild_refusal, +) from sqlbuild.compiler.planner.main.execution.cursor_replay_policy import ( apply_typed_cursor_replay_policy, ) @@ -22,6 +31,7 @@ CursorInputEvidence, CursorInputRelation, Duration, + EmptyCursorInputDecision, MaximumStartPolicyInputs, ModelCursorSnapshot, ModelPlanEntry, @@ -55,6 +65,7 @@ ) from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import RuntimeCursorInputRelation, RuntimeCursorSpec from sqlbuild.executor.run.types import RuntimeCursorWatermarkMode, WatermarkResolver from sqlbuild.spec.contracts.constants import ZERO_DAY_CURSOR_DURATION @@ -92,6 +103,7 @@ def build_runtime_cursor_spec( RuntimeCursorInputRelation( relation=relation.relation, cursor_column=relation.cursor_column, + input_name=relation.input_name, cursor_grain=relation.cursor_grain, is_model_backed=relation.is_model_backed, is_runtime_produced=relation.is_runtime_produced, @@ -222,10 +234,6 @@ def resolve_runtime_cursor_bounds( if read_minimum and minimum is None and terminal_start is not None: minimum = terminal_start if maximum is None or (read_minimum and minimum is None): - if spec.cursor_watermark_mode == RuntimeCursorWatermarkMode.ALL: - raise ExecutorInputError( - f"required cursor watermark is empty: {relation_column[0]}.{relation_column[1]}" - ) continue if minimum is not None: upstream_mins.append(minimum) @@ -252,6 +260,11 @@ def resolve_runtime_cursor_bounds( maximum=maximum, ) ) + _raise_for_empty_inputs( + physical_inputs=physical_inputs, + input_evidence=input_evidence, + waits_for_every_input=_waits_for_every_input(spec=spec), + ) if not upstream_maxes: return None upstream_min: CursorScalar | None = ( @@ -376,6 +389,76 @@ def resolve_runtime_cursor_bounds( ) +def cursor_window_unavailable_error(*, entry: ModelPlanEntry) -> ExecutorInputError: + """Report a missing cursor window in terms of the model and its cursor inputs.""" + + input_names: str = ", ".join( + _input_display_name(relation=relation) for relation in entry.cursor_input_relations + ) + return ExecutorInputError( + f"model '{entry.name}' cannot work out its cursor window: no cursor bounds were found " + f"for its inputs ({input_names or 'none'})", + help=( + "check that the inputs exist and have their cursor columns, or set the window " + f"explicitly with {cursor_window_override_flags(cursor_type=entry.cursor_type)}" + ), + ) + + +def empty_input_rebuild_error( + *, entry: ModelPlanEntry, input_names: tuple[str, ...] +) -> ExecutorInputError: + """Refuse to replace an existing destination with the result of empty cursor inputs.""" + + message, help_text = empty_input_rebuild_refusal( + model_name=entry.name, input_names=input_names, cursor_type=entry.cursor_type + ) + return ExecutorInputError(message, code="S302", help=help_text) + + +def _raise_for_empty_inputs( + *, + physical_inputs: tuple[tuple[tuple[str, str], RuntimeCursorInputRelation], ...], + input_evidence: list[CursorInputEvidence], + waits_for_every_input: bool, +) -> None: + """Signal empty inputs with the same decision the planner applies to its own windows.""" + + with_rows: set[tuple[str, str]] = { + (evidence.relation, evidence.cursor_column) for evidence in input_evidence + } + decision: EmptyCursorInputDecision = decide_empty_cursor_inputs( + empty_input_names=tuple( + _input_display_name(relation=cursor_input) + for relation_column, cursor_input in physical_inputs + if relation_column not in with_rows + ), + waits_for_every_input=waits_for_every_input, + inputs_with_rows=bool(with_rows), + has_window_start=bool(with_rows), + ) + if decision.input_names: + raise EmptyCursorInputsError( + input_names=decision.input_names, + waiting_on_empty_inputs=decision.waiting_on_empty_inputs, + ) + + +def _waits_for_every_input(*, spec: RuntimeCursorSpec) -> bool: + """Return whether every cursor input must have rows, matching planner-owned windows.""" + + if spec.cursor_watermark_mode == RuntimeCursorWatermarkMode.ANY: + return False + return spec.microbatch_strategy != MicrobatchStrategy.ROLLING_WINDOW + + +def _input_display_name(*, relation: CursorInputRelation | RuntimeCursorInputRelation) -> str: + """Label one cursor input by its project name, falling back to the physical relation.""" + + relation_name: str = relation.input_name or relation.relation.rsplit(".", 1)[-1].strip('"`[]') + return f"{relation_name}.{relation.cursor_column}" + + def _clamp_cursor_end( *, bounds: CursorBounds, cursor_end: CursorScalar | None, cursor_type: CursorType | None ) -> CursorBounds: diff --git a/src/sqlbuild/executor/run/exceptions.py b/src/sqlbuild/executor/run/exceptions.py new file mode 100644 index 0000000000..33b100f774 --- /dev/null +++ b/src/sqlbuild/executor/run/exceptions.py @@ -0,0 +1,16 @@ +"""Model execution errors.""" + +from __future__ import annotations + +from sqlbuild.errors.contracts.exceptions import ExecutorInputError + + +class EmptyCursorInputsError(ExecutorInputError): + """Raised when cursor inputs have no rows, so there is no cursor window to process.""" + + def __init__( + self, *, input_names: tuple[str, ...], waiting_on_empty_inputs: bool = False + ) -> None: + super().__init__(f"cursor inputs have no rows: {', '.join(input_names)}") + self.input_names: tuple[str, ...] = input_names + self.waiting_on_empty_inputs: bool = waiting_on_empty_inputs diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 14dbccf334..9843ae0a42 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -31,6 +31,9 @@ build_failed_result, build_skipped_result, ) +from sqlbuild.executor.run._helpers.execution.results import ( + with_waiting_on_empty_inputs_warning as with_waiting_on_empty_inputs_warning, +) from sqlbuild.executor.run._helpers.execution.schema import inspect_runtime_relation_schema from sqlbuild.executor.run._helpers.execution.staging import create_staging_relation from sqlbuild.executor.run._helpers.execution.table_targets import resolve_table_targets @@ -107,6 +110,13 @@ def execute_table_entry( if cursor_resolution.bounds is not None: entry = replace(entry, cursor_bounds=cursor_resolution.bounds) context = replace(context, entry=entry) + if cursor_resolution.empty_cursor_inputs: + entry = replace( + entry, + empty_cursor_inputs=cursor_resolution.empty_cursor_inputs, + waiting_on_empty_inputs=cursor_resolution.waiting_on_empty_inputs, + ) + context = replace(context, entry=entry) if cursor_resolution.warning is not None: warnings.append(cursor_resolution.warning) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 61cdfde382..3f1c9488c6 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -169,6 +169,8 @@ class ModelExecutionResult: cursor_range_end: str | None = None cursor_type: str | None = None cursor_grain: str | None = None + empty_cursor_inputs: tuple[str, ...] = () + waiting_on_empty_inputs: bool = False future_cursor_safety: FutureCursorSafetyEvidence | None = None maximum_start_safety: MaximumStartSafetyEvidence | None = None audit_results: tuple[AuditExecutionResult, ...] = field(default_factory=tuple) @@ -320,6 +322,7 @@ class RuntimeCursorInputRelation: is_runtime_produced: bool = False terminal_cursor_start: CursorScalar | None = None terminal_cursor_end: CursorScalar | None = None + input_name: str | None = None def __init__( self, @@ -331,7 +334,9 @@ def __init__( is_runtime_produced: bool = False, terminal_cursor_start: CursorScalar | str | None = None, terminal_cursor_end: CursorScalar | str | None = None, + input_name: str | None = None, ) -> None: + object.__setattr__(self, "input_name", input_name) object.__setattr__(self, "relation", relation) object.__setattr__(self, "cursor_column", cursor_column) object.__setattr__( @@ -438,6 +443,7 @@ def __init__(self, **values: Any) -> None: RuntimeCursorInputRelation( relation=relation.relation, cursor_column=relation.cursor_column, + input_name=relation.input_name, cursor_grain=( CursorGrain(str(relation.cursor_grain)) if relation.cursor_grain is not None @@ -618,6 +624,8 @@ class TableCursorResolution: bounds: CursorBounds | None warning: str | None = None error: str | None = None + empty_cursor_inputs: tuple[str, ...] = () + waiting_on_empty_inputs: bool = False @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 0b5e71ea1f..02b1451654 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1217,3 +1217,91 @@ class TerminalBuildFailureE2ETestCase: model_replacements: tuple[tuple[str, str, str], ...] expected_screen_fragments: tuple[str, ...] columns: int = 120 + + +@dataclass(frozen=True) +class EmptyCursorInputsFirstBuildE2ETestCase: + """A first build over empty cursor inputs, followed by a build once data arrives.""" + + description: str + model_sql: str + rows_sql: str + expected_reason: str + expected_rows_after_load: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class EmptyCursorInputsIncrementalE2ETestCase: + """An incremental run after the cursor inputs were emptied.""" + + description: str + command: tuple[str, ...] + expected_output_fragments: tuple[str, ...] + expected_json_empty_inputs: tuple[str, ...] | None + + +@dataclass(frozen=True) +class CursorModelWithoutInputsCompileE2ETestCase: + """A cursor-based incremental model that reads no inputs.""" + + description: str + command: tuple[str, ...] + expected_output_fragments: tuple[str, ...] + expected_json_location: tuple[int, int, int] | None + + +@dataclass(frozen=True) +class DeltaFailureRetentionE2ETestCase: + """A real incremental build failure after the delta relation was created.""" + + description: str + failing_rows_sql: str + expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class EmptyCursorInputsRebuildRefusalE2ETestCase: + """A rebuild that would replace a loaded destination with the result of empty inputs.""" + + description: str + model_sql: str + changed_model_sql: str + command: tuple[str, ...] + rows_sql: str + expected_rows: tuple[tuple[object, ...], ...] + expected_output_fragments: tuple[str, ...] + expected_json_error_code: str | None + + +@dataclass(frozen=True) +class WaitingOnEmptyInputsE2ETestCase: + """An incremental run where one cursor input is empty while another has new rows.""" + + description: str + command: tuple[str, ...] + expected_output_fragments: tuple[str, ...] + expected_json_empty_inputs: tuple[str, ...] | None + + +@dataclass(frozen=True) +class EmptyRuntimeInputLabelE2ETestCase: + """An empty cursor input rebuilt in the same run, labelled by its project name.""" + + description: str + changed_upstream_sql: str + expected_output_fragments: tuple[str, ...] + unexpected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SameRunEmptyInputsE2ETestCase: + """Empty cursor inputs whose staging views are rebuilt earlier in the same run.""" + + description: str + emptying_sql: str + command: tuple[str, ...] + expected_output_fragments: tuple[str, ...] + unexpected_output_fragments: tuple[str, ...] + expected_json_empty_inputs: tuple[str, ...] | None + expected_json_waiting_on_empty_inputs: bool | None + expected_json_warnings: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 5347d40f26..514ba07286 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -4,6 +4,7 @@ import subprocess from pathlib import Path from textwrap import dedent +from typing import Any from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( DeferCloneBuildE2ETestCase, @@ -883,3 +884,175 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), } + + +_EMPTY_CURSOR_DATABASE: str = "warehouse.duckdb" +_EMPTY_CURSOR_PROJECT_TOML: str = ( + 'name = "empty_cursor_inputs"\nadapter = "duckdb"\n\n' + f'[connection]\ndatabase = "{_EMPTY_CURSOR_DATABASE}"\n' +) +_EMPTY_CURSOR_SOURCES_YML: str = dedent( + """ + sources: + - name: raw_payments + description: Raw payments, one row per payment. + schema: main + table: raw_payments + - name: raw_refunds + description: Raw refunds, one row per refund. + schema: main + table: raw_refunds + """ +).lstrip() +_CREATE_PAYMENTS_SQL: str = ( + "CREATE TABLE main.raw_payments (payment_id INTEGER, paid_at TIMESTAMP, amount_cents INTEGER); " + "CREATE TABLE main.raw_refunds (refund_id INTEGER, refunded_at TIMESTAMP, amount_cents INTEGER)" +) +_INSERT_PAYMENTS_SQL: str = ( + "INSERT INTO main.raw_payments VALUES " + "(1, '2026-04-01 09:00:00', 1200), (2, '2026-04-02 10:00:00', 1650)" +) + + +def prepare_empty_cursor_inputs_project(*, tmp_path: Path, models: dict[str, str]) -> Path: + """Write a project whose models read an initially empty payments source.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="empty_cursor_inputs", + repo_files={ + "sqlbuild_project.toml": _EMPTY_CURSOR_PROJECT_TOML, + "sources/raw.yml": _EMPTY_CURSOR_SOURCES_YML, + **models, + }, + ) + execute_duckdb(db_path=project_dir / _EMPTY_CURSOR_DATABASE, sql=_CREATE_PAYMENTS_SQL) + return project_dir + + +def execute_empty_cursor_sql(*, project_dir: Path, sql: str) -> None: + """Run warehouse SQL against the empty-cursor-inputs project database.""" + + execute_duckdb(db_path=project_dir / _EMPTY_CURSOR_DATABASE, sql=sql) + + +def load_empty_cursor_payments(*, project_dir: Path) -> None: + """Load two days of payments into the source.""" + + execute_empty_cursor_sql(project_dir=project_dir, sql=_INSERT_PAYMENTS_SQL) + + +def run_empty_cursor_command( + *, project_dir: Path, command: tuple[str, ...] +) -> subprocess.CompletedProcess[str]: + """Run one sqb command without colour against the project.""" + + return run_sqb(command=("--no-color", *command), project_dir=project_dir) + + +def build_empty_cursor_project(*, project_dir: Path) -> None: + """Run a build that must succeed.""" + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=("build",) + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def prepare_emptied_cursor_project( + *, tmp_path: Path, model_sql: str, changed_model_sql: str +) -> Path: + """Build a model over loaded payments, then empty the source and write the next model SQL.""" + + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models={"models/daily_revenue.sql": model_sql} + ) + load_empty_cursor_payments(project_dir=project_dir) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql(project_dir=project_dir, sql="DELETE FROM main.raw_payments") + (project_dir / "models" / "daily_revenue.sql").write_text(changed_model_sql, encoding="utf-8") + return project_dir + + +def prepare_waiting_on_empty_input_project(*, tmp_path: Path, model_sql: str) -> Path: + """Build over payments and refunds, then empty refunds while new payments arrive.""" + + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models={"models/daily_revenue.sql": model_sql} + ) + execute_empty_cursor_sql( + project_dir=project_dir, + sql=( + "INSERT INTO main.raw_payments VALUES (1, '2026-04-01 09:00:00', 1200); " + "INSERT INTO main.raw_refunds VALUES (9, '2026-04-01 12:00:00', 300)" + ), + ) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql( + project_dir=project_dir, + sql=( + "DELETE FROM main.raw_refunds; " + "INSERT INTO main.raw_payments VALUES (2, '2026-04-02 10:00:00', 1650)" + ), + ) + return project_dir + + +def prepare_same_run_empty_inputs_project( + *, tmp_path: Path, models: dict[str, str], emptying_sql: str +) -> Path: + """Build over payments and refunds, change the data, then touch the staging views.""" + + project_dir: Path = prepare_empty_cursor_inputs_project(tmp_path=tmp_path, models=models) + execute_empty_cursor_sql( + project_dir=project_dir, + sql=( + "INSERT INTO main.raw_payments VALUES (1, '2026-04-01 09:00:00', 1200); " + "INSERT INTO main.raw_refunds VALUES (9, '2026-04-01 12:00:00', 300)" + ), + ) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql(project_dir=project_dir, sql=emptying_sql) + view_path: Path + for view_path in sorted((project_dir / "models").glob("stg_*.sql")): + view_path.write_text( + view_path.read_text(encoding="utf-8").replace( + "amount_cents FROM", "amount_cents * 1 AS amount_cents FROM" + ), + encoding="utf-8", + ) + return project_dir + + +def empty_cursor_rows(*, project_dir: Path, sql: str) -> tuple[tuple[object, ...], ...]: + """Return warehouse rows as tuples.""" + + return tuple( + tuple(row) for row in query_duckdb(db_path=project_dir / _EMPTY_CURSOR_DATABASE, sql=sql) + ) + + +def empty_cursor_relation_exists(*, project_dir: Path, name: str) -> bool: + """Return whether a relation exists in the project's main schema.""" + + return bool( + empty_cursor_rows( + project_dir=project_dir, + sql=( + "SELECT 1 FROM information_schema.tables " + f"WHERE table_schema = 'main' AND table_name = '{name}'" + ), + ) + ) + + +def named_json_entry(*, entries: list[dict[str, Any]], name: str) -> dict[str, Any]: + """Return the JSON entry with the given name.""" + + return {entry["name"]: entry for entry in entries}[name] + + +def coded_json_diagnostic(*, document: dict[str, Any], code: str) -> dict[str, Any]: + """Return the first compile diagnostic with the given code.""" + + return {item["code"]: item for item in document["diagnostics"]}[code] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_empty_cursor_inputs.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_empty_cursor_inputs.py new file mode 100644 index 0000000000..873f07bcc4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_empty_cursor_inputs.py @@ -0,0 +1,880 @@ +"""E2E tests for cursor-based incremental models whose inputs are missing or empty.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from textwrap import dedent +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + CursorModelWithoutInputsCompileE2ETestCase, + DeltaFailureRetentionE2ETestCase, + EmptyCursorInputsFirstBuildE2ETestCase, + EmptyCursorInputsIncrementalE2ETestCase, + EmptyCursorInputsRebuildRefusalE2ETestCase, + EmptyRuntimeInputLabelE2ETestCase, + SameRunEmptyInputsE2ETestCase, + WaitingOnEmptyInputsE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + build_empty_cursor_project, + coded_json_diagnostic, + empty_cursor_relation_exists, + empty_cursor_rows, + execute_empty_cursor_sql, + load_empty_cursor_payments, + named_json_entry, + prepare_emptied_cursor_project, + prepare_empty_cursor_inputs_project, + prepare_same_run_empty_inputs_project, + prepare_waiting_on_empty_input_project, + run_empty_cursor_command, +) + +_DAILY_REVENUE_SQL: str = dedent( + """ + MODEL ( + description "Daily revenue from payments", + materialized incremental, + incremental_strategy delete_insert, + cursor revenue_date, + cursor_type timestamp, + cursor_grain day, + cursor_inputs (raw_payments paid_at), + columns ( + total_revenue_cents (audits [not_null]), + ), + ); + + SELECT CAST(paid_at AS DATE) AS revenue_date, SUM(amount_cents) AS total_revenue_cents + FROM __source("raw_payments") + GROUP BY 1 + """ +).lstrip() +_MICROBATCH_PAYMENTS_SQL: str = dedent( + """ + MODEL ( + description "Payments processed in daily batches", + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor_watermark_mode all, + batch_size 1d, + cursor paid_at, + cursor_type timestamp, + cursor_grain hour, + cursor_inputs (raw_payments (column paid_at, roles [filter, watermark])), + ); + + SELECT payment_id, paid_at, amount_cents FROM __source("raw_payments") + """ +).lstrip() +_REVENUE_SUMMARY_SQL: str = dedent( + """ + MODEL (description "Revenue across all days", materialized table); + + SELECT COUNT(*) AS revenue_days, SUM(total_revenue_cents) AS total_revenue_cents + FROM __ref("daily_revenue") + """ +).lstrip() +_NO_INPUTS_SQL: str = dedent( + """ + MODEL ( + description "Daily revenue from successful payments", + materialized incremental, + incremental_strategy delete_insert, + cursor revenue_date, + cursor_type timestamp, + cursor_grain day, + ); + + SELECT CAST('2026-04-01' AS DATE) AS revenue_date, 2850 AS total_revenue_cents + """ +).lstrip() +_REVENUE_ROWS_SQL: str = ( + "SELECT CAST(revenue_date AS VARCHAR), total_revenue_cents " + "FROM main.daily_revenue ORDER BY revenue_date" +) +_LOADED_REVENUE_ROWS: tuple[tuple[object, ...], ...] = (("2026-04-01", 1200), ("2026-04-02", 1650)) +_EMPTIED_MODELS: dict[str, str] = { + "models/daily_revenue.sql": _DAILY_REVENUE_SQL, + "models/revenue_summary.sql": _REVENUE_SUMMARY_SQL, +} +_ALIASED_REPLAY_SQL: str = dedent( + """ + MODEL ( + description "Daily revenue published under a versioned name", + materialized incremental, + incremental_strategy delete_insert, + alias daily_revenue_v2, + replay_on_change full, + cursor revenue_date, + cursor_type timestamp, + cursor_grain day, + cursor_inputs (raw_payments paid_at), + ); + + SELECT CAST(paid_at AS DATE) AS revenue_date, SUM(amount_cents) AS total_revenue_cents + FROM __source("raw_payments") + GROUP BY 1 + """ +).lstrip() +_ANY_MODE_MICROBATCH_SQL: str = _MICROBATCH_PAYMENTS_SQL.replace( + "cursor_watermark_mode all", "cursor_watermark_mode any" +) +_NET_REVENUE_SQL: str = dedent( + """ + MODEL ( + description "Daily revenue net of refunds", + materialized incremental, + incremental_strategy delete_insert, + cursor revenue_date, + cursor_type timestamp, + cursor_grain day, + cursor_inputs (raw_payments paid_at, raw_refunds refunded_at), + ); + + SELECT CAST(paid_at AS DATE) AS revenue_date, SUM(amount_cents) AS total_revenue_cents + FROM __source("raw_payments") + WHERE payment_id NOT IN (SELECT refund_id FROM __source("raw_refunds")) + GROUP BY 1 + """ +).lstrip() +_STG_PAYMENTS_SQL: str = dedent( + """ + MODEL (description "Payments staged for reporting", materialized view, alias payments_clean); + + SELECT payment_id, paid_at, amount_cents FROM __source("raw_payments") + """ +).lstrip() +_STAGED_REVENUE_SQL: str = dedent( + """ + MODEL ( + description "Daily revenue from staged payments", + materialized incremental, + incremental_strategy delete_insert, + cursor revenue_date, + cursor_type timestamp, + cursor_grain day, + cursor_inputs (stg_payments paid_at), + ); + + SELECT CAST(paid_at AS DATE) AS revenue_date, SUM(amount_cents) AS total_revenue_cents + FROM __ref("stg_payments") + GROUP BY 1 + """ +).lstrip() +_SAME_RUN_MODELS: dict[str, str] = { + "models/stg_payments.sql": dedent( + """ + MODEL (description "Payments staged for reporting", materialized view); + + SELECT payment_id, paid_at, amount_cents FROM __source("raw_payments") + """ + ).lstrip(), + "models/stg_refunds.sql": dedent( + """ + MODEL (description "Refunds staged for reporting", materialized view); + + SELECT refund_id, refunded_at, amount_cents FROM __source("raw_refunds") + """ + ).lstrip(), + "models/daily_revenue.sql": dedent( + """ + MODEL ( + description "Daily revenue net of refunds, from staged inputs", + materialized incremental, + incremental_strategy delete_insert, + cursor revenue_date, + cursor_type timestamp, + cursor_grain day, + cursor_inputs (stg_payments paid_at, stg_refunds refunded_at), + ); + + SELECT CAST(paid_at AS DATE) AS revenue_date, SUM(amount_cents) AS total_revenue_cents + FROM __ref("stg_payments") + WHERE payment_id NOT IN (SELECT refund_id FROM __ref("stg_refunds")) + GROUP BY 1 + """ + ).lstrip(), +} +_EMPTY_ALL_SQL: str = "DELETE FROM main.raw_payments; DELETE FROM main.raw_refunds" +_EMPTY_REFUNDS_SQL: str = ( + "DELETE FROM main.raw_refunds; " + "INSERT INTO main.raw_payments VALUES (2, '2026-04-02 10:00:00', 1650)" +) +_NO_INPUTS_MODELS: dict[str, str] = {"models/finance/daily_revenue.sql": _NO_INPUTS_SQL} + + +@pytest.mark.parametrize( + "test_case", + [ + EmptyCursorInputsFirstBuildE2ETestCase( + description="delete_insert model", + model_sql=_DAILY_REVENUE_SQL, + rows_sql=_REVENUE_ROWS_SQL, + expected_reason="no input rows (raw_payments.paid_at)", + expected_rows_after_load=_LOADED_REVENUE_ROWS, + ), + EmptyCursorInputsFirstBuildE2ETestCase( + description="watermark microbatch model", + model_sql=_MICROBATCH_PAYMENTS_SQL, + rows_sql="SELECT payment_id, amount_cents FROM main.daily_revenue ORDER BY 1", + expected_reason="no input rows (raw_payments.paid_at)", + expected_rows_after_load=((1, 1200), (2, 1650)), + ), + ], + ids=lambda case: case.description, +) +def test_given_empty_source_when_building_first_then_creates_empty_table_and_next_build_loads( + test_case: EmptyCursorInputsFirstBuildE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models={"models/daily_revenue.sql": test_case.model_sql} + ) + + first: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=("build",) + ) + empty_rows: tuple[tuple[object, ...], ...] = empty_cursor_rows( + project_dir=project_dir, sql="SELECT COUNT(*) FROM main.daily_revenue" + ) + load_empty_cursor_payments(project_dir=project_dir) + second: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=("build",) + ) + + assert first.returncode == 0, first.stdout + first.stderr + assert test_case.expected_reason in first.stdout + assert empty_rows == ((0,),) + assert second.returncode == 0, second.stdout + second.stderr + assert test_case.expected_reason not in second.stdout + assert ( + empty_cursor_rows(project_dir=project_dir, sql=test_case.rows_sql) + == test_case.expected_rows_after_load + ) + assert not empty_cursor_relation_exists(project_dir=project_dir, name="daily_revenue__delta") + + +@pytest.mark.parametrize( + "test_case", + [ + EmptyCursorInputsIncrementalE2ETestCase( + description="plan and build text output", + command=("build",), + expected_output_fragments=( + "daily_revenue (delete_insert)", + "no input rows (raw_payments.paid_at)", + "audit not_null (total_revenue_cents) PASS", + "Completed successfully PASS=3", + ), + expected_json_empty_inputs=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_emptied_source_when_building_incrementally_then_destination_is_unchanged( + test_case: EmptyCursorInputsIncrementalE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models=_EMPTIED_MODELS + ) + load_empty_cursor_payments(project_dir=project_dir) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql(project_dir=project_dir, sql="DELETE FROM main.raw_payments") + execute_empty_cursor_sql(project_dir=project_dir, sql="DROP TABLE main.revenue_summary") + + plan: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=("plan",) + ) + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert "window no input rows (raw_payments.paid_at)" in plan.stdout + assert result.returncode == 0, result.stdout + result.stderr + assert all(fragment in result.stdout for fragment in test_case.expected_output_fragments), ( + result.stdout + ) + assert empty_cursor_rows(project_dir=project_dir, sql=_REVENUE_ROWS_SQL) == ( + _LOADED_REVENUE_ROWS + ) + assert empty_cursor_rows(project_dir=project_dir, sql="SELECT * FROM main.revenue_summary") == ( + (2, 2850), + ) + assert not empty_cursor_relation_exists(project_dir=project_dir, name="daily_revenue__delta") + + +@pytest.mark.parametrize( + "test_case", + [ + EmptyCursorInputsIncrementalE2ETestCase( + description="build json output", + command=("build", "--json"), + expected_output_fragments=(), + expected_json_empty_inputs=("raw_payments.paid_at",), + ), + ], + ids=lambda case: case.description, +) +def test_given_emptied_source_when_building_json_then_reports_empty_cursor_inputs( + test_case: EmptyCursorInputsIncrementalE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models=_EMPTIED_MODELS + ) + load_empty_cursor_payments(project_dir=project_dir) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql(project_dir=project_dir, sql="DELETE FROM main.raw_payments") + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert result.returncode == 0, result.stdout + result.stderr + asset: dict[str, Any] = named_json_entry( + entries=json.loads(result.stdout)["assets"], name="daily_revenue" + ) + assert asset["status"] == "success" + assert asset["empty_cursor_inputs"] == list(test_case.expected_json_empty_inputs or ()) + assert empty_cursor_rows(project_dir=project_dir, sql=_REVENUE_ROWS_SQL) == ( + _LOADED_REVENUE_ROWS + ) + assert not empty_cursor_relation_exists(project_dir=project_dir, name="daily_revenue__delta") + + +@pytest.mark.parametrize( + "test_case", + [ + EmptyCursorInputsIncrementalE2ETestCase( + description="plan json output", + command=("plan", "--json"), + expected_output_fragments=(), + expected_json_empty_inputs=("raw_payments.paid_at",), + ), + ], + ids=lambda case: case.description, +) +def test_given_emptied_source_when_planning_json_then_reports_no_input_rows( + test_case: EmptyCursorInputsIncrementalE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models={"models/daily_revenue.sql": _DAILY_REVENUE_SQL} + ) + load_empty_cursor_payments(project_dir=project_dir) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql(project_dir=project_dir, sql="DELETE FROM main.raw_payments") + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert result.returncode == 0, result.stdout + result.stderr + model: dict[str, Any] = named_json_entry( + entries=json.loads(result.stdout)["models"], name="daily_revenue" + ) + assert model["cursor"] == { + **model["cursor"], + "resolution_status": "no_input_rows", + "empty_inputs": list(test_case.expected_json_empty_inputs or ()), + } + + +@pytest.mark.parametrize( + "test_case", + [ + CursorModelWithoutInputsCompileE2ETestCase( + description="compile text output", + command=("compile",), + expected_output_fragments=( + "error[P011]: incremental model 'daily_revenue' reads no inputs, so builds " + "after the first cannot work out their cursor window", + "--> models/finance/daily_revenue.sql:5:3", + "cursor revenue_date,", + "^^^^^^^^^^^^^^^^^^^", + "help: read the data through __ref(), __source() or __seed() so its cursor can " + "bound the window, or use materialized table if the model has no upstream data", + ), + expected_json_location=None, + ), + CursorModelWithoutInputsCompileE2ETestCase( + description="build fails before warehouse work", + command=("build",), + expected_output_fragments=("error[P011]",), + expected_json_location=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_cursor_model_without_inputs_when_compiling_then_fails_with_p011( + test_case: CursorModelWithoutInputsCompileE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models=_NO_INPUTS_MODELS + ) + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert all( + fragment in result.stdout + result.stderr + for fragment in test_case.expected_output_fragments + ) + assert not empty_cursor_relation_exists(project_dir=project_dir, name="daily_revenue") + + +@pytest.mark.parametrize( + "test_case", + [ + CursorModelWithoutInputsCompileE2ETestCase( + description="compile json output", + command=("compile", "--json"), + expected_output_fragments=(), + expected_json_location=(5, 3, 22), + ), + ], + ids=lambda case: case.description, +) +def test_given_cursor_model_without_inputs_when_compiling_json_then_locates_cursor( + test_case: CursorModelWithoutInputsCompileE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models=_NO_INPUTS_MODELS + ) + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert result.returncode == 1, result.stdout + result.stderr + diagnostic: dict[str, Any] = coded_json_diagnostic( + document=json.loads(result.stdout), code="P011" + ) + assert diagnostic["resource_name"] == "daily_revenue" + assert test_case.expected_json_location is not None + line, column, end_column = test_case.expected_json_location + assert diagnostic["location"] == { + "path": "models/finance/daily_revenue.sql", + "line": line, + "column": column, + "end_line": line, + "end_column": end_column, + } + + +@pytest.mark.parametrize( + "test_case", + [ + DeltaFailureRetentionE2ETestCase( + description="delta audit failure keeps the delta relation", + failing_rows_sql="INSERT INTO main.raw_payments VALUES (3, '2026-04-03 08:00:00', NULL)", + expected_output_fragments=( + "daily_revenue", + "delta table kept for inspection: main.daily_revenue__delta", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_incremental_build_when_building_then_delta_relation_is_kept( + test_case: DeltaFailureRetentionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models={"models/daily_revenue.sql": _DAILY_REVENUE_SQL} + ) + load_empty_cursor_payments(project_dir=project_dir) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql(project_dir=project_dir, sql=test_case.failing_rows_sql) + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=("build",) + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert all( + fragment in result.stdout + result.stderr + for fragment in test_case.expected_output_fragments + ) + assert empty_cursor_relation_exists(project_dir=project_dir, name="daily_revenue__delta") + assert empty_cursor_rows(project_dir=project_dir, sql=_REVENUE_ROWS_SQL) == ( + _LOADED_REVENUE_ROWS + ) + + +@pytest.mark.parametrize( + "test_case", + [ + EmptyCursorInputsRebuildRefusalE2ETestCase( + description="aliased replay_on_change full model after a query change", + model_sql=_ALIASED_REPLAY_SQL, + changed_model_sql=_ALIASED_REPLAY_SQL.replace( + "SUM(amount_cents) AS", "SUM(amount_cents) * 1 AS" + ), + command=("build",), + rows_sql=( + "SELECT CAST(revenue_date AS VARCHAR), total_revenue_cents " + "FROM main.daily_revenue_v2 ORDER BY revenue_date" + ), + expected_rows=_LOADED_REVENUE_ROWS, + expected_output_fragments=( + "error[S302]: model 'daily_revenue' was not rebuilt: its cursor inputs have no " + "rows (raw_payments.paid_at), so rebuilding it now would leave it empty", + "help: load the inputs first, or set the window explicitly with " + "--start-cursor-ts and --end-cursor-ts", + ), + expected_json_error_code=None, + ), + EmptyCursorInputsRebuildRefusalE2ETestCase( + description="watermark microbatch full refresh in any mode", + model_sql=_ANY_MODE_MICROBATCH_SQL, + changed_model_sql=_ANY_MODE_MICROBATCH_SQL, + command=("build", "--full-refresh"), + rows_sql="SELECT payment_id, amount_cents FROM main.daily_revenue ORDER BY 1", + expected_rows=((1, 1200), (2, 1650)), + expected_output_fragments=( + "[S302] model 'daily_revenue' was not rebuilt: its cursor inputs have no rows " + "(raw_payments.paid_at), so rebuilding it now would leave it empty", + "help: load the inputs first, or set the window explicitly with " + "--start-cursor-ts and --end-cursor-ts", + ), + expected_json_error_code=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_loaded_destination_when_rebuilding_from_empty_source_then_refuses_and_keeps_rows( + test_case: EmptyCursorInputsRebuildRefusalE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_emptied_cursor_project( + tmp_path=tmp_path, + model_sql=test_case.model_sql, + changed_model_sql=test_case.changed_model_sql, + ) + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert all( + fragment in result.stdout + result.stderr + for fragment in test_case.expected_output_fragments + ), result.stdout + result.stderr + assert empty_cursor_rows(project_dir=project_dir, sql=test_case.rows_sql) == ( + test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + EmptyCursorInputsRebuildRefusalE2ETestCase( + description="watermark microbatch full refresh in any mode json output", + model_sql=_ANY_MODE_MICROBATCH_SQL, + changed_model_sql=_ANY_MODE_MICROBATCH_SQL, + command=("build", "--full-refresh", "--json"), + rows_sql="SELECT payment_id, amount_cents FROM main.daily_revenue ORDER BY 1", + expected_rows=((1, 1200), (2, 1650)), + expected_output_fragments=(), + expected_json_error_code="S302", + ), + ], + ids=lambda case: case.description, +) +def test_given_loaded_destination_when_rebuilding_json_from_empty_source_then_reports_s302( + test_case: EmptyCursorInputsRebuildRefusalE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_emptied_cursor_project( + tmp_path=tmp_path, + model_sql=test_case.model_sql, + changed_model_sql=test_case.changed_model_sql, + ) + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert result.returncode == 1, result.stdout + result.stderr + asset: dict[str, Any] = named_json_entry( + entries=json.loads(result.stdout)["assets"], name="daily_revenue" + ) + assert asset["status"] == "failed" + assert asset["error_code"] == test_case.expected_json_error_code + assert empty_cursor_rows(project_dir=project_dir, sql=test_case.rows_sql) == ( + test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + WaitingOnEmptyInputsE2ETestCase( + description="plan text output", + command=("plan",), + expected_output_fragments=( + "window waiting on empty input raw_refunds.refunded_at; " + "other inputs have new rows", + ), + expected_json_empty_inputs=None, + ), + WaitingOnEmptyInputsE2ETestCase( + description="build text output", + command=("build",), + expected_output_fragments=( + "waiting on empty input raw_refunds.refunded_at; other inputs have new rows", + "Warnings:", + "Completed with warnings PASS=0 WARN=1", + ), + expected_json_empty_inputs=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_one_empty_input_when_others_have_rows_then_warns_that_model_is_waiting( + test_case: WaitingOnEmptyInputsE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_waiting_on_empty_input_project( + tmp_path=tmp_path, model_sql=_NET_REVENUE_SQL + ) + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert all(fragment in result.stdout for fragment in test_case.expected_output_fragments), ( + result.stdout + ) + assert "no input rows" not in result.stdout + assert empty_cursor_rows(project_dir=project_dir, sql=_REVENUE_ROWS_SQL) == ( + ("2026-04-01", 1200), + ) + + +@pytest.mark.parametrize( + "test_case", + [ + WaitingOnEmptyInputsE2ETestCase( + description="build and plan json output", + command=("build", "--json"), + expected_output_fragments=(), + expected_json_empty_inputs=("raw_refunds.refunded_at",), + ), + ], + ids=lambda case: case.description, +) +def test_given_one_empty_input_when_reporting_json_then_flags_waiting_on_empty_inputs( + test_case: WaitingOnEmptyInputsE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_waiting_on_empty_input_project( + tmp_path=tmp_path, model_sql=_NET_REVENUE_SQL + ) + + plan: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=("plan", "--json") + ) + build: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert build.returncode == 0, build.stdout + build.stderr + expected_inputs: list[str] = list(test_case.expected_json_empty_inputs or ()) + model: dict[str, Any] = named_json_entry( + entries=json.loads(plan.stdout)["models"], name="daily_revenue" + ) + assert model["cursor"] == { + **model["cursor"], + "resolution_status": "no_input_rows", + "empty_inputs": expected_inputs, + "waiting_on_empty_inputs": True, + } + asset: dict[str, Any] = named_json_entry( + entries=json.loads(build.stdout)["assets"], name="daily_revenue" + ) + assert asset["status"] == "success" + assert asset["empty_cursor_inputs"] == expected_inputs + assert asset["waiting_on_empty_inputs"] is True + assert asset["warnings"] == [ + "waiting on empty input raw_refunds.refunded_at; other inputs have new rows" + ] + assert json.loads(build.stdout)["summary"]["warning_count"] == 1 + + +@pytest.mark.parametrize( + "test_case", + [ + EmptyRuntimeInputLabelE2ETestCase( + description="aliased staging view rebuilt in the same run", + changed_upstream_sql=_STG_PAYMENTS_SQL.replace( + 'FROM __source("raw_payments")', + 'FROM __source("raw_payments") WHERE amount_cents IS NOT NULL', + ), + expected_output_fragments=("no input rows (stg_payments.paid_at)",), + unexpected_output_fragments=("payments_clean.paid_at",), + ), + ], + ids=lambda case: case.description, +) +def test_given_empty_input_built_in_same_run_when_building_then_labels_it_by_project_name( + test_case: EmptyRuntimeInputLabelE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, + models={ + "models/stg_payments.sql": _STG_PAYMENTS_SQL, + "models/daily_revenue.sql": _STAGED_REVENUE_SQL, + }, + ) + load_empty_cursor_payments(project_dir=project_dir) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql(project_dir=project_dir, sql="DELETE FROM main.raw_payments") + (project_dir / "models" / "stg_payments.sql").write_text( + test_case.changed_upstream_sql, encoding="utf-8" + ) + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=("build",) + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert all(fragment in result.stdout for fragment in test_case.expected_output_fragments), ( + result.stdout + ) + assert not any(fragment in result.stdout for fragment in test_case.unexpected_output_fragments) + assert empty_cursor_rows(project_dir=project_dir, sql=_REVENUE_ROWS_SQL) == ( + _LOADED_REVENUE_ROWS + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SameRunEmptyInputsE2ETestCase( + description="all inputs empty", + emptying_sql=_EMPTY_ALL_SQL, + command=("build",), + expected_output_fragments=( + "no input rows (stg_payments.paid_at, stg_refunds.refunded_at)", + "Completed successfully PASS=3 WARN=0", + ), + unexpected_output_fragments=("waiting on empty input", "Warnings:"), + expected_json_empty_inputs=None, + expected_json_waiting_on_empty_inputs=None, + expected_json_warnings=(), + ), + SameRunEmptyInputsE2ETestCase( + description="one input empty while the other has new rows", + emptying_sql=_EMPTY_REFUNDS_SQL, + command=("build",), + expected_output_fragments=( + "waiting on empty input stg_refunds.refunded_at; other inputs have new rows", + "Completed with warnings PASS=2 WARN=1", + ), + unexpected_output_fragments=("no input rows",), + expected_json_empty_inputs=None, + expected_json_waiting_on_empty_inputs=None, + expected_json_warnings=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_inputs_built_in_same_run_when_empty_then_matches_planned_empty_input_handling( + test_case: SameRunEmptyInputsE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_same_run_empty_inputs_project( + tmp_path=tmp_path, models=_SAME_RUN_MODELS, emptying_sql=test_case.emptying_sql + ) + + plan: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=("plan",) + ) + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert "bounds computed at run time" in plan.stdout + assert result.returncode == 0, result.stdout + result.stderr + assert all(fragment in result.stdout for fragment in test_case.expected_output_fragments), ( + result.stdout + ) + assert not any(fragment in result.stdout for fragment in test_case.unexpected_output_fragments) + assert empty_cursor_rows(project_dir=project_dir, sql=_REVENUE_ROWS_SQL) == ( + ("2026-04-01", 1200), + ) + assert not empty_cursor_relation_exists(project_dir=project_dir, name="daily_revenue__delta") + + +@pytest.mark.parametrize( + "test_case", + [ + SameRunEmptyInputsE2ETestCase( + description="all inputs empty json output", + emptying_sql=_EMPTY_ALL_SQL, + command=("build", "--json"), + expected_output_fragments=(), + unexpected_output_fragments=(), + expected_json_empty_inputs=("stg_payments.paid_at", "stg_refunds.refunded_at"), + expected_json_waiting_on_empty_inputs=None, + expected_json_warnings=(), + ), + SameRunEmptyInputsE2ETestCase( + description="one input empty json output", + emptying_sql=_EMPTY_REFUNDS_SQL, + command=("build", "--json"), + expected_output_fragments=(), + unexpected_output_fragments=(), + expected_json_empty_inputs=("stg_refunds.refunded_at",), + expected_json_waiting_on_empty_inputs=True, + expected_json_warnings=( + "waiting on empty input stg_refunds.refunded_at; other inputs have new rows", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_inputs_built_in_same_run_when_empty_then_json_reports_empty_inputs( + test_case: SameRunEmptyInputsE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_same_run_empty_inputs_project( + tmp_path=tmp_path, models=_SAME_RUN_MODELS, emptying_sql=test_case.emptying_sql + ) + + result: subprocess.CompletedProcess[str] = run_empty_cursor_command( + project_dir=project_dir, command=test_case.command + ) + + assert result.returncode == 0, result.stdout + result.stderr + asset: dict[str, Any] = named_json_entry( + entries=json.loads(result.stdout)["assets"], name="daily_revenue" + ) + assert asset["status"] == "success" + assert asset["empty_cursor_inputs"] == list(test_case.expected_json_empty_inputs or ()) + assert asset.get("waiting_on_empty_inputs") == test_case.expected_json_waiting_on_empty_inputs + assert asset["warnings"] == list(test_case.expected_json_warnings) + assert empty_cursor_rows(project_dir=project_dir, sql=_REVENUE_ROWS_SQL) == ( + ("2026-04-01", 1200), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py index fc0f0bbf94..0975748e5b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py @@ -186,7 +186,10 @@ ), expected_absent_runtime_fragments=("__SQB_CURSOR_START__", "__SQB_CURSOR_END__"), expected_full_refresh_exit_code=1, - expected_full_refresh_error_fragment="required cursor watermark is empty", + expected_full_refresh_error_fragment=( + "model 'hourly_order_activity' was not rebuilt: its cursor inputs have no rows " + "(fact_orders.ordered_at)" + ), expected_full_refresh_counts=( ("fact_orders", 0), ("order_status_index", 0), diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py b/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py index 713f303186..dbef4d15bb 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py @@ -135,3 +135,13 @@ class DagsterSelectorParityE2ETestCase: description: str select: str expected_model_names: tuple[str, ...] + + +@dataclass(frozen=True) +class DagsterEmptyCursorInputsCheckE2ETestCase: + """Test case for audit checks on an incremental asset whose cursor inputs were emptied.""" + + description: str + model_sql: str + asset_key: tuple[str, ...] + expected_check_names: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py index ee0fa7932e..817c9a5bd4 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py @@ -31,6 +31,12 @@ sqlbuild_assets, sqlbuild_scenario_checks, ) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + build_empty_cursor_project, + execute_empty_cursor_sql, + load_empty_cursor_payments, + prepare_empty_cursor_inputs_project, +) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( REPO_ROOT, prepare_inline_project, @@ -41,6 +47,7 @@ ) from tests.e2e.src.sqlbuild.integrations.dagster._test_types import ( DagsterColumnAuditChecksE2ETestCase, + DagsterEmptyCursorInputsCheckE2ETestCase, DagsterPlaygroundE2ETestCase, DagsterPythonNodesArtifactE2ETestCase, DagsterSqlBuildE2ETestCase, @@ -998,3 +1005,68 @@ def sqlbuild_waffle_shop_scenarios(context: AssetCheckExecutionContext) -> Itera asset_key=test_case.scenario_order_prices_asset_key, ) ) + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterEmptyCursorInputsCheckE2ETestCase( + description="incremental run with no input rows still reports declared audits", + model_sql=( + "MODEL (\n" + ' description "Daily revenue from payments",\n' + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor revenue_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (raw_payments paid_at),\n" + " columns (total_revenue_cents (audits [not_null])),\n" + ");\n\n" + "SELECT CAST(paid_at AS DATE) AS revenue_date, " + "SUM(amount_cents) AS total_revenue_cents\n" + 'FROM __source("raw_payments")\n' + "GROUP BY 1\n" + ), + asset_key=("main", "daily_revenue"), + expected_check_names=("audit__not_null__total_revenue_cents",), + ) + ], + ids=lambda case: case.description, +) +def test_given_emptied_cursor_inputs_when_executing_dagster_then_emits_declared_checks( + test_case: DagsterEmptyCursorInputsCheckE2ETestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_dir: Path = prepare_empty_cursor_inputs_project( + tmp_path=tmp_path, models={"models/daily_revenue.sql": test_case.model_sql} + ) + load_empty_cursor_payments(project_dir=project_dir) + build_empty_cursor_project(project_dir=project_dir) + execute_empty_cursor_sql(project_dir=project_dir, sql="DELETE FROM main.raw_payments") + sqb_executable: Path = REPO_ROOT / ".venv" / "bin" / "sqb" + sqlbuild_project: SqlBuildProject = SqlBuildProject( + project_dir=project_dir, + sqb_command=(str(sqb_executable),), + ) + monkeypatch.setenv("DAGSTER_IS_DEV_CLI", "1") + sqlbuild_project.prepare_if_dev() + + @sqlbuild_assets(project=sqlbuild_project, required_resource_keys={"sqb"}) + def sqlbuild_empty_inputs(context: AssetExecutionContext) -> Iterator[object]: + yield from context.resources.sqb.cli(["build"], context=context).stream() + + result: ExecuteInProcessResult = materialize( + [sqlbuild_empty_inputs], + resources={"sqb": SqlBuildCliResource(project_dir=sqlbuild_project)}, + ) + + assert result.success + assert AssetKey(list(test_case.asset_key)) in { + event.asset_key for event in result.get_asset_materialization_events() + } + assert check_names_for_asset(result=result, asset_key=test_case.asset_key) == frozenset( + test_case.expected_check_names + ) + assert all(evaluation.passed for evaluation in result.get_asset_check_evaluations()) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index aa79df4ae8..2da79324d3 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -2,9 +2,9 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass -from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, field from sqlbuild.spec.contracts.models import SourceEntry @@ -38,6 +38,17 @@ class ResolveWatermarkFailureTestCase: expected_error_fragment: str +@dataclass(frozen=True) +class ResolveEmptyCursorInputsTestCase: + description: str + target_max: str | None + target_relation: str | None + existing_relations: dict[str, RelationInfo] + replaces_relation: bool + cursor_start: str | None + expected_window_fragment: str + + @dataclass(frozen=True) class ResolveBoundedOverrideTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 1c59015b03..f8c84b04ed 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any, cast -from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -20,6 +20,7 @@ from sqlbuild.compiler.planner.models import ( BackfillResult, CursorOverridePair, + ModelCursorSnapshot, ModelPlanContext, WarehouseSnapshot, ) @@ -80,6 +81,7 @@ def resolve_and_execute( full_refresh: bool = False, start_cursor_override: str | None = None, end_cursor_override: str | None = None, + replaces_relation: bool = False, ) -> _ResolveResult: """Resolve model SQL and execute it against a real connection.""" @@ -103,9 +105,69 @@ def resolve_and_execute( start_cursor_override=start_cursor_override, end_cursor_override=end_cursor_override, ), + replaces_relation=replaces_relation, ) result: Any = connection.execute(resolved_sql) rows: list[Any] = result.fetchall() column_types: dict[str, str] = {desc[0]: str(desc[1]) for desc in result.description} return _ResolveResult(resolved_sql=resolved_sql, rows=rows, column_types=column_types) + + +def build_empty_input_model(*, cursor_start: str | None) -> CompiledModel: + """Build a delete_insert cursor model reading the raw_events source.""" + + config: dict[str, object] = { + "materialized": "incremental", + "incremental_strategy": "delete_insert", + "cursor": "event_time", + "cursor_type": "timestamp", + "cursor_inputs": {"raw_events": "event_time"}, + "cursor_start": cursor_start, + } + return build_model( + name="fact_events", + query_sql='SELECT event_id, event_time FROM __source("raw_events")', + config=config, + ref_names=(), + ) + + +def build_empty_input_snapshot( + *, + target_max: str | None, + target_relation: str | None, + existing_relations: dict[str, RelationInfo], +) -> WarehouseSnapshot: + """Build a planning snapshot in which the raw_events cursor input has no rows.""" + + return WarehouseSnapshot( + existing_relations=existing_relations, + cursor_snapshots={ + "fact_events": ModelCursorSnapshot( + target_max=target_max, + upstream_mins=(), + upstream_maxes=(), + target_relation=target_relation, + expected_watermark_count=1, + unavailable_watermark_tags=("fact_events__raw_events__max",), + empty_input_names=("raw_events.event_time",), + ) + }, + ) + + +def raw_events_source_map() -> dict[str, SourceEntry]: + """Return the raw_events source declaration.""" + + return {"raw_events": SourceEntry(name="raw_events", schema="staging", table="raw_events")} + + +def fact_events_relation(*, schema: str) -> dict[str, RelationInfo]: + """Return an existing fact_events table in one schema, keyed as the planner keys it.""" + + return { + "fact_events": RelationInfo( + database=None, schema=schema, name="fact_events", relation_type="table" + ) + } diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py index 54ea574ea1..bf5731aa2a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py @@ -18,12 +18,17 @@ from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ResolveAndExecuteTestCase, ResolveBoundedOverrideTestCase, + ResolveEmptyCursorInputsTestCase, ResolveSourceTestCase, ResolveWatermarkFailureTestCase, ) from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve.helpers import ( _ResolveResult, + build_empty_input_model, + build_empty_input_snapshot, build_model, + fact_events_relation, + raw_events_source_map, resolve_and_execute, ) @@ -363,7 +368,7 @@ def test_given_view_filter_and_physical_watermark_when_resolving_then_filters_on ResolveWatermarkFailureTestCase( description="partial planning watermark results fail before SQL execution", unavailable_tags=("fact_events__raw_events__max",), - expected_error_fragment="required cursor watermark bounds are unavailable", + expected_error_fragment="cannot work out its cursor window", ) ], ids=lambda case: case.description, @@ -408,6 +413,109 @@ def test_given_unavailable_planning_watermark_when_resolving_then_fails_closed( ) +@pytest.mark.parametrize( + "test_case", + [ + ResolveEmptyCursorInputsTestCase( + description="incremental run reads no rows from the empty listed input", + target_max="2024-01-15 00:00:00", + target_relation="staging.fact_events", + existing_relations=fact_events_relation(schema="staging"), + replaces_relation=False, + cursor_start=None, + expected_window_fragment=( + "event_time >= TIMESTAMP '1970-01-01T00:00:00' " + "AND event_time < TIMESTAMP '1970-01-01T00:00:00'" + ), + ), + ResolveEmptyCursorInputsTestCase( + description="first build uses a zero-width window at cursor_start", + target_max=None, + target_relation=None, + existing_relations={}, + replaces_relation=True, + cursor_start="2024-01-01", + expected_window_fragment=( + "event_time >= TIMESTAMP '2024-01-01' AND event_time < TIMESTAMP '2024-01-01'" + ), + ), + ResolveEmptyCursorInputsTestCase( + description="first build ignores a same-named table in another schema", + target_max=None, + target_relation="staging.fact_events", + existing_relations=fact_events_relation(schema="archive"), + replaces_relation=True, + cursor_start="2024-01-01", + expected_window_fragment=( + "event_time >= TIMESTAMP '2024-01-01' AND event_time < TIMESTAMP '2024-01-01'" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_empty_listed_input_when_resolving_then_window_selects_no_rows( + test_case: ResolveEmptyCursorInputsTestCase, + connection: Any, +) -> None: + connection.execute("CREATE TABLE staging.raw_events (event_id INTEGER, event_time TIMESTAMP)") + connection.execute("INSERT INTO staging.raw_events VALUES (1, '2024-01-20 00:00:00')") + + result: _ResolveResult = resolve_and_execute( + model=build_empty_input_model(cursor_start=test_case.cursor_start), + snapshot=build_empty_input_snapshot( + target_max=test_case.target_max, + target_relation=test_case.target_relation, + existing_relations=test_case.existing_relations, + ), + model_locations={}, + source_map=raw_events_source_map(), + source_warehouse_columns={}, + connection=connection, + replaces_relation=test_case.replaces_relation, + ) + + assert test_case.expected_window_fragment in result.resolved_sql + assert result.rows == [] + + +@pytest.mark.parametrize( + "test_case", + [ + ResolveWatermarkFailureTestCase( + description="rebuilding an existing destination from empty inputs fails", + unavailable_tags=("fact_events__raw_events__max",), + expected_error_fragment=( + "model 'fact_events' was not rebuilt: its cursor inputs have no rows " + r"\(raw_events.event_time\)" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_empty_listed_input_when_replacing_existing_destination_then_fails( + test_case: ResolveWatermarkFailureTestCase, + connection: Any, +) -> None: + with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment) as exc_info: + resolve_and_execute( + model=build_empty_input_model(cursor_start=None), + snapshot=build_empty_input_snapshot( + target_max="2024-01-15 00:00:00", + target_relation="staging.fact_events", + existing_relations=fact_events_relation(schema="staging"), + ), + model_locations={}, + source_map=raw_events_source_map(), + source_warehouse_columns={}, + connection=connection, + replaces_relation=True, + ) + + assert exc_info.value.code == "S302" + assert exc_info.value.help is not None + assert "--start-cursor-ts and --end-cursor-ts" in exc_info.value.help + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py index b565a28996..f7a346009a 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py @@ -770,7 +770,11 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( " cursor_type timestamp,\n" " cursor_grain second,\n" ");\n\n" - "SELECT * FROM raw_orders" + 'SELECT * FROM __source("raw_orders")' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), }, setup_sql=( diff --git a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py index 08d4ea960a..b35c539ac7 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py +++ b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py @@ -36,7 +36,11 @@ " cursor_type timestamp,\n" " cursor_grain second,\n" ");\n\n" - "SELECT id, customer_id, amount, updated_at FROM raw_orders" + 'SELECT id, customer_id, amount, updated_at FROM __source("raw_orders")' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), } _SETUP_SQL: str = ( @@ -55,7 +59,7 @@ conversion_error=None, expected_exit_code=0, expected_conversions=(("orders", 2),), - expected_order_rows=5, + expected_order_rows=4, ) ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index 7455032d85..be63cd2796 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -19,7 +19,6 @@ from sqlbuild.compiler.planner.types import BackfillAction, CursorWatermarkMode from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.cursor_algebra.models import IntegerValue -from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.materializations.microbatch import ( _clamp_intervals_to_model_domain, _count_unaccounted_intervals, @@ -30,6 +29,7 @@ execute_microbatch_entry, ) from sqlbuild.executor.run._helpers.validation.cursor_bounds import resolve_runtime_cursor_bounds +from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import ( MicrobatchLifecycleState, MicrobatchTargets, @@ -345,13 +345,13 @@ def test_given_any_watermarks_when_one_input_is_empty_then_runtime_uses_populate "test_case", ( MicrobatchBehaviorTestCase( - description="all_watermarks_when_one_input_is_empty_then_runtime_fails_closed", - expected_outcome="required cursor watermark is empty", + description="all_watermarks_when_one_input_is_empty_then_runtime_reports_it", + expected_outcome="cursor inputs have no rows: empty_events.event_time", ), ), ids=lambda case: case.description, ) -def test_given_all_watermarks_when_one_input_is_empty_then_runtime_fails_closed( +def test_given_all_watermarks_when_one_input_is_empty_then_runtime_reports_it( adapter: DuckDbAdapter, connection: Any, test_case: MicrobatchBehaviorTestCase, @@ -361,7 +361,7 @@ def test_given_all_watermarks_when_one_input_is_empty_then_runtime_fails_closed( connection.execute("CREATE TABLE main.live_events (event_time TIMESTAMP)") connection.execute("INSERT INTO main.live_events VALUES ('2026-07-02')") - with pytest.raises(ExecutorInputError, match=str(test_case.expected_outcome)): + with pytest.raises(EmptyCursorInputsError, match=str(test_case.expected_outcome)): resolve_runtime_cursor_bounds( adapter=adapter, connection=connection, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index cae6af4a7f..b45d310063 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -1137,3 +1137,14 @@ class DialectSetOperationSplitTestCase: sql: str syntax: SqlLexicalSyntax | None expected_branches: tuple[str, ...] + + +@dataclass(frozen=True) +class CursorModelWithoutInputsCase: + """One model header and query checked by the cursor-input compile check (P011).""" + + description: str + config: dict[str, object] + reference_kinds: tuple[str, ...] + expected_codes: tuple[str, ...] + expected_locations: tuple[tuple[str, int, int, int], ...] = () diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 18a979f4fb..20adf26aff 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -37,12 +37,14 @@ CompiledProject, CompiledRelationLocation, CompiledSqlTest, + CompileModelConfig, CompileModelInput, CompileProjectInputs, CompilerDiagnostic, CompileSeedInput, CompileSourceInput, CompileSqlFunctionInput, + CompileSqlReference, DeclarationExpansionContext, DeclarationResolutionContext, DeclarationRuntimeProjection, @@ -90,6 +92,7 @@ SchemaModelEntry, SchemaSeedEntry, SourceEntry, + SourceLocation, ) from sqlbuild.sql_values.types import CollectionRendering @@ -761,3 +764,54 @@ def required_description_diagnostics( source_inputs=inputs.source_inputs, function_inputs=inputs.function_inputs, ) + + +_CURSOR_MODEL_PATH: Path = Path("models/finance/daily_revenue.sql") +_CURSOR_MODEL_CONTENTS: str = ( + "MODEL (\n" + ' description "Daily revenue from successful payments",\n' + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor revenue_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ");\n\n" + "SELECT CAST('2026-04-01' AS DATE) AS revenue_date, 2850 AS total_revenue_cents\n" +) + + +def cursor_model_input( + *, config: dict[str, object], reference_kinds: tuple[str, ...] +) -> CompileModelInput: + """Build the daily_revenue model input with the given config and reference kinds.""" + + return CompileModelInput( + model_file=DiscoveredSqlModelFile( + file_path=Path("/project") / _CURSOR_MODEL_PATH, + relative_path=_CURSOR_MODEL_PATH, + contents=_CURSOR_MODEL_CONTENTS, + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql="", + ), + config=CompileModelConfig(values=dict(config)), + references=tuple( + CompileSqlReference(ref_kind=kind, ref_name=f"input_{index}") + for index, kind in enumerate(reference_kinds) + ), + ) + + +def diagnostic_spans( + diagnostics: tuple[CompilerDiagnostic, ...], +) -> tuple[tuple[str, int, int, int | None], ...]: + """Return each located diagnostic's path, line, column and end column.""" + + locations: tuple[SourceLocation, ...] = tuple( + cast(SourceLocation, diagnostic.location) for diagnostic in diagnostics + ) + return tuple( + (location.path.as_posix(), location.line, location.column, location.end_column) + for location in locations + ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_model_inputs.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_model_inputs.py new file mode 100644 index 0000000000..02041d5323 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_model_inputs.py @@ -0,0 +1,113 @@ +"""Unit tests for the compile check that rejects cursor models without inputs (P011).""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.compile._helpers.config.cursor_inputs import ( + cursor_model_without_inputs_diagnostics, +) +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + CursorModelWithoutInputsCase, +) +from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( + cursor_model_input, + diagnostic_spans, +) + +_CURSOR_CONFIG: dict[str, object] = { + "materialized": "incremental", + "incremental_strategy": "delete_insert", + "cursor": "revenue_date", + "cursor_type": "timestamp", + "cursor_grain": "day", +} +_CURSOR_LOCATION: tuple[str, int, int, int] = ("models/finance/daily_revenue.sql", 5, 3, 22) +_MESSAGE: str = ( + "incremental model 'daily_revenue' reads no inputs, so builds after the first " + "cannot work out their cursor window" +) + + +@pytest.mark.parametrize( + "test_case", + [ + CursorModelWithoutInputsCase( + description="cursor model reading only constants", + config=_CURSOR_CONFIG, + reference_kinds=(), + expected_codes=("P011",), + expected_locations=(_CURSOR_LOCATION,), + ), + CursorModelWithoutInputsCase( + description="cursor model calling only a function", + config=_CURSOR_CONFIG, + reference_kinds=("udf",), + expected_codes=("P011",), + expected_locations=(_CURSOR_LOCATION,), + ), + CursorModelWithoutInputsCase( + description="cursor model reading a ref", + config=_CURSOR_CONFIG, + reference_kinds=("ref",), + expected_codes=(), + ), + CursorModelWithoutInputsCase( + description="cursor model reading a source", + config=_CURSOR_CONFIG, + reference_kinds=("source",), + expected_codes=(), + ), + CursorModelWithoutInputsCase( + description="cursor model reading a seed", + config=_CURSOR_CONFIG, + reference_kinds=("seed",), + expected_codes=(), + ), + CursorModelWithoutInputsCase( + description="cursor model with declared cursor inputs", + config={**_CURSOR_CONFIG, "cursor_inputs": {"payments": "paid_at"}}, + reference_kinds=(), + expected_codes=(), + ), + CursorModelWithoutInputsCase( + description="rolling window microbatch model", + config={ + **_CURSOR_CONFIG, + "incremental_mode": "microbatch", + "microbatch_strategy": "rolling_window", + }, + reference_kinds=(), + expected_codes=(), + ), + CursorModelWithoutInputsCase( + description="incremental model without a cursor", + config={"materialized": "incremental", "incremental_strategy": "append"}, + reference_kinds=(), + expected_codes=(), + ), + CursorModelWithoutInputsCase( + description="table model", + config={"materialized": "table"}, + reference_kinds=(), + expected_codes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_model_inputs_when_checking_cursor_inputs_then_reports_p011_only_without_inputs( + test_case: CursorModelWithoutInputsCase, +) -> None: + diagnostics: tuple[CompilerDiagnostic, ...] = cursor_model_without_inputs_diagnostics( + model_inputs=( + cursor_model_input(config=test_case.config, reference_kinds=test_case.reference_kinds), + ) + ) + + assert tuple(diagnostic.code for diagnostic in diagnostics) == test_case.expected_codes + assert all(diagnostic.message == _MESSAGE for diagnostic in diagnostics) + assert all( + "__ref(), __source() or __seed()" in (diagnostic.help or "") for diagnostic in diagnostics + ) + assert diagnostic_spans(diagnostics) == test_case.expected_locations diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 00bdbdcd28..70b1c9411c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -294,6 +294,11 @@ ), expected_model_references=((),), expected_audit_references=(), + expected_diagnostic_codes=("P011",), + expected_diagnostic_messages=( + "incremental model 'orders' reads no inputs, so builds after the first " + "cannot work out their cursor window", + ), ), BuildCompileInputsTestCase( description="does not require snapshot generated validity columns in enforced contract", @@ -893,6 +898,11 @@ expected_effective_vars={}, expected_model_references=((),), expected_audit_references=(), + expected_diagnostic_codes=("P011",), + expected_diagnostic_messages=( + "incremental model 'orders' reads no inputs, so builds after the first " + "cannot work out their cursor window", + ), ), BuildCompileInputsTestCase( description="merges row diff config from project defaults and model header", @@ -1208,6 +1218,11 @@ environment_variables={"CI": "1", "APPEND_INCLUSIVE": "0"}, expected_model_references=((),), expected_audit_references=(), + expected_diagnostic_codes=("P011",), + expected_diagnostic_messages=( + "incremental model 'orders' reads no inputs, so builds after the first " + "cannot work out their cursor window", + ), ), BuildCompileInputsTestCase( description="expands config templates in model header metadata fields", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index e54708cf40..e294ba57b9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -16,6 +16,7 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( CursorBounds, + EmptyCursorInputDecision, GraphIdentityNode, GraphNodeKey, MissingUpstream, @@ -145,6 +146,7 @@ class CursorQueryFailureTestCase: expected_results: dict[str, str] expected_failure_progress: str expected_success_progress: str + expected_failed_tags: frozenset[str] = frozenset() @dataclass(frozen=True) @@ -918,3 +920,13 @@ class MissingUpstreamMessageTestCase: missing: tuple[MissingUpstream, ...] edge_origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str] expected_message: str + + +@dataclass(frozen=True) +class EmptyCursorInputsSnapshotTestCase: + description: str + results: dict[str, str] + failed_tags: frozenset[str] + expected_empty_input_names: tuple[str, ...] + expected_unreadable_input_names: tuple[str, ...] + expected_decision: EmptyCursorInputDecision diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py index 6a56a78e09..d1a1f4d0ac 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_queries.py @@ -4,16 +4,18 @@ import pytest +from sqlbuild.compiler.planner._helpers.resolve.cursor import cursor_inputs_without_rows from sqlbuild.compiler.planner._helpers.warehouse import snapshot as snapshot_module from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( _assemble_cursor_snapshots, _build_cursor_queries, _CursorModelInfo, + _CursorQueryResults, _execute_cursor_queries, _PhysicalCursorQuery, _UpstreamCursorInfo, ) -from sqlbuild.compiler.planner.models import ModelCursorSnapshot +from sqlbuild.compiler.planner.models import EmptyCursorInputDecision, ModelCursorSnapshot from sqlbuild.compiler.planner.types import CursorType from sqlbuild.cursor_algebra.main.parse import parse from sqlbuild.cursor_algebra.main.render import render @@ -24,6 +26,7 @@ CursorQueryGroupingTestCase, CursorQueryShapeTestCase, CursorSnapshotAvailabilityTestCase, + EmptyCursorInputsSnapshotTestCase, ) @@ -235,7 +238,7 @@ def _execute(*, connection: Any, sql: str) -> _RowsResult: connection=object(), execute=_execute, on_progress=progress.append, - ) + ).values assert { key: render(value=value) for key, value in results.items() @@ -260,6 +263,7 @@ def _execute(*, connection: Any, sql: str) -> _RowsResult: expected_success_progress=( "Inspected cursor bounds (2/2): schema.good.ts [min,max] (0.50s)" ), + expected_failed_tags=frozenset({"bad__max"}), ) ], ids=lambda case: case.description, @@ -287,7 +291,7 @@ def test_given_one_failed_physical_read_when_executing_then_reports_failure_and_ ), ] - results: dict[str, CursorScalar] = _execute_cursor_queries( + query_results: _CursorQueryResults = _execute_cursor_queries( queries=queries, connection=object(), execute=execute, @@ -295,8 +299,9 @@ def test_given_one_failed_physical_read_when_executing_then_reports_failure_and_ ) assert { - key: render(value=value) for key, value in results.items() + key: render(value=value) for key, value in query_results.values.items() } == test_case.expected_results + assert query_results.failed_tags == test_case.expected_failed_tags assert len(execute.sql) == 2 assert progress[1] == test_case.expected_failure_progress assert progress[3] == test_case.expected_success_progress @@ -346,7 +351,7 @@ def test_given_fetch_failure_when_executing_then_reports_failure_and_continues( connection=object(), execute=execute, on_progress=progress.append, - ) + ).values assert { key: render(value=value) for key, value in results.items() @@ -436,5 +441,92 @@ def test_given_partial_watermark_results_when_assembling_then_model_snapshot_fai assert snapshot.unavailable_watermark_tags == test_case.expected_unavailable_tags +@pytest.mark.parametrize( + "test_case", + [ + EmptyCursorInputsSnapshotTestCase( + description="one empty input leaves no cursor window", + results={"model__good__min": "2024-01-01", "model__good__max": "2024-02-01"}, + failed_tags=frozenset(), + expected_empty_input_names=("bad_events.event_time",), + expected_unreadable_input_names=(), + expected_decision=EmptyCursorInputDecision( + input_names=("bad_events.event_time",), waiting_on_empty_inputs=True + ), + ), + EmptyCursorInputsSnapshotTestCase( + description="every input empty leaves no cursor window", + results={}, + failed_tags=frozenset(), + expected_empty_input_names=("good_events.event_time", "bad_events.event_time"), + expected_unreadable_input_names=(), + expected_decision=EmptyCursorInputDecision( + input_names=("good_events.event_time", "bad_events.event_time") + ), + ), + EmptyCursorInputsSnapshotTestCase( + description="an unreadable input is a failure rather than an empty input", + results={}, + failed_tags=frozenset({"model__bad__min", "model__bad__max"}), + expected_empty_input_names=("good_events.event_time",), + expected_unreadable_input_names=("bad_events.event_time",), + expected_decision=EmptyCursorInputDecision(), + ), + EmptyCursorInputsSnapshotTestCase( + description="inputs with rows are neither empty nor unreadable", + results={ + "model__good__min": "2024-01-01", + "model__good__max": "2024-02-01", + "model__bad__min": "2024-01-02", + "model__bad__max": "2024-02-02", + }, + failed_tags=frozenset(), + expected_empty_input_names=(), + expected_unreadable_input_names=(), + expected_decision=EmptyCursorInputDecision(), + ), + ], + ids=lambda case: case.description, +) +def test_given_empty_or_failed_watermark_reads_when_assembling_then_names_the_inputs( + test_case: EmptyCursorInputsSnapshotTestCase, +) -> None: + info: _CursorModelInfo = _CursorModelInfo( + model_name="model", + target_tag=None, + target_relation=None, + cursor_column="event_time", + upstreams=( + _UpstreamCursorInfo( + tag_min="model__good__min", + tag_max="model__good__max", + relation="raw.good_events", + cursor_column="event_time", + input_name="good_events", + ), + _UpstreamCursorInfo( + tag_min="model__bad__min", + tag_max="model__bad__max", + relation="raw.bad_events", + cursor_column="event_time", + input_name="bad_events", + ), + ), + ) + + snapshot: ModelCursorSnapshot = _assemble_cursor_snapshots( + cursor_models=[info], + results={ + key: parse(raw=value, cursor_type=CursorType.TIMESTAMP) + for key, value in test_case.results.items() + }, + failed_tags=test_case.failed_tags, + )["model"] + + assert snapshot.empty_input_names == test_case.expected_empty_input_names + assert snapshot.unreadable_input_names == test_case.expected_unreadable_input_names + assert cursor_inputs_without_rows(cursor_snapshot=snapshot) == test_case.expected_decision + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py index 218993e9f6..3987a610cb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_warehouse_cursor_concurrency.py @@ -64,7 +64,7 @@ def test_given_concurrency_when_reading_cursor_bounds_then_matches_sequential_re ) sequential: dict[str, CursorScalar] = _execute_cursor_queries( queries=queries, connection=None, execute=sequential_execute, on_progress=None - ) + ).values parallel_execute: ConcurrencyTrackingCursorExecute = ConcurrencyTrackingCursorExecute( failing_relations=test_case.failing_relations, latency_seconds=_LATENCY_SECONDS ) @@ -76,7 +76,7 @@ def test_given_concurrency_when_reading_cursor_bounds_then_matches_sequential_re execute=parallel_execute, on_progress=progress.append, concurrency=test_case.concurrency, - ) + ).values failures: tuple[str, ...] = tuple( filter(lambda message: message.startswith("Failed"), progress) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index 0f2eb18658..da728214e4 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -2,7 +2,12 @@ from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent from sqlbuild.compiler.planner.types import PlanReason -from sqlbuild.executor.run.types import AuditGateReuseReason, AuditGateStatus, ExecutionPhase +from sqlbuild.executor.run.types import ( + AuditGateReuseReason, + AuditGateStatus, + ExecutionPhase, + RuntimeCursorWatermarkMode, +) from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -224,8 +229,10 @@ class RuntimeCursorPolicyTestCase: @dataclass(frozen=True) class RuntimeCursorFailureTestCase: description: str - slow_input_setup_sql: str - expected_error_fragment: str + fast_input_setup_sql: str + expected_input_names: tuple[str, ...] + expected_waiting_on_empty_inputs: bool + cursor_watermark_mode: RuntimeCursorWatermarkMode = RuntimeCursorWatermarkMode.ALL @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index e31192f644..4bca9ee616 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -36,7 +36,9 @@ resolve_runtime_cursor_bounds, substitute_cursor_sentinels, ) +from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import RuntimeCursorInputRelation, RuntimeCursorSpec +from sqlbuild.executor.run.types import RuntimeCursorWatermarkMode from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig from sqlbuild.spec.contracts.types import FutureCursorAction from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( @@ -903,23 +905,37 @@ def test_given_multiple_runtime_inputs_when_resolving_then_uses_common_watermark "test_case", [ RuntimeCursorFailureTestCase( - description="all mode fails closed for an empty required input", - slow_input_setup_sql="SELECT 1", - expected_error_fragment="required cursor watermark is empty", + description="all mode reports an empty required input", + fast_input_setup_sql="INSERT INTO fast_input VALUES (10), (200)", + expected_input_names=("stg_refunds.cursor_value",), + expected_waiting_on_empty_inputs=True, + ), + RuntimeCursorFailureTestCase( + description="legacy mode waits on an empty input like planned windows", + fast_input_setup_sql="INSERT INTO fast_input VALUES (10), (200)", + expected_input_names=("stg_refunds.cursor_value",), + expected_waiting_on_empty_inputs=True, + cursor_watermark_mode=RuntimeCursorWatermarkMode.LEGACY, + ), + RuntimeCursorFailureTestCase( + description="legacy mode reports inputs that are all empty", + fast_input_setup_sql="SELECT 1", + expected_input_names=("stg_payments.cursor_value", "stg_refunds.cursor_value"), + expected_waiting_on_empty_inputs=False, + cursor_watermark_mode=RuntimeCursorWatermarkMode.LEGACY, ), ], ids=lambda case: case.description, ) -def test_given_all_runtime_inputs_when_one_is_empty_then_resolution_fails_closed( +def test_given_empty_runtime_inputs_when_resolving_then_reports_inputs_without_rows( test_case: RuntimeCursorFailureTestCase, ) -> None: connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") connection.execute("CREATE TABLE fast_input (cursor_value INTEGER)") - connection.execute("INSERT INTO fast_input VALUES (10), (200)") + connection.execute(test_case.fast_input_setup_sql) connection.execute("CREATE TABLE slow_input (cursor_value INTEGER)") - connection.execute(test_case.slow_input_setup_sql) - with pytest.raises(ExecutorInputError, match=test_case.expected_error_fragment): + with pytest.raises(EmptyCursorInputsError) as exc_info: resolve_runtime_cursor_bounds( adapter=cast(BaseAdapter, FakeCursorAdapter()), connection=connection, @@ -932,13 +948,25 @@ def test_given_all_runtime_inputs_when_one_is_empty_then_resolution_fails_closed cursor_type=CursorType.INTEGER, cursor_grain=None, cursor_start=None, + cursor_watermark_mode=test_case.cursor_watermark_mode, cursor_input_relations=( - RuntimeCursorInputRelation(relation="fast_input", cursor_column="cursor_value"), - RuntimeCursorInputRelation(relation="slow_input", cursor_column="cursor_value"), + RuntimeCursorInputRelation( + relation="fast_input", + cursor_column="cursor_value", + input_name="stg_payments", + ), + RuntimeCursorInputRelation( + relation="slow_input", + cursor_column="cursor_value", + input_name="stg_refunds", + ), ), ), ) + assert exc_info.value.input_names == test_case.expected_input_names + assert exc_info.value.waiting_on_empty_inputs is test_case.expected_waiting_on_empty_inputs + @pytest.mark.parametrize( "test_case", diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index f98ac33f7b..83ea632849 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -151,7 +151,33 @@ MODEL ( SQLBuild uses these to determine the replay window. With multiple listed inputs, the end is the conservative common watermark: the minimum of their maxima. This prevents a faster input from advancing the model beyond data available from a slower input. -On a first build, SQLBuild derives the interval from the declared cursor inputs and cursor policy. If it cannot establish a valid interval, the build fails before mutating the destination. +On a first build, SQLBuild derives the interval from the declared cursor inputs and cursor policy. SQLBuild decides the window while planning, before any staging or delta relation is created. When an input is built earlier in the same run, the window is decided just before the model runs, still before any staging work. + +#### Models with no inputs + +A cursor-based model must read at least one input that can bound its window, either listed in `cursor_inputs` or read through `__ref()`, `__source()` or `__seed()`. A model that reads none of them could build once but could never work out a window afterwards, so compile rejects it with `P011`: + +``` +error[P011]: incremental model 'daily_revenue' reads no inputs, so builds after the first cannot work out their cursor window + --> models/finance/daily_revenue.sql:5:3 + | + 5 | cursor revenue_date, + | ^^^^^^^^^^^^^^^^^^^ + = help: read the data through __ref(), __source() or __seed() so its cursor can bound the window, or use materialized table if the model has no upstream data +``` + +A `rolling_window` microbatch model takes its window from the invocation time, so it may read no inputs. + +#### Inputs with no rows + +Empty inputs are normal: a new development or CI environment, a source in the middle of a reload, or an upstream model that is legitimately empty. When a listed input has no rows, there is no new data to process: + +- On an incremental run, the model succeeds without changing its destination, and the plan and build output show the reason, for example `no input rows (stg_payments.paid_at)`. The model's audits still run against the unchanged destination, and downstream models continue. The `--json` build output lists the inputs as `empty_cursor_inputs`, and `sqb plan --json` reports `"resolution_status": "no_input_rows"` with the same inputs in `empty_inputs`. +- When only some listed inputs are empty, the window still waits for them, so the model makes no progress even though other inputs have new rows. The plan and build output show this as a warning, for example `waiting on empty input stg_refunds.refunded_at; other inputs have new rows`. The build counts it in the summary's `WARN=` total and lists it in the asset's `warnings` in `--json` output, and the JSON output adds `"waiting_on_empty_inputs": true`. The exit code and model status do not change. A watermark microbatch model with `cursor_watermark_mode any` instead progresses from the inputs that have rows. +- On a first build, the destination is created empty so that models that reference it can build. The next build derives its window from the inputs as usual. +- A rebuild that would replace an existing destination, such as a query change with `replay_on_change full` or a microbatch `--full-refresh`, fails instead of leaving the table empty. Load the inputs first, or set the window explicitly with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms). + +Use [source freshness](/docs/concepts/sources) checks and source audits to catch an input that should not be empty. If SQLBuild cannot read the cursor bounds of an input at all, the plan fails with `S302`, naming the model and its inputs. ### Cursor bounds in model SQL From c4906940e0c4a11a560fa9bb596e0f4bcc4f170f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:07:08 +0100 Subject: [PATCH 2/2] test(e2e): give cursor incremental fixtures a declared input --- .../commands/main/providers/test_providers.py | 9 ++++++--- .../main/rules/test_sql_quality_rules.py | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index 1836ed8fa5..709b490aa1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -1016,7 +1016,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): [ ProviderHookMaterializationE2ETestCase( description="incremental python hooks use provider injection", - command=("--no-color", "build", "--select", "incremental_orders"), + command=("--no-color", "build", "--select", "+incremental_orders"), model_relative_path="models/incremental_orders.sql", model_sql=dedent( """ @@ -1029,11 +1029,14 @@ def mark_post(ctx, marker_provider: MarkerProvider): post_hooks [python("mark_hook")] ); - SELECT 1 AS order_id + SELECT order_id FROM __ref("order_rows") """ ).strip() + "\n", - extra_repo_files={}, + extra_repo_files={ + "models/order_rows.sql": 'MODEL (description "Test model order_rows.");\n\n' + "SELECT 1 AS order_id\n" + }, expected_marker_entries=( "setup", "incremental_orders:pre_hooks", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py index ede09a9dba..76d4d67345 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py @@ -78,6 +78,19 @@ "SELECT 1 AS order_id, 7 AS customer_id, 3 AS amount," " TIMESTAMP '2026-01-01 00:00:00' AS order_date\n" ) +_CURSOR_INCREMENTAL_SQL: str = ( + 'SELECT order_id, customer_id, amount, order_date FROM __source("landing_order_events")\n' +) +_CURSOR_SOURCE_YML: str = ( + "sources:\n" + " - name: landing_order_events\n description: Test source landing_order_events.\n" + " schema: landing\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: customer_id\n type: INTEGER\n" + " - name: amount\n type: INTEGER\n" + " - name: order_date\n type: TIMESTAMP\n" +) _SNAPSHOT_HEADER: str = ( "MODEL (\n" ' description "Raw orders",\n' @@ -192,9 +205,10 @@ def test_given_quality_rule_violations_when_compiling_and_fixing_then_findings_a upstream_header=_INCREMENTAL_HEADER_TEMPLATE.format( strategy="merge", cursor=_CURSOR_CONFIG ), - upstream_sql=_INCREMENTAL_SQL, + upstream_sql=_CURSOR_INCREMENTAL_SQL, query_sql=_RENAMED_CHAIN_SQL, expected_codes=(), + extra_files=(("sources/landing_events.yml", _CURSOR_SOURCE_YML),), ), RankingKeyProofCase( description="a key-matched delete_insert unique_key proves the order", @@ -210,9 +224,10 @@ def test_given_quality_rule_violations_when_compiling_and_fixing_then_findings_a upstream_header=_INCREMENTAL_HEADER_TEMPLATE.format( strategy="delete_insert", cursor=_CURSOR_CONFIG ), - upstream_sql=_INCREMENTAL_SQL, + upstream_sql=_CURSOR_INCREMENTAL_SQL, query_sql=_RENAMED_CHAIN_SQL, expected_codes=("SQBRSQL018",), + extra_files=(("sources/landing_events.yml", _CURSOR_SOURCE_YML),), ), RankingKeyProofCase( description="a snapshot unique_key identifies entities, not versions",