From 3e669ff89d5d8d32bbe182f7aebc22791dcbb4e4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:00:49 +0100 Subject: [PATCH 1/8] fix: name the resolved target in the migrate_from removal notice --- .../planner/_helpers/migrations/display.py | 4 +- .../planner/_helpers/migrations/planning.py | 2 +- src/sqlbuild/compiler/planner/models.py | 6 +++ .../main/model_migrations/_test_types.py | 7 +++ .../test_manual_migrations.py | 49 +++++++++++++++++++ 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py index fa5e1d07fb..bc06a199be 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/display.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/display.py @@ -145,9 +145,7 @@ def _migration_lines(*, entry: ModelMigrationPlanEntry, style: CliStyle) -> list rows.append( _property_row( label="completed", - value=( - f"{entry.completed_at.isoformat()} on target '{entry.target_name or 'default'}'" - ), + value=f"{entry.completed_at.isoformat()} on target '{entry.target_label}'", style=style, ) ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 5f23334a2b..50f3af0a86 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -731,7 +731,7 @@ def _entry_warning(entry: ModelMigrationPlanEntry) -> PlanWarning | None: severity=WarningSeverity.WARNING, message=( f"migration from {origin} completed at {completed} on target " - f"'{entry.target_name}'; migrate_from can be removed from '{entry.model_name}'" + f"'{entry.target_label}'; migrate_from can be removed from '{entry.model_name}'" ), code="M101", ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index bb7c757098..73641ba3c8 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -929,6 +929,12 @@ def storage_transition(self) -> str | None: labels: tuple[str, str] = ("permanent", "transient") return f"{labels[self.origin_is_transient]} -> {labels[self.stage_is_transient]}" + @property + def target_label(self) -> str: + """Return the target name shown in migration text, naming the unnamed target 'default'.""" + + return self.target_name or "default" + @property def blocks_build(self) -> bool: """Return whether this migration must stop a build before any execution.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index 30c5a84dd9..1fb66bd9b3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -125,3 +125,10 @@ class IdentityHandoverTestCase: class RenamedPlanTextTestCase: description: str expected_fragment: str + + +@dataclass(frozen=True) +class CompletedMigrationPlanTextTestCase: + description: str + project_toml: str + expected_target_label: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index 14423a36bd..c8ed592d5f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -9,6 +9,7 @@ from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( BackAndForthMigrationTestCase, + CompletedMigrationPlanTextTestCase, MigrationCompatibilityTestCase, MigrationCompileErrorTestCase, MigrationInterruptionTestCase, @@ -17,6 +18,8 @@ from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( DESTINATION_MODEL, ORIGIN_MODEL, + PROJECT_TOML, + TARGETS_PROJECT_TOML, CliRun, build, build_ok, @@ -118,6 +121,52 @@ def test_given_completed_migration_when_rebuilding_then_warns_and_skips( ) +@pytest.mark.parametrize( + "test_case", + [ + CompletedMigrationPlanTextTestCase( + description="unnamed target", + project_toml=PROJECT_TOML, + expected_target_label="default", + ), + CompletedMigrationPlanTextTestCase( + description="named default target", + project_toml=TARGETS_PROJECT_TOML, + expected_target_label="dev", + ), + ], + ids=lambda case: case.description, +) +def test_given_completed_migration_when_planning_then_removal_notice_names_the_target( + test_case: CompletedMigrationPlanTextTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project( + project_dir=tmp_path, + models={ORIGIN_MODEL: incremental_orders_sql()}, + project_toml=test_case.project_toml, + ) + load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + write_project( + project_dir=tmp_path, + models={DESTINATION_MODEL: incremental_orders_sql(migrate_from=ORIGIN_MODEL)}, + project_toml=test_case.project_toml, + ) + _ = build_ok(project_dir=tmp_path, capsys=capsys) + + result: CliRun = run_sqb(project_dir=tmp_path, args=("plan",), capsys=capsys) + + assert result.exit_code == 0, result.output + assert f"on target '{test_case.expected_target_label}'\n" in result.output + assert ( + f"on target '{test_case.expected_target_label}'; " + f"migrate_from can be removed from '{DESTINATION_MODEL}'" + ) in result.output + assert "'None'" not in result.output + + @pytest.mark.parametrize( "test_case", [ From 7a222cfcd79573460ceb8e03f9ff91495a4a18b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:04:46 +0100 Subject: [PATCH 2/8] fix: count source, seed and end-of-build audits in the build summary --- .../classes/build_progress_callbacks.py | 24 ++++++---- .../cli/commands/main/build/_test_types.py | 9 ++++ .../main/build/test_attached_audit_gates.py | 45 +++++++++++++++++++ .../main/build/test_audit_execution_errors.py | 18 +++++++- 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py index 3a3ff52e7f..0fa259a237 100644 --- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py +++ b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py @@ -804,6 +804,14 @@ def _write_error_detail( self._stream.write(f"{continuation_pad}{line}\n") +def _build_audit_results(result: BuildExecutionResult) -> tuple[AuditExecutionResult, ...]: + audits: list[AuditExecutionResult] = [] + model_result: ModelExecutionResult + for model_result in result.model_results: + audits.extend(model_result.audit_results) + return (*audits, *result.source_audit_results, *result.end_audit_results) + + def _count_build_footer_results( *, result: BuildExecutionResult, @@ -824,14 +832,14 @@ def _count_build_footer_results( fail_count += 1 elif model_result.status == ExecutionStatus.SKIPPED: skip_count += 1 - audit_r: AuditExecutionResult - for audit_r in model_result.audit_results: - if audit_r.outcome == AuditOutcome.PASS: - pass_count += 1 - elif audit_r.outcome == AuditOutcome.WARN: - warn_count += 1 - elif audit_r.outcome == AuditOutcome.ERROR: - fail_count += 1 + audit_r: AuditExecutionResult + for audit_r in _build_audit_results(result): + if audit_r.outcome == AuditOutcome.PASS: + pass_count += 1 + elif audit_r.outcome == AuditOutcome.WARN: + warn_count += 1 + elif audit_r.outcome == AuditOutcome.ERROR: + fail_count += 1 seed_result: SeedExecutionResult for seed_result in result.seed_results: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 93c7d59100..8e51024b8d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -950,6 +950,15 @@ class AttachedAuditGateBuildE2ETestCase: concurrency: int = 1 +@dataclass(frozen=True) +class AttachedAuditGateSummaryE2ETestCase: + description: str + target_kind: str + order_code: str + expected_exit_code: int + expected_summary: str + + @dataclass(frozen=True) class AttachedAuditGateCycleE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py index 1ef5b96ae6..a147362453 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py @@ -12,6 +12,7 @@ AttachedAuditGateCycleE2ETestCase, AttachedAuditGateNoAuditsE2ETestCase, AttachedAuditGatePartialBuildE2ETestCase, + AttachedAuditGateSummaryE2ETestCase, AuditReadPlanE2ETestCase, NestedSourceGateE2ETestCase, ) @@ -163,6 +164,50 @@ def test_given_attached_audit_reading_another_resource_when_building_then_it_gat ) +@pytest.mark.parametrize( + "test_case", + [ + AttachedAuditGateSummaryE2ETestCase( + description="passing source audit counts as a pass", + target_kind="source", + order_code="A", + expected_exit_code=0, + expected_summary="PASS=3 WARN=0 FAIL=0 INSUFFICIENT=0 SKIP=0 TOTAL=3", + ), + AttachedAuditGateSummaryE2ETestCase( + description="failing source audit counts as a failure", + target_kind="source", + order_code="Z", + expected_exit_code=1, + expected_summary="PASS=1 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=3", + ), + AttachedAuditGateSummaryE2ETestCase( + description="failing seed audit counts as a failure", + target_kind="seed", + order_code="Z", + expected_exit_code=1, + expected_summary="PASS=2 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=4", + ), + ], + ids=lambda case: case.description, +) +def test_given_attached_audit_outcome_when_building_in_terminal_then_summary_counts_it( + test_case: AttachedAuditGateSummaryE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_attached_audit_gate_project( + tmp_path=tmp_path, target_kind=test_case.target_kind, order_code=test_case.order_code + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + assert test_case.expected_summary in output + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py index ddd4b15699..d8f87abd44 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py @@ -129,12 +129,28 @@ def test_given_audit_query_raises_when_building_then_it_reports_a_failed_audit( "Failures:", "broken_check on raw_orders (audit)", "missing_lookup", + "PASS=0 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=2", + ), + ), + AuditExecutionErrorTerminalE2ETestCase( + description="seed audit error is listed and counted as a failure", + audit_kind="seed", + expected_fragments=( + "Failures:", + "broken_check on order_codes (audit)", + "missing_lookup", + "PASS=1 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=3", ), ), AuditExecutionErrorTerminalE2ETestCase( description="end-of-build audit error is listed with its message", audit_kind="end", - expected_fragments=("Failures:", "broken_check (audit)", "missing_lookup"), + expected_fragments=( + "Failures:", + "broken_check (audit)", + "missing_lookup", + "PASS=2 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=0 TOTAL=3", + ), ), ], ids=lambda case: case.description, From d94a783ea2d04c4ef1443c59424f62ef55b6a154 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:17:45 +0100 Subject: [PATCH 3/8] fix: plan sources declared by selected Python nodes --- .../pipeline/_helpers/python_plan_entries.py | 18 ++++ .../compiler/pipeline/main/compile.py | 10 ++- .../compiler/planner/_helpers/graph/scope.py | 2 + .../planner/_helpers/output/plan_entry.py | 14 +++- .../planner/_helpers/planning/scopes.py | 1 + .../_helpers/warehouse/source_deferral.py | 16 ++++ src/sqlbuild/compiler/planner/models.py | 2 + .../cli/commands/main/build/_test_types.py | 9 ++ .../main/build/test_python_source_deferral.py | 44 ++++++++++ .../sqlbuild/compiler/pipeline/_test_types.py | 7 ++ .../pipeline/test_python_source_reads.py | 83 +++++++++++++++++++ 11 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py diff --git a/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py b/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py index c2b8470137..8d7f564267 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py @@ -23,6 +23,24 @@ PythonNodeKind, PythonRunPhase, ) +from sqlbuild.python_nodes.types import SqlResourceRefKind + + +def python_read_source_names_for( + *, discovered_inputs: DiscoveredProjectInputs, python_node_names: frozenset[str] +) -> frozenset[str]: + """Return sources declared by the selected Python nodes and their Python upstreams.""" + + if not python_node_names: + return frozenset() + python_graph: PythonNodeGraph = build_discovered_python_node_graph( + discovered_inputs=discovered_inputs + ) + return frozenset( + ref.name + for ref in python_graph.selected_sql_refs(selected_names=python_node_names) + if ref.kind == SqlResourceRefKind.SOURCE + ) def build_python_run_plan_outputs( diff --git a/src/sqlbuild/compiler/pipeline/main/compile.py b/src/sqlbuild/compiler/pipeline/main/compile.py index 03bb146649..89643c926c 100644 --- a/src/sqlbuild/compiler/pipeline/main/compile.py +++ b/src/sqlbuild/compiler/pipeline/main/compile.py @@ -32,7 +32,10 @@ ) from sqlbuild.compiler.pipeline._helpers.graph import build_static_all_keys from sqlbuild.compiler.pipeline._helpers.materializations import load_custom_materializations -from sqlbuild.compiler.pipeline._helpers.python_plan_entries import build_python_run_plan_outputs +from sqlbuild.compiler.pipeline._helpers.python_plan_entries import ( + build_python_run_plan_outputs, + python_read_source_names_for, +) from sqlbuild.compiler.pipeline.main._compile_phase import compile_project_phase from sqlbuild.compiler.pipeline.models import ( CompiledProjectPhaseResult, @@ -203,6 +206,7 @@ def _build_result( selected_sql_keys: frozenset[CompiledObjectKey] | None = None selected_python_node_names: frozenset[str] = frozenset() + python_read_source_names: frozenset[str] = frozenset() run_selection: PythonSqlRunSelection | None = None if options.resolve_python_run_selectors: run_selection = resolve_python_sql_run_selection_from_inputs( @@ -213,6 +217,9 @@ def _build_result( ) selected_sql_keys = run_selection.sql_keys selected_python_node_names = run_selection.python_node_names + python_read_source_names = python_read_source_names_for( + discovered_inputs=discovered_inputs, python_node_names=selected_python_node_names + ) plan_output: PlanOutput = build_execution_plan( project=project, @@ -222,6 +229,7 @@ def _build_result( select=select, exclude=exclude, selected_keys=selected_sql_keys, + python_read_source_names=python_read_source_names, ), overrides=PlannerOverrides( cursor_overrides=options.cursor_overrides, diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py index 19fd984b5a..69abcb5a09 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py @@ -43,6 +43,7 @@ def build_planner_scope( exclude: tuple[str, ...], auto_load_sources: bool, selected_keys: frozenset[CompiledObjectKey] | None = None, + python_read_source_names: frozenset[str] = frozenset(), ) -> PlannerScope: upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( build_execution_upstream_deps(project) @@ -100,6 +101,7 @@ def build_planner_scope( injected_edge_origins=build_execution_edge_origins(project), ), user_selected_keys=resolved_selected_keys, + python_read_source_names=python_read_source_names, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 75a309a111..ffeb2240a4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -70,7 +70,10 @@ from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( get_semantic_source_columns, ) -from sqlbuild.compiler.planner._helpers.warehouse.source_deferral import build_source_read_map +from sqlbuild.compiler.planner._helpers.warehouse.source_deferral import ( + build_source_read_map, + with_declared_source_reads, +) from sqlbuild.compiler.planner.constants import ( METADATA_NAME_FILTER_LIMIT, ) @@ -249,18 +252,21 @@ def build_planner_relations_context( if effective_deferral.source_deferral_enabled else source_map ) + python_source_map: dict[str, SourceEntry] = with_declared_source_reads( + project=project, source_map=source_map, source_names=scope.python_read_source_names + ) python_source_read_map: dict[str, SourceEntry] = ( build_source_read_map( project=project, - source_map=source_map, + source_map=python_source_map, selected_keys=scope.selected_keys, project_config=project_config, local_config=local_config, defer_sources_to=effective_deferral.defer_sources_to, - read_source_names=frozenset(source_map), + read_source_names=frozenset(python_source_map), ) if effective_deferral.source_deferral_enabled - else source_map + else python_source_map ) return PlannerRelationsContext( model_locations=model_locations, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py index 333af898ce..61e5a2aef4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py @@ -29,6 +29,7 @@ def resolve_planner_scopes( exclude=selection.exclude, auto_load_sources=policies.auto_load_sources, selected_keys=selection.selected_keys, + python_read_source_names=selection.python_read_source_names, ) full_scope: PlannerScope = build_planner_scope( project=project, diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py index e55dedb709..870466580b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py @@ -96,6 +96,22 @@ def build_source_read_map( return result +def with_declared_source_reads( + *, project: CompiledProject, source_map: dict[str, SourceEntry], source_names: frozenset[str] +) -> dict[str, SourceEntry]: + """Add declared project sources that no selected SQL resource reads to a source map.""" + + missing: frozenset[str] = source_names - frozenset(source_map) + if not missing: + return source_map + result: dict[str, SourceEntry] = dict(source_map) + source: CompiledSource + for source in project.sources: + if source.source_entry.name in missing: + result[source.source_entry.name] = source.source_entry + return result + + def _sql_hook_source_reads(model: CompiledModel) -> tuple[str, ...]: names: list[str] = [] for hook_key in (MODEL_PRE_HOOKS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY): diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 73641ba3c8..98ac8acc53 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -729,6 +729,7 @@ class PlannerScope: selected_keys: frozenset[CompiledObjectKey] execution_order: tuple[CompiledObjectKey, ...] user_selected_keys: frozenset[CompiledObjectKey] = frozenset() + python_read_source_names: frozenset[str] = frozenset() @dataclass(frozen=True) @@ -1458,6 +1459,7 @@ class PlannerSelection: select: tuple[str, ...] = () exclude: tuple[str, ...] = () selected_keys: frozenset[CompiledObjectKey] | None = None + python_read_source_names: frozenset[str] = frozenset() @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 8e51024b8d..40ad9ccbd1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1098,6 +1098,15 @@ class PythonSourceDeferralE2ETestCase: expected_check_pattern: str +@dataclass(frozen=True) +class PythonSourceTaskOnlyE2ETestCase: + """A task selected alone whose declared source no selected SQL resource reads.""" + + description: str + target_config: str + expected_task_pattern: str + + @dataclass(frozen=True) class UnrenderedMacroArgumentE2ETestCase: """A model passing typed references its macro does not render.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py index 9f7f8020cb..54b3e96270 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( PythonSourceDeferralE2ETestCase, + PythonSourceTaskOnlyE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, @@ -73,6 +74,9 @@ "INSERT INTO raw_prod.raw_orders VALUES (1), (2);" ) _CHECK_PASSED: str = r"check\s+two_raw_orders\s+PASS" +_TASK_WAITS_FOR_SOURCE: str = ( + r"count_task\s+SKIP\s+Upstream SQL resource did not complete: raw_orders" +) @pytest.mark.parametrize( @@ -119,5 +123,45 @@ def test_given_deferred_source_reads_when_python_reads_the_source_then_it_reads_ assert re.search(test_case.expected_check_pattern, checked.stdout), checked.stdout +@pytest.mark.parametrize( + "test_case", + [ + PythonSourceTaskOnlyE2ETestCase( + description="with source deferral", + target_config='defer_sources_to = "prod"\n', + expected_task_pattern=_TASK_WAITS_FOR_SOURCE, + ), + PythonSourceTaskOnlyE2ETestCase( + description="without source deferral", + target_config="", + expected_task_pattern=_TASK_WAITS_FOR_SOURCE, + ), + ], + ids=lambda case: case.description, +) +def test_given_only_a_task_selected_when_building_then_its_declared_source_is_planned( + test_case: PythonSourceTaskOnlyE2ETestCase, + tmp_path: Path, +) -> None: + files: dict[str, str] = dict(_PROJECT_FILES) + files["sqlbuild_project.toml"] = files["sqlbuild_project.toml"].replace( + 'defer_sources_to = "prod"\n', test_case.target_config + ) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="deferred_sources", repo_files=files + ) + execute_duckdb(db_path=project_dir / _DATABASE, sql=_ENVIRONMENT_ROWS) + + build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "task:count_task"), project_dir=project_dir + ) + + output: str = build.stdout + build.stderr + assert build.returncode == 0, output + assert "G000" not in output + assert "missing from the resolved plan source map" not in output + assert re.search(test_case.expected_task_pattern, output), output + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index 600ddefb7a..807d7529bc 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -373,3 +373,10 @@ class SingularAuditAttachmentIntegrationTestCase: orders_header: str generic_audit_files: dict[str, str] expected_attachment: tuple[str, str | None] + + +@dataclass(frozen=True) +class PythonSourceReadPlanTestCase: + description: str + target_config: str + expected_schema: str diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py new file mode 100644 index 0000000000..c7e54f315a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py @@ -0,0 +1,83 @@ +"""Integration coverage for sources declared by selected Python nodes through the real pipeline.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.pipeline.models import CompilePipelineResult +from sqlbuild.spec.contracts.models import SourceEntry +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + PythonSourceReadPlanTestCase, +) +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( + run_compile_pipeline_for_project, +) + +_PROJECT_TOML: str = ( + 'name = "python_reads"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = ":memory:"\n\n' + '[targets.dev]\nschema = "dev"\nloader_schema = "raw_dev"\n{target_config}\n' + '[targets.prod]\nschema = "prod"\nloader_schema = "raw_prod"\n' +) +_PROJECT_FILES: dict[str, str] = { + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + "python/loaders/raw.py": ( + "from sqlbuild.loaders import loader\n\n\n" + "@loader\ndef raw_orders(ctx):\n return [{'order_id': 1}]\n" + ), + "models/order_marker.sql": "MODEL (materialized table);\nSELECT 1 AS n\n", + "python/tasks/count.py": ( + "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" + '@task(depends_on=source("raw_orders"))\n' + "def count_task(ctx):\n" + " ctx.query(f\"SELECT count(*) FROM {ctx.relation(source('raw_orders'))}\")\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + PythonSourceReadPlanTestCase( + description="deferred source read resolves to the deferral target", + target_config='defer_sources_to = "prod"\n', + expected_schema="raw_prod", + ), + PythonSourceReadPlanTestCase( + description="undeferred source read resolves to the active target", + target_config="", + expected_schema="raw_dev", + ), + ], + ids=lambda case: case.description, +) +def test_given_only_a_task_selected_when_planning_then_its_source_resolves_like_a_sql_read( + test_case: PythonSourceReadPlanTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files( + tmp_path, + { + "sqlbuild_project.toml": _PROJECT_TOML.format(target_config=test_case.target_config), + **_PROJECT_FILES, + }, + ) + + result: CompilePipelineResult = run_compile_pipeline_for_project( + project_dir=tmp_path, + adapter=DuckDbAdapter(), + select=("task:count_task",), + resolve_python_run_selectors=True, + ) + + source: SourceEntry = result.plan_output.python_source_entries["raw_orders"] + assert source.schema == test_case.expected_schema + assert "raw_orders" not in result.plan_output.source_map From 1bd38799d6662885784c7299bcdd831399c01009 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:28:30 +0100 Subject: [PATCH 4/8] fix: warn about hard-coded relation names in loaders under sqb load --- .../concepts/python-nodes/sql-references.md | 5 +- .../cli/commands/_helpers/load/execution.py | 12 ++- .../cli/commands/_helpers/load/invocation.py | 4 + .../cli/commands/_helpers/load/outputs.py | 3 +- .../cli/commands/_helpers/load/planning.py | 29 ++++++ .../cli/commands/_helpers/load/selection.py | 11 +++ .../classes/load_progress_reporter.py | 33 +++++-- src/sqlbuild/cli/commands/models.py | 3 + .../cli/commands/main/load/_test_types.py | 8 ++ .../main/load/test_load_relation_guard.py | 97 +++++++++++++++++++ .../concepts/python-nodes/sql-references.mdx | 5 +- 11 files changed, 196 insertions(+), 14 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md index 96ab6b2873..2d73271471 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md @@ -104,8 +104,9 @@ Both checks ignore: - SQL that cannot be parsed; it is never blocked, only logged at debug level This is a guard against accidental hard-coding, not a sandbox. SQL sent directly through -`ctx.connection` or `ctx.adapter` is not checked. Run-time checks apply to `sqb build` and -`sqb check`. To turn both checks off while migrating a project, +`ctx.connection` or `ctx.adapter` is not checked. Run-time checks apply to `sqb build`, +`sqb check`, and `sqb load`. `sqb load` does not compile models, so it checks loaders against +the project's sources. To turn both checks off while migrating a project, set [`[references] enforce_explicit = false`](../project-configuration.md#references). ## model vs source diff --git a/src/sqlbuild/cli/commands/_helpers/load/execution.py b/src/sqlbuild/cli/commands/_helpers/load/execution.py index eb68b2428d..ef463fc56d 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/load/execution.py @@ -79,6 +79,7 @@ def on_load_complete(result: LoadExecutionResult) -> None: end_cursor_int=parse_cursor_integer(preparation.effective_cursor_overrides.end_int), use_color=invocation.use_color, providers=preparation.provider_session.providers, + project_relations=preparation.project_relations, ), callbacks=LoadCallbacks( on_load_start=load_progress.on_start, @@ -104,7 +105,16 @@ def on_load_complete(result: LoadExecutionResult) -> None: return LoadRunOutcome( results=results, elapsed=time.monotonic() - start, - success_count=sum(1 for result in results if result.status == ExecutionStatus.SUCCESS), + success_count=sum( + 1 + for result in results + if result.status == ExecutionStatus.SUCCESS and not result.warning_messages + ), fail_count=sum(1 for result in results if result.status == ExecutionStatus.FAILED), skip_count=sum(1 for result in results if result.status == ExecutionStatus.SKIPPED), + warn_count=sum( + 1 + for result in results + if result.status == ExecutionStatus.SUCCESS and result.warning_messages + ), ) diff --git a/src/sqlbuild/cli/commands/_helpers/load/invocation.py b/src/sqlbuild/cli/commands/_helpers/load/invocation.py index f04fd93137..5620b315d8 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/load/invocation.py @@ -10,6 +10,7 @@ from sqlbuild.cli.commands._helpers.load.selection import ( select_load_entries, select_load_reference_entries, + select_load_relation_entries, ) from sqlbuild.cli.commands.models import LoadCommandRequest, LoadInvocation from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config @@ -65,6 +66,9 @@ def resolve_load_invocation(*, request: LoadCommandRequest) -> LoadInvocation: discovered_inputs=discovered_inputs, selected_sources=selected_sources, reference_sources=reference_sources, + relation_sources=select_load_relation_entries( + discovered_inputs=discovered_inputs, target_config=target_config + ), use_color=use_color, progress_stream=progress_stream, ) diff --git a/src/sqlbuild/cli/commands/_helpers/load/outputs.py b/src/sqlbuild/cli/commands/_helpers/load/outputs.py index d1e2cb0f42..b2e6b2f851 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/load/outputs.py @@ -108,10 +108,11 @@ def write_load_completion_output( invocation.progress_stream.write( format_load_footer( + results=outcome.results, success_count=outcome.success_count, + warn_count=outcome.warn_count, fail_count=outcome.fail_count, skip_count=outcome.skip_count, - total_count=len(outcome.results), elapsed=outcome.elapsed, use_color=invocation.use_color, ) diff --git a/src/sqlbuild/cli/commands/_helpers/load/planning.py b/src/sqlbuild/cli/commands/_helpers/load/planning.py index 7adad80232..a7ff6a2802 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/load/planning.py @@ -16,10 +16,14 @@ from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config from sqlbuild.compiler.planner.models import CursorOverrides +from sqlbuild.compiler.references.main.render_source_relation import render_source_relation from sqlbuild.provider.main.session import build_provider_session +from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) +from sqlbuild.spec.contracts.models import SourceEntry def prepare_load_execution( @@ -75,4 +79,29 @@ def prepare_load_execution( provider_session=build_provider_session( discovered_providers=invocation.discovered_inputs.providers ), + project_relations=_load_project_relations(adapter=adapter, invocation=invocation), ) + + +def _load_project_relations( + *, adapter: BaseAdapter, invocation: LoadInvocation +) -> dict[SqlResourceRef, str] | None: + """Return the source relations loaders must not hard-code, or None when enforcement is off.""" + + if not invocation.discovered_inputs.project_config.references.enforce_explicit: + return None + sources: dict[str, SourceEntry] = { + source.name: source + for source in ( + *invocation.relation_sources, + *invocation.reference_sources, + *invocation.selected_sources, + ) + if source.expression is None + } + return { + SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=name): render_source_relation( + entry=source, adapter=adapter + ) + for name, source in sources.items() + } diff --git a/src/sqlbuild/cli/commands/_helpers/load/selection.py b/src/sqlbuild/cli/commands/_helpers/load/selection.py index 7c275b0b27..c62da708df 100644 --- a/src/sqlbuild/cli/commands/_helpers/load/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/load/selection.py @@ -165,6 +165,17 @@ def select_load_reference_entries( ) +def select_load_relation_entries( + *, discovered_inputs: DiscoveredProjectInputs, target_config: TargetConfig | None +) -> tuple[SourceEntry, ...]: + """Return every declared source resolved for the target, for hard-coded name warnings.""" + + discovered_sources: list[SourceEntry] = [] + for source_file in discovered_inputs.source_files: + discovered_sources.extend(source_file.source_entries) + return _environment_sources(sources=tuple(discovered_sources), target_config=target_config) + + def sources_order( *, sources: tuple[SourceEntry, ...], selected_sources: set[str] ) -> tuple[str, ...]: diff --git a/src/sqlbuild/cli/commands/classes/load_progress_reporter.py b/src/sqlbuild/cli/commands/classes/load_progress_reporter.py index d65d1ca9ca..22aecbf6fa 100644 --- a/src/sqlbuild/cli/commands/classes/load_progress_reporter.py +++ b/src/sqlbuild/cli/commands/classes/load_progress_reporter.py @@ -220,10 +220,11 @@ def _write_log_block(self, message: str) -> None: def format_load_footer( *, + results: tuple[LoadExecutionResult, ...], success_count: int, + warn_count: int, fail_count: int, skip_count: int, - total_count: int, elapsed: float, use_color: bool, ) -> str: @@ -231,18 +232,34 @@ def format_load_footer( counts_summary: str = format_summary_footer( counts=( ("PASS", success_count), - ("WARN", 0), + ("WARN", warn_count), ("FAIL", fail_count), ("SKIP", skip_count), - ("TOTAL", total_count), + ("TOTAL", len(results)), ), use_color=use_color, elapsed=f"{elapsed:.2f}s", ) + state: CompletionState = CompletionState.OK + label: str = "Completed successfully" + if fail_count: + state, label = CompletionState.FAIL, "Completed with errors" + elif warn_count: + state, label = CompletionState.WARN, "Completed with warnings" completion_message: str = format_completion_line( - style=style, - state=CompletionState.OK if fail_count == 0 else CompletionState.FAIL, - label="Completed successfully" if fail_count == 0 else "Completed with errors", - summary=counts_summary, + style=style, state=state, label=label, summary=counts_summary ) - return f"\n{completion_message}\n" + return "".join(_load_warning_lines(results=results, style=style)) + f"\n{completion_message}\n" + + +def _load_warning_lines(*, results: tuple[LoadExecutionResult, ...], style: CliStyle) -> list[str]: + lines: list[str] = [] + result: LoadExecutionResult + for result in results: + if not result.warning_messages: + continue + if not lines: + lines.append(f"\n{style.warning_strong('Warnings:')}\n\n") + lines.append(f" {result.source_name} (loader {result.loader_name})\n") + lines.extend(f" {message}\n" for message in result.warning_messages) + return lines diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index e22960ed9d..2f80ebb3de 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -792,6 +792,7 @@ class LoadInvocation: reference_sources: tuple[SourceEntry, ...] use_color: bool progress_stream: TextIO + relation_sources: tuple[SourceEntry, ...] @dataclass(frozen=True) @@ -807,6 +808,7 @@ class LoadExecutionPreparation: effective_cursor_overrides: CursorOverrides effective_concurrency: int provider_session: Any + project_relations: dict[SqlResourceRef, str] | None @dataclass(frozen=True) @@ -818,6 +820,7 @@ class LoadRunOutcome: success_count: int fail_count: int skip_count: int + warn_count: int @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py index 8f79ca8b22..79bad71942 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py @@ -119,3 +119,11 @@ class DltLocalSourceE2ETestCase: description: str expected_loaded_rows: tuple[tuple[object, ...], ...] expected_model_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class LoaderRelationGuardE2ETestCase: + description: str + enforce_explicit: bool + expected_warnings: tuple[str, ...] + expected_final_line_prefix: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py new file mode 100644 index 0000000000..7e0da24b9e --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py @@ -0,0 +1,97 @@ +"""Real-CLI coverage of the run-time hard-coded relation-name guard under sqb load.""" + +from __future__ import annotations + +import json +import subprocess +from itertools import chain +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.load._test_types import ( + LoaderRelationGuardE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_SOURCES: str = ( + "sources:\n" + " - name: raw_regions\n managed: true\n write_strategy: table\n" + " - name: raw_customers\n managed: true\n write_strategy: table\n" +) +_LOADERS: str = ( + "from sqlbuild.loaders import loader\n\n\n" + "@loader\n" + "def raw_regions(ctx):\n" + " return [{'id': 1}]\n\n\n" + "@loader(depends_on=[raw_regions])\n" + "def raw_customers(ctx):\n" + " table = 'raw_' + 'regions'\n" + " ctx.query(f'SELECT count(*) FROM {table}')\n" + " return [{'id': 1}]\n" +) +_HARD_CODED_WARNING: str = ( + "[P008] loader 'raw_customers' named source:raw_regions as 'raw_regions' in SQL; " + "use ctx.loader(raw_regions) instead of the relation name" +) + + +@pytest.mark.parametrize( + "test_case", + [ + LoaderRelationGuardE2ETestCase( + description="dynamic SQL naming a source warns while the load succeeds", + enforce_explicit=True, + expected_warnings=(_HARD_CODED_WARNING,), + expected_final_line_prefix="\u2713 Completed with warnings PASS=1 WARN=1 FAIL=0", + ), + LoaderRelationGuardE2ETestCase( + description="enforcement disabled loads without warnings", + enforce_explicit=False, + expected_warnings=(), + expected_final_line_prefix="\u2713 Completed successfully PASS=2 WARN=0 FAIL=0", + ), + ], + ids=lambda case: case.description, +) +def test_given_loader_hard_codes_a_source_when_loading_then_it_warns_like_build( + test_case: LoaderRelationGuardE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="guarded_loads", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "guarded_loads"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "warehouse.duckdb"\n\n' + f"[references]\nenforce_explicit = {str(test_case.enforce_explicit).lower()}\n" + ), + "sources/raw.yml": _SOURCES, + "python/loaders/raw.py": _LOADERS, + }, + ) + + text: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "load"), project_dir=project_dir + ) + machine: subprocess.CompletedProcess[str] = run_sqb( + command=("load", "--json"), project_dir=project_dir + ) + + assert text.returncode == 0, text.stdout + text.stderr + assert machine.returncode == 0, machine.stdout + machine.stderr + assert all(warning in text.stdout for warning in test_case.expected_warnings) + assert ("Warnings:" in text.stdout) is bool(test_case.expected_warnings) + assert text.stdout.strip().splitlines()[-1].startswith(test_case.expected_final_line_prefix) + assert all(warning in machine.stderr for warning in test_case.expected_warnings) + stored: list[str] = list( + chain.from_iterable( + asset.get("warnings") or () for asset in json.loads(machine.stdout)["assets"] + ) + ) + assert len(stored) == len(test_case.expected_warnings) + assert all( + warning in stored_warning + for stored_warning, warning in zip(stored, test_case.expected_warnings, strict=True) + ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx index bf7391dfd1..16d94f7902 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx @@ -100,8 +100,9 @@ Both checks ignore: - SQL that cannot be parsed; it is never blocked, only logged at debug level This is a guard against accidental hard-coding, not a sandbox. SQL sent directly through -`ctx.connection` or `ctx.adapter` is not checked. Run-time checks apply to `sqb build` and -`sqb check`. To turn both checks off while migrating a project, +`ctx.connection` or `ctx.adapter` is not checked. Run-time checks apply to `sqb build`, +`sqb check`, and `sqb load`. `sqb load` does not compile models, so it checks loaders against +the project's sources. To turn both checks off while migrating a project, set [`[references] enforce_explicit = false`](/docs/concepts/project-configuration#references). ## model vs source From 435a06089218dbdfeae86684b56dd56e055f1654 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:39:46 +0100 Subject: [PATCH 5/8] fix: count microbatches for aware cursors fed by DATE inputs and allow ); in quoted header values --- Makefile | 1 + .../cli/output/_helpers/cursor_plan.py | 16 +- .../discovery/_helpers/sql/functions.py | 3 +- .../discovery/_helpers/sql/model_files.py | 3 +- .../discovery/_helpers/sql/scenarios.py | 7 +- .../compiler/discovery/_helpers/sql/tests.py | 5 +- src/sqlbuild/compiler/discovery/constants.py | 8 + .../_helpers/output/microbatch_bounds.py | 14 +- .../_helpers/output/microbatch_count.py | 16 +- .../execution/aligned_timestamp_bounds.py | 28 ++++ .../_helpers/materializations/microbatch.py | 14 +- .../cli/commands/main/build/_test_types.py | 8 + ...test_microbatch_aware_cursor_date_input.py | 143 ++++++++++++++++++ .../discovery/_helpers/test_sql_models.py | 28 ++++ 14 files changed, 248 insertions(+), 46 deletions(-) create mode 100644 src/sqlbuild/compiler/planner/main/execution/aligned_timestamp_bounds.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py diff --git a/Makefile b/Makefile index 4f16add339..e76ffecaab 100644 --- a/Makefile +++ b/Makefile @@ -99,6 +99,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py \ diff --git a/src/sqlbuild/cli/output/_helpers/cursor_plan.py b/src/sqlbuild/cli/output/_helpers/cursor_plan.py index 9bc11a0b7a..927176cc72 100644 --- a/src/sqlbuild/cli/output/_helpers/cursor_plan.py +++ b/src/sqlbuild/cli/output/_helpers/cursor_plan.py @@ -6,6 +6,9 @@ from sqlbuild.cli.output.models import CursorPlanDetails from sqlbuild.cli.output.types import CursorBoundsOwner, CursorResolutionStatus +from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( + aligned_timestamp_bounds, +) from sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size import ( resolve_effective_microbatch_batch_size, ) @@ -110,16 +113,9 @@ def _count_batches(*, bounds: CursorBounds, batch_size: str, cursor_type: str) - bounds.end, DateValue | TimestampValue ): return None - current: datetime = ( - bounds.start.value - if isinstance(bounds.start, TimestampValue) - else datetime.combine(bounds.start.value, datetime.min.time()) - ) - end: datetime = ( - bounds.end.value - if isinstance(bounds.end, TimestampValue) - else datetime.combine(bounds.end.value, datetime.min.time()) - ) + current: datetime + end: datetime + current, end = aligned_timestamp_bounds(start=bounds.start, end=bounds.end) count: int = 0 while current < end: current = min(duration.add_to(current), end) diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py b/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py index 967d6d952a..8e1fc1298c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py @@ -6,10 +6,11 @@ from pathlib import Path from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values +from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError _FUNCTION_HEADER_PATTERN: re.Pattern[str] = re.compile( - r"^\s*FUNCTION\s*\((?P
.*?)\)\s*;\s*(?P.*)\Z", + r"^\s*FUNCTION\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P.*)\Z", re.DOTALL, ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index ab0d0cb15a..3d347d4242 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -10,6 +10,7 @@ from typing import cast import sqlbuild._native as _native +from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN from sqlbuild.compiler.discovery.exceptions import ( DiscoveryError, ModelHeaderSyntaxError, @@ -66,7 +67,7 @@ _NATIVE_MODEL_HEADER_SYMBOL_TOKEN: _MODEL_HEADER_SYMBOL_TOKEN, } _MODEL_HEADER_PATTERN: re.Pattern[str] = re.compile( - r"^\s*MODEL\s*\((?P
.*?)\)\s*;\s*(?P.*)\Z", + r"^\s*MODEL\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P.*)\Z", re.DOTALL, ) _SELECT_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"['\"`()sSfFuU]") diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py index da701024b0..f6ed07d547 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py @@ -7,12 +7,15 @@ from pathlib import Path from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values -from sqlbuild.compiler.discovery.constants import SQL_SCENARIOS_OWNERSHIP_ROOT +from sqlbuild.compiler.discovery.constants import ( + SQL_SCENARIOS_OWNERSHIP_ROOT, + STATEMENT_HEADER_BODY_PATTERN, +) from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile _SCENARIO_HEADER_PATTERN: re.Pattern[str] = re.compile( - r"^\s*SCENARIO\s*\((?P
.*?)\)\s*;\s*(?P.*)\Z", + r"^\s*SCENARIO\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P.*)\Z", re.DOTALL, ) _SCENARIO_DESCRIPTION_HEADER_KEY: str = "description" diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py index a99deb0368..15654b9358 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py @@ -14,6 +14,7 @@ parse_header_values, prepare_model_header_tokens, ) +from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN from sqlbuild.compiler.discovery.exceptions import SqlTestParseError from sqlbuild.compiler.discovery.models import ( DiscoveredSqlTestBlock, @@ -26,11 +27,11 @@ from sqlbuild.sql_values.types import SqlValueKind _TEST_HEADER_PATTERN: re.Pattern[str] = re.compile( - r"^\s*TEST\s*\((?P
.*?)\)\s*;\s*(?P.*)\Z", + r"^\s*TEST\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P.*)\Z", re.DOTALL, ) _TEST_HEADER_ONLY_PATTERN: re.Pattern[str] = re.compile( - r"^\s*TEST\s*\((?P
.*?)\)\s*;\s*", + r"^\s*TEST\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*", re.DOTALL | re.MULTILINE, ) _TEST_NAME_HEADER_KEY: str = "name" diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index dc2c1548f1..8b46cf1ba7 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -68,3 +68,11 @@ "description", ) SQL_HOOK_IDENTITY_FIELDS: tuple[str, ...] = ("statement", "name", "definition_sql", "kwargs") + +STATEMENT_HEADER_BODY_PATTERN: str = ( + r"(?P
(?>[^\"')]++" + r"|\"(?:[^\"\\]|\\.)*+\"" + r"|'(?:[^'\\]|\\.)*+'" + r"|[\"']" + r"|\)(?!\s*;))*+)" +) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py index 755f2ad32b..44032415a4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py @@ -12,6 +12,9 @@ CompileSqlReference, ) from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference +from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( + aligned_timestamp_bounds, +) from sqlbuild.compiler.planner.main.execution.microbatch_limit import ( _resolve_microbatch_limit_config, ) @@ -73,16 +76,7 @@ def cap_microbatch_bounds( bounds.end, DateValue | TimestampValue ): return bounds - start_at: datetime = ( - bounds.start.value - if isinstance(bounds.start, TimestampValue) - else datetime.combine(bounds.start.value, datetime.min.time()) - ) - end_at: datetime = ( - bounds.end.value - if isinstance(bounds.end, TimestampValue) - else datetime.combine(bounds.end.value, datetime.min.time()) - ) + start_at, end_at = aligned_timestamp_bounds(start=bounds.start, end=bounds.end) if action == MicrobatchLimitAction.CAP_FROM_START: capped_end: datetime = start_at for _ in range(max_batches): diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py index 78da69c234..8189affa91 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_count.py @@ -4,6 +4,9 @@ from datetime import datetime +from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( + aligned_timestamp_bounds, +) from sqlbuild.compiler.planner.models import CursorBounds, Duration from sqlbuild.compiler.planner.types import CursorType from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue @@ -37,16 +40,9 @@ def count_microbatches( bounds.end, DateValue | TimestampValue ): return 0 - current: datetime = ( - bounds.start.value - if isinstance(bounds.start, TimestampValue) - else datetime.combine(bounds.start.value, datetime.min.time()) - ) - end_at: datetime = ( - bounds.end.value - if isinstance(bounds.end, TimestampValue) - else datetime.combine(bounds.end.value, datetime.min.time()) - ) + current: datetime + end_at: datetime + current, end_at = aligned_timestamp_bounds(start=bounds.start, end=bounds.end) if equal_bounds_are_batch and current == end_at: return 1 count: int = 0 diff --git a/src/sqlbuild/compiler/planner/main/execution/aligned_timestamp_bounds.py b/src/sqlbuild/compiler/planner/main/execution/aligned_timestamp_bounds.py new file mode 100644 index 0000000000..5bd37f83a2 --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/execution/aligned_timestamp_bounds.py @@ -0,0 +1,28 @@ +"""Project temporal cursor bounds onto comparable datetimes.""" + +from datetime import UTC, datetime + +from sqlbuild.cursor_algebra.models import DateValue, TimestampValue + + +def aligned_timestamp_bounds( + *, start: DateValue | TimestampValue, end: DateValue | TimestampValue +) -> tuple[datetime, datetime]: + """Return comparable bounds, presuming a naive side UTC when the other is aware.""" + + start_at: datetime = _as_datetime(value=start) + end_at: datetime = _as_datetime(value=end) + if (start_at.tzinfo is None) != (end_at.tzinfo is None): + start_at = _presume_utc(value=start_at) + end_at = _presume_utc(value=end_at) + return start_at, end_at + + +def _as_datetime(*, value: DateValue | TimestampValue) -> datetime: + if isinstance(value, TimestampValue): + return value.value + return datetime.combine(value.value, datetime.min.time()) + + +def _presume_utc(*, value: datetime) -> datetime: + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 3b2eae3edd..570af4625b 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -21,6 +21,9 @@ resolve_relation_location_qualified_name, ) from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash +from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( + aligned_timestamp_bounds, +) from sqlbuild.compiler.planner.main.execution.cursor_bound_display import cursor_bound_display from sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size import ( resolve_effective_microbatch_batch_size, @@ -3450,16 +3453,7 @@ def _compute_timestamp_batches( end, TimestampValue | DateValue ): return () - start_dt: datetime = ( - start.value - if isinstance(start, TimestampValue) - else datetime.combine(start.value, datetime.min.time()) - ) - end_dt: datetime = ( - end.value - if isinstance(end, TimestampValue) - else datetime.combine(end.value, datetime.min.time()) - ) + start_dt, end_dt = aligned_timestamp_bounds(start=start, end=end) if start_dt >= end_dt: return () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 40ad9ccbd1..e46aa02b7e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1134,3 +1134,11 @@ class SqlHookReadCycleE2ETestCase: description: str overrides: dict[str, str] expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class AwareCursorDateWatermarkE2ETestCase: + description: str + plan_args: tuple[str, ...] + expected_plan_fragment: str + expected_event_ids: tuple[int, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py new file mode 100644 index 0000000000..8af804249c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py @@ -0,0 +1,143 @@ +"""DuckDB E2E coverage for timezone-aware microbatch cursors fed by DATE inputs.""" + +import subprocess +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + AwareCursorDateWatermarkE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": dedent( + """ + name = "aware_cursor_orders" + adapter = "duckdb" + + [connection] + database = "orders.duckdb" + """ + ).strip() + + "\n", + "sources/raw.yml": dedent( + """ + sources: + - name: raw_events + schema: main + table: raw_events + columns: + - name: event_id + type: INTEGER + - name: event_date + type: DATE + """ + ).strip() + + "\n", + "models/events.sql": dedent( + """ + MODEL ( + description "Events (web, store or phone); one row per event.", + materialized incremental, + incremental_strategy delete_insert, + unique_key event_id, + cursor observed_at, + cursor_type timestamp, + cursor_grain day, + cursor_start "2026-01-01", + incremental_mode microbatch, + microbatch_strategy watermark, + cursor_watermark_mode all, + cursor_inputs ( + raw_events (column event_date, roles [filter, watermark]), + ), + batch_size 1d, + lookback 1d, + columns ( + event_id (type INTEGER), + observed_at (type TIMESTAMP_TZ), + ), + ); + + SELECT event_id, CAST(event_date AS TIMESTAMPTZ) AS observed_at + FROM __source("raw_events") + """ + ).strip() + + "\n", +} + + +@pytest.mark.parametrize( + "test_case", + ( + AwareCursorDateWatermarkE2ETestCase( + description="plan and rebuild without a batch limit", + plan_args=(), + expected_plan_fragment="batches 4 x 1d", + expected_event_ids=(1, 2, 3, 4, 5), + ), + AwareCursorDateWatermarkE2ETestCase( + description="plan and rebuild with a batch limit", + plan_args=("--max-microbatches", "50"), + expected_plan_fragment="batches 4 x 1d", + expected_event_ids=(1, 2, 3, 4, 5), + ), + ), + ids=lambda case: case.description, +) +def test_given_aware_cursor_with_date_watermark_when_planning_and_rebuilding_then_batches_are_counted( + tmp_path: Path, + test_case: AwareCursorDateWatermarkE2ETestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="aware_cursor_orders", repo_files=_PROJECT_FILES + ) + db_path: Path = project_dir / "orders.duckdb" + execute_duckdb( + db_path=db_path, + sql=( + "CREATE TABLE main.raw_events (event_id INTEGER, event_date DATE);" + "INSERT INTO main.raw_events VALUES " + "(1, '2026-01-01'), (2, '2026-01-02'), (3, '2026-01-03')" + ), + ) + first_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", *test_case.plan_args), project_dir=project_dir + ) + assert first_build.returncode == 0, first_build.stdout + first_build.stderr + execute_duckdb( + db_path=db_path, + sql="INSERT INTO main.raw_events VALUES (4, '2026-01-04'), (5, '2026-01-05')", + ) + + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--select", "events", *test_case.plan_args), + project_dir=project_dir, + ) + rebuild: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", *test_case.plan_args), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert test_case.expected_plan_fragment in plan.stdout + assert rebuild.returncode == 0, rebuild.stdout + rebuild.stderr + assert ( + tuple( + row[0] + for row in query_duckdb( + db_path=db_path, sql="SELECT event_id FROM main.events ORDER BY event_id" + ) + ) + == test_case.expected_event_ids + ) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-vv"])) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index aabfd31b0c..8e1bd07bef 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -574,6 +574,34 @@ def test_given_deferred_output_locations_when_discovering_models_then_projection }, expected_query="SELECT 1", ), + ParseModelSqlHeaderTestCase( + description="keeps a closing parenthesis and semicolon inside a quoted description", + contents=""" + MODEL ( + description "Orders (online, store or phone); one row per order.", + materialized table, + ); + + SELECT 1 + """, + expected_header_values={ + "description": "Orders (online, store or phone); one row per order.", + "materialized": "table", + }, + expected_query="SELECT 1", + ), + ParseModelSqlHeaderTestCase( + description="keeps a closing parenthesis and semicolon inside a single quoted value", + contents=""" + MODEL ( + tags ['orders);archive'], + ); + + SELECT 1 + """, + expected_header_values={"tags": ["orders);archive"]}, + expected_query="SELECT 1", + ), ], ids=lambda case: case.description, ) From db97294a09a801938486b447fac48be78da5f5fe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:05:01 +0100 Subject: [PATCH 6/8] fix: scan unterminated quoted header values in linear time --- src/sqlbuild/compiler/discovery/constants.py | 2 +- .../sqlbuild/compiler/discovery/_helpers/test_sql_models.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 8b46cf1ba7..cb660865e4 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -73,6 +73,6 @@ r"(?P
(?>[^\"')]++" r"|\"(?:[^\"\\]|\\.)*+\"" r"|'(?:[^'\\]|\\.)*+'" - r"|[\"']" + r"|[\"'](?:[^\\)]|\\.)*+" r"|\)(?!\s*;))*+)" ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 8e1bd07bef..f056379a4c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -621,6 +621,11 @@ def test_given_sql_model_header_variants_when_parsing_then_it_returns_expected_h @pytest.mark.parametrize( "test_case", [ + ParseModelSqlErrorTestCase( + description="reports an unterminated string with many escaped quotes", + contents='MODEL (description "' + '\\"' * 20000 + "); SELECT 1", + expected_error_fragment="unterminated double-quoted string", + ), ParseModelSqlErrorTestCase( description="raises when the model header is missing", contents="SELECT 1\n", From 095f5403f71a9c2106791662b69b34dcd4c55eb8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:05:01 +0100 Subject: [PATCH 7/8] fix: clone into the destination model's table type on Snowflake --- .../_helpers/build_planning/defer_clone.py | 6 ++ .../cli/commands/_helpers/clone/execution.py | 6 ++ .../clone/_helpers/execution_items.py | 4 ++ .../executor/clone/_helpers/operations.py | 12 ++-- .../build_destination_transient_models.py | 14 ++++ src/sqlbuild/executor/clone/models.py | 1 + .../adapters/snowflake/_test_types.py | 8 +++ .../snowflake/test_table_type_conversion.py | 71 +++++++++++++++++++ .../executor/clone/_helpers/_test_types.py | 1 + .../clone/_helpers/test_operations.py | 13 ++++ .../executor/clone/main/_test_types.py | 8 +++ .../executor/clone/main/test_execute.py | 52 ++++++++++++++ 12 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 src/sqlbuild/executor/clone/main/build_destination_transient_models.py diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index f2cac8ecb2..c433987284 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -45,6 +45,9 @@ from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope from sqlbuild.compiler.planner.models import PlannerScope from sqlbuild.compiler.planner.types import MaterializationType +from sqlbuild.executor.clone.main.build_destination_transient_models import ( + build_destination_transient_models, +) from sqlbuild.executor.clone.main.build_retention_requests import ( build_destination_retention_requests, ) @@ -293,6 +296,9 @@ def run_clone(on_item: Any) -> CloneExecutionResult: hard_copy=False, run_id=clone_pipeline.destination_project.run_id, query_change_tracking=clone_pipeline.destination_project.settings.query_change_tracking, + destination_transient_models=build_destination_transient_models( + project=clone_pipeline.destination_project + ), destination_retention_requests=build_destination_retention_requests( project=clone_pipeline.destination_project, adapter_name=adapter.adapter_name, diff --git a/src/sqlbuild/cli/commands/_helpers/clone/execution.py b/src/sqlbuild/cli/commands/_helpers/clone/execution.py index dc7899c07c..0a369f1528 100644 --- a/src/sqlbuild/cli/commands/_helpers/clone/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/clone/execution.py @@ -29,6 +29,9 @@ ModelPlanEntry, SeedPlanEntry, ) +from sqlbuild.executor.clone.main.build_destination_transient_models import ( + build_destination_transient_models, +) from sqlbuild.executor.clone.main.build_retention_requests import ( build_destination_retention_requests, ) @@ -90,6 +93,9 @@ def execute_clone_plan( preparation=preparation, adapter=invocation.adapter, ), + destination_transient_models=build_destination_transient_models( + project=preparation.pipeline_result.destination_project + ), destination_retention_requests=build_destination_retention_requests( project=preparation.pipeline_result.destination_project, adapter_name=invocation.adapter_name, diff --git a/src/sqlbuild/executor/clone/_helpers/execution_items.py b/src/sqlbuild/executor/clone/_helpers/execution_items.py index 4db5292496..f4ab512551 100644 --- a/src/sqlbuild/executor/clone/_helpers/execution_items.py +++ b/src/sqlbuild/executor/clone/_helpers/execution_items.py @@ -94,6 +94,10 @@ def execute_clone_relation_item( destination_connection=inputs.destination_connection, hard_copy=inputs.hard_copy, origin_lookup=relation_lookup, + destination_is_transient=( + isinstance(destination_entry, ModelPlanEntry) + and destination_entry.name in inputs.destination_transient_models + ), ) return _with_destination_retention(result=result, inputs=inputs) missing_dependencies: tuple[CompiledObjectKey, ...] = _missing_destination_dependencies( diff --git a/src/sqlbuild/executor/clone/_helpers/operations.py b/src/sqlbuild/executor/clone/_helpers/operations.py index 42e9d7d58e..0c88295635 100644 --- a/src/sqlbuild/executor/clone/_helpers/operations.py +++ b/src/sqlbuild/executor/clone/_helpers/operations.py @@ -23,7 +23,13 @@ def clone_relation( destination_connection: Any, hard_copy: bool, origin_lookup: RelationLookup, + destination_is_transient: bool = False, ) -> CloneItemResult: + clone_as_transient: bool = destination_is_transient or origin_lookup.is_transient( + database=origin_entry.destination.database, + schema=origin_entry.destination.schema, + name=origin_entry.destination.name, + ) return clone_relation_by_names( name=destination_entry.name, origin_relation=qualified_name(adapter=adapter, entry=origin_entry), @@ -36,11 +42,7 @@ def clone_relation( adapter=adapter, connection=destination_connection, hard_copy=hard_copy, - origin_is_transient=origin_lookup.is_transient( - database=origin_entry.destination.database, - schema=origin_entry.destination.schema, - name=origin_entry.destination.name, - ), + origin_is_transient=clone_as_transient, ) diff --git a/src/sqlbuild/executor/clone/main/build_destination_transient_models.py b/src/sqlbuild/executor/clone/main/build_destination_transient_models.py new file mode 100644 index 0000000000..26675fb4e4 --- /dev/null +++ b/src/sqlbuild/executor/clone/main/build_destination_transient_models.py @@ -0,0 +1,14 @@ +"""Collect destination models whose resolved table type is transient.""" + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.spec.contracts.types import TableType + + +def build_destination_transient_models(*, project: CompiledProject) -> frozenset[str]: + """Return destination model names that clones should create as transient tables.""" + + return frozenset( + model.name + for model in project.models + if model.config.table_type.value == TableType.TRANSIENT + ) diff --git a/src/sqlbuild/executor/clone/models.py b/src/sqlbuild/executor/clone/models.py index 524ddda90b..2dc71e87b7 100644 --- a/src/sqlbuild/executor/clone/models.py +++ b/src/sqlbuild/executor/clone/models.py @@ -48,6 +48,7 @@ class CloneExecutionInput: default_factory=dict ) destination_retention_requests: dict[str, RetentionRequest] = field(default_factory=dict) + destination_transient_models: frozenset[str] = frozenset() allow_namespace_retention_decrease: bool = False on_item: CloneItemCallback | None = None diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py index 881db656fa..75e2928189 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py @@ -130,3 +130,11 @@ class SnowflakeFunctionDiscoveryTestCase: description: str function_name: str expected_function_count: int + + +@dataclass(frozen=True) +class SnowflakeCloneTableTypeTestCase: + description: str + origin_table_kind: str + clone_as_transient: bool + expected_is_transient: bool diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py index daed05fd6e..9cb255c9cc 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py @@ -11,8 +11,10 @@ from sqlbuild.compiler.planner._helpers.planning.retention import table_type_copy_name from sqlbuild.compiler.planner.models import TableTypePlanEntry from sqlbuild.executor.build._helpers.retention import apply_table_type_conversion +from sqlbuild.executor.clone.main._clone_relation_operation import clone_relation_by_names from sqlbuild.spec.contracts.types import TableType from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeCloneTableTypeTestCase, SnowflakeTableTypeConversionTestCase, ) from tests.integration.src.sqlbuild.adapters.snowflake.helpers import ( @@ -120,5 +122,74 @@ def test_given_existing_snowflake_table_when_applying_table_type_then_metadata_a assert conversion_statement_count == test_case.expected_conversion_statement_count +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCloneTableTypeTestCase( + description="permanent origin clones as transient for a transient destination", + origin_table_kind="TABLE", + clone_as_transient=True, + expected_is_transient=True, + ), + SnowflakeCloneTableTypeTestCase( + description="permanent origin clones as permanent for a permanent destination", + origin_table_kind="TABLE", + clone_as_transient=False, + expected_is_transient=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_permanent_snowflake_origin_when_cloning_then_clone_has_requested_type_and_rows( + test_case: SnowflakeCloneTableTypeTestCase, + recording_adapter: RecordingSnowflakeAdapter, + recording_connection: Any, + snowflake_database: str, + snowflake_schema: str, +) -> None: + adapter: RecordingSnowflakeAdapter = recording_adapter + connection: Any = recording_connection + origin: str = qualified_name( + database=snowflake_database, schema=snowflake_schema, name="clone_type_orders_origin" + ) + destination_name: str = "clone_type_orders_clone" + destination: str = qualified_name( + database=snowflake_database, schema=snowflake_schema, name=destination_name + ) + adapter.execute( + connection=connection, + sql=f"CREATE OR REPLACE {test_case.origin_table_kind} {origin} (id NUMBER, status VARCHAR)", + ) + adapter.execute( + connection=connection, + sql=f"INSERT INTO {origin} VALUES (1, 'ready'), (2, 'complete')", + ) + + clone_relation_by_names( + name=destination_name, + origin_relation=origin, + destination_relation=destination, + origin_exists=True, + adapter=adapter, + connection=connection, + hard_copy=False, + origin_is_transient=test_case.clone_as_transient, + ) + relations: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=snowflake_database, + schemas=(snowflake_schema,), + names=(destination_name,), + ) + rows: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT id, status FROM {destination} ORDER BY id", + ) + + assert relations[0].is_transient is test_case.expected_is_transient + assert rows == ((1, "ready"), (2, "complete")) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py index 90323e58b9..7312f01590 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py @@ -13,6 +13,7 @@ class CloneRelationExecutionTestCase: expected_status: CloneStatus expected_statements: tuple[str, ...] origin_is_transient: bool = False + destination_is_transient: bool = False @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py index 35ac554a05..f15a4301e8 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py @@ -66,6 +66,18 @@ "CREATE TRANSIENT TABLE dev.fact_orders CLONE prod.fact_orders", ), ), + CloneRelationExecutionTestCase( + description="clones a permanent origin as transient for a transient destination model", + hard_copy=False, + supports_zero_copy_clone=True, + destination_is_transient=True, + expected_action=CloneAction.CLONED, + expected_status=CloneStatus.SUCCESS, + expected_statements=( + "DROP TABLE IF EXISTS dev.fact_orders", + "CREATE TRANSIENT TABLE dev.fact_orders CLONE prod.fact_orders", + ), + ), ], ids=lambda case: case.description, ) @@ -91,6 +103,7 @@ def test_given_clone_relation_when_executing_then_records_sql_and_reports_copy_m destination_connection=object(), hard_copy=test_case.hard_copy, origin_lookup=origin_lookup, + destination_is_transient=test_case.destination_is_transient, ) assert result.action == test_case.expected_action diff --git a/tests/unit/src/sqlbuild/executor/clone/main/_test_types.py b/tests/unit/src/sqlbuild/executor/clone/main/_test_types.py index 212e08f4e1..bb72836a18 100644 --- a/tests/unit/src/sqlbuild/executor/clone/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/clone/main/_test_types.py @@ -65,3 +65,11 @@ class CloneRunIdentityTestCase: expected_run_id: str expected_operation_names: tuple[str, ...] expected_resource_terminal: str + + +@dataclass(frozen=True) +class CloneDestinationTableTypeTestCase: + description: str + model_names: tuple[str, ...] + transient_model_names: frozenset[str] + expected_clone_statements: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py b/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py index b93202aab3..fb15f5b492 100644 --- a/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py @@ -21,6 +21,7 @@ build_clone_model_entry, ) from tests.unit.src.sqlbuild.executor.clone.main._test_types import ( + CloneDestinationTableTypeTestCase, CloneFunctionDependencyTestCase, CloneRunIdentityTestCase, CloneSchemaPreparationTestCase, @@ -96,6 +97,57 @@ def _on_item(index: int, *, total: int, item: CloneItemResult) -> None: assert all(connection is destination_connection for connection in adapter.used_connections) +@pytest.mark.parametrize( + "test_case", + [ + CloneDestinationTableTypeTestCase( + description="permanent origins clone as transient only for transient destination models", + model_names=("orders", "payments"), + transient_model_names=frozenset({"orders"}), + expected_clone_statements=( + "CREATE SCHEMA IF NOT EXISTS dev", + "DROP TABLE IF EXISTS dev.orders", + "CREATE TRANSIENT TABLE dev.orders CLONE prod.orders", + "DROP TABLE IF EXISTS dev.payments", + "CREATE TABLE dev.payments CLONE prod.payments", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_transient_destination_models_when_cloning_then_clone_uses_destination_table_type( + test_case: CloneDestinationTableTypeTestCase, +) -> None: + adapter: FakeCloneAdapter = FakeCloneAdapter( + supports_zero_copy=True, origin_names=test_case.model_names + ) + destination_entries: tuple[ModelPlanEntry, ...] = tuple( + build_clone_model_entry(schema="dev", name=name) for name in test_case.model_names + ) + + execute_clone( + inputs=CloneExecutionInput( + source_entries=CloneSourceEntries(), + origin_model_entries=tuple( + build_clone_model_entry(schema="prod", name=name) for name in test_case.model_names + ), + destination_model_entries=destination_entries, + origin_seed_entries=(), + destination_seed_entries=(), + destination_function_entries=(), + execution_order=tuple(entry.key for entry in destination_entries), + adapter=adapter, + destination_connection=object(), + hard_copy=False, + run_id="clone-run", + query_change_tracking=False, + destination_transient_models=test_case.transient_model_names, + ) + ) + + assert tuple(adapter.executed_statements) == test_case.expected_clone_statements + + @pytest.mark.parametrize( "test_case", ( From c4afded52b192d007cb4f338925247fe8b612500 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:08:14 +0100 Subject: [PATCH 8/8] fix: pair backslash escapes outside quoted header values --- src/sqlbuild/compiler/discovery/constants.py | 2 +- .../sqlbuild/compiler/discovery/_helpers/test_sql_models.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index cb660865e4..14cc1a4330 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -70,7 +70,7 @@ SQL_HOOK_IDENTITY_FIELDS: tuple[str, ...] = ("statement", "name", "definition_sql", "kwargs") STATEMENT_HEADER_BODY_PATTERN: str = ( - r"(?P
(?>[^\"')]++" + r"(?P
(?>(?:[^\"')\\]|\\.)++" r"|\"(?:[^\"\\]|\\.)*+\"" r"|'(?:[^'\\]|\\.)*+'" r"|[\"'](?:[^\\)]|\\.)*+" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index f056379a4c..0d3ddeefcd 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -621,6 +621,11 @@ def test_given_sql_model_header_variants_when_parsing_then_it_returns_expected_h @pytest.mark.parametrize( "test_case", [ + ParseModelSqlErrorTestCase( + description="reports an unterminated string with escaped quotes before parentheses", + contents='MODEL (description "' + '\\")' * 20000 + "); SELECT 1", + expected_error_fragment="unterminated double-quoted string", + ), ParseModelSqlErrorTestCase( description="reports an unterminated string with many escaped quotes", contents='MODEL (description "' + '\\"' * 20000 + "); SELECT 1",