fix: small fixes for audits, selection, loads, microbatch cursors, headers and clones

Makefile

@@ -99,6 +99,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py \

src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md

@@ -104,8 +104,9 @@ Both checks ignore:
 - SQL that cannot be parsed; it is never blocked, only logged at debug level
 
 This is a guard against accidental hard-coding, not a sandbox. SQL sent directly through
-`ctx.connection` or `ctx.adapter` is not checked. Run-time checks apply to `sqb build` and
-`sqb check`. To turn both checks off while migrating a project,
+`ctx.connection` or `ctx.adapter` is not checked. Run-time checks apply to `sqb build`,
+`sqb check`, and `sqb load`. `sqb load` does not compile models, so it checks loaders against
+the project's sources. To turn both checks off while migrating a project,
 set [`[references] enforce_explicit = false`](../project-configuration.md#references).
 
 ## model vs source

src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py

@@ -45,6 +45,9 @@
 from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope
 from sqlbuild.compiler.planner.models import PlannerScope
 from sqlbuild.compiler.planner.types import MaterializationType
+from sqlbuild.executor.clone.main.build_destination_transient_models import (
+    build_destination_transient_models,
+)
 from sqlbuild.executor.clone.main.build_retention_requests import (
     build_destination_retention_requests,
 )
@@ -293,6 +296,9 @@ def run_clone(on_item: Any) -> CloneExecutionResult:
                     hard_copy=False,
                     run_id=clone_pipeline.destination_project.run_id,
                     query_change_tracking=clone_pipeline.destination_project.settings.query_change_tracking,
+                    destination_transient_models=build_destination_transient_models(
+                        project=clone_pipeline.destination_project
+                    ),
                     destination_retention_requests=build_destination_retention_requests(
                         project=clone_pipeline.destination_project,
                         adapter_name=adapter.adapter_name,

src/sqlbuild/cli/commands/_helpers/clone/execution.py

@@ -29,6 +29,9 @@
     ModelPlanEntry,
     SeedPlanEntry,
 )
+from sqlbuild.executor.clone.main.build_destination_transient_models import (
+    build_destination_transient_models,
+)
 from sqlbuild.executor.clone.main.build_retention_requests import (
     build_destination_retention_requests,
 )
@@ -90,6 +93,9 @@ def execute_clone_plan(
                     preparation=preparation,
                     adapter=invocation.adapter,
                 ),
+                destination_transient_models=build_destination_transient_models(
+                    project=preparation.pipeline_result.destination_project
+                ),
                 destination_retention_requests=build_destination_retention_requests(
                     project=preparation.pipeline_result.destination_project,
                     adapter_name=invocation.adapter_name,

src/sqlbuild/cli/commands/_helpers/load/execution.py

@@ -79,6 +79,7 @@ def on_load_complete(result: LoadExecutionResult) -> None:
                 end_cursor_int=parse_cursor_integer(preparation.effective_cursor_overrides.end_int),
                 use_color=invocation.use_color,
                 providers=preparation.provider_session.providers,
+                project_relations=preparation.project_relations,
             ),
             callbacks=LoadCallbacks(
                 on_load_start=load_progress.on_start,
@@ -104,7 +105,16 @@ def on_load_complete(result: LoadExecutionResult) -> None:
     return LoadRunOutcome(
         results=results,
         elapsed=time.monotonic() - start,
-        success_count=sum(1 for result in results if result.status == ExecutionStatus.SUCCESS),
+        success_count=sum(
+            1
+            for result in results
+            if result.status == ExecutionStatus.SUCCESS and not result.warning_messages
+        ),
         fail_count=sum(1 for result in results if result.status == ExecutionStatus.FAILED),
         skip_count=sum(1 for result in results if result.status == ExecutionStatus.SKIPPED),
+        warn_count=sum(
+            1
+            for result in results
+            if result.status == ExecutionStatus.SUCCESS and result.warning_messages
+        ),
     )

src/sqlbuild/cli/commands/_helpers/load/invocation.py

@@ -10,6 +10,7 @@
 from sqlbuild.cli.commands._helpers.load.selection import (
     select_load_entries,
     select_load_reference_entries,
+    select_load_relation_entries,
 )
 from sqlbuild.cli.commands.models import LoadCommandRequest, LoadInvocation
 from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config
@@ -65,6 +66,9 @@ def resolve_load_invocation(*, request: LoadCommandRequest) -> LoadInvocation:
         discovered_inputs=discovered_inputs,
         selected_sources=selected_sources,
         reference_sources=reference_sources,
+        relation_sources=select_load_relation_entries(
+            discovered_inputs=discovered_inputs, target_config=target_config
+        ),
         use_color=use_color,
         progress_stream=progress_stream,
     )

src/sqlbuild/cli/commands/_helpers/load/outputs.py

@@ -108,10 +108,11 @@ def write_load_completion_output(
 
     invocation.progress_stream.write(
         format_load_footer(
+            results=outcome.results,
             success_count=outcome.success_count,
+            warn_count=outcome.warn_count,
             fail_count=outcome.fail_count,
             skip_count=outcome.skip_count,
-            total_count=len(outcome.results),
             elapsed=outcome.elapsed,
             use_color=invocation.use_color,
         )

src/sqlbuild/cli/commands/_helpers/load/planning.py

@@ -16,10 +16,14 @@
 from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config
 from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config
 from sqlbuild.compiler.planner.models import CursorOverrides
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
 from sqlbuild.provider.main.session import build_provider_session
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
 from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import (
     resolve_effective_adapter_name,
 )
+from sqlbuild.spec.contracts.models import SourceEntry
 
 
 def prepare_load_execution(
@@ -75,4 +79,29 @@ def prepare_load_execution(
         provider_session=build_provider_session(
             discovered_providers=invocation.discovered_inputs.providers
         ),
+        project_relations=_load_project_relations(adapter=adapter, invocation=invocation),
     )
+
+
+def _load_project_relations(
+    *, adapter: BaseAdapter, invocation: LoadInvocation
+) -> dict[SqlResourceRef, str] | None:
+    """Return the source relations loaders must not hard-code, or None when enforcement is off."""
+
+    if not invocation.discovered_inputs.project_config.references.enforce_explicit:
+        return None
+    sources: dict[str, SourceEntry] = {
+        source.name: source
+        for source in (
+            *invocation.relation_sources,
+            *invocation.reference_sources,
+            *invocation.selected_sources,
+        )
+        if source.expression is None
+    }
+    return {
+        SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=name): render_source_relation(
+            entry=source, adapter=adapter
+        )
+        for name, source in sources.items()
+    }

src/sqlbuild/cli/commands/_helpers/load/selection.py

@@ -165,6 +165,17 @@ def select_load_reference_entries(
     )
 
 
+def select_load_relation_entries(
+    *, discovered_inputs: DiscoveredProjectInputs, target_config: TargetConfig | None
+) -> tuple[SourceEntry, ...]:
+    """Return every declared source resolved for the target, for hard-coded name warnings."""
+
+    discovered_sources: list[SourceEntry] = []
+    for source_file in discovered_inputs.source_files:
+        discovered_sources.extend(source_file.source_entries)
+    return _environment_sources(sources=tuple(discovered_sources), target_config=target_config)
+
+
 def sources_order(
     *, sources: tuple[SourceEntry, ...], selected_sources: set[str]
 ) -> tuple[str, ...]:

src/sqlbuild/cli/commands/classes/build_progress_callbacks.py

@@ -804,6 +804,14 @@ def _write_error_detail(
                 self._stream.write(f"{continuation_pad}{line}\n")
 
 
+def _build_audit_results(result: BuildExecutionResult) -> tuple[AuditExecutionResult, ...]:
+    audits: list[AuditExecutionResult] = []
+    model_result: ModelExecutionResult
+    for model_result in result.model_results:
+        audits.extend(model_result.audit_results)
+    return (*audits, *result.source_audit_results, *result.end_audit_results)
+
+
 def _count_build_footer_results(
     *,
     result: BuildExecutionResult,
@@ -824,14 +832,14 @@ def _count_build_footer_results(
             fail_count += 1
         elif model_result.status == ExecutionStatus.SKIPPED:
             skip_count += 1
-        audit_r: AuditExecutionResult
-        for audit_r in model_result.audit_results:
-            if audit_r.outcome == AuditOutcome.PASS:
-                pass_count += 1
-            elif audit_r.outcome == AuditOutcome.WARN:
-                warn_count += 1
-            elif audit_r.outcome == AuditOutcome.ERROR:
-                fail_count += 1
+    audit_r: AuditExecutionResult
+    for audit_r in _build_audit_results(result):
+        if audit_r.outcome == AuditOutcome.PASS:
+            pass_count += 1
+        elif audit_r.outcome == AuditOutcome.WARN:
+            warn_count += 1
+        elif audit_r.outcome == AuditOutcome.ERROR:
+            fail_count += 1
 
     seed_result: SeedExecutionResult
     for seed_result in result.seed_results:

src/sqlbuild/cli/commands/classes/load_progress_reporter.py

@@ -220,29 +220,46 @@ def _write_log_block(self, message: str) -> None:
 
 def format_load_footer(
     *,
+    results: tuple[LoadExecutionResult, ...],
     success_count: int,
+    warn_count: int,
     fail_count: int,
     skip_count: int,
-    total_count: int,
     elapsed: float,
     use_color: bool,
 ) -> str:
     style: CliStyle = CliStyle(use_color=use_color)
     counts_summary: str = format_summary_footer(
         counts=(
             ("PASS", success_count),
-            ("WARN", 0),
+            ("WARN", warn_count),
             ("FAIL", fail_count),
             ("SKIP", skip_count),
-            ("TOTAL", total_count),
+            ("TOTAL", len(results)),
         ),
         use_color=use_color,
         elapsed=f"{elapsed:.2f}s",
     )
+    state: CompletionState = CompletionState.OK
+    label: str = "Completed successfully"
+    if fail_count:
+        state, label = CompletionState.FAIL, "Completed with errors"
+    elif warn_count:
+        state, label = CompletionState.WARN, "Completed with warnings"
     completion_message: str = format_completion_line(
-        style=style,
-        state=CompletionState.OK if fail_count == 0 else CompletionState.FAIL,
-        label="Completed successfully" if fail_count == 0 else "Completed with errors",
-        summary=counts_summary,
+        style=style, state=state, label=label, summary=counts_summary
     )
-    return f"\n{completion_message}\n"
+    return "".join(_load_warning_lines(results=results, style=style)) + f"\n{completion_message}\n"
+
+
+def _load_warning_lines(*, results: tuple[LoadExecutionResult, ...], style: CliStyle) -> list[str]:
+    lines: list[str] = []
+    result: LoadExecutionResult
+    for result in results:
+        if not result.warning_messages:
+            continue
+        if not lines:
+            lines.append(f"\n{style.warning_strong('Warnings:')}\n\n")
+        lines.append(f"  {result.source_name}  (loader {result.loader_name})\n")
+        lines.extend(f"    {message}\n" for message in result.warning_messages)
+    return lines

src/sqlbuild/cli/commands/models.py

@@ -792,6 +792,7 @@ class LoadInvocation:
     reference_sources: tuple[SourceEntry, ...]
     use_color: bool
     progress_stream: TextIO
+    relation_sources: tuple[SourceEntry, ...]
 
 
 @dataclass(frozen=True)
@@ -807,6 +808,7 @@ class LoadExecutionPreparation:
     effective_cursor_overrides: CursorOverrides
     effective_concurrency: int
     provider_session: Any
+    project_relations: dict[SqlResourceRef, str] | None
 
 
 @dataclass(frozen=True)
@@ -818,6 +820,7 @@ class LoadRunOutcome:
     success_count: int
     fail_count: int
     skip_count: int
+    warn_count: int
 
 
 @dataclass(frozen=True)

src/sqlbuild/cli/output/_helpers/cursor_plan.py

@@ -6,6 +6,9 @@
 
 from sqlbuild.cli.output.models import CursorPlanDetails
 from sqlbuild.cli.output.types import CursorBoundsOwner, CursorResolutionStatus
+from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import (
+    aligned_timestamp_bounds,
+)
 from sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size import (
     resolve_effective_microbatch_batch_size,
 )
@@ -110,16 +113,9 @@ def _count_batches(*, bounds: CursorBounds, batch_size: str, cursor_type: str) -
             bounds.end, DateValue | TimestampValue
         ):
             return None
-        current: datetime = (
-            bounds.start.value
-            if isinstance(bounds.start, TimestampValue)
-            else datetime.combine(bounds.start.value, datetime.min.time())
-        )
-        end: datetime = (
-            bounds.end.value
-            if isinstance(bounds.end, TimestampValue)
-            else datetime.combine(bounds.end.value, datetime.min.time())
-        )
+        current: datetime
+        end: datetime
+        current, end = aligned_timestamp_bounds(start=bounds.start, end=bounds.end)
         count: int = 0
         while current < end:
             current = min(duration.add_to(current), end)

src/sqlbuild/compiler/discovery/_helpers/sql/functions.py

@@ -6,10 +6,11 @@
 from pathlib import Path
 
 from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values
+from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN
 from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError
 
 _FUNCTION_HEADER_PATTERN: re.Pattern[str] = re.compile(
-    r"^\s*FUNCTION\s*\((?P<header>.*?)\)\s*;\s*(?P<sql>.*)\Z",
+    r"^\s*FUNCTION\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P<sql>.*)\Z",
     re.DOTALL,
 )
 

src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py

@@ -10,6 +10,7 @@
 from typing import cast
 
 import sqlbuild._native as _native
+from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN
 from sqlbuild.compiler.discovery.exceptions import (
     DiscoveryError,
     ModelHeaderSyntaxError,
@@ -66,7 +67,7 @@
     _NATIVE_MODEL_HEADER_SYMBOL_TOKEN: _MODEL_HEADER_SYMBOL_TOKEN,
 }
 _MODEL_HEADER_PATTERN: re.Pattern[str] = re.compile(
-    r"^\s*MODEL\s*\((?P<header>.*?)\)\s*;\s*(?P<sql>.*)\Z",
+    r"^\s*MODEL\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P<sql>.*)\Z",
     re.DOTALL,
 )
 _SELECT_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"['\"`()sSfFuU]")

src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py

@@ -7,12 +7,15 @@
 from pathlib import Path
 
 from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values
-from sqlbuild.compiler.discovery.constants import SQL_SCENARIOS_OWNERSHIP_ROOT
+from sqlbuild.compiler.discovery.constants import (
+    SQL_SCENARIOS_OWNERSHIP_ROOT,
+    STATEMENT_HEADER_BODY_PATTERN,
+)
 from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError
 from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile
 
 _SCENARIO_HEADER_PATTERN: re.Pattern[str] = re.compile(
-    r"^\s*SCENARIO\s*\((?P<header>.*?)\)\s*;\s*(?P<sql>.*)\Z",
+    r"^\s*SCENARIO\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P<sql>.*)\Z",
     re.DOTALL,
 )
 _SCENARIO_DESCRIPTION_HEADER_KEY: str = "description"

src/sqlbuild/compiler/discovery/_helpers/sql/tests.py

@@ -14,6 +14,7 @@
     parse_header_values,
     prepare_model_header_tokens,
 )
+from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN
 from sqlbuild.compiler.discovery.exceptions import SqlTestParseError
 from sqlbuild.compiler.discovery.models import (
     DiscoveredSqlTestBlock,
@@ -26,11 +27,11 @@
 from sqlbuild.sql_values.types import SqlValueKind
 
 _TEST_HEADER_PATTERN: re.Pattern[str] = re.compile(
-    r"^\s*TEST\s*\((?P<header>.*?)\)\s*;\s*(?P<sql>.*)\Z",
+    r"^\s*TEST\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P<sql>.*)\Z",
     re.DOTALL,
 )
 _TEST_HEADER_ONLY_PATTERN: re.Pattern[str] = re.compile(
-    r"^\s*TEST\s*\((?P<header>.*?)\)\s*;\s*",
+    r"^\s*TEST\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*",
     re.DOTALL | re.MULTILINE,
 )
 _TEST_NAME_HEADER_KEY: str = "name"