From 7aab43946da01b97db39780ab14d33a0a720f249 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Sun, 27 Sep 2026 20:21:01 +0100
Subject: [PATCH 01/13] feat!: accept typed reference arguments in macros and
reject macro-generated references
Macros now accept __ref(), __source() and __seed() as arguments, including
inside lists and dictionaries, and receive sqlbuild.refs typed values. The
reference renders at the call site, so the graph, lineage and scopes see it
as written in the model. References that appear in a macro's expanded output
fail compile with P006 unless [references] enforce_explicit = false.
BREAKING CHANGE: macros that emit __ref(), __source() or __seed() in their
output now fail compile; pass the reference in as a typed argument.
---
.../compile/_helpers/assembly/project.py | 1 +
.../_helpers/render/macro_references.py | 126 +++++++++
.../compile/_helpers/render/macros.py | 50 +++-
src/sqlbuild/compiler/compile/constants.py | 1 +
.../compile/main/_build_compile_inputs.py | 1 +
.../compile/main/sql_expansion_context.py | 3 +
src/sqlbuild/compiler/compile/models.py | 1 +
.../_helpers/validation/discovery.py | 6 +
.../discovery/_helpers/yml/project.py | 20 ++
src/sqlbuild/python_nodes/constants.py | 8 +
.../python_nodes/main/build_model_ref.py | 10 -
.../python_nodes/main/build_source_ref.py | 10 -
src/sqlbuild/python_nodes/models.py | 9 +
src/sqlbuild/python_nodes/types.py | 3 +-
src/sqlbuild/refs.py | 24 +-
src/sqlbuild/spec/contracts/models.py | 8 +
.../compiler/compile/_helpers/_test_types.py | 48 ++++
.../compiler/compile/_helpers/helpers.py | 59 +++-
.../_helpers/test_typed_macro_references.py | 265 ++++++++++++++++++
.../discovery/_helpers/_test_types.py | 1 +
.../discovery/_helpers/test_yml_project.py | 27 ++
.../sqlbuild/compiler/discovery/test_main.py | 22 ++
22 files changed, 671 insertions(+), 32 deletions(-)
create mode 100644 src/sqlbuild/compiler/compile/_helpers/render/macro_references.py
create mode 100644 src/sqlbuild/python_nodes/constants.py
delete mode 100644 src/sqlbuild/python_nodes/main/build_model_ref.py
delete mode 100644 src/sqlbuild/python_nodes/main/build_source_ref.py
create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
index a881ea5a9a..add8658a13 100644
--- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
+++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
@@ -1928,6 +1928,7 @@ def _build_test_model_query_overrides(
sql_analysis_enabled=inputs.effective_settings.sql_analysis,
target_name=inputs.effective_target_name,
vars=inputs.effective_vars,
+ _enforce_explicit_references=inputs.project_config.references.enforce_explicit,
)
overrides: dict[str, str] = {}
model_input: CompileModelInput
diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macro_references.py b/src/sqlbuild/compiler/compile/_helpers/render/macro_references.py
new file mode 100644
index 0000000000..183261fb3d
--- /dev/null
+++ b/src/sqlbuild/compiler/compile/_helpers/render/macro_references.py
@@ -0,0 +1,126 @@
+"""Typed reference arguments and explicit-reference enforcement for macro expansion."""
+
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+
+from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references
+from sqlbuild.compiler.compile.constants import MACRO_GENERATED_REFERENCE_CODE
+from sqlbuild.compiler.compile.exceptions import CompileInputError
+from sqlbuild.compiler.compile.models import CompileSqlReference, LoadedMacro
+from sqlbuild.compiler.references.types import SqlReferenceKind
+from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity
+from sqlbuild.compiler.scopes.types import ResourceKind
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
+
+_TYPED_REFERENCE_KINDS: dict[str, SqlResourceRefKind] = {
+ "__ref": SqlResourceRefKind.MODEL,
+ "__source": SqlResourceRefKind.SOURCE,
+ "__seed": SqlResourceRefKind.SEED,
+}
+_REFERENCE_FUNCTION_BY_KIND: dict[SqlResourceRefKind, str] = {
+ kind: function for function, kind in _TYPED_REFERENCE_KINDS.items()
+}
+_EXPLICIT_REFERENCE_KINDS: dict[str, SqlResourceRefKind] = {
+ SqlReferenceKind.REF.value: SqlResourceRefKind.MODEL,
+ SqlReferenceKind.SOURCE.value: SqlResourceRefKind.SOURCE,
+ SqlReferenceKind.SEED.value: SqlResourceRefKind.SEED,
+}
+_REFERENCE_CALL_MARKERS: tuple[str, ...] = tuple(f"{name}(" for name in _TYPED_REFERENCE_KINDS)
+_RELATION_PLACEHOLDER_PREFIX: str = "__sqlbuild_relation_"
+_RELATION_PLACEHOLDER_PATTERN: re.Pattern[str] = re.compile(
+ rf"{_RELATION_PLACEHOLDER_PREFIX}(\d+)__"
+)
+
+
+def relation_placeholder_text(index: int) -> str:
+ """Return the in-expansion placeholder text for the typed reference at ``index``."""
+
+ return f"{_RELATION_PLACEHOLDER_PREFIX}{index}__"
+
+
+def render_relation_placeholders(*, sql: str, relations: dict[SqlResourceRef, int]) -> str:
+ """Replace placeholders with the reference call written at the macro call site."""
+
+ if not relations or _RELATION_PLACEHOLDER_PREFIX not in sql:
+ return sql
+ ordered: tuple[SqlResourceRef, ...] = tuple(relations)
+ return _RELATION_PLACEHOLDER_PATTERN.sub(
+ lambda match: _reference_call_text(ordered[int(match.group(1))]), sql
+ )
+
+
+def _reference_call_text(ref: SqlResourceRef) -> str:
+ return f'{_REFERENCE_FUNCTION_BY_KIND[ref.kind]}("{ref.name}")'
+
+
+def reject_macro_generated_references(
+ *,
+ loaded_macro: LoadedMacro,
+ macro_result: str,
+ file_path: Path,
+ consumer: ResourceIdentity | DeclarationIdentity | None,
+) -> None:
+ if not any(marker in macro_result for marker in _REFERENCE_CALL_MARKERS):
+ return
+ generated: SqlResourceRef | None = _first_generated_reference(macro_result)
+ if generated is None:
+ return
+ consumer_label: str = (
+ f"{consumer.kind.value}:{consumer.name}" if consumer is not None else f"'{file_path}'"
+ )
+ location: str = (
+ "the model"
+ if isinstance(consumer, ResourceIdentity) and consumer.kind is ResourceKind.MODEL
+ else "the calling SQL"
+ )
+ raise CompileInputError(
+ f"{consumer_label} depends on {generated.kind.value}:{generated.name} through macro "
+ f"{loaded_macro.name}()\n --> {loaded_macro.relative_path.as_posix()}",
+ code=MACRO_GENERATED_REFERENCE_CODE,
+ help=(
+ f"write the reference in {location}, or pass it in: "
+ f"@{loaded_macro.name}({_reference_call_text(generated)})\n"
+ " = help: while migrating a project, allow macro-generated references with "
+ "[references] enforce_explicit = false in sqlbuild_project.toml"
+ ),
+ )
+
+
+def _first_generated_reference(sql: str) -> SqlResourceRef | None:
+ try:
+ references: tuple[CompileSqlReference, ...] = extract_sql_references(sql)
+ except CompileInputError:
+ return None
+ for reference in references:
+ kind: SqlResourceRefKind | None = _EXPLICIT_REFERENCE_KINDS.get(str(reference.ref_kind))
+ if kind is not None:
+ return SqlResourceRef(kind=kind, name=reference.ref_name)
+ return None
+
+
+def evaluate_typed_reference(*, node: ast.Call, file_path: Path) -> SqlResourceRef:
+ function_name: str | None = node.func.id if isinstance(node.func, ast.Name) else None
+ kind: SqlResourceRefKind | None = (
+ _TYPED_REFERENCE_KINDS.get(function_name) if function_name is not None else None
+ )
+ if kind is None:
+ raise CompileInputError(
+ f"Macro arguments in '{file_path}' must use only Python literals, nested macro "
+ "calls, and __ref(), __source(), or __seed() references"
+ )
+ if (
+ node.keywords
+ or len(node.args) != 1
+ or not isinstance(node.args[0], ast.Constant)
+ or not isinstance(node.args[0].value, str)
+ or not node.args[0].value
+ ):
+ raise CompileInputError(
+ f"Macro argument {function_name}() in '{file_path}' must contain exactly one "
+ "quoted resource name"
+ )
+ return SqlResourceRef(kind=kind, name=node.args[0].value)
diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py
index 000bb0802b..e79b2e3a51 100644
--- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py
+++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py
@@ -10,11 +10,18 @@
import sys
import threading
from collections.abc import Callable, Iterator, Mapping
+from contextvars import Token
from dataclasses import dataclass, field, replace
from pathlib import Path
from types import MappingProxyType, ModuleType
from typing import cast
+from sqlbuild.compiler.compile._helpers.render.macro_references import (
+ evaluate_typed_reference,
+ reject_macro_generated_references,
+ relation_placeholder_text,
+ render_relation_placeholders,
+)
from sqlbuild.compiler.compile.constants import (
DECLARATION_REFERENCE_NAMES,
MACRO_CONTEXT_PARAMETER_NAME,
@@ -61,6 +68,7 @@
from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment
from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment
from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import skip_quoted_text
+from sqlbuild.python_nodes.models import RELATION_RENDERER, SqlResourceRef
from sqlbuild.sql_values.models import SqlValue
from sqlbuild.sql_values.types import SqlValueKind
@@ -97,6 +105,7 @@ class _MacroAnalysisInputs:
class _ExpansionFacts:
dependencies: list[DeclarationIdentity] = field(default_factory=list)
usages: list[UsageRecord] = field(default_factory=list)
+ relations: dict[SqlResourceRef, int] = field(default_factory=dict)
def add_dependency(self, identity: DeclarationIdentity) -> _ExpansionFacts:
"""Record a resolved dependency in encounter order."""
@@ -110,6 +119,18 @@ def add_usage(self, usage: UsageRecord) -> _ExpansionFacts:
self.usages.append(usage)
return self
+ def relation_placeholder(self, ref: object) -> str:
+ """Return the stable in-expansion placeholder for one typed reference argument."""
+
+ if not isinstance(ref, SqlResourceRef):
+ raise CompileInputError("Only typed resource references render as relations")
+ return relation_placeholder_text(self.relations.setdefault(ref, len(self.relations)))
+
+ def render_relation_placeholders(self, sql: str) -> str:
+ """Replace typed reference placeholders with the reference call written at the call site."""
+
+ return render_relation_placeholders(sql=sql, relations=self.relations)
+
@dataclass(frozen=True)
class _ExpansionState:
@@ -1269,6 +1290,7 @@ def _expand_sql_macros(
f"Macro '@{_parse_macro_name(sql=sql, call_start_index=macro_start_index)}' in "
f"'{consumer_path}' must return a SQL string when used directly in SQL"
)
+ macro_result = state.facts.render_relation_placeholders(macro_result)
rendered_sql_parts.append(macro_result)
spans.append(
ExpansionSpan(
@@ -1431,12 +1453,18 @@ def _evaluate_macro_call(
file_path=file_path,
state=state,
)
- macro_result: object = _call_loaded_macro(
- loaded_macro=loaded_macro,
- macro_context=invocation_context,
- args=args,
- kwargs=kwargs,
+ renderer_token: Token[Callable[[object], str] | None] = RELATION_RENDERER.set(
+ state.facts.relation_placeholder
)
+ try:
+ macro_result: object = _call_loaded_macro(
+ loaded_macro=loaded_macro,
+ macro_context=invocation_context,
+ args=args,
+ kwargs=kwargs,
+ )
+ finally:
+ RELATION_RENDERER.reset(renderer_token)
except CompileInputError:
raise
except TypeError as error:
@@ -1456,6 +1484,13 @@ def _evaluate_macro_call(
_validate_final_macro_sql(
macro_name=macro_name, file_path=file_path, macro_result=macro_result
)
+ if state.macro_context._enforce_explicit_references:
+ reject_macro_generated_references(
+ loaded_macro=loaded_macro,
+ macro_result=macro_result,
+ file_path=file_path,
+ consumer=state.consumer,
+ )
return macro_result, closing_paren_index + 1
@@ -1752,6 +1787,8 @@ def _evaluate_literal_ast_node(
) -> object:
if isinstance(node, ast.Constant):
return node.value
+ if isinstance(node, ast.Call):
+ return evaluate_typed_reference(node=node, file_path=file_path)
if isinstance(node, ast.Name):
if node.id in placeholder_values:
return placeholder_values[node.id]
@@ -1798,7 +1835,8 @@ def _evaluate_literal_ast_node(
raise CompileInputError(f"Macro arguments in '{file_path}' use unsupported unary value")
return -operand if isinstance(node.op, ast.USub) else operand
raise CompileInputError(
- f"Macro arguments in '{file_path}' must use only Python literals and nested macro calls"
+ f"Macro arguments in '{file_path}' must use only Python literals, nested macro calls, "
+ "and __ref(), __source(), or __seed() references"
)
diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py
index a2d472e75a..d583d5c430 100644
--- a/src/sqlbuild/compiler/compile/constants.py
+++ b/src/sqlbuild/compiler/compile/constants.py
@@ -11,6 +11,7 @@
SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE: str = "P004"
ATTACHED_AUDIT_READS_OWN_DEPENDANT_CODE: str = "P005"
+MACRO_GENERATED_REFERENCE_CODE: str = "P006"
HOOK_DIRECTORY_NAME: str = "hooks"
MODEL_DIRECTORY_NAME: str = "models"
NOT_NULL_AUDIT_NAME: str = "not_null"
diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
index 9135b18805..60453da560 100644
--- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
+++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
@@ -115,6 +115,7 @@ def build_compile_inputs(
vars=effective_vars,
_value_renderer=adapter_context.value_renderer,
_collection_rendering=adapter_context.collection_rendering,
+ _enforce_explicit_references=(discovered_inputs.project_config.references.enforce_explicit),
)
resolved_run_id: str = resolve_run_id(selected_run_id=run_id)
loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files)
diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py
index 104000a4b7..69d20e7db9 100644
--- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py
+++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py
@@ -88,6 +88,9 @@ def build_sql_expansion_context(
project_config=effective_discovered_inputs.project_config,
declaration_override=None,
),
+ _enforce_explicit_references=(
+ effective_discovered_inputs.project_config.references.enforce_explicit
+ ),
),
enums=enums,
constants=constants,
diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py
index 6ccb97ed87..5524faa97c 100644
--- a/src/sqlbuild/compiler/compile/models.py
+++ b/src/sqlbuild/compiler/compile/models.py
@@ -330,6 +330,7 @@ class MacroContext:
_constant_declarations: Mapping[str, ConstantDeclaration] = field(
default_factory=dict, repr=False, compare=False
)
+ _enforce_explicit_references: bool = field(default=True, repr=False, compare=False)
def render_constant(self, name: str) -> str:
"""Render one visible constant as an adapter-safe SQL value."""
diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py
index 92063a11a1..bb2800cac5 100644
--- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py
+++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py
@@ -40,6 +40,7 @@
SqlResourceRef,
TaskDefinition,
)
+from sqlbuild.python_nodes.types import SqlResourceRefKind
from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry, SourceEntry
@@ -355,6 +356,11 @@ def _validate_python_node_dependencies(
f"'{dependency.name}'; loaders "
"may depend on Python loader/task/asset functions only"
)
+ if dependency.kind is SqlResourceRefKind.SEED:
+ raise DiscoveryConflictError(
+ f"Python node '{node.name}' depends on seed '{dependency.name}'; "
+ "Python nodes may depend on models and sources"
+ )
continue
if _python_node_dependency_key(dependency) not in node_by_dependency_key:
if isinstance(node, DiscoveredLoaderFunction):
diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py
index 06d0b4b09f..bee4de9067 100644
--- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py
+++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py
@@ -66,6 +66,7 @@
MicrobatchesConfig,
MicrobatchLimitsConfig,
ProjectConfig,
+ ReferencesConfig,
ScenarioConfig,
ScenarioSnapshotLimitsConfig,
ScopesConfig,
@@ -127,6 +128,9 @@ def load_project_config(*, project_dir: Path) -> ProjectConfig:
)
settings: SettingsConfig = _load_settings(payload=payload.get("settings"), file_path=file_path)
scopes: ScopesConfig = _load_scopes(payload=payload.get("scopes"), file_path=file_path)
+ references: ReferencesConfig = _load_references(
+ payload=payload.get("references"), file_path=file_path
+ )
cost: CostConfig = _load_cost(payload=payload.get("cost"), file_path=file_path)
constants: ConstantsConfig = _load_constants(
payload=payload.get("constants"), file_path=file_path
@@ -172,6 +176,7 @@ def load_project_config(*, project_dir: Path) -> ProjectConfig:
connections=connections,
settings=settings,
scopes=scopes,
+ references=references,
cost=cost,
constants=constants,
cursors=cursors,
@@ -687,6 +692,21 @@ def _load_scopes(*, payload: object, file_path: Path) -> ScopesConfig:
)
+def _load_references(*, payload: object, file_path: Path) -> ReferencesConfig:
+ mapping: dict[str, object] = _coerce_mapping(
+ payload=payload, label="references", file_path=file_path
+ )
+ _validate_allowed_keys(
+ mapping=mapping,
+ allowed_keys=frozenset({"enforce_explicit"}),
+ label="references",
+ file_path=file_path,
+ )
+ return ReferencesConfig(
+ enforce_explicit=_optional_bool(mapping=mapping, key="enforce_explicit", default=True)
+ )
+
+
def _load_cost(*, payload: object, file_path: Path) -> CostConfig:
mapping: dict[str, object] = _coerce_mapping(payload=payload, label="cost", file_path=file_path)
_validate_allowed_keys(
diff --git a/src/sqlbuild/python_nodes/constants.py b/src/sqlbuild/python_nodes/constants.py
new file mode 100644
index 0000000000..89bdee63df
--- /dev/null
+++ b/src/sqlbuild/python_nodes/constants.py
@@ -0,0 +1,8 @@
+"""Stable constants for typed SQL resource references."""
+
+from collections.abc import Callable
+from contextvars import ContextVar
+
+RELATION_RENDERER: ContextVar[Callable[[object], str] | None] = ContextVar(
+ "sqlbuild_relation_renderer", default=None
+)
diff --git a/src/sqlbuild/python_nodes/main/build_model_ref.py b/src/sqlbuild/python_nodes/main/build_model_ref.py
deleted file mode 100644
index 71c5d28601..0000000000
--- a/src/sqlbuild/python_nodes/main/build_model_ref.py
+++ /dev/null
@@ -1,10 +0,0 @@
-"""Construct typed references to SQLBuild models."""
-
-from sqlbuild.python_nodes.models import SqlResourceRef
-from sqlbuild.python_nodes.types import SqlResourceRefKind
-
-
-def build_model_ref(name: str) -> SqlResourceRef:
- """Return a typed dependency reference to a SQLBuild model."""
-
- return SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=name)
diff --git a/src/sqlbuild/python_nodes/main/build_source_ref.py b/src/sqlbuild/python_nodes/main/build_source_ref.py
deleted file mode 100644
index 573a9a5fed..0000000000
--- a/src/sqlbuild/python_nodes/main/build_source_ref.py
+++ /dev/null
@@ -1,10 +0,0 @@
-"""Construct typed references to SQLBuild sources."""
-
-from sqlbuild.python_nodes.models import SqlResourceRef
-from sqlbuild.python_nodes.types import SqlResourceRefKind
-
-
-def build_source_ref(name: str) -> SqlResourceRef:
- """Return a typed dependency reference to a SQLBuild source."""
-
- return SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=name)
diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py
index 232f38ce25..b9def447a2 100644
--- a/src/sqlbuild/python_nodes/models.py
+++ b/src/sqlbuild/python_nodes/models.py
@@ -9,6 +9,7 @@
from sqlbuild.compiler.auditing.models import MeasurementThresholds
from sqlbuild.compiler.auditing.types import AuditSeverity
from sqlbuild.errors.contracts.exceptions import SharedInputError
+from sqlbuild.python_nodes.constants import RELATION_RENDERER
from sqlbuild.python_nodes.types import PythonCheckSeverity, SqlResourceRefKind
from sqlbuild.spec.contracts.models import SourceColumnEntry
from sqlbuild.spec.contracts.types import SourceWriteStrategy
@@ -21,6 +22,14 @@ class SqlResourceRef:
kind: SqlResourceRefKind
name: str
+ def __str__(self) -> str:
+ """Render as SQL inside a macro invocation, or as the authoring call elsewhere."""
+
+ renderer: Callable[[object], str] | None = RELATION_RENDERER.get()
+ if renderer is None:
+ return f'{self.kind.value}("{self.name}")'
+ return renderer(self)
+
@dataclass(frozen=True)
class ColumnLineageRef:
diff --git a/src/sqlbuild/python_nodes/types.py b/src/sqlbuild/python_nodes/types.py
index b5b41d53e5..ba01ad5969 100644
--- a/src/sqlbuild/python_nodes/types.py
+++ b/src/sqlbuild/python_nodes/types.py
@@ -39,7 +39,8 @@ class PythonCheckSeverity(StrEnum):
class SqlResourceRefKind(StrEnum):
- """Python-node typed dependency target for SQL graph resources."""
+ """Typed reference target for SQL graph resources."""
MODEL = "model"
SOURCE = "source"
+ SEED = "seed"
diff --git a/src/sqlbuild/refs.py b/src/sqlbuild/refs.py
index 9ea1a50b26..ff596e8d74 100644
--- a/src/sqlbuild/refs.py
+++ b/src/sqlbuild/refs.py
@@ -1,8 +1,24 @@
-"""Public typed dependency references for SQL graph resources."""
+"""Public typed references for SQL graph resources."""
-from sqlbuild.python_nodes.main.build_model_ref import build_model_ref as model
-from sqlbuild.python_nodes.main.build_source_ref import build_source_ref as source
from sqlbuild.python_nodes.models import SqlResourceRef as SqlResourceRef
from sqlbuild.python_nodes.types import SqlResourceRefKind as SqlResourceRefKind
-__all__ = ("SqlResourceRef", "SqlResourceRefKind", "model", "source")
+__all__ = ("SqlResourceRef", "SqlResourceRefKind", "model", "seed", "source")
+
+
+def model(name: str) -> SqlResourceRef:
+ """Return a typed reference to a SQLBuild model."""
+
+ return SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=name)
+
+
+def source(name: str) -> SqlResourceRef:
+ """Return a typed reference to a SQLBuild source."""
+
+ return SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=name)
+
+
+def seed(name: str) -> SqlResourceRef:
+ """Return a typed reference to a SQLBuild seed."""
+
+ return SqlResourceRef(kind=SqlResourceRefKind.SEED, name=name)
diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py
index 5fc5b240df..812bab2ff7 100644
--- a/src/sqlbuild/spec/contracts/models.py
+++ b/src/sqlbuild/spec/contracts/models.py
@@ -178,6 +178,13 @@ class ScopesConfig:
enforce_placement: bool = True
+@dataclass(frozen=True)
+class ReferencesConfig:
+ """Explicit-reference enforcement policy configuration."""
+
+ enforce_explicit: bool = True
+
+
@dataclass(frozen=True)
class CostConfig:
"""Snowflake compute estimate configuration."""
@@ -377,6 +384,7 @@ class ProjectConfig:
connections: dict[str, dict[str, object]] = field(default_factory=dict)
settings: SettingsConfig = field(default_factory=SettingsConfig)
scopes: ScopesConfig = field(default_factory=ScopesConfig)
+ references: ReferencesConfig = field(default_factory=ReferencesConfig)
cost: CostConfig = field(default_factory=CostConfig)
constants: ConstantsConfig = field(default_factory=ConstantsConfig)
cursors: CursorsConfig = field(default_factory=CursorsConfig)
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
index 6e80a19b2d..38006f78a5 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
@@ -937,3 +937,51 @@ class AttachedAuditGateCycleTestCase:
description: str
files: dict[str, str]
expected_error_fragments: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class TypedMacroReferenceTestCase:
+ """A macro call passing typed references, and the SQL it expands to."""
+
+ description: str
+ macro_file_contents: str
+ sql: str
+ expected_sql: str
+
+
+@dataclass(frozen=True)
+class MacroGeneratedReferenceErrorTestCase:
+ """A macro call whose expanded output contains a reference it was not given."""
+
+ description: str
+ macro_file_contents: str
+ sql: str
+ expected_error_fragments: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class MacroGeneratedReferenceSwitchTestCase:
+ """A macro emitting a reference while explicit-reference enforcement is disabled."""
+
+ description: str
+ macro_file_contents: str
+ sql: str
+ expected_sql: str
+
+
+@dataclass(frozen=True)
+class MalformedTypedMacroReferenceTestCase:
+ """A macro argument that looks like a reference call but is not a valid typed reference."""
+
+ description: str
+ sql: str
+ expected_error_fragment: str
+
+
+@dataclass(frozen=True)
+class TypedReferenceAdapterRenderingTestCase:
+ """An adapter resolving a typed macro reference with its own relation quoting."""
+
+ description: str
+ adapter_name: str
+ expected_resolved_sql: str
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
index 29eaf81144..e3c654405c 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
@@ -4,9 +4,17 @@
from types import MappingProxyType
from typing import cast
+from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
+from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter
from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter
+from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter
+from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter
+from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter
from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core
-from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros
+from sqlbuild.compiler.compile._helpers.render.macros import (
+ expand_sql_macros,
+ load_project_macros,
+)
from sqlbuild.compiler.compile.main._assemble_project import assemble_project
from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs
from sqlbuild.compiler.compile.models import (
@@ -19,6 +27,7 @@
CompiledModelSqlTestPayload,
CompiledObjectKey,
CompiledProject,
+ CompiledRelationLocation,
CompiledSqlTest,
CompileProjectInputs,
DeclarationExpansionContext,
@@ -26,6 +35,7 @@
DeclarationRuntimeProjection,
DeclarationScopeResolver,
LoadedMacro,
+ MacroContext,
)
from sqlbuild.compiler.compile.types import CompiledResourceType
from sqlbuild.compiler.discovery.main.discover import discover_project_inputs
@@ -33,6 +43,7 @@
from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, ColumnTransformKind
from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project
from sqlbuild.compiler.planner._helpers.graph.core import build_execution_upstream_deps
+from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references
from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup
from sqlbuild.compiler.scopes.models import (
DeclarationIdentity,
@@ -290,3 +301,49 @@ def execution_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, st
for key, deps in build_execution_upstream_deps(project).items():
edges.update((key.name, dep.name) for dep in deps)
return frozenset(edges)
+
+
+def expand_typed_macro_sql(
+ *, tmp_path: Path, macro_file_contents: str, sql: str, enforce_explicit: bool = True
+) -> str:
+ """Expand model SQL for the ``order_summary`` model against one macro module."""
+
+ loaded_macros: dict[str, LoadedMacro] = build_loaded_macros(tmp_path, macro_file_contents)
+ return expand_sql_macros(
+ sql=sql,
+ file_path=tmp_path / "models" / "order_summary.sql",
+ loaded_macros=loaded_macros,
+ macro_context=MacroContext(
+ adapter_name="duckdb",
+ sql_analysis_enabled=True,
+ target_name="dev",
+ _enforce_explicit_references=enforce_explicit,
+ ),
+ consumer=ResourceIdentity(kind=ResourceKind.MODEL, name="order_summary"),
+ )
+
+
+def resolve_model_references_for_adapter(*, sql: str, adapter_name: str) -> str:
+ """Resolve ``__ref("orders")`` to ``analytics.sales.orders`` with one adapter's quoting."""
+
+ adapters: dict[str, Callable[[], BaseAdapter]] = {
+ "bigquery": BigQueryAdapter,
+ "duckdb": DuckDbAdapter,
+ "postgres": PostgresAdapter,
+ "snowflake": SnowflakeAdapter,
+ "sqlserver": SqlServerAdapter,
+ }
+ return resolve_ref_references(
+ query_sql=sql,
+ model_locations={
+ "orders": CompiledRelationLocation(
+ database="analytics", schema="sales", name="orders", qualified_name=None
+ )
+ },
+ seed_locations={},
+ cursor_bounds=None,
+ cursor_filter_inputs={},
+ adapter=adapters[adapter_name](),
+ cursor_type=None,
+ lower_bound_inclusive=True,
+ )
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
new file mode 100644
index 0000000000..1c5126b885
--- /dev/null
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+from sqlbuild.compiler.compile.exceptions import CompileInputError
+from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import (
+ MacroGeneratedReferenceErrorTestCase,
+ MacroGeneratedReferenceSwitchTestCase,
+ MalformedTypedMacroReferenceTestCase,
+ TypedMacroReferenceTestCase,
+ TypedReferenceAdapterRenderingTestCase,
+)
+from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import (
+ expand_typed_macro_sql,
+ resolve_model_references_for_adapter,
+)
+
+_UNION_MACRO: str = (
+ "def union_all(relations):\n"
+ " return ' UNION ALL '.join(f'SELECT order_id FROM {relation}' for relation in relations)\n"
+)
+_BASE_MACRO: str = "def base(relation):\n return f'SELECT * FROM {relation}'\n"
+_EMITTING_MACRO: str = "def orders_base():\n return 'SELECT * FROM __ref(\"stg_orders\")'\n"
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ TypedMacroReferenceTestCase(
+ description="single model reference renders as the call-site reference",
+ macro_file_contents=_BASE_MACRO,
+ sql='@base(__ref("orders"))',
+ expected_sql='SELECT * FROM __ref("orders")',
+ ),
+ TypedMacroReferenceTestCase(
+ description="references inside a list render in order",
+ macro_file_contents=_UNION_MACRO,
+ sql='@union_all([__ref("orders_eu"), __ref("orders_us")])',
+ expected_sql=(
+ 'SELECT order_id FROM __ref("orders_eu") UNION ALL '
+ 'SELECT order_id FROM __ref("orders_us")'
+ ),
+ ),
+ TypedMacroReferenceTestCase(
+ description="references inside a dictionary and keyword arguments",
+ macro_file_contents=(
+ "def pick(relations, key):\n return f'SELECT * FROM {relations[key]}'\n"
+ ),
+ sql='@pick(relations={"eu": __ref("orders_eu"), "us": __ref("orders_us")}, key="us")',
+ expected_sql='SELECT * FROM __ref("orders_us")',
+ ),
+ TypedMacroReferenceTestCase(
+ description="source and seed references keep their kind",
+ macro_file_contents=(
+ "def join_lookup(orders, countries):\n"
+ " return f'SELECT * FROM {orders} JOIN {countries} USING (country_code)'\n"
+ ),
+ sql='@join_lookup(__source("raw_orders"), __seed("country_codes"))',
+ expected_sql=(
+ 'SELECT * FROM __source("raw_orders") JOIN __seed("country_codes") '
+ "USING (country_code)"
+ ),
+ ),
+ TypedMacroReferenceTestCase(
+ description="macro receives typed values it can inspect",
+ macro_file_contents=(
+ "from sqlbuild.refs import SqlResourceRef, SqlResourceRefKind\n\n"
+ "def describe(relation):\n"
+ " assert isinstance(relation, SqlResourceRef)\n"
+ " assert relation.kind is SqlResourceRefKind.SEED\n"
+ " return f\"SELECT '{relation.name}' AS seed_name FROM {relation}\"\n"
+ ),
+ sql='@describe(__seed("country_codes"))',
+ expected_sql="SELECT 'country_codes' AS seed_name FROM __seed(\"country_codes\")",
+ ),
+ TypedMacroReferenceTestCase(
+ description="nested macro output carrying a reference renders once at the top level",
+ macro_file_contents=(
+ f"{_BASE_MACRO}\n"
+ "def wrap(sql):\n return f'SELECT order_id FROM ({sql}) AS wrapped'\n"
+ ),
+ sql='@wrap(@base(__ref("orders")))',
+ expected_sql='SELECT order_id FROM (SELECT * FROM __ref("orders")) AS wrapped',
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_typed_reference_arguments_when_expanding_then_renders_call_site_references(
+ test_case: TypedMacroReferenceTestCase, tmp_path: Path
+) -> None:
+ expanded: str = expand_typed_macro_sql(
+ tmp_path=tmp_path, macro_file_contents=test_case.macro_file_contents, sql=test_case.sql
+ )
+
+ assert expanded == test_case.expected_sql
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ MacroGeneratedReferenceErrorTestCase(
+ description="macro code that emits a model reference",
+ macro_file_contents=_EMITTING_MACRO,
+ sql="@orders_base()",
+ expected_error_fragments=(
+ "model:order_summary depends on model:stg_orders through macro orders_base()",
+ "--> macros/common.py",
+ 'pass it in: @orders_base(__ref("stg_orders"))',
+ "[references] enforce_explicit = false",
+ ),
+ ),
+ MacroGeneratedReferenceErrorTestCase(
+ description="quoted string argument smuggling a reference",
+ macro_file_contents="def wrap(sql):\n return f'SELECT * FROM ({sql})'\n",
+ sql="@wrap('__ref(\"stg_orders\")')",
+ expected_error_fragments=(
+ "model:order_summary depends on model:stg_orders through macro wrap()",
+ "[references] enforce_explicit = false",
+ ),
+ ),
+ MacroGeneratedReferenceErrorTestCase(
+ description="macro code that emits a source reference",
+ macro_file_contents=(
+ "def raw_orders():\n return 'SELECT * FROM __source(\"raw_orders\")'\n"
+ ),
+ sql="@raw_orders()",
+ expected_error_fragments=("depends on source:raw_orders through macro raw_orders()",),
+ ),
+ MacroGeneratedReferenceErrorTestCase(
+ description="nested macro that emits a seed reference",
+ macro_file_contents=(
+ "def countries():\n return '__seed(\"country_codes\")'\n\n"
+ "def wrap(sql):\n return f'SELECT * FROM {sql}'\n"
+ ),
+ sql="@wrap(@countries())",
+ expected_error_fragments=("depends on seed:country_codes through macro countries()",),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_macro_output_reference_when_expanding_then_raises_explicit_reference_error(
+ test_case: MacroGeneratedReferenceErrorTestCase, tmp_path: Path
+) -> None:
+ with pytest.raises(CompileInputError) as error:
+ expand_typed_macro_sql(
+ tmp_path=tmp_path, macro_file_contents=test_case.macro_file_contents, sql=test_case.sql
+ )
+
+ assert error.value.code == "P006"
+ rendered: str = f"{error.value.message}\n{error.value.help}"
+ assert all(fragment in rendered for fragment in test_case.expected_error_fragments)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ MacroGeneratedReferenceSwitchTestCase(
+ description="macro code that emits a model reference",
+ macro_file_contents=_EMITTING_MACRO,
+ sql="@orders_base()",
+ expected_sql='SELECT * FROM __ref("stg_orders")',
+ ),
+ MacroGeneratedReferenceSwitchTestCase(
+ description="quoted string argument smuggling a reference",
+ macro_file_contents="def wrap(sql):\n return f'SELECT * FROM {sql}'\n",
+ sql="@wrap('__ref(\"stg_orders\")')",
+ expected_sql='SELECT * FROM __ref("stg_orders")',
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_enforcement_disabled_when_macro_emits_reference_then_expands_unchanged(
+ test_case: MacroGeneratedReferenceSwitchTestCase, tmp_path: Path
+) -> None:
+ expanded: str = expand_typed_macro_sql(
+ tmp_path=tmp_path,
+ macro_file_contents=test_case.macro_file_contents,
+ sql=test_case.sql,
+ enforce_explicit=False,
+ )
+
+ assert expanded == test_case.expected_sql
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ MalformedTypedMacroReferenceTestCase(
+ description="non-string name",
+ sql="@base(__ref(1))",
+ expected_error_fragment="must contain exactly one quoted resource name",
+ ),
+ MalformedTypedMacroReferenceTestCase(
+ description="two names",
+ sql='@base(__ref("a", "b"))',
+ expected_error_fragment="must contain exactly one quoted resource name",
+ ),
+ MalformedTypedMacroReferenceTestCase(
+ description="keyword name",
+ sql='@base(__ref(name="a"))',
+ expected_error_fragment="must contain exactly one quoted resource name",
+ ),
+ MalformedTypedMacroReferenceTestCase(
+ description="unsupported reference function",
+ sql='@base(__udf("a"))',
+ expected_error_fragment="__ref(), __source(), or __seed() references",
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_malformed_reference_argument_when_expanding_then_raises(
+ test_case: MalformedTypedMacroReferenceTestCase, tmp_path: Path
+) -> None:
+ with pytest.raises(CompileInputError, match=re.escape(test_case.expected_error_fragment)):
+ expand_typed_macro_sql(
+ tmp_path=tmp_path, macro_file_contents=_BASE_MACRO, sql=test_case.sql
+ )
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ TypedReferenceAdapterRenderingTestCase(
+ description="bigquery",
+ adapter_name="bigquery",
+ expected_resolved_sql="SELECT * FROM `analytics.sales.orders`",
+ ),
+ TypedReferenceAdapterRenderingTestCase(
+ description="duckdb",
+ adapter_name="duckdb",
+ expected_resolved_sql="SELECT * FROM analytics.sales.orders",
+ ),
+ TypedReferenceAdapterRenderingTestCase(
+ description="postgres",
+ adapter_name="postgres",
+ expected_resolved_sql="SELECT * FROM analytics.sales.orders",
+ ),
+ TypedReferenceAdapterRenderingTestCase(
+ description="snowflake",
+ adapter_name="snowflake",
+ expected_resolved_sql="SELECT * FROM analytics.sales.orders",
+ ),
+ TypedReferenceAdapterRenderingTestCase(
+ description="sqlserver",
+ adapter_name="sqlserver",
+ expected_resolved_sql="SELECT * FROM analytics.sales.orders",
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_adapter_when_resolving_typed_macro_reference_then_uses_adapter_quoting(
+ test_case: TypedReferenceAdapterRenderingTestCase, tmp_path: Path
+) -> None:
+ expanded: str = expand_typed_macro_sql(
+ tmp_path=tmp_path, macro_file_contents=_BASE_MACRO, sql='@base(__ref("orders"))'
+ )
+
+ resolved: str = resolve_model_references_for_adapter(
+ sql=expanded, adapter_name=test_case.adapter_name
+ )
+
+ assert resolved == test_case.expected_resolved_sql
diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py
index ee06de1b02..83ad6e4e03 100644
--- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py
@@ -291,6 +291,7 @@ class LoadProjectConfigTestCase:
expected_dbt_vars: dict[str, object] = field(default_factory=dict)
expected_auto_load_sources: bool = True
expected_enforce_placement: bool = True
+ expected_enforce_explicit_references: bool = True
@dataclass(frozen=True)
diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py
index dee0663c5d..998232e4db 100644
--- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py
+++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py
@@ -689,6 +689,9 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise
[scopes]
enforce_placement = false
+[references]
+enforce_explicit = false
+
[defaults]
materialized = "table"
seed_database = "seed_db"
@@ -774,6 +777,7 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise
expected_max_concurrency=8,
expected_auto_load_sources=False,
expected_enforce_placement=False,
+ expected_enforce_explicit_references=False,
expected_materialized="table",
expected_row_diff_exclude_columns=("loaded_at",),
expected_row_diff_tolerances={
@@ -853,6 +857,7 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e
assert config.settings.concurrency == test_case.expected_max_concurrency
assert config.settings.auto_load_sources is test_case.expected_auto_load_sources
assert config.scopes.enforce_placement is test_case.expected_enforce_placement
+ assert config.references.enforce_explicit is test_case.expected_enforce_explicit_references
assert config.defaults.materialized == test_case.expected_materialized
assert config.defaults.row_diff_exclude_columns == test_case.expected_row_diff_exclude_columns
assert config.defaults.row_diff_tolerances == test_case.expected_row_diff_tolerances
@@ -1226,6 +1231,28 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed(
""".strip(),
expected_error_fragment="Expected 'enforce_placement' to be a boolean when provided",
),
+ LoadProjectConfigErrorTestCase(
+ description="raises when explicit reference enforcement is not a boolean",
+ project_file_contents="""
+name = "demo"
+adapter = "duckdb"
+
+[references]
+enforce_explicit = "no"
+""".strip(),
+ expected_error_fragment="Expected 'enforce_explicit' to be a boolean when provided",
+ ),
+ LoadProjectConfigErrorTestCase(
+ description="raises when the references section has an unknown key",
+ project_file_contents="""
+name = "demo"
+adapter = "duckdb"
+
+[references]
+enforce = false
+""".strip(),
+ expected_error_fragment="references",
+ ),
LoadProjectConfigErrorTestCase(
description="raises when settings sql_analysis is not a boolean",
project_file_contents="""
diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py
index 4cac20c490..450abc6e1d 100644
--- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py
+++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py
@@ -836,6 +836,28 @@ def check_orders(ctx):
},
expected_error_fragment="Check 'check_orders' depends on SQL resource 'orders'",
),
+ DiscoverFactoryValidationTestCase(
+ description="generated task seed dependency fails validation",
+ repo_files=base_repo_files()
+ | {
+ "python/tasks/generated.py": """
+from sqlbuild.factories import factory
+from sqlbuild.refs import seed
+from sqlbuild.tasks import task
+
+
+@factory
+def generated_tasks():
+ @task(name="export_countries", depends_on=seed("country_codes"))
+ def export_countries(ctx):
+ return None
+ return export_countries
+""",
+ },
+ expected_error_fragment=(
+ "Python node 'export_countries' depends on seed 'country_codes'"
+ ),
+ ),
DiscoverFactoryValidationTestCase(
description="generated managed source loader mismatch fails validation",
repo_files=base_repo_files()
From 5f2fe3055b63bf3569da6e5cade56ddff3f3d4a5 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Sun, 27 Sep 2026 20:52:11 +0100
Subject: [PATCH 02/13] feat!: add Python hook reads and reject hard-coded
project relation names
@hook(reads=...) declares the models, sources and seeds a Python hook
reads; ctx.relation(...) resolves them and raises for undeclared refs.
Declared reads order the hooked model after the resources it reads, and a
read that depends on the hooked model fails compile with P007.
Literal SQL passed to ctx.query or ctx.execute_sql in tasks, assets and
hooks that names a project relation fails compile with P008. At run time
the same check parses SQL in the adapter dialect and records a P008
warning in run output, the summary and the JSON run results; the query
still runs. [references] enforce_explicit = false disables both.
Build summaries now print warning details before the completion line and
count a node that succeeded with warnings as WARN.
BREAKING CHANGE: Python tasks, assets and hooks that hard-code project
relation names in literal SQL now fail compile; declare the dependency and
use ctx.relation(...).
---
Makefile | 1 +
.../build_python_nodes/python_lifecycle.py | 9 +
.../classes/build_progress_callbacks.py | 40 ++-
.../_helpers/execution_result_document.py | 8 +-
src/sqlbuild/compiler/auditing/main/render.py | 2 +-
.../compile/_helpers/analysis/validation.py | 2 +-
.../compile/_helpers/assembly/project.py | 1 +
.../explicit_references/hook_reads.py | 47 ++++
.../macro_arguments.py} | 0
.../explicit_references/python_sql.py | 218 +++++++++++++++++
.../compile/_helpers/render/macros.py | 2 +-
src/sqlbuild/compiler/compile/constants.py | 2 +
.../compile/main/_assemble_project.py | 7 +
src/sqlbuild/compiler/compile/models.py | 1 +
.../discovery/_helpers/filesystem/core.py | 1 +
src/sqlbuild/compiler/discovery/models.py | 1 +
.../compiler/graph/_helpers/hook_reads.py | 78 ++++++
.../compiler/graph/main/_hook_read_cycle.py | 13 +
.../compiler/graph/main/_hook_read_edges.py | 13 +
.../graph/main/_model_python_hook_names.py | 12 +
src/sqlbuild/compiler/graph/models.py | 9 +
.../pipeline/main/relation_targets.py | 2 +-
.../_helpers/command_planning/static.py | 1 +
.../compiler/planner/_helpers/graph/core.py | 18 +-
.../planner/_helpers/output/plan_entry.py | 2 +-
.../planner/_helpers/output/plan_output.py | 1 +
.../planner/_helpers/resolve/sources.py | 2 +-
.../planner/_helpers/scenario/relations.py | 2 +-
.../planner/_helpers/warehouse/snapshot.py | 2 +-
src/sqlbuild/compiler/planner/models.py | 1 +
.../references/_helpers/project_relations.py | 79 ++++++
.../references/_helpers/relation_names.py | 138 +++++++++++
.../main/_compiled_project_relations.py | 13 +
.../references/main/extract_relation_names.py | 14 ++
.../references/main/match_project_relation.py | 23 ++
..._relation.py => render_source_relation.py} | 0
.../main/runtime_project_relations.py | 17 ++
src/sqlbuild/compiler/references/models.py | 31 +++
.../source_freshness/_helpers/observation.py | 2 +-
.../executor/build/classes/build_scheduler.py | 1 +
.../python_nodes/_helpers/execution.py | 30 +++
.../classes/runtime_relation_guard.py | 104 ++++++++
src/sqlbuild/executor/python_nodes/models.py | 10 +
.../run/_helpers/execution/hook_phases.py | 18 +-
.../executor/run/_helpers/execution/hooks.py | 115 +++++++--
.../run/_helpers/materializations/custom.py | 26 +-
.../run/_helpers/materializations/view.py | 31 +--
src/sqlbuild/executor/run/models.py | 45 +++-
src/sqlbuild/hooks.py | 5 +-
src/sqlbuild/hooks.pyi | 6 +-
.../_helpers/dependency_normalization.py | 16 ++
src/sqlbuild/python_nodes/main/apply_hook.py | 7 +-
src/sqlbuild/python_nodes/models.py | 1 +
.../cli/commands/main/build/_test_types.py | 35 +++
.../cli/commands/main/build/helpers.py | 54 +++++
.../main/build/test_explicit_references.py | 227 ++++++++++++++++++
.../commands/shared/_helpers/test_progress.py | 23 ++
.../compiler/compile/_helpers/_test_types.py | 36 +++
.../compiler/compile/_helpers/helpers.py | 24 ++
.../compile/_helpers/test_hook_reads.py | 133 ++++++++++
.../_helpers/test_python_sql_references.py | 199 +++++++++++++++
.../compiler/references/main/_test_types.py | 33 +++
.../references/main/test_relation_names.py | 135 +++++++++++
.../executor/python_nodes/classes/__init__.py | 0
.../python_nodes/classes/_test_types.py | 16 ++
.../classes/test_runtime_relation_guard.py | 92 +++++++
.../executor/run/_helpers/_test_types.py | 30 +++
.../sqlbuild/executor/run/_helpers/helpers.py | 62 ++++-
.../executor/run/_helpers/test_hooks.py | 108 ++++++++-
69 files changed, 2350 insertions(+), 87 deletions(-)
create mode 100644 src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
rename src/sqlbuild/compiler/compile/_helpers/{render/macro_references.py => explicit_references/macro_arguments.py} (100%)
create mode 100644 src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
create mode 100644 src/sqlbuild/compiler/graph/_helpers/hook_reads.py
create mode 100644 src/sqlbuild/compiler/graph/main/_hook_read_cycle.py
create mode 100644 src/sqlbuild/compiler/graph/main/_hook_read_edges.py
create mode 100644 src/sqlbuild/compiler/graph/main/_model_python_hook_names.py
create mode 100644 src/sqlbuild/compiler/references/_helpers/project_relations.py
create mode 100644 src/sqlbuild/compiler/references/_helpers/relation_names.py
create mode 100644 src/sqlbuild/compiler/references/main/_compiled_project_relations.py
create mode 100644 src/sqlbuild/compiler/references/main/extract_relation_names.py
create mode 100644 src/sqlbuild/compiler/references/main/match_project_relation.py
rename src/sqlbuild/compiler/references/main/{_render_source_relation.py => render_source_relation.py} (100%)
create mode 100644 src/sqlbuild/compiler/references/main/runtime_project_relations.py
create mode 100644 src/sqlbuild/compiler/references/models.py
create mode 100644 src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py
create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
create mode 100644 tests/unit/src/sqlbuild/compiler/references/main/test_relation_names.py
create mode 100644 tests/unit/src/sqlbuild/executor/python_nodes/classes/__init__.py
create mode 100644 tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py
create mode 100644 tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py
diff --git a/Makefile b/Makefile
index 991ff782bf..9f50994d7a 100644
--- a/Makefile
+++ b/Makefile
@@ -69,6 +69,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py \
+ tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py \
diff --git a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py
index ccc2bcf242..ca24146c94 100644
--- a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py
+++ b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py
@@ -100,6 +100,13 @@ def prepare_direct_python_lifecycle(
),
),
)
+ project_relations: dict[SqlResourceRef, str] | None = (
+ build_python_relation_targets(
+ adapter=adapter, project=pipeline_result.project, plan_output=plan_output
+ )
+ if pipeline_result.project.enforce_explicit_references
+ else None
+ )
default_database: str | None = pipeline_result.project.effective_target_database
if default_database is None:
default_database = adapter.default_database()
@@ -141,6 +148,7 @@ def prepare_direct_python_lifecycle(
start_cursor_int=start_cursor_int,
end_cursor_int=end_cursor_int,
relation_targets=relation_targets,
+ project_relations=project_relations,
providers=providers,
),
callbacks=IngressCallbacks(
@@ -181,6 +189,7 @@ def prepare_direct_python_lifecycle(
default_database=default_database,
default_schema=default_schema,
relation_targets=relation_targets,
+ project_relations=project_relations,
start_cursor_ts=start_cursor_ts,
end_cursor_ts=end_cursor_ts,
start_cursor_int=start_cursor_int,
diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py
index 8fc7d340f7..4b90820806 100644
--- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py
+++ b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py
@@ -816,7 +816,9 @@ def _count_build_footer_results(
model_result: ModelExecutionResult
for model_result in result.model_results:
- if model_result.status == ExecutionStatus.SUCCESS:
+ if model_result.status == ExecutionStatus.SUCCESS and model_result.warning_messages:
+ warn_count += 1
+ elif model_result.status == ExecutionStatus.SUCCESS:
pass_count += 1
elif model_result.status == ExecutionStatus.FAILED:
fail_count += 1
@@ -868,7 +870,9 @@ def _count_build_footer_results(
python_result: PythonNodeExecutionResult
for python_result in python_node_results:
- if python_result.status == PythonNodeStatus.SUCCESS:
+ if python_result.status == PythonNodeStatus.SUCCESS and python_result.warning_messages:
+ warn_count += 1
+ elif python_result.status == PythonNodeStatus.SUCCESS:
pass_count += 1
elif python_result.status == PythonNodeStatus.FAILED:
fail_count += 1
@@ -918,12 +922,17 @@ def format_build_footer(
if result.status == BuildStatus.FAILED or python_fail_count:
state: CompletionState = CompletionState.FAIL
label: str = "Completed with errors"
- elif result.warning_count > 0:
+ elif result.warning_count > 0 or counts.warn_count > 0:
state = CompletionState.WARN
label = "Completed with warnings"
else:
state = CompletionState.OK
label = "Completed successfully"
+ warning_lines: list[str] = _format_warning_details(
+ result=result, python_node_results=python_node_results, style=style
+ )
+ if warning_lines:
+ lines.extend(warning_lines)
lines.append(
format_completion_line(style=style, state=state, label=label, summary=counts_summary)
)
@@ -939,10 +948,6 @@ def format_build_footer(
if skipped_lines:
lines.extend(skipped_lines)
- warning_lines: list[str] = _format_warning_details(result=result, style=style)
- if warning_lines:
- lines.extend(warning_lines)
-
return "\n".join(lines)
@@ -1170,7 +1175,12 @@ def _format_audit_error_details(
return lines
-def _format_warning_details(*, result: BuildExecutionResult, style: CliStyle) -> list[str]:
+def _format_warning_details(
+ *,
+ result: BuildExecutionResult,
+ python_node_results: tuple[PythonNodeExecutionResult, ...],
+ style: CliStyle,
+) -> list[str]:
lines: list[str] = []
has_warnings: bool = False
@@ -1220,6 +1230,20 @@ def _format_warning_details(*, result: BuildExecutionResult, style: CliStyle) ->
lines.append(f" {warning_msg}")
lines.append("")
+ python_result: PythonNodeExecutionResult
+ for python_result in python_node_results:
+ if not python_result.warning_messages:
+ continue
+ if not has_warnings:
+ lines.append("")
+ lines.append(style.warning_strong("Warnings:"))
+ lines.append("")
+ has_warnings = True
+ lines.append(f" {python_result.node_name} ({python_result.kind.value})")
+ for warning_msg in python_result.warning_messages:
+ lines.append(f" {warning_msg}")
+ lines.append("")
+
return lines
diff --git a/src/sqlbuild/cli/output/_helpers/execution_result_document.py b/src/sqlbuild/cli/output/_helpers/execution_result_document.py
index 03f85a939e..feb08cfeef 100644
--- a/src/sqlbuild/cli/output/_helpers/execution_result_document.py
+++ b/src/sqlbuild/cli/output/_helpers/execution_result_document.py
@@ -156,6 +156,7 @@ def format_build_execution_json(
)
python_check_fail_count: int = sum(1 for result in python_check_results if result.failed)
python_check_warn_count: int = sum(1 for result in python_check_results if result.warned)
+ python_warning_count: int = sum(len(result.warning_messages) for result in python_node_results)
python_check_pass_count: int = sum(1 for result in python_check_results if result.passed)
status: BuildStatus = (
BuildStatus.FAILED
@@ -194,7 +195,7 @@ def format_build_execution_json(
"success_count": result.success_count + python_success_count,
"failure_count": result.failure_count + python_fail_count + python_check_fail_count,
"skipped_count": result.skipped_count + python_skipped_count,
- "warning_count": result.warning_count + python_check_warn_count,
+ "warning_count": (result.warning_count + python_check_warn_count + python_warning_count),
"python_check_pass_count": python_check_pass_count,
"python_check_warn_count": python_check_warn_count,
"python_check_fail_count": python_check_fail_count,
@@ -219,7 +220,9 @@ def format_build_execution_json(
retained_result.failure_count + python_fail_count + python_check_fail_count
),
"skipped_count": retained_result.skipped_count + python_skipped_count,
- "warning_count": retained_result.warning_count + python_check_warn_count,
+ "warning_count": (
+ retained_result.warning_count + python_check_warn_count + python_warning_count
+ ),
"python_check_pass_count": python_check_pass_count,
"python_check_warn_count": python_check_warn_count,
"python_check_fail_count": python_check_fail_count,
@@ -760,6 +763,7 @@ def _format_python_node_assets(
"skip_mode": result.skip_mode.value if result.skip_mode else None,
"skip_reason": result.skip_reason,
"error_message": result.error_message,
+ "warnings": result.warning_messages or None,
}
)
for result in results
diff --git a/src/sqlbuild/compiler/auditing/main/render.py b/src/sqlbuild/compiler/auditing/main/render.py
index 256eaf858a..b5e2a2ef48 100644
--- a/src/sqlbuild/compiler/auditing/main/render.py
+++ b/src/sqlbuild/compiler/auditing/main/render.py
@@ -7,7 +7,7 @@
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
from sqlbuild.compiler.compile.models import CompiledRelationLocation
from sqlbuild.compiler.references.constants import REF_PATTERN, SEED_PATTERN, SOURCE_PATTERN
-from sqlbuild.compiler.references.main._render_source_relation import render_source_relation
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.spec.contracts.models import SourceEntry
diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py
index a697e0e16b..7277f7950a 100644
--- a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py
+++ b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py
@@ -147,7 +147,7 @@ def validate_source_expression_syntax(
) -> None:
"""Validate that a source expression is parseable as a FROM target."""
- from sqlbuild.compiler.references.main._render_source_relation import render_source_relation
+ from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.spec.contracts.models import SourceEntry
rendered: str = render_source_relation(
diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
index add8658a13..aaca60446f 100644
--- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
+++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
@@ -444,6 +444,7 @@ def assemble_compiled_project(
),
loader_functions=inputs.discovered_inputs.loader_functions,
hook_functions=inputs.discovered_inputs.hook_functions,
+ enforce_explicit_references=inputs.project_config.references.enforce_explicit,
sql_hook_files=inputs.discovered_inputs.sql_hook_files,
materialization_files=inputs.discovered_inputs.materialization_files,
public_enums=inputs.public_enums,
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
new file mode 100644
index 0000000000..726fbd2c3c
--- /dev/null
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
@@ -0,0 +1,47 @@
+"""Compile validation for resources Python hooks declare they read."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.constants import HOOK_READS_OWN_DEPENDANT_CODE
+from sqlbuild.compiler.compile.exceptions import CompileInputError
+from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject
+from sqlbuild.compiler.graph.main._hook_read_cycle import hook_read_cycle
+from sqlbuild.compiler.graph.models import HookReadEdge
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
+
+
+def validate_hook_reads(*, project: CompiledProject) -> None:
+ """Reject unknown declared hook reads and reads that would wait for the hook's own model."""
+
+ known: dict[SqlResourceRefKind, frozenset[str]] = {
+ SqlResourceRefKind.MODEL: frozenset(model.name for model in project.models),
+ SqlResourceRefKind.SOURCE: frozenset(source.name for source in project.sources),
+ SqlResourceRefKind.SEED: frozenset(seed.name for seed in project.seeds),
+ }
+ for hook in sorted(project.hook_functions, key=lambda item: item.name):
+ ref: SqlResourceRef
+ for ref in hook.reads:
+ if ref.name not in known[ref.kind]:
+ raise CompileInputError(
+ f"Python hook '{hook.name}' in '{hook.relative_path.as_posix()}' reads unknown "
+ f"{ref.kind.value} '{ref.name}'"
+ )
+ cycle: HookReadEdge | None = hook_read_cycle(project=project)
+ if cycle is None:
+ return
+ raise CompileInputError(
+ f"Python hook '{cycle.hook_name}' on {_describe(cycle.gated)} reads "
+ f"{_describe(cycle.read)}, "
+ f"which depends on '{cycle.gated.name}'; a hook runs as part of its model, so it cannot "
+ "wait for a resource built from that model",
+ code=HOOK_READS_OWN_DEPENDANT_CODE,
+ help=(
+ f"attach the hook to a model built after '{cycle.read.name}', or drop the read and "
+ f"use ctx.destination for '{cycle.gated.name}' itself"
+ ),
+ )
+
+
+def _describe(key: CompiledObjectKey) -> str:
+ return f"{key.resource_type} '{key.name}'"
diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macro_references.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
similarity index 100%
rename from src/sqlbuild/compiler/compile/_helpers/render/macro_references.py
rename to src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
new file mode 100644
index 0000000000..e006aaaecd
--- /dev/null
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
@@ -0,0 +1,218 @@
+"""Compile-time rejection of project relation names hard-coded in Python SQL."""
+
+from __future__ import annotations
+
+import ast
+import inspect
+import textwrap
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from sqlbuild.compiler.compile.constants import HARD_CODED_PROJECT_RELATION_CODE
+from sqlbuild.compiler.compile.exceptions import CompileInputError
+from sqlbuild.compiler.compile.models import CompiledProject
+from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs
+from sqlbuild.compiler.graph.main._model_python_hook_names import model_python_hook_names
+from sqlbuild.compiler.references.main._compiled_project_relations import (
+ compiled_project_relations,
+)
+from sqlbuild.compiler.references.main.extract_relation_names import extract_relation_names
+from sqlbuild.compiler.references.main.match_project_relation import match_project_relation
+from sqlbuild.compiler.references.models import (
+ ProjectRelation,
+ ProjectRelationIndex,
+ RelationName,
+)
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
+
+_SQL_METHOD_NAMES: frozenset[str] = frozenset({"query", "execute_sql"})
+_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset({"ctx", "context", "_ctx", "hook_context"})
+_VALUE_PLACEHOLDER: str = "__sqlbuild_python_value_"
+
+
+@dataclass(frozen=True)
+class _PythonSqlOwner:
+ label: str
+ function: Callable[..., object]
+ relative_path: Path
+ declare_help: str
+ own_refs: frozenset[SqlResourceRef]
+ relation_kinds: frozenset[SqlResourceRefKind]
+
+
+@dataclass(frozen=True)
+class _LiteralSql:
+ method: str
+ sql: str
+ line: int
+
+
+def validate_python_sql_references(
+ *, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs
+) -> None:
+ """Reject literal SQL in tasks, assets, and hooks that names a project relation."""
+
+ owners: tuple[_PythonSqlOwner, ...] = _owners(
+ project=project, discovered_inputs=discovered_inputs
+ )
+ if not owners:
+ return
+ index: ProjectRelationIndex = compiled_project_relations(project=project)
+ if not index.relations:
+ return
+ for owner in owners:
+ temporary: frozenset[str] = frozenset()
+ for literal in sorted(_literal_sql_calls(owner.function), key=lambda item: item.line):
+ temporary = _reject_hard_coded_relation(
+ owner=owner,
+ literal=literal,
+ index=index,
+ dialect=project.sql_analysis_dialect,
+ temporary=temporary,
+ )
+
+
+def _owners(
+ *, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs
+) -> tuple[_PythonSqlOwner, ...]:
+ node_kinds: frozenset[SqlResourceRefKind] = frozenset(
+ {SqlResourceRefKind.MODEL, SqlResourceRefKind.SOURCE}
+ )
+ owners: list[_PythonSqlOwner] = []
+ for label, nodes in (
+ ("task", discovered_inputs.task_functions),
+ ("asset", discovered_inputs.asset_functions),
+ ):
+ owners.extend(
+ _PythonSqlOwner(
+ label=f"{label}:{node.name}",
+ function=node.function,
+ relative_path=node.relative_path,
+ declare_help="declare it with depends_on={typed}",
+ own_refs=frozenset(),
+ relation_kinds=node_kinds,
+ )
+ for node in nodes
+ )
+ attached_models: dict[str, set[SqlResourceRef]] = {}
+ for model in project.models:
+ for hook_name in model_python_hook_names(model=model):
+ attached_models.setdefault(hook_name, set()).add(
+ SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=model.name)
+ )
+ owners.extend(
+ _PythonSqlOwner(
+ label=f"hook:{hook.name}",
+ function=hook.function,
+ relative_path=hook.relative_path,
+ declare_help="declare it with @hook(reads={typed})",
+ own_refs=frozenset(attached_models.get(hook.name, ())),
+ relation_kinds=frozenset(SqlResourceRefKind),
+ )
+ for hook in project.hook_functions
+ )
+ return tuple(owners)
+
+
+def _literal_sql_calls(function: Callable[..., object]) -> Iterator[_LiteralSql]:
+ try:
+ lines: list[str]
+ start_line: int
+ lines, start_line = inspect.getsourcelines(function)
+ tree: ast.Module = ast.parse(textwrap.dedent("".join(lines)))
+ except (OSError, TypeError, SyntaxError):
+ return
+ definition: ast.stmt | None = tree.body[0] if tree.body else None
+ if not isinstance(definition, ast.FunctionDef | ast.AsyncFunctionDef):
+ return
+ parameters: list[str] = [
+ argument.arg
+ for argument in (
+ *definition.args.posonlyargs,
+ *definition.args.args,
+ *definition.args.kwonlyargs,
+ )
+ ]
+ context_name: str | None = next(
+ (name for name in parameters if name in _CONTEXT_PARAMETER_NAMES),
+ parameters[0] if parameters else None,
+ )
+ if context_name is None:
+ return
+ for node in ast.walk(definition):
+ if not (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr in _SQL_METHOD_NAMES
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id == context_name
+ and node.args
+ ):
+ continue
+ sql: str | None = _literal_text(node.args[0])
+ if sql is not None:
+ yield _LiteralSql(method=node.func.attr, sql=sql, line=start_line + node.lineno - 1)
+
+
+def _literal_text(node: ast.expr) -> str | None:
+ if isinstance(node, ast.Constant) and isinstance(node.value, str):
+ return node.value
+ if not isinstance(node, ast.JoinedStr):
+ return None
+ parts: list[str] = []
+ for index, value in enumerate(node.values):
+ if isinstance(value, ast.Constant) and isinstance(value.value, str):
+ parts.append(value.value)
+ else:
+ parts.append(f"{_VALUE_PLACEHOLDER}{index}")
+ return "".join(parts)
+
+
+def _reject_hard_coded_relation(
+ *,
+ owner: _PythonSqlOwner,
+ literal: _LiteralSql,
+ index: ProjectRelationIndex,
+ dialect: str | None,
+ temporary: frozenset[str],
+) -> frozenset[str]:
+ extracted: tuple[tuple[RelationName, ...], frozenset[str]] | None = extract_relation_names(
+ sql=literal.sql, dialect=dialect
+ )
+ if extracted is None:
+ return temporary
+ relations, created = extracted
+ temporary = temporary | created
+ for relation in relations:
+ if _VALUE_PLACEHOLDER in "".join(
+ part for part in (relation.database, relation.schema, relation.name) if part
+ ):
+ continue
+ if relation.schema is None and relation.name.casefold() in temporary:
+ continue
+ match: ProjectRelation | None = match_project_relation(index=index, relation=relation)
+ if (
+ match is None
+ or match.ref in owner.own_refs
+ or match.ref.kind not in owner.relation_kinds
+ ):
+ continue
+ typed: str = f'{match.ref.kind.value}("{match.ref.name}")'
+ written: str = ".".join(
+ part for part in (relation.database, relation.schema, relation.name) if part
+ )
+ raise CompileInputError(
+ f"{owner.label} names {match.ref.kind.value}:{match.ref.name} as '{written}' in SQL "
+ f"passed to ctx.{literal.method}()\n --> {owner.relative_path.as_posix()}:"
+ f"{literal.line}",
+ code=HARD_CODED_PROJECT_RELATION_CODE,
+ help=(
+ f"{owner.declare_help.format(typed=typed)} and use ctx.relation({typed}) instead "
+ "of the relation name\n"
+ " = help: while migrating a project, allow hard-coded relation names with "
+ "[references] enforce_explicit = false in sqlbuild_project.toml"
+ ),
+ )
+ return temporary
diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py
index e79b2e3a51..e5b17b1d4d 100644
--- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py
+++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py
@@ -16,7 +16,7 @@
from types import MappingProxyType, ModuleType
from typing import cast
-from sqlbuild.compiler.compile._helpers.render.macro_references import (
+from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import (
evaluate_typed_reference,
reject_macro_generated_references,
relation_placeholder_text,
diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py
index d583d5c430..59fa1c641d 100644
--- a/src/sqlbuild/compiler/compile/constants.py
+++ b/src/sqlbuild/compiler/compile/constants.py
@@ -12,6 +12,8 @@
SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE: str = "P004"
ATTACHED_AUDIT_READS_OWN_DEPENDANT_CODE: str = "P005"
MACRO_GENERATED_REFERENCE_CODE: str = "P006"
+HOOK_READS_OWN_DEPENDANT_CODE: str = "P007"
+HARD_CODED_PROJECT_RELATION_CODE: str = "P008"
HOOK_DIRECTORY_NAME: str = "hooks"
MODEL_DIRECTORY_NAME: str = "models"
NOT_NULL_AUDIT_NAME: str = "not_null"
diff --git a/src/sqlbuild/compiler/compile/main/_assemble_project.py b/src/sqlbuild/compiler/compile/main/_assemble_project.py
index 1815d86254..a7e8b7ba23 100644
--- a/src/sqlbuild/compiler/compile/main/_assemble_project.py
+++ b/src/sqlbuild/compiler/compile/main/_assemble_project.py
@@ -7,6 +7,10 @@
from sqlbuild.adapter.contract.models import ExpressionInferenceProfile
from sqlbuild.compiler.compile._helpers.assembly.audit_gates import validate_attached_audit_gates
from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project
+from sqlbuild.compiler.compile._helpers.explicit_references.hook_reads import validate_hook_reads
+from sqlbuild.compiler.compile._helpers.explicit_references.python_sql import (
+ validate_python_sql_references,
+)
from sqlbuild.compiler.compile.models import (
CompiledProject,
CompileProjectInputs,
@@ -34,4 +38,7 @@ def assemble_project(
analysis_model_names=analysis_model_names,
)
validate_attached_audit_gates(project=project)
+ validate_hook_reads(project=project)
+ if inputs.project_config.references.enforce_explicit:
+ validate_python_sql_references(project=project, discovered_inputs=inputs.discovered_inputs)
return project
diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py
index 5524faa97c..70d55d4cd5 100644
--- a/src/sqlbuild/compiler/compile/models.py
+++ b/src/sqlbuild/compiler/compile/models.py
@@ -1047,6 +1047,7 @@ class CompiledProject:
sql_scenarios: tuple[CompiledSqlScenario, ...] = field(default_factory=tuple)
loader_functions: tuple[DiscoveredLoaderFunction, ...] = field(default_factory=tuple)
hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple)
+ enforce_explicit_references: bool = True
sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = field(default_factory=tuple)
materialization_files: tuple[DiscoveredMaterializationFile, ...] = field(default_factory=tuple)
public_enums: dict[str, EnumDeclaration] = field(default_factory=dict)
diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py
index 37f8eb90df..3cf840d014 100644
--- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py
+++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py
@@ -980,6 +980,7 @@ def discover_hook_functions(
name=hook_definition.name,
function=value,
description=hook_definition.description,
+ reads=hook_definition.reads,
provider_usages=_provider_usages(
function=value,
provider_by_name=provider_by_name,
diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py
index 6d32670776..abf27116dc 100644
--- a/src/sqlbuild/compiler/discovery/models.py
+++ b/src/sqlbuild/compiler/discovery/models.py
@@ -501,6 +501,7 @@ class DiscoveredHookFunction:
name: str
function: Callable[..., object]
description: str | None = None
+ reads: tuple[SqlResourceRef, ...] = field(default_factory=tuple)
provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple)
scope_kind: ScopeKind = ScopeKind.GLOBAL
ownership_root: Path | None = None
diff --git a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
new file mode 100644
index 0000000000..baec15d9ac
--- /dev/null
+++ b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
@@ -0,0 +1,78 @@
+"""Ordering edges from the resources Python hooks declare they read."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, CompiledProject
+from sqlbuild.compiler.compile.types import CompiledResourceType
+from sqlbuild.compiler.discovery.models import PythonHookEntry
+from sqlbuild.compiler.graph._helpers.algorithms import transitive_closure_many_impl
+from sqlbuild.compiler.graph._helpers.audit_gates import attached_audit_gate_edges_impl
+from sqlbuild.compiler.graph._helpers.lineage import build_lineage_upstream_deps_impl
+from sqlbuild.compiler.graph.models import HookReadEdge
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
+
+_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks")
+_RESOURCE_TYPE_BY_REF_KIND: dict[SqlResourceRefKind, CompiledResourceType] = {
+ SqlResourceRefKind.MODEL: CompiledResourceType.MODEL,
+ SqlResourceRefKind.SOURCE: CompiledResourceType.SOURCE,
+ SqlResourceRefKind.SEED: CompiledResourceType.SEED,
+}
+
+
+def hook_read_key(ref: SqlResourceRef) -> CompiledObjectKey:
+ """Return the compiled graph key a declared hook read names."""
+
+ return CompiledObjectKey(resource_type=_RESOURCE_TYPE_BY_REF_KIND[ref.kind], name=ref.name)
+
+
+def model_python_hook_names_impl(model: CompiledModel) -> tuple[str, ...]:
+ """Return the Python hooks attached to one model, in lifecycle order."""
+
+ names: list[str] = []
+ for hook_key in _MODEL_HOOK_KEYS:
+ entries: object = model.config.values.get(hook_key)
+ if isinstance(entries, PythonHookEntry):
+ names.append(entries.name)
+ elif isinstance(entries, list | tuple):
+ names.extend(entry.name for entry in entries if isinstance(entry, PythonHookEntry))
+ return tuple(dict.fromkeys(names))
+
+
+def hook_read_edges_impl(*, project: CompiledProject) -> tuple[HookReadEdge, ...]:
+ """Return edges from each declared hook read to the model that runs the hook."""
+
+ reads_by_hook: dict[str, tuple[SqlResourceRef, ...]] = {
+ hook.name: hook.reads for hook in project.hook_functions if hook.reads
+ }
+ if not reads_by_hook:
+ return ()
+ edges: dict[HookReadEdge, None] = {}
+ for model in project.models:
+ for hook_name in model_python_hook_names_impl(model):
+ for ref in reads_by_hook.get(hook_name, ()):
+ read: CompiledObjectKey = hook_read_key(ref)
+ if read != model.key:
+ edges[HookReadEdge(hook_name=hook_name, gated=model.key, read=read)] = None
+ return tuple(edges)
+
+
+def hook_read_cycle_impl(*, project: CompiledProject) -> HookReadEdge | None:
+ """Return the first declared hook read that depends on the model running the hook."""
+
+ edges: tuple[HookReadEdge, ...] = hook_read_edges_impl(project=project)
+ if not edges:
+ return None
+ upstream: dict[CompiledObjectKey, list[CompiledObjectKey]] = {
+ key: list(deps) for key, deps in build_lineage_upstream_deps_impl(project).items()
+ }
+ for gate in attached_audit_gate_edges_impl(project=project):
+ upstream.setdefault(gate.gated, []).append(gate.read)
+ for edge in edges:
+ upstream.setdefault(edge.gated, []).append(edge.read)
+ for edge in sorted(edges, key=lambda item: (item.gated.name, item.hook_name, item.read.name)):
+ if edge.gated in transitive_closure_many_impl(
+ starts=(edge.read,), edges=upstream, include_starts=False
+ ):
+ return edge
+ return None
diff --git a/src/sqlbuild/compiler/graph/main/_hook_read_cycle.py b/src/sqlbuild/compiler/graph/main/_hook_read_cycle.py
new file mode 100644
index 0000000000..7e57d62226
--- /dev/null
+++ b/src/sqlbuild/compiler/graph/main/_hook_read_cycle.py
@@ -0,0 +1,13 @@
+"""Python hook read cycle detection entrypoint."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.models import CompiledProject
+from sqlbuild.compiler.graph._helpers.hook_reads import hook_read_cycle_impl
+from sqlbuild.compiler.graph.models import HookReadEdge
+
+
+def hook_read_cycle(*, project: CompiledProject) -> HookReadEdge | None:
+ """Return the first declared hook read that depends on the model running the hook."""
+
+ return hook_read_cycle_impl(project=project)
diff --git a/src/sqlbuild/compiler/graph/main/_hook_read_edges.py b/src/sqlbuild/compiler/graph/main/_hook_read_edges.py
new file mode 100644
index 0000000000..c4448480c4
--- /dev/null
+++ b/src/sqlbuild/compiler/graph/main/_hook_read_edges.py
@@ -0,0 +1,13 @@
+"""Python hook read ordering edge entrypoint."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.models import CompiledProject
+from sqlbuild.compiler.graph._helpers.hook_reads import hook_read_edges_impl
+from sqlbuild.compiler.graph.models import HookReadEdge
+
+
+def hook_read_edges(*, project: CompiledProject) -> tuple[HookReadEdge, ...]:
+ """Return edges from resources a Python hook declares it reads to the model running it."""
+
+ return hook_read_edges_impl(project=project)
diff --git a/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py b/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py
new file mode 100644
index 0000000000..7505329945
--- /dev/null
+++ b/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py
@@ -0,0 +1,12 @@
+"""Python hooks attached to one model."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.models import CompiledModel
+from sqlbuild.compiler.graph._helpers.hook_reads import model_python_hook_names_impl
+
+
+def model_python_hook_names(*, model: CompiledModel) -> tuple[str, ...]:
+ """Return the Python hooks attached to one model, in lifecycle order."""
+
+ return model_python_hook_names_impl(model)
diff --git a/src/sqlbuild/compiler/graph/models.py b/src/sqlbuild/compiler/graph/models.py
index 86df593339..021b31386e 100644
--- a/src/sqlbuild/compiler/graph/models.py
+++ b/src/sqlbuild/compiler/graph/models.py
@@ -15,3 +15,12 @@ class AttachedAuditGateEdge:
target: CompiledObjectKey
gated: CompiledObjectKey
read: CompiledObjectKey
+
+
+@dataclass(frozen=True)
+class HookReadEdge:
+ """One ordering edge: a model waits for a resource its Python hook declares it reads."""
+
+ hook_name: str
+ gated: CompiledObjectKey
+ read: CompiledObjectKey
diff --git a/src/sqlbuild/compiler/pipeline/main/relation_targets.py b/src/sqlbuild/compiler/pipeline/main/relation_targets.py
index 1b399999d5..77f01a2d72 100644
--- a/src/sqlbuild/compiler/pipeline/main/relation_targets.py
+++ b/src/sqlbuild/compiler/pipeline/main/relation_targets.py
@@ -12,7 +12,7 @@
CompiledRelationLocation,
)
from sqlbuild.compiler.planner.models import PlanOutput
-from sqlbuild.compiler.references.main._render_source_relation import render_source_relation
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.errors.contracts.exceptions import SharedInputError
from sqlbuild.python_nodes.models import SqlResourceRef
from sqlbuild.python_nodes.types import SqlResourceRefKind
diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py
index 6ebdb1f67d..1f4f79b2d6 100644
--- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py
+++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py
@@ -248,4 +248,5 @@ def _base_plan_output(
source_map=relations.source_map,
source_read_map=relations.source_read_map,
hook_functions=project.hook_functions,
+ enforce_explicit_references=project.enforce_explicit_references,
)
diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/core.py b/src/sqlbuild/compiler/planner/_helpers/graph/core.py
index a29d12ce41..fe2394afb3 100644
--- a/src/sqlbuild/compiler/planner/_helpers/graph/core.py
+++ b/src/sqlbuild/compiler/planner/_helpers/graph/core.py
@@ -13,10 +13,11 @@
)
from sqlbuild.compiler.graph.main._attached_audit_gate_edges import attached_audit_gate_edges
from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps
+from sqlbuild.compiler.graph.main._hook_read_edges import hook_read_edges
from sqlbuild.compiler.graph.main.invert_edges import invert_edges
from sqlbuild.compiler.graph.main.path_nodes import path_nodes
from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure
-from sqlbuild.compiler.graph.models import AttachedAuditGateEdge
+from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge
from sqlbuild.compiler.planner.exceptions import PlannerInputError
@@ -51,6 +52,12 @@ def build_execution_upstream_deps(
if edge.read not in gated_upstream:
gated_upstream.append(edge.read)
+ hook_edge: HookReadEdge
+ for hook_edge in hook_read_edges(project=project):
+ hook_upstream: list[CompiledObjectKey] = upstream.setdefault(hook_edge.gated, [])
+ if hook_edge.read not in hook_upstream:
+ hook_upstream.append(hook_edge.read)
+
return {k: tuple(v) for k, v in upstream.items()}
@@ -77,6 +84,15 @@ def build_execution_edge_origins(
(edge.gated, edge.read),
f"audit '{edge.audit_name}' on '{edge.target.name}' reads '{edge.read.name}'",
)
+ hook_edge: HookReadEdge
+ for hook_edge in hook_read_edges(project=project):
+ if hook_edge.read in lineage_upstream.get(hook_edge.gated, ()):
+ continue
+ origins.setdefault(
+ (hook_edge.gated, hook_edge.read),
+ f"hook '{hook_edge.hook_name}' on '{hook_edge.gated.name}' reads "
+ f"'{hook_edge.read.name}'",
+ )
return origins
diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
index ec42700c22..3034767498 100644
--- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
+++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
@@ -124,7 +124,7 @@
PlanReason,
WarningSeverity,
)
-from sqlbuild.compiler.references.main._render_source_relation import render_source_relation
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind
from sqlbuild.cursor_algebra.main.parse import parse
from sqlbuild.cursor_algebra.main.render import render
diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py
index 212ac7f200..2304d0cc74 100644
--- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py
+++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py
@@ -166,6 +166,7 @@ def build_plan_output(
source_map=relations.source_map,
source_read_map=relations.source_read_map,
hook_functions=project.hook_functions,
+ enforce_explicit_references=project.enforce_explicit_references,
provider_usages=_build_provider_usages(
project=project,
model_entries=model_entry_results.entries,
diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py
index 64de45f52f..c2f6fc1ea4 100644
--- a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py
+++ b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py
@@ -19,7 +19,7 @@
from sqlbuild.compiler.references.main._quoted_reference_call_pattern import (
quoted_reference_call_pattern,
)
-from sqlbuild.compiler.references.main._render_source_relation import render_source_relation
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.compiler.references.types import SqlReferenceKind
from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token
from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry
diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py
index f05bd0aac3..4a80df2b60 100644
--- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py
+++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py
@@ -74,10 +74,10 @@
RelationMarkerTargetResolver,
ScenarioArtifactKind,
)
-from sqlbuild.compiler.references.main._render_source_relation import render_source_relation
from sqlbuild.compiler.references.main.reference_call_prefix_pattern_text import (
reference_call_prefix_pattern_text,
)
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.compiler.references.types import SqlReferenceKind
from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql
from sqlbuild.diagnostics.main.log_debug_event import log_debug_event
diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py
index e65d4fb085..69da86940f 100644
--- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py
+++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py
@@ -75,7 +75,7 @@
MaterializationType,
MicrobatchStrategy,
)
-from sqlbuild.compiler.references.main._render_source_relation import render_source_relation
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.compiler.references.types import SqlReferenceKind
from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME
from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError
diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py
index d63023d0d0..bb1e2c493c 100644
--- a/src/sqlbuild/compiler/planner/models.py
+++ b/src/sqlbuild/compiler/planner/models.py
@@ -1356,6 +1356,7 @@ class PlanOutput:
source_map: dict[str, SourceEntry] = field(default_factory=dict)
source_read_map: dict[str, SourceEntry] = field(default_factory=dict)
hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple)
+ enforce_explicit_references: bool = True
provider_usages: tuple[PlanProviderUsage, ...] = field(default_factory=tuple)
source_freshness: DirectSourceFreshnessPlanningResult | None = None
python_identity_fingerprints: dict[tuple[str, str], Fingerprint] = field(default_factory=dict)
diff --git a/src/sqlbuild/compiler/references/_helpers/project_relations.py b/src/sqlbuild/compiler/references/_helpers/project_relations.py
new file mode 100644
index 0000000000..ef51c8de91
--- /dev/null
+++ b/src/sqlbuild/compiler/references/_helpers/project_relations.py
@@ -0,0 +1,79 @@
+"""The single owner of which relation names count as project relations."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+
+from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation
+from sqlbuild.compiler.references._helpers.relation_names import extract_relation_names_impl
+from sqlbuild.compiler.references.models import ProjectRelation, ProjectRelationIndex, RelationName
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
+from sqlbuild.spec.contracts.models import SourceEntry
+
+
+def build_project_relation_index_impl(
+ relations: Iterable[ProjectRelation],
+) -> ProjectRelationIndex:
+ """Return the deduplicated project-relation index for hard-coded name checks."""
+
+ return ProjectRelationIndex(relations=tuple(dict.fromkeys(relations)))
+
+
+def compiled_project_relations_impl(project: CompiledProject) -> ProjectRelationIndex:
+ """Return the project relations a compiled project declares."""
+
+ relations: list[ProjectRelation] = [
+ _location_relation(
+ ref=SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=model.name),
+ location=model.destination,
+ )
+ for model in project.models
+ ]
+ relations.extend(
+ _location_relation(
+ ref=SqlResourceRef(kind=SqlResourceRefKind.SEED, name=seed.name),
+ location=seed.destination,
+ )
+ for seed in project.seeds
+ )
+ for source in project.sources:
+ entry: SourceEntry = source.source_entry
+ if entry.expression is not None:
+ continue
+ relations.append(
+ ProjectRelation(
+ ref=SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source.name),
+ relation=RelationName(
+ name=entry.table or entry.name, schema=entry.schema, database=entry.database
+ ),
+ )
+ )
+ return build_project_relation_index_impl(relations)
+
+
+def runtime_project_relations_impl(
+ *, relations: Mapping[SqlResourceRef, str], dialect: str | None
+) -> ProjectRelationIndex:
+ """Return project relations from adapter-rendered runtime relation names."""
+
+ resolved: list[ProjectRelation] = []
+ for ref, qualified in relations.items():
+ extracted: tuple[tuple[RelationName, ...], frozenset[str]] | None = (
+ extract_relation_names_impl(sql=f"SELECT 1 FROM {qualified}", dialect=dialect)
+ )
+ if extracted is None or len(extracted[0]) != 1:
+ continue
+ resolved.append(ProjectRelation(ref=ref, relation=extracted[0][0]))
+ return build_project_relation_index_impl(resolved)
+
+
+def _location_relation(
+ *, ref: SqlResourceRef, location: CompiledRelationLocation
+) -> ProjectRelation:
+ return ProjectRelation(
+ ref=ref,
+ relation=RelationName(
+ name=location.name, schema=location.schema, database=location.database
+ ),
+ )
diff --git a/src/sqlbuild/compiler/references/_helpers/relation_names.py b/src/sqlbuild/compiler/references/_helpers/relation_names.py
new file mode 100644
index 0000000000..ca9f188d6f
--- /dev/null
+++ b/src/sqlbuild/compiler/references/_helpers/relation_names.py
@@ -0,0 +1,138 @@
+"""Relation-name extraction and project-relation matching for hard-coded name checks."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any, cast
+
+from sqlbuild.compiler.references.models import ProjectRelation, ProjectRelationIndex, RelationName
+from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql
+
+_GENERIC_DIALECT: str = "generic"
+_TABLE_KEYS: frozenset[str] = frozenset({"name", "schema", "catalog", "alias", "hints"})
+_CTE_LIST_KEY: str = "ctes"
+_CTE_ALIAS_KEY: str = "alias"
+_CREATE_TABLE_KEY: str = "create_table"
+_TEMPORARY_KEY: str = "temporary"
+_QUALIFIER_SEPARATOR: str = "."
+_SCHEMA_QUALIFIED_PART_COUNT: int = 2
+
+
+def extract_relation_names_impl(
+ *, sql: str, dialect: str | None
+) -> tuple[tuple[RelationName, ...], frozenset[str]] | None:
+ """Return relations named in ``sql`` and temporary tables it creates, or None if unparseable."""
+
+ polyglot: Any = import_polyglot_sql()
+ try:
+ statements: list[Any] = polyglot.parse(sql, dialect=dialect or _GENERIC_DIALECT)
+ except polyglot.PolyglotError:
+ return None
+ names: dict[RelationName, None] = {}
+ temporary: set[str] = set()
+ for statement in statements:
+ tree: object = statement.to_dict()
+ cte_names: frozenset[str] = frozenset(_cte_names(tree))
+ temporary.update(_temporary_table_names(tree))
+ for relation in _table_names(tree):
+ if relation.schema is None and relation.database is None:
+ if relation.name.casefold() in cte_names:
+ continue
+ names[relation] = None
+ return tuple(names), frozenset(temporary)
+
+
+def match_project_relation_impl(
+ *,
+ index: ProjectRelationIndex,
+ relation: RelationName,
+ default_database: str | None,
+ default_schema: str | None,
+) -> ProjectRelation | None:
+ """Return the project relation ``relation`` names, resolving missing qualifiers."""
+
+ schema: str | None = relation.schema if relation.schema is not None else default_schema
+ database: str | None = relation.database if relation.database is not None else default_database
+ for candidate in index.relations:
+ target: RelationName = candidate.relation
+ if target.name.casefold() != relation.name.casefold():
+ continue
+ if not _parts_compatible(left=schema, right=target.schema):
+ continue
+ if not _parts_compatible(left=database, right=target.database):
+ continue
+ return candidate
+ return None
+
+
+def _parts_compatible(*, left: str | None, right: str | None) -> bool:
+ return left is None or right is None or left.casefold() == right.casefold()
+
+
+def _walk(node: object) -> Iterator[dict[str, object]]:
+ pending: list[object] = [node]
+ while pending:
+ current: object = pending.pop()
+ if isinstance(current, dict):
+ mapping: dict[str, object] = cast(dict[str, object], current)
+ yield mapping
+ pending.extend(mapping.values())
+ elif isinstance(current, list | tuple):
+ pending.extend(current)
+
+
+def _identifier(value: object) -> str | None:
+ if isinstance(value, dict):
+ name: object = cast(dict[str, object], value).get("name")
+ return name if isinstance(name, str) else None
+ return None
+
+
+def _table_names(tree: object) -> Iterator[RelationName]:
+ for node in _walk(tree):
+ if not _TABLE_KEYS.issubset(node.keys()):
+ continue
+ name: str | None = _identifier(node.get("name"))
+ if name is None:
+ continue
+ schema: str | None = _identifier(node.get("schema"))
+ database: str | None = _identifier(node.get("catalog"))
+ if schema is None and database is None and _QUALIFIER_SEPARATOR in name:
+ parts: list[str] = name.split(_QUALIFIER_SEPARATOR)
+ name = parts[-1]
+ schema = parts[-2]
+ database = parts[-3] if len(parts) > _SCHEMA_QUALIFIED_PART_COUNT else None
+ yield RelationName(name=name, schema=schema, database=database)
+
+
+def _cte_names(tree: object) -> Iterator[str]:
+ for node in _walk(tree):
+ ctes: object = node.get(_CTE_LIST_KEY)
+ if not isinstance(ctes, list):
+ continue
+ for cte in ctes:
+ alias: str | None = (
+ _identifier(cast(dict[str, object], cte).get(_CTE_ALIAS_KEY))
+ if isinstance(cte, dict)
+ else None
+ )
+ if alias is not None:
+ yield alias.casefold()
+
+
+def _temporary_table_names(tree: object) -> Iterator[str]:
+ for node in _walk(tree):
+ create: object = node.get(_CREATE_TABLE_KEY)
+ if not isinstance(create, dict):
+ continue
+ create_table: dict[str, object] = cast(dict[str, object], create)
+ if create_table.get(_TEMPORARY_KEY) is not True:
+ continue
+ table: object = create_table.get("name")
+ name: str | None = (
+ _identifier(cast(dict[str, object], table).get("name"))
+ if isinstance(table, dict)
+ else None
+ )
+ if name is not None:
+ yield name.casefold()
diff --git a/src/sqlbuild/compiler/references/main/_compiled_project_relations.py b/src/sqlbuild/compiler/references/main/_compiled_project_relations.py
new file mode 100644
index 0000000000..c4596c1288
--- /dev/null
+++ b/src/sqlbuild/compiler/references/main/_compiled_project_relations.py
@@ -0,0 +1,13 @@
+"""Compile-time project-relation index entrypoint."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.models import CompiledProject
+from sqlbuild.compiler.references._helpers.project_relations import compiled_project_relations_impl
+from sqlbuild.compiler.references.models import ProjectRelationIndex
+
+
+def compiled_project_relations(*, project: CompiledProject) -> ProjectRelationIndex:
+ """Return the project relations a compiled project declares."""
+
+ return compiled_project_relations_impl(project)
diff --git a/src/sqlbuild/compiler/references/main/extract_relation_names.py b/src/sqlbuild/compiler/references/main/extract_relation_names.py
new file mode 100644
index 0000000000..165d17f5ae
--- /dev/null
+++ b/src/sqlbuild/compiler/references/main/extract_relation_names.py
@@ -0,0 +1,14 @@
+"""Relation-name extraction entrypoint for hard-coded name checks."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.references._helpers.relation_names import extract_relation_names_impl
+from sqlbuild.compiler.references.models import RelationName
+
+
+def extract_relation_names(
+ *, sql: str, dialect: str | None
+) -> tuple[tuple[RelationName, ...], frozenset[str]] | None:
+ """Return relations named in ``sql`` and temporary tables it creates, or None if unparseable."""
+
+ return extract_relation_names_impl(sql=sql, dialect=dialect)
diff --git a/src/sqlbuild/compiler/references/main/match_project_relation.py b/src/sqlbuild/compiler/references/main/match_project_relation.py
new file mode 100644
index 0000000000..1ca1320a1c
--- /dev/null
+++ b/src/sqlbuild/compiler/references/main/match_project_relation.py
@@ -0,0 +1,23 @@
+"""Project-relation matching entrypoint for hard-coded name checks."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.references._helpers.relation_names import match_project_relation_impl
+from sqlbuild.compiler.references.models import ProjectRelation, ProjectRelationIndex, RelationName
+
+
+def match_project_relation(
+ *,
+ index: ProjectRelationIndex,
+ relation: RelationName,
+ default_database: str | None = None,
+ default_schema: str | None = None,
+) -> ProjectRelation | None:
+ """Return the project relation ``relation`` names, resolving missing qualifiers."""
+
+ return match_project_relation_impl(
+ index=index,
+ relation=relation,
+ default_database=default_database,
+ default_schema=default_schema,
+ )
diff --git a/src/sqlbuild/compiler/references/main/_render_source_relation.py b/src/sqlbuild/compiler/references/main/render_source_relation.py
similarity index 100%
rename from src/sqlbuild/compiler/references/main/_render_source_relation.py
rename to src/sqlbuild/compiler/references/main/render_source_relation.py
diff --git a/src/sqlbuild/compiler/references/main/runtime_project_relations.py b/src/sqlbuild/compiler/references/main/runtime_project_relations.py
new file mode 100644
index 0000000000..3d2f052022
--- /dev/null
+++ b/src/sqlbuild/compiler/references/main/runtime_project_relations.py
@@ -0,0 +1,17 @@
+"""Run-time project-relation index entrypoint."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from sqlbuild.compiler.references._helpers.project_relations import runtime_project_relations_impl
+from sqlbuild.compiler.references.models import ProjectRelationIndex
+from sqlbuild.python_nodes.models import SqlResourceRef
+
+
+def runtime_project_relations(
+ *, relations: Mapping[SqlResourceRef, str], dialect: str | None
+) -> ProjectRelationIndex:
+ """Return project relations from adapter-rendered runtime relation names."""
+
+ return runtime_project_relations_impl(relations=relations, dialect=dialect)
diff --git a/src/sqlbuild/compiler/references/models.py b/src/sqlbuild/compiler/references/models.py
new file mode 100644
index 0000000000..3945155419
--- /dev/null
+++ b/src/sqlbuild/compiler/references/models.py
@@ -0,0 +1,31 @@
+"""Structured facts for explicit-reference enforcement."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from sqlbuild.python_nodes.models import SqlResourceRef
+
+
+@dataclass(frozen=True)
+class RelationName:
+ """A relation named in SQL text, with any qualifier parts that were written."""
+
+ name: str
+ schema: str | None = None
+ database: str | None = None
+
+
+@dataclass(frozen=True)
+class ProjectRelation:
+ """A relation SQLBuild manages for one project resource."""
+
+ ref: SqlResourceRef
+ relation: RelationName
+
+
+@dataclass(frozen=True)
+class ProjectRelationIndex:
+ """Every relation name that counts as a project relation for hard-coded name checks."""
+
+ relations: tuple[ProjectRelation, ...] = field(default_factory=tuple)
diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/observation.py b/src/sqlbuild/compiler/source_freshness/_helpers/observation.py
index 0e34ebe1bd..64674e5b64 100644
--- a/src/sqlbuild/compiler/source_freshness/_helpers/observation.py
+++ b/src/sqlbuild/compiler/source_freshness/_helpers/observation.py
@@ -11,7 +11,7 @@
TableFreshnessMetadata,
TableFreshnessRequest,
)
-from sqlbuild.compiler.references.main._render_source_relation import render_source_relation
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.compiler.source_freshness._helpers.datetime import normalize_presumed_utc_datetime
from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError
from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation
diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py
index 46f789558d..822653ded1 100644
--- a/src/sqlbuild/executor/build/classes/build_scheduler.py
+++ b/src/sqlbuild/executor/build/classes/build_scheduler.py
@@ -1053,6 +1053,7 @@ def _execute_model_node(
query_change_tracking=self._query_change_tracking,
schema_prepared=self._schema_prepared,
hook_functions=self._plan.hook_functions,
+ enforce_explicit_references=self._plan.enforce_explicit_references,
effective_target_name=self._target,
effective_vars=self._effective_vars,
providers=self._providers,
diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py
index 272468d85f..ea63510a98 100644
--- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py
+++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py
@@ -5,6 +5,7 @@
import logging
import time
from collections.abc import Callable
+from dataclasses import replace
from typing import Any
from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder
@@ -24,6 +25,7 @@
evaluate_python_node_fan_in,
normalize_python_node_return,
)
+from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
from sqlbuild.executor.python_nodes.models import (
AssetContext,
PythonNodeExecutionResult,
@@ -177,6 +179,7 @@ def _execute_ready_node_in_cost_scope(
)
_persist_python_node_result(result_store=result_store, result=result, run_id=run_id)
return result, None
+ warnings: list[str] = []
context: TaskContext | AssetContext = _build_context(
node=node,
node_kind=node_kind,
@@ -185,6 +188,7 @@ def _execute_ready_node_in_cost_scope(
logger=logger,
run_state=run_state,
result_store=result_store,
+ warnings=warnings,
)
try:
result: PythonNodeExecutionResult = _call_node_with_retry(
@@ -201,8 +205,10 @@ def _execute_ready_node_in_cost_scope(
kind=node_kind,
error=error,
)
+ result = replace(result, warning_messages=tuple(warnings))
_persist_python_node_result(result_store=result_store, result=result, run_id=run_id)
return result, None
+ result = replace(result, warning_messages=tuple(warnings))
_persist_python_node_result(result_store=result_store, result=result, run_id=run_id)
return result, "explicit" if result.status == PythonNodeStatus.SKIPPED else None
@@ -348,6 +354,7 @@ def _build_context(
logger: logging.Logger | None,
run_state: PythonNodeRunState,
result_store: Any | None,
+ warnings: list[str],
) -> TaskContext | AssetContext:
context_logger: logging.Logger = logger or logging.getLogger(
f"sqlbuild.{node_kind.value}.{node.name}"
@@ -377,6 +384,9 @@ def _build_context(
default_schema=runtime.default_schema,
relation_targets=runtime.resolved_relation_targets,
allowed_sql_refs=allowed_sql_refs,
+ relation_guard=_node_relation_guard(
+ node=node, node_kind=node_kind, runtime=runtime, warnings=warnings
+ ),
providers=providers,
start_cursor_ts=runtime.start_cursor_ts,
end_cursor_ts=runtime.end_cursor_ts,
@@ -385,6 +395,26 @@ def _build_context(
)
+def _node_relation_guard(
+ *,
+ node: ExecutablePythonNode,
+ node_kind: PythonNodeKind,
+ runtime: PythonNodeRuntime,
+ warnings: list[str],
+) -> RuntimeRelationGuard | None:
+ if runtime.project_relations is None:
+ return None
+ return RuntimeRelationGuard(
+ owner_label=f"{node_kind.value} '{node.name}'",
+ declare_help="declare it with depends_on={typed}",
+ project_relations=runtime.project_relations,
+ dialect=runtime.adapter.sql_analysis_dialect(),
+ default_database=runtime.default_database,
+ default_schema=runtime.default_schema,
+ warnings=warnings,
+ )
+
+
def _node_kind(node: ExecutablePythonNode) -> PythonNodeKind:
if isinstance(node, DiscoveredAssetFunction):
return PythonNodeKind.ASSET
diff --git a/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py b/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py
new file mode 100644
index 0000000000..a1e68af7ea
--- /dev/null
+++ b/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py
@@ -0,0 +1,104 @@
+"""Run-time detection of project relation names hard-coded in Python SQL."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping
+
+from sqlbuild.compiler.references.main.extract_relation_names import extract_relation_names
+from sqlbuild.compiler.references.main.match_project_relation import match_project_relation
+from sqlbuild.compiler.references.main.runtime_project_relations import (
+ runtime_project_relations,
+)
+from sqlbuild.compiler.references.models import (
+ ProjectRelation,
+ ProjectRelationIndex,
+ RelationName,
+)
+from sqlbuild.python_nodes.models import SqlResourceRef
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
+
+class RuntimeRelationGuard:
+ """Warn when ctx.query() or ctx.execute_sql() SQL hard-codes a project relation name."""
+
+ def __init__( # noqa: PLR0913
+ self,
+ *,
+ owner_label: str,
+ declare_help: str,
+ project_relations: Mapping[SqlResourceRef, str],
+ dialect: str | None,
+ default_database: str | None,
+ default_schema: str | None,
+ warnings: list[str],
+ own_refs: frozenset[SqlResourceRef] = frozenset(),
+ ) -> None:
+ self._owner_label: str = owner_label
+ self._declare_help: str = declare_help
+ self._project_relations: Mapping[SqlResourceRef, str] = project_relations
+ self._dialect: str | None = dialect
+ self._default_database: str | None = default_database
+ self._default_schema: str | None = default_schema
+ self._warnings: list[str] = warnings
+ self._resolved_refs: set[SqlResourceRef] = set(own_refs)
+ self._reported_refs: set[SqlResourceRef] = set()
+ self._temporary_names: set[str] = set()
+ self._index: ProjectRelationIndex | None = None
+
+ def record_resolved(self, ref: SqlResourceRef) -> None:
+ """Record a relation the node obtained through ctx.relation()."""
+
+ self._resolved_refs.add(ref)
+
+ def check(self, sql: str) -> None:
+ """Warn once per project relation that ``sql`` names without ctx.relation()."""
+
+ extracted: tuple[tuple[RelationName, ...], frozenset[str]] | None = extract_relation_names(
+ sql=sql, dialect=self._dialect
+ )
+ if extracted is None:
+ _LOGGER.debug(
+ "%s sent SQL that could not be parsed for hard-coded relation names",
+ self._owner_label,
+ )
+ return
+ relations, temporary = extracted
+ self._temporary_names.update(temporary)
+ for relation in relations:
+ if relation.schema is None and relation.name.casefold() in self._temporary_names:
+ continue
+ match: ProjectRelation | None = match_project_relation(
+ index=self._relation_index(),
+ relation=relation,
+ default_database=self._default_database,
+ default_schema=self._default_schema,
+ )
+ if (
+ match is None
+ or match.ref in self._resolved_refs
+ or match.ref in self._reported_refs
+ ):
+ continue
+ self._reported_refs.add(match.ref)
+ self._warnings.append(self._warning(match=match, relation=relation))
+
+ def _relation_index(self) -> ProjectRelationIndex:
+ if self._index is None:
+ self._index = runtime_project_relations(
+ relations=self._project_relations, dialect=self._dialect
+ )
+ return self._index
+
+ def _warning(self, *, match: ProjectRelation, relation: RelationName) -> str:
+ typed: str = f'{match.ref.kind.value}("{match.ref.name}")'
+ written: str = ".".join(
+ part for part in (relation.database, relation.schema, relation.name) if part
+ )
+ return (
+ f"[P008] {self._owner_label} named {match.ref.kind.value}:{match.ref.name} as "
+ f"'{written}' in SQL without ctx.relation(); "
+ f"{self._declare_help.format(typed=typed)} and use ctx.relation({typed}), or set "
+ "[references] enforce_explicit = false"
+ )
diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py
index e4139cdbe2..71644dfd30 100644
--- a/src/sqlbuild/executor/python_nodes/models.py
+++ b/src/sqlbuild/executor/python_nodes/models.py
@@ -22,6 +22,7 @@
from sqlbuild.errors.contracts.exceptions import ExecutorInputError
from sqlbuild.executor.load.models import LoadExecutionResult
from sqlbuild.executor.node_results.models import NodeResultEnvelope
+from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
from sqlbuild.executor.python_nodes.constants import (
MISSING_DEFAULT,
PYTHON_NODE_RELATION_QUALIFIER_SEPARATOR,
@@ -98,6 +99,7 @@ class PythonNodeExecutionResult:
skip_mode: SkipMode | None = None
skip_reason: str | None = None
error_message: str | None = None
+ warning_messages: tuple[str, ...] = field(default_factory=tuple)
@dataclass(frozen=True)
@@ -145,6 +147,7 @@ class PythonNodeRuntime:
default_database: str | None = None
default_schema: str | None = None
relation_targets: dict[SqlResourceRef, str] | None = None
+ project_relations: dict[SqlResourceRef, str] | None = None
start_cursor_ts: datetime | None = None
end_cursor_ts: datetime | None = None
start_cursor_int: int | None = None
@@ -189,6 +192,7 @@ class BasePythonNodeContext:
default_schema: str | None = None
relation_targets: dict[SqlResourceRef, str] = field(default_factory=dict)
allowed_sql_refs: frozenset[SqlResourceRef] = frozenset()
+ relation_guard: RuntimeRelationGuard | None = field(default=None, repr=False)
providers: ProviderContainer = field(default_factory=_empty_provider_container)
use_color: bool = False
start_cursor_ts: datetime | None = None
@@ -197,10 +201,14 @@ class BasePythonNodeContext:
end_cursor_int: int | None = None
def execute_sql(self, sql: str) -> Any:
+ if self.relation_guard is not None:
+ self.relation_guard.check(sql)
self.statement_recorder.record(sql)
return self.adapter.execute(connection=self.connection, sql=sql)
def query(self, sql: str) -> Any:
+ if self.relation_guard is not None:
+ self.relation_guard.check(sql)
self.statement_recorder.record(sql)
return self.adapter.execute(connection=self.connection, sql=sql)
@@ -236,6 +244,8 @@ def relation(self, ref: SqlResourceRef) -> str:
relation: str | None = self.relation_targets.get(ref)
if relation is None:
raise ExecutorInputError(f"No runtime relation found for SQL ref '{ref.name}'")
+ if self.relation_guard is not None:
+ self.relation_guard.record_resolved(ref)
return relation
def skip(
diff --git a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py
index 7e86a7a9bd..0c2cbbaca8 100644
--- a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py
+++ b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+from collections.abc import Mapping
+
from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder
from sqlbuild.compiler.planner.models import ModelPlanEntry
from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context
@@ -13,6 +15,7 @@
)
from sqlbuild.executor.run.models import (
HookExecutionResult,
+ HookRelationLookup,
HookRunContext,
ModelExecutionResult,
ModelMaterializationContext,
@@ -45,7 +48,7 @@ def run_pre_hook_phase(
hook_functions=context.hook_functions,
hook_results=hook_results,
hook_run=build_model_hook_run(
- context=context, statement_recorder=statement_recorder
+ context=context, statement_recorder=statement_recorder, warnings=warnings
),
)
if pre_hook_skipped:
@@ -95,7 +98,7 @@ def run_post_hook_phase(
hook_functions=context.hook_functions,
hook_results=hook_results,
hook_run=build_model_hook_run(
- context=context, statement_recorder=statement_recorder
+ context=context, statement_recorder=statement_recorder, warnings=warnings
),
)
except Exception as exc:
@@ -119,6 +122,8 @@ def build_model_hook_run(
*,
context: ModelMaterializationContext,
statement_recorder: StatementRecorder,
+ warnings: list[str],
+ effective_vars: Mapping[str, object] | None = None,
) -> HookRunContext:
"""Build the hook run context for one model's lifecycle hooks."""
@@ -127,8 +132,15 @@ def build_model_hook_run(
destination=context.entry.destination,
run_id=context.run_id,
target=context.effective_target_name,
- effective_vars=context.effective_vars,
+ effective_vars=context.effective_vars if effective_vars is None else effective_vars,
statement_recorder=statement_recorder,
providers=context.providers,
python_identity_recorder=context.python_identity_recorder,
+ relation_lookup=HookRelationLookup(
+ model_locations=context.model_locations,
+ seed_locations=context.seed_locations,
+ source_map=context.source_map,
+ ),
+ warnings=warnings,
+ enforce_explicit_references=context.enforce_explicit_references,
)
diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py
index 865bb32249..5810b046fa 100644
--- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py
+++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from collections.abc import Mapping
from typing import Any
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
@@ -19,9 +20,11 @@
from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity
from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity
from sqlbuild.compiler.python_nodes.types import SkipMode
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
from sqlbuild.cost.classes.cost_context import CostContext
from sqlbuild.cost.models import CostResourceContext
from sqlbuild.errors.contracts.exceptions import ExecutorInputError
+from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
from sqlbuild.executor.python_nodes.main._fingerprinting import (
try_write_python_node_identity_fingerprint,
)
@@ -29,7 +32,9 @@
from sqlbuild.executor.run.models import (
HookContext,
HookExecutionResult,
+ HookInvocation,
HookRelation,
+ HookRelationLookup,
HookRunContext,
HookSkipResult,
)
@@ -39,7 +44,10 @@
_empty_provider_container,
invoke_with_providers,
)
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle
+from sqlbuild.spec.contracts.models import SourceEntry
def execute_hooks(
@@ -215,16 +223,27 @@ def _invoke_python_hook(
)
raise ExecutorInputError(error_message)
- context: HookContext = build_hook_context(
- connection=connection,
- adapter=adapter,
- hook_entry=hook_entry,
- hook_index=hook_index,
- phase=phase,
- model_name=model_name,
- destination=destination,
- hook_run=hook_run,
- )
+ try:
+ context: HookContext = build_hook_context(
+ connection=connection,
+ adapter=adapter,
+ hook_function=hook_function,
+ invocation=HookInvocation(entry=hook_entry, index=hook_index, phase=phase),
+ model_name=model_name,
+ destination=destination,
+ hook_run=hook_run,
+ )
+ except ExecutorInputError as exc:
+ _record_hook_result(
+ hook_results=hook_results,
+ phase=phase,
+ hook_index=hook_index,
+ hook_type="python",
+ label=hook_entry.name,
+ status=ExecutionStatus.FAILED,
+ error_message=f"{hook_label} failed: {exc}",
+ )
+ raise ExecutorInputError(f"{hook_label} failed: {exc}") from exc
with OperationLifecycle(
operation_kind="python_node",
operation_name="python_hook",
@@ -423,13 +442,12 @@ def _sql_hook_preview(statement: str) -> str:
return normalized[:77] + "..."
-def build_hook_context(
+def build_hook_context( # noqa: PLR0913
*,
connection: Any,
adapter: BaseAdapter,
- hook_entry: PythonHookEntry,
- hook_index: int,
- phase: HookPhase,
+ hook_function: DiscoveredHookFunction,
+ invocation: HookInvocation,
model_name: str,
destination: CompiledRelationLocation,
hook_run: HookRunContext,
@@ -442,9 +460,9 @@ def build_hook_context(
)
return HookContext(
model_name=model_name,
- phase=phase,
- hook_name=hook_entry.name,
- hook_index=hook_index,
+ phase=invocation.phase,
+ hook_name=invocation.entry.name,
+ hook_index=invocation.index,
run_id=hook_run.run_id,
target=hook_run.target,
vars=hook_run.effective_vars if hook_run.effective_vars is not None else {},
@@ -458,7 +476,70 @@ def build_hook_context(
providers=hook_run.providers
if hook_run.providers is not None
else _empty_provider_container(),
+ relations=_resolve_hook_relations(
+ adapter=adapter, refs=hook_function.reads, lookup=hook_run.relation_lookup
+ ),
+ relation_guard=_hook_relation_guard(
+ adapter=adapter,
+ hook_name=hook_function.name,
+ model_name=model_name,
+ hook_run=hook_run,
+ ),
+ )
+
+
+def _hook_relation_guard(
+ *, adapter: BaseAdapter, hook_name: str, model_name: str, hook_run: HookRunContext
+) -> RuntimeRelationGuard | None:
+ if not hook_run.enforce_explicit_references or hook_run.warnings is None:
+ return None
+ lookup: HookRelationLookup = hook_run.relation_lookup
+ all_refs: tuple[SqlResourceRef, ...] = (
+ *(
+ SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=name)
+ for name in lookup.model_locations
+ ),
+ *(
+ SqlResourceRef(kind=SqlResourceRefKind.SEED, name=name)
+ for name in lookup.seed_locations
+ ),
+ *(SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=name) for name in lookup.source_map),
)
+ return RuntimeRelationGuard(
+ owner_label=f"hook '{hook_name}' on model '{model_name}'",
+ declare_help="declare it with @hook(reads={typed})",
+ project_relations=_resolve_hook_relations(adapter=adapter, refs=all_refs, lookup=lookup),
+ dialect=adapter.sql_analysis_dialect(),
+ default_database=adapter.default_database(),
+ default_schema=adapter.default_schema(),
+ warnings=hook_run.warnings,
+ own_refs=frozenset({SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=model_name)}),
+ )
+
+
+def _resolve_hook_relations(
+ *, adapter: BaseAdapter, refs: tuple[SqlResourceRef, ...], lookup: HookRelationLookup
+) -> dict[SqlResourceRef, str]:
+ relations: dict[SqlResourceRef, str] = {}
+ for ref in refs:
+ if ref.kind is SqlResourceRefKind.SOURCE:
+ entry: SourceEntry | None = lookup.source_map.get(ref.name)
+ if entry is None:
+ raise ExecutorInputError(f"No runtime relation found for source '{ref.name}'")
+ relations[ref] = render_source_relation(entry=entry, adapter=adapter)
+ continue
+ locations: Mapping[str, CompiledRelationLocation] = (
+ lookup.model_locations
+ if ref.kind is SqlResourceRefKind.MODEL
+ else lookup.seed_locations
+ )
+ location: CompiledRelationLocation | None = locations.get(ref.name)
+ if location is None:
+ raise ExecutorInputError(f"No runtime relation found for {ref.kind.value} '{ref.name}'")
+ relations[ref] = resolve_relation_location_qualified_name(
+ adapter=adapter, location=location
+ )
+ return relations
def _find_hook_function(
diff --git a/src/sqlbuild/executor/run/_helpers/materializations/custom.py b/src/sqlbuild/executor/run/_helpers/materializations/custom.py
index 379705f9ef..a5dc8b2c4d 100644
--- a/src/sqlbuild/executor/run/_helpers/materializations/custom.py
+++ b/src/sqlbuild/executor/run/_helpers/materializations/custom.py
@@ -26,6 +26,7 @@
MaterializationResult,
)
from sqlbuild.executor.run._helpers.execution.final_audits import run_final_scope_audits
+from sqlbuild.executor.run._helpers.execution.hook_phases import build_model_hook_run
from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks
from sqlbuild.executor.run._helpers.execution.results import (
build_failed_result,
@@ -38,7 +39,6 @@
CustomMaterializationPhaseOutcome,
CustomMaterializationSetup,
FinalAuditRun,
- HookRunContext,
ModelExecutionResult,
ModelMaterializationContext,
)
@@ -150,15 +150,11 @@ def _run_custom_pre_hooks(
phase=HookPhase.PRE_HOOKS,
hook_functions=context.hook_functions,
hook_results=state.hook_results,
- hook_run=HookRunContext(
- model_name=entry.name,
- destination=entry.destination,
- run_id=context.run_id,
- target=context.effective_target_name,
- effective_vars=effective_vars,
+ hook_run=build_model_hook_run(
+ context=context,
statement_recorder=state.statement_recorder,
- providers=context.providers,
- python_identity_recorder=context.python_identity_recorder,
+ warnings=state.warnings,
+ effective_vars=effective_vars,
),
)
if pre_hook_skipped:
@@ -336,15 +332,11 @@ def _run_custom_post_hooks(
phase=HookPhase.POST_HOOKS,
hook_functions=context.hook_functions,
hook_results=state.hook_results,
- hook_run=HookRunContext(
- model_name=entry.name,
- destination=entry.destination,
- run_id=context.run_id,
- target=context.effective_target_name,
- effective_vars=effective_vars,
+ hook_run=build_model_hook_run(
+ context=context,
statement_recorder=state.statement_recorder,
- providers=context.providers,
- python_identity_recorder=context.python_identity_recorder,
+ warnings=state.warnings,
+ effective_vars=effective_vars,
),
)
if post_hook_skipped:
diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py
index efc87f234f..d26eb5257d 100644
--- a/src/sqlbuild/executor/run/_helpers/materializations/view.py
+++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py
@@ -17,7 +17,7 @@
from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context
from sqlbuild.executor.auditing.main._execute import execute_audit
from sqlbuild.executor.auditing.models import AuditExecutionResult
-from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder
+from sqlbuild.executor.run._helpers.execution.hook_phases import build_model_hook_run
from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks
from sqlbuild.executor.run._helpers.execution.results import (
build_failed_result,
@@ -26,13 +26,11 @@
from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint
from sqlbuild.executor.run.models import (
HookExecutionResult,
- HookRunContext,
ModelExecutionResult,
ModelMaterializationContext,
)
from sqlbuild.executor.run.types import ExecutionPhase, HookPhase
from sqlbuild.executor.scheduling.types import ExecutionStatus
-from sqlbuild.provider.main.runtime import ProviderContainer
from sqlbuild.spec.contracts.models import SourceEntry
@@ -52,10 +50,7 @@ def execute_view_entry(
run_id: str = context.run_id
query_change_tracking: bool = context.query_change_tracking
hook_functions: tuple[DiscoveredHookFunction, ...] = context.hook_functions
- effective_target_name: str | None = context.effective_target_name
effective_vars: Mapping[str, object] | None = context.effective_vars
- providers: ProviderContainer | None = context.providers
- python_identity_recorder: PythonIdentityRecorder | None = context.python_identity_recorder
target_database: str | None = entry.destination.database
target_schema: str | None = entry.destination.schema
target_qualified: str = resolve_relation_location_qualified_name(
@@ -78,15 +73,11 @@ def execute_view_entry(
phase=HookPhase.PRE_HOOKS,
hook_functions=hook_functions,
hook_results=hook_results,
- hook_run=HookRunContext(
- model_name=entry.name,
- destination=entry.destination,
- run_id=run_id,
- target=effective_target_name,
- effective_vars=effective_vars,
+ hook_run=build_model_hook_run(
+ context=context,
statement_recorder=statement_recorder,
- providers=providers,
- python_identity_recorder=python_identity_recorder,
+ warnings=warnings,
+ effective_vars=effective_vars,
),
)
if pre_hook_skipped:
@@ -179,15 +170,11 @@ def execute_view_entry(
phase=HookPhase.POST_HOOKS,
hook_functions=hook_functions,
hook_results=hook_results,
- hook_run=HookRunContext(
- model_name=entry.name,
- destination=entry.destination,
- run_id=run_id,
- target=effective_target_name,
- effective_vars=effective_vars,
+ hook_run=build_model_hook_run(
+ context=context,
statement_recorder=statement_recorder,
- providers=providers,
- python_identity_recorder=python_identity_recorder,
+ warnings=warnings,
+ effective_vars=effective_vars,
),
)
if post_hook_skipped:
diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py
index 52c4b70829..3e1194b094 100644
--- a/src/sqlbuild/executor/run/models.py
+++ b/src/sqlbuild/executor/run/models.py
@@ -11,7 +11,7 @@
from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder
from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult
from sqlbuild.compiler.compile.models import CompiledRelationLocation
-from sqlbuild.compiler.discovery.models import DiscoveredHookFunction
+from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry
from sqlbuild.compiler.planner.models import (
AuditPlanEntry,
CursorBounds,
@@ -27,8 +27,10 @@
from sqlbuild.cursor_algebra.main.parse import parse
from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue
from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar
+from sqlbuild.errors.contracts.exceptions import ExecutorInputError
from sqlbuild.executor.auditing.models import AuditExecutionResult
from sqlbuild.executor.custom.models import MaterializationResult
+from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder
from sqlbuild.executor.run.types import (
AuditGateMetadataParseFailure,
@@ -43,6 +45,7 @@
from sqlbuild.microbatches.models import MicrobatchScope
from sqlbuild.microbatches.types import MicrobatchEventStore
from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container
+from sqlbuild.python_nodes.models import SqlResourceRef
from sqlbuild.spec.contracts.models import FutureCursorsConfig, SourceEntry, StartCursorsConfig
from sqlbuild.spec.contracts.types import MicrobatchLimitAction
@@ -91,16 +94,34 @@ class HookContext:
connection: Any = field(repr=False)
statement_recorder: StatementRecorder = field(repr=False)
providers: ProviderContainer = field(default_factory=_empty_provider_container, repr=False)
+ relations: Mapping[SqlResourceRef, str] = field(default_factory=dict, repr=False)
+ relation_guard: RuntimeRelationGuard | None = field(default=None, repr=False)
def execute_sql(self, sql: str) -> None:
+ if self.relation_guard is not None:
+ self.relation_guard.check(sql)
self.statement_recorder.record(sql)
self.adapter.execute(connection=self.connection, sql=sql)
def query(self, sql: str) -> list[tuple[object, ...]]:
+ if self.relation_guard is not None:
+ self.relation_guard.check(sql)
self.statement_recorder.record(sql)
result: QueryResult = self.adapter.query(connection=self.connection, sql=sql, limit=None)
return list(result.rows)
+ def relation(self, ref: SqlResourceRef) -> str:
+ """Return the adapter-qualified runtime relation for a declared hook read."""
+
+ relation: str | None = self.relations.get(ref)
+ if relation is None:
+ raise ExecutorInputError(
+ f"SQL relation ref '{ref.name}' must be declared in @hook(reads=...) before use"
+ )
+ if self.relation_guard is not None:
+ self.relation_guard.record_resolved(ref)
+ return relation
+
def log(self, message: str) -> None:
self.statement_recorder.log(message)
@@ -273,6 +294,7 @@ class ModelMaterializationContext:
microbatch_global_concurrency: int = 1
microbatch_batch_runner: MicrobatchBatchRunner | None = None
watermark_resolver: WatermarkResolver | None = None
+ enforce_explicit_references: bool = False
@dataclass(frozen=True, kw_only=True, init=False)
@@ -445,6 +467,24 @@ def __init__(self, **values: Any) -> None:
object.__setattr__(self, field_name, defaults[field_name])
+@dataclass(frozen=True)
+class HookInvocation:
+ """One Python hook entry being invoked in a lifecycle phase."""
+
+ entry: PythonHookEntry
+ index: int
+ phase: HookPhase
+
+
+@dataclass(frozen=True)
+class HookRelationLookup:
+ """Planned relation locations Python hooks resolve declared reads against."""
+
+ model_locations: Mapping[str, CompiledRelationLocation] = field(default_factory=dict)
+ seed_locations: Mapping[str, CompiledRelationLocation] = field(default_factory=dict)
+ source_map: Mapping[str, SourceEntry] = field(default_factory=dict)
+
+
@dataclass(frozen=True)
class HookRunContext:
"""Model-scoped runtime inputs shared by lifecycle hook execution."""
@@ -457,6 +497,9 @@ class HookRunContext:
statement_recorder: StatementRecorder | None = None
providers: ProviderContainer | None = None
python_identity_recorder: PythonIdentityRecorder | None = None
+ relation_lookup: HookRelationLookup = field(default_factory=HookRelationLookup)
+ warnings: list[str] | None = None
+ enforce_explicit_references: bool = False
@dataclass(frozen=True)
diff --git a/src/sqlbuild/hooks.py b/src/sqlbuild/hooks.py
index 4753acb897..f2ef0a686d 100644
--- a/src/sqlbuild/hooks.py
+++ b/src/sqlbuild/hooks.py
@@ -7,7 +7,7 @@
from sqlbuild.executor.run.models import HookContext, HookSkipResult
from sqlbuild.python_nodes.main.apply_hook import apply_hook
from sqlbuild.python_nodes.main.read_hook_definition import read_hook_definition
-from sqlbuild.python_nodes.models import HookDefinition
+from sqlbuild.python_nodes.models import HookDefinition, SqlResourceRef
__all__ = ("HookContext", "HookDefinition", "HookSkipResult", "get_hook_definition", "hook")
@@ -17,10 +17,11 @@ def hook(
*,
name: str | None = None,
description: str | None = None,
+ reads: SqlResourceRef | tuple[SqlResourceRef, ...] | list[SqlResourceRef] = (),
) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]:
"""Mark a Python function as a SQLBuild model lifecycle hook."""
- return apply_hook(function=function, name=name, description=description)
+ return apply_hook(function=function, name=name, description=description, reads=reads)
def get_hook_definition(function: Callable[..., object]) -> HookDefinition | None:
diff --git a/src/sqlbuild/hooks.pyi b/src/sqlbuild/hooks.pyi
index 0190d2fc54..26861edd6f 100644
--- a/src/sqlbuild/hooks.pyi
+++ b/src/sqlbuild/hooks.pyi
@@ -4,6 +4,7 @@ from typing import overload
from sqlbuild.executor.run.models import HookContext as HookContext
from sqlbuild.executor.run.models import HookSkipResult as HookSkipResult
from sqlbuild.python_nodes.models import HookDefinition as HookDefinition
+from sqlbuild.python_nodes.models import SqlResourceRef
__all__ = ("HookContext", "HookDefinition", "HookSkipResult", "get_hook_definition", "hook")
@@ -11,6 +12,9 @@ __all__ = ("HookContext", "HookDefinition", "HookSkipResult", "get_hook_definiti
def hook(function: Callable[..., object]) -> Callable[..., object]: ...
@overload
def hook(
- *, name: str | None = None, description: str | None = None
+ *,
+ name: str | None = None,
+ description: str | None = None,
+ reads: SqlResourceRef | tuple[SqlResourceRef, ...] | list[SqlResourceRef] = (),
) -> Callable[[Callable[..., object]], Callable[..., object]]: ...
def get_hook_definition(function: Callable[..., object]) -> HookDefinition | None: ...
diff --git a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py
index ec6adc9837..6af73f223e 100644
--- a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py
+++ b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py
@@ -2,6 +2,7 @@
from collections.abc import Callable
+from sqlbuild.errors.contracts.exceptions import SharedInputError
from sqlbuild.python_nodes.models import SqlResourceRef
@@ -20,3 +21,18 @@ def normalize_loader_dependencies(
value: tuple[Callable[..., object], ...] | list[Callable[..., object]],
) -> tuple[Callable[..., object], ...]:
return tuple(value)
+
+
+def normalize_hook_reads(
+ value: SqlResourceRef | tuple[SqlResourceRef, ...] | list[SqlResourceRef],
+) -> tuple[SqlResourceRef, ...]:
+ reads: tuple[object, ...] = (value,) if isinstance(value, SqlResourceRef) else tuple(value)
+ invalid: object | None = next(
+ (item for item in reads if not isinstance(item, SqlResourceRef)), None
+ )
+ if invalid is not None:
+ raise SharedInputError(
+ "@hook(reads=...) accepts model(...), source(...), and seed(...) references only; "
+ f"got {type(invalid).__name__}"
+ )
+ return tuple(dict.fromkeys(item for item in reads if isinstance(item, SqlResourceRef)))
diff --git a/src/sqlbuild/python_nodes/main/apply_hook.py b/src/sqlbuild/python_nodes/main/apply_hook.py
index 351e98188c..c74abe3473 100644
--- a/src/sqlbuild/python_nodes/main/apply_hook.py
+++ b/src/sqlbuild/python_nodes/main/apply_hook.py
@@ -6,8 +6,9 @@
from typing import Any, cast
from sqlbuild.python_nodes._helpers.attachment import attach_definition
+from sqlbuild.python_nodes._helpers.dependency_normalization import normalize_hook_reads
from sqlbuild.python_nodes._helpers.description_resolution import resolve_description
-from sqlbuild.python_nodes.models import HookDefinition
+from sqlbuild.python_nodes.models import HookDefinition, SqlResourceRef
def apply_hook(
@@ -15,14 +16,18 @@ def apply_hook(
function: Callable[..., object] | None = None,
name: str | None = None,
description: str | None = None,
+ reads: SqlResourceRef | tuple[SqlResourceRef, ...] | list[SqlResourceRef] = (),
) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]:
"""Apply SQLBuild lifecycle-hook metadata to a Python function."""
+ normalized_reads: tuple[SqlResourceRef, ...] = normalize_hook_reads(reads)
+
def decorate(inner: Callable[..., object]) -> Callable[..., object]:
inner_function: Any = cast(Any, inner)
definition: HookDefinition = HookDefinition(
name=name or inner_function.__name__,
description=resolve_description(function=inner, description=description),
+ reads=normalized_reads,
)
return attach_definition(
function=inner,
diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py
index b9def447a2..ede8bc1cae 100644
--- a/src/sqlbuild/python_nodes/models.py
+++ b/src/sqlbuild/python_nodes/models.py
@@ -128,6 +128,7 @@ class HookDefinition:
name: str
description: str | None = None
+ reads: tuple[SqlResourceRef, ...] = ()
@dataclass(frozen=True)
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
index 5990f584c6..d90d49cb7a 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
@@ -1009,3 +1009,38 @@ class AuditReadPlanE2ETestCase:
description: str
select: str
expected_error_fragment: str
+
+
+@dataclass(frozen=True)
+class ExplicitReferenceBuildE2ETestCase:
+ """A build of a project using typed macro references and Python hook reads."""
+
+ description: str
+ overrides: dict[str, str]
+ expected_exit_code: int
+ expected_dag_edges: tuple[tuple[str, str], ...]
+ expected_customer_counts: tuple[tuple[object, ...], ...]
+ expected_order_before: tuple[str, str]
+
+
+@dataclass(frozen=True)
+class ExplicitReferenceFailureE2ETestCase:
+ """A project whose references are not explicit, and the failure it reports."""
+
+ description: str
+ overrides: dict[str, str]
+ command: tuple[str, ...]
+ expected_exit_code: int
+ expected_output_fragments: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class ExplicitReferenceRuntimeWarningE2ETestCase:
+ """A build whose Python SQL hard-codes a project relation only at run time."""
+
+ description: str
+ overrides: dict[str, str]
+ enforce_explicit: bool
+ expected_exit_code: int
+ expected_warning_fragments: tuple[str, ...]
+ expected_final_line_prefix: str
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py
index 67cfe04111..4d3d22b2f5 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py
@@ -682,3 +682,57 @@ def prepare_audit_read_plan_project(*, tmp_path: Path) -> Path:
),
},
)
+
+
+EXPLICIT_REFERENCE_MODELS: str = "models/sales"
+EXPLICIT_REFERENCE_MACRO_PATH: str = "models/sales/_sqlbuild/_macros/unions.py"
+EXPLICIT_REFERENCE_HOOK_PATH: str = "models/sales/_sqlbuild/_hooks/python/lookups.py"
+EXPLICIT_REFERENCE_MACROS: str = (
+ "def union_all(relations):\n"
+ " return ' UNION ALL '.join(\n"
+ " f'SELECT order_id, amount FROM {relation}' for relation in relations\n"
+ " )\n"
+)
+EXPLICIT_REFERENCE_HOOKS: str = (
+ "from sqlbuild.hooks import hook\n"
+ "from sqlbuild.refs import model\n\n\n"
+ '@hook(reads=model("stg_customers"))\n'
+ "def record_customers(ctx):\n"
+ ' customers = ctx.relation(model("stg_customers"))\n'
+ " ctx.execute_sql(\n"
+ ' f"CREATE OR REPLACE TABLE customer_counts AS SELECT count(*) AS n FROM {customers}"\n'
+ " )\n"
+)
+
+
+def explicit_reference_project_files(
+ *, overrides: dict[str, str], enforce_explicit: bool = True
+) -> dict[str, str]:
+ """Return a DuckDB project using typed macro references and a hook with declared reads."""
+
+ references: str = f"\n[references]\nenforce_explicit = {str(enforce_explicit).lower()}\n"
+ return {
+ "sqlbuild_project.toml": (
+ 'name = "explicit_refs"\nadapter = "duckdb"\n\n'
+ '[connection]\ndatabase = "warehouse.duckdb"\n' + references
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_eu.sql": (
+ "MODEL (materialized table);\nSELECT 1 AS order_id, 10 AS amount\n"
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_us.sql": (
+ "MODEL (materialized table);\nSELECT 2 AS order_id, 20 AS amount\n"
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/stg_customers.sql": (
+ "MODEL (materialized table);\nSELECT 1 AS customer_id\n"
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/all_orders.sql": (
+ "MODEL (materialized table);\n"
+ '@union_all([__ref("stg_orders_eu"), __ref("stg_orders_us")])\n'
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/orders_summary.sql": (
+ 'MODEL (materialized table, post_hooks [python("record_customers")]);\n'
+ 'SELECT count(*) AS order_count FROM __ref("all_orders")\n'
+ ),
+ EXPLICIT_REFERENCE_MACRO_PATH: EXPLICIT_REFERENCE_MACROS,
+ EXPLICIT_REFERENCE_HOOK_PATH: EXPLICIT_REFERENCE_HOOKS,
+ } | overrides
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
new file mode 100644
index 0000000000..6e808adb18
--- /dev/null
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
@@ -0,0 +1,227 @@
+"""E2E tests for explicit, typed references in macros, Python hooks, and Python SQL."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from itertools import chain
+from pathlib import Path
+
+import pytest
+
+from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
+ ExplicitReferenceBuildE2ETestCase,
+ ExplicitReferenceFailureE2ETestCase,
+ ExplicitReferenceRuntimeWarningE2ETestCase,
+)
+from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import (
+ EXPLICIT_REFERENCE_HOOK_PATH,
+ EXPLICIT_REFERENCE_HOOKS,
+ EXPLICIT_REFERENCE_MACRO_PATH,
+ EXPLICIT_REFERENCE_MACROS,
+ EXPLICIT_REFERENCE_MODELS,
+ explicit_reference_project_files,
+)
+from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import (
+ prepare_inline_project,
+ query_duckdb,
+ run_sqb,
+)
+
+_EMITTING_MACROS: str = (
+ EXPLICIT_REFERENCE_MACROS
+ + "\n\ndef orders_base():\n return 'SELECT * FROM __ref(\"stg_orders_eu\")'\n"
+)
+_EMITTING_MODEL: str = "MODEL (materialized table);\n@orders_base()\n"
+_UNDECLARED_HOOKS: str = EXPLICIT_REFERENCE_HOOKS + ' ctx.relation(model("all_orders"))\n'
+_LITERAL_TASK: str = (
+ "from sqlbuild.refs import model\n"
+ "from sqlbuild.tasks import task\n\n\n"
+ '@task(depends_on=model("all_orders"))\n'
+ "def export_orders(ctx):\n"
+ ' ctx.query("SELECT count(*) FROM stg_customers")\n'
+)
+_RUNTIME_TASK: str = (
+ "from sqlbuild.refs import model\n"
+ "from sqlbuild.tasks import task\n\n\n"
+ '@task(depends_on=model("all_orders"))\n'
+ "def export_orders(ctx):\n"
+ ' orders = ctx.relation(model("all_orders"))\n'
+ ' ctx.query(f"SELECT count(*) FROM {orders}")\n'
+ ' customers = "stg_" + "customers"\n'
+ ' ctx.query(f"SELECT count(*) FROM {customers}")\n'
+)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ ExplicitReferenceBuildE2ETestCase(
+ description="typed macro references record lineage and hook reads order the build",
+ overrides={},
+ expected_exit_code=0,
+ expected_dag_edges=(
+ ("model:stg_orders_eu", "model:all_orders"),
+ ("model:stg_orders_us", "model:all_orders"),
+ ),
+ expected_customer_counts=((1,),),
+ expected_order_before=("stg_customers", "orders_summary"),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_and_order_hold(
+ test_case: ExplicitReferenceBuildE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="explicit_refs",
+ repo_files=explicit_reference_project_files(overrides=test_case.overrides),
+ )
+
+ dag: subprocess.CompletedProcess[str] = run_sqb(
+ command=("dag", "--json"), project_dir=project_dir
+ )
+ build: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build", "--concurrency", "4"), project_dir=project_dir
+ )
+
+ assert dag.returncode == 0, dag.stdout + dag.stderr
+ edges: set[tuple[str, str]] = {
+ (edge["from_id"], edge["to_id"]) for edge in json.loads(dag.stdout)["edges"]
+ }
+ assert set(test_case.expected_dag_edges) <= edges
+ assert build.returncode == test_case.expected_exit_code, build.stdout + build.stderr
+ assert query_duckdb(
+ db_path=project_dir / "warehouse.duckdb", sql="SELECT n FROM customer_counts"
+ ) == list(test_case.expected_customer_counts)
+ first, second = test_case.expected_order_before
+ assert build.stdout.index(f" {first} ") < build.stdout.index(f" {second} ")
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ ExplicitReferenceFailureE2ETestCase(
+ description="macro emitting a reference fails compile",
+ overrides={
+ EXPLICIT_REFERENCE_MACRO_PATH: _EMITTING_MACROS,
+ f"{EXPLICIT_REFERENCE_MODELS}/hidden.sql": _EMITTING_MODEL,
+ },
+ command=("--no-color", "compile"),
+ expected_exit_code=1,
+ expected_output_fragments=(
+ "error[P006]: model:hidden depends on model:stg_orders_eu through macro "
+ "orders_base()",
+ "--> models/sales/_sqlbuild/_macros/unions.py",
+ '@orders_base(__ref("stg_orders_eu"))',
+ "[references] enforce_explicit = false",
+ ),
+ ),
+ ExplicitReferenceFailureE2ETestCase(
+ description="hook resolving an undeclared relation fails the build",
+ overrides={EXPLICIT_REFERENCE_HOOK_PATH: _UNDECLARED_HOOKS},
+ command=("--no-color", "build"),
+ expected_exit_code=1,
+ expected_output_fragments=(
+ "SQL relation ref 'all_orders' must be declared in @hook(reads=...)",
+ ),
+ ),
+ ExplicitReferenceFailureE2ETestCase(
+ description="literal SQL naming a project relation fails compile",
+ overrides={"python/tasks/export.py": _LITERAL_TASK},
+ command=("--no-color", "compile"),
+ expected_exit_code=1,
+ expected_output_fragments=(
+ "error[P008]: task:export_orders names model:stg_customers as 'stg_customers'",
+ "--> python/tasks/export.py:7",
+ "[references] enforce_explicit = false",
+ ),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_implicit_reference_when_running_command_then_it_fails_with_guidance(
+ test_case: ExplicitReferenceFailureE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="explicit_refs",
+ repo_files=explicit_reference_project_files(overrides=test_case.overrides),
+ )
+
+ result: subprocess.CompletedProcess[str] = run_sqb(
+ command=test_case.command, project_dir=project_dir
+ )
+
+ output: str = result.stdout + result.stderr
+ assert result.returncode == test_case.expected_exit_code, output
+ assert all(fragment in output for fragment in test_case.expected_output_fragments)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ ExplicitReferenceRuntimeWarningE2ETestCase(
+ description="run-time hard-coded name warns while the build succeeds",
+ overrides={"python/tasks/export.py": _RUNTIME_TASK},
+ enforce_explicit=True,
+ expected_exit_code=0,
+ expected_warning_fragments=(
+ "[P008] task 'export_orders' named model:stg_customers as 'stg_customers'",
+ ),
+ expected_final_line_prefix="\u2713 Completed with warnings",
+ ),
+ ExplicitReferenceRuntimeWarningE2ETestCase(
+ description="enforcement disabled allows every implicit reference",
+ overrides={
+ "python/tasks/export.py": _RUNTIME_TASK,
+ EXPLICIT_REFERENCE_MACRO_PATH: _EMITTING_MACROS,
+ f"{EXPLICIT_REFERENCE_MODELS}/hidden.sql": _EMITTING_MODEL,
+ "python/tasks/literal.py": _LITERAL_TASK.replace(
+ "export_orders", "count_customers"
+ ),
+ },
+ enforce_explicit=False,
+ expected_exit_code=0,
+ expected_warning_fragments=(),
+ expected_final_line_prefix="\u2713 Completed successfully",
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_runtime_hard_coded_relation_when_building_then_it_warns_without_failing(
+ test_case: ExplicitReferenceRuntimeWarningE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="explicit_refs",
+ repo_files=explicit_reference_project_files(
+ overrides=test_case.overrides, enforce_explicit=test_case.enforce_explicit
+ ),
+ )
+
+ text: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build"), project_dir=project_dir
+ )
+ machine: subprocess.CompletedProcess[str] = run_sqb(
+ command=("build", "--json"), project_dir=project_dir
+ )
+
+ assert text.returncode == test_case.expected_exit_code, text.stdout + text.stderr
+ assert machine.returncode == test_case.expected_exit_code, machine.stdout + machine.stderr
+ assert all(fragment in text.stdout for fragment in test_case.expected_warning_fragments)
+ assert text.stdout.strip().splitlines()[-1].startswith(test_case.expected_final_line_prefix)
+ stored: list[str] = list(
+ chain.from_iterable(
+ asset.get("warnings") or () for asset in json.loads(machine.stdout)["assets"]
+ )
+ )
+ assert len(stored) == len(test_case.expected_warning_fragments)
+ assert all(
+ fragment in warning
+ for warning, fragment in zip(stored, test_case.expected_warning_fragments, strict=True)
+ )
diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py
index be8a84b858..d76c121ae8 100644
--- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py
+++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py
@@ -622,6 +622,29 @@ def test_given_audit_results_when_aggregating_then_produces_expected_entries(
),
unexpected_fragments=("\033[",),
),
+ BuildFooterTestCase(
+ description="model warning counts the model as warned and precedes the completion line",
+ result=BuildExecutionResult(
+ status=BuildStatus.SUCCESS,
+ model_results=(
+ ModelExecutionResult(
+ model_name="orders",
+ status=ExecutionStatus.SUCCESS,
+ warning_messages=("[P008] hook 'refresh_lookup' named model:customers",),
+ ),
+ ),
+ success_count=1,
+ warning_count=1,
+ ),
+ expected_fragments=(
+ "Warnings:",
+ "[P008] hook 'refresh_lookup' named model:customers\n\n\u2713 Completed with warnings",
+ "PASS=0",
+ "WARN=1",
+ "TOTAL=1",
+ ),
+ unexpected_fragments=("\033[",),
+ ),
BuildFooterTestCase(
description="warning footer uses semantic colors",
result=BuildExecutionResult(
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
index 38006f78a5..8f774763e1 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
@@ -985,3 +985,39 @@ class TypedReferenceAdapterRenderingTestCase:
description: str
adapter_name: str
expected_resolved_sql: str
+
+
+@dataclass(frozen=True)
+class HookReadEdgeTestCase:
+ """A Python hook declaring reads, and the ordering edges its model gains."""
+
+ description: str
+ files: dict[str, str]
+ expected_edges: tuple[tuple[str, str], ...]
+
+
+@dataclass(frozen=True)
+class HookReadErrorTestCase:
+ """A Python hook declaring reads that cannot be satisfied."""
+
+ description: str
+ files: dict[str, str]
+ expected_error_fragments: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class PythonSqlReferenceErrorTestCase:
+ """A task, asset, or hook whose literal SQL hard-codes a project relation name."""
+
+ description: str
+ files: dict[str, str]
+ expected_error_fragments: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class PythonSqlReferenceAllowedTestCase:
+ """Python SQL that compiles because it names no undeclared project relation literally."""
+
+ description: str
+ files: dict[str, str]
+ expected_model_names: tuple[str, ...]
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
index e3c654405c..3682e528bf 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
@@ -347,3 +347,27 @@ def resolve_model_references_for_adapter(*, sql: str, adapter_name: str) -> str:
cursor_type=None,
lower_bound_inclusive=True,
)
+
+
+def python_hook_source(*, reads: str = "()", body: str = " return None\n") -> str:
+ """Return a Python hook module defining ``refresh_lookup`` with declared reads."""
+
+ return (
+ "from sqlbuild.hooks import hook\n"
+ "from sqlbuild.refs import model, seed, source\n\n\n"
+ f"@hook(reads={reads})\n"
+ "def refresh_lookup(ctx):\n"
+ f"{body}"
+ )
+
+
+def python_task_source(*, body: str) -> str:
+ """Return a Python task module defining ``export_orders`` that depends on ``orders``."""
+
+ return (
+ "from sqlbuild.refs import model\n"
+ "from sqlbuild.tasks import task\n\n\n"
+ '@task(depends_on=model("orders"))\n'
+ "def export_orders(ctx):\n"
+ f"{body}"
+ )
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
new file mode 100644
index 0000000000..2d80815ede
--- /dev/null
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
@@ -0,0 +1,133 @@
+"""Python hooks that declare reads order their model after them and cannot read its dependants."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+
+from sqlbuild.compiler.compile.models import CompiledProject
+from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import (
+ HookReadEdgeTestCase,
+ HookReadErrorTestCase,
+)
+from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import (
+ compile_and_assemble,
+ execution_edge_names,
+ gate_model,
+ python_hook_source,
+)
+
+_PROJECT_FILE: str = """
+name = "demo"
+adapter = "duckdb"
+
+[settings]
+sql_analysis = false
+sql_validation = false
+"""
+_HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/python/lookups.py"
+_HOOKED_HEADER: str = 'MODEL (post_hooks [python("refresh_lookup")]);'
+_COUNTRY_SEED: str = (
+ "seeds:\n - name: country_codes\n columns:\n - name: code\n type: VARCHAR\n"
+)
+_SOURCES: str = "sources:\n - name: raw_orders\n table: orders\n"
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ HookReadEdgeTestCase(
+ description="hook reading a model makes the hooked model wait for it",
+ files={
+ _HOOK_PATH: python_hook_source(reads='model("customers")'),
+ "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"),
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id", header=_HOOKED_HEADER
+ ),
+ },
+ expected_edges=(("orders", "customers"),),
+ ),
+ HookReadEdgeTestCase(
+ description="hook reading a list of a seed and a source adds one edge per read",
+ files={
+ _HOOK_PATH: python_hook_source(
+ reads='[seed("country_codes"), source("raw_orders")]'
+ ),
+ "seeds/country_codes.yml": _COUNTRY_SEED,
+ "seeds/country_codes.csv": "code\nGB\n",
+ "sources/raw.yml": _SOURCES,
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id", header=_HOOKED_HEADER
+ ),
+ },
+ expected_edges=(("orders", "country_codes"), ("orders", "raw_orders")),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_hook_reads_when_compiling_then_hooked_model_runs_after_reads(
+ test_case: HookReadEdgeTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
+
+ project: CompiledProject = compile_and_assemble(project_dir=tmp_path)
+
+ assert set(test_case.expected_edges) <= execution_edge_names(project=project)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ HookReadErrorTestCase(
+ description="hook reading a model built from its own model",
+ files={
+ _HOOK_PATH: python_hook_source(reads='model("order_rollup")'),
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id", header=_HOOKED_HEADER
+ ),
+ "models/marts/order_rollup.sql": gate_model(
+ sql='SELECT count(*) AS n FROM __ref("orders")'
+ ),
+ },
+ expected_error_fragments=(
+ "[P007]",
+ "Python hook 'refresh_lookup' on model 'orders' reads model 'order_rollup', "
+ "which depends on 'orders'",
+ ),
+ ),
+ HookReadErrorTestCase(
+ description="hook reading an unknown model",
+ files={
+ _HOOK_PATH: python_hook_source(reads='model("missing_customers")'),
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id", header=_HOOKED_HEADER
+ ),
+ },
+ expected_error_fragments=(
+ "Python hook 'refresh_lookup'",
+ "reads unknown model 'missing_customers'",
+ ),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_unsatisfiable_hook_reads_when_compiling_then_it_fails(
+ test_case: HookReadErrorTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
+
+ with pytest.raises(ValueError) as error:
+ compile_and_assemble(project_dir=tmp_path)
+
+ rendered: str = f"[{getattr(error.value, 'code', '')}] {error.value}"
+ assert all(fragment in rendered for fragment in test_case.expected_error_fragments)
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-vv"])
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
new file mode 100644
index 0000000000..a6d1925b59
--- /dev/null
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
@@ -0,0 +1,199 @@
+"""Literal SQL in tasks, assets, and hooks must not hard-code project relation names."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+
+from sqlbuild.compiler.compile.models import CompiledProject
+from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import (
+ PythonSqlReferenceAllowedTestCase,
+ PythonSqlReferenceErrorTestCase,
+)
+from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import (
+ compile_and_assemble,
+ gate_model,
+ python_hook_source,
+ python_task_source,
+)
+
+_PROJECT_FILE: str = """
+name = "demo"
+adapter = "duckdb"
+
+[settings]
+sql_analysis = false
+sql_validation = false
+"""
+_DISABLED_PROJECT_FILE: str = _PROJECT_FILE + "\n[references]\nenforce_explicit = false\n"
+_HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/python/lookups.py"
+_TASK_PATH: str = "python/tasks/export.py"
+_HOOKED_HEADER: str = 'MODEL (post_hooks [python("refresh_lookup")]);'
+_MODELS: dict[str, str] = {
+ "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"),
+ "models/marts/orders.sql": gate_model(sql="SELECT 1 AS order_id", header=_HOOKED_HEADER),
+}
+_COUNTRY_SEED: dict[str, str] = {
+ "seeds/country_codes.yml": (
+ "seeds:\n - name: country_codes\n columns:\n - name: code\n type: VARCHAR\n"
+ ),
+ "seeds/country_codes.csv": "code\nGB\n",
+}
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ PythonSqlReferenceErrorTestCase(
+ description="task literal SQL naming a model",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(),
+ _TASK_PATH: python_task_source(
+ body=' ctx.query("SELECT count(*) FROM customers")\n'
+ ),
+ },
+ expected_error_fragments=(
+ "[P008]",
+ "task:export_orders names model:customers as 'customers' in SQL passed to "
+ "ctx.query()",
+ "python/tasks/export.py:7",
+ 'depends_on=model("customers")',
+ "[references] enforce_explicit = false",
+ ),
+ ),
+ PythonSqlReferenceErrorTestCase(
+ description="hook f-string with a literal seed table part",
+ files={
+ **_MODELS,
+ **_COUNTRY_SEED,
+ _HOOK_PATH: python_hook_source(
+ body=(
+ " limit = 10\n"
+ ' ctx.execute_sql(f"SELECT * FROM country_codes LIMIT {limit}")\n'
+ )
+ ),
+ },
+ expected_error_fragments=(
+ "hook:refresh_lookup names seed:country_codes",
+ "ctx.execute_sql()",
+ '@hook(reads=seed("country_codes"))',
+ ),
+ ),
+ PythonSqlReferenceErrorTestCase(
+ description="hook literal SQL naming a schema-qualified model",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(
+ body=' ctx.query("SELECT * FROM main.customers")\n'
+ ),
+ },
+ expected_error_fragments=("names model:customers as 'main.customers'",),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_hard_coded_project_relation_when_compiling_then_it_fails(
+ test_case: PythonSqlReferenceErrorTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
+
+ with pytest.raises(ValueError) as error:
+ compile_and_assemble(project_dir=tmp_path)
+
+ rendered: str = (
+ f"[{getattr(error.value, 'code', '')}] {error.value} {getattr(error.value, 'help', '')}"
+ )
+ assert all(fragment in rendered for fragment in test_case.expected_error_fragments)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ PythonSqlReferenceAllowedTestCase(
+ description="non-project relations and a temporary table created by the hook",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(
+ body=(
+ ' ctx.execute_sql("CREATE TEMP TABLE customers AS SELECT 1 AS id")\n'
+ ' ctx.query("SELECT * FROM customers")\n'
+ ' ctx.query("SELECT * FROM information_schema.tables")\n'
+ )
+ ),
+ },
+ expected_model_names=("customers", "orders"),
+ ),
+ PythonSqlReferenceAllowedTestCase(
+ description="relation formatted from ctx.relation and a CTE named like a model",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(),
+ _TASK_PATH: python_task_source(
+ body=(
+ ' orders = ctx.relation(model("orders"))\n'
+ ' ctx.query(f"SELECT * FROM {orders}")\n'
+ ' ctx.query("WITH customers AS (SELECT 1 AS id) SELECT * FROM customers")\n'
+ )
+ ),
+ },
+ expected_model_names=("customers", "orders"),
+ ),
+ PythonSqlReferenceAllowedTestCase(
+ description="hook naming the model it runs on",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(body=' ctx.query("SELECT * FROM orders")\n'),
+ },
+ expected_model_names=("customers", "orders"),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_explicit_or_non_project_relations_when_compiling_then_it_succeeds(
+ test_case: PythonSqlReferenceAllowedTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
+
+ project: CompiledProject = compile_and_assemble(project_dir=tmp_path)
+
+ assert tuple(sorted(model.name for model in project.models)) == test_case.expected_model_names
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ PythonSqlReferenceAllowedTestCase(
+ description="enforcement disabled allows hard-coded names",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(),
+ _TASK_PATH: python_task_source(
+ body=' ctx.query("SELECT count(*) FROM customers")\n'
+ ),
+ },
+ expected_model_names=("customers", "orders"),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_enforcement_disabled_when_compiling_hard_coded_names_then_it_succeeds(
+ test_case: PythonSqlReferenceAllowedTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(tmp_path, {"sqlbuild_project.toml": _DISABLED_PROJECT_FILE} | test_case.files)
+
+ project: CompiledProject = compile_and_assemble(project_dir=tmp_path)
+
+ assert tuple(sorted(model.name for model in project.models)) == test_case.expected_model_names
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-vv"])
diff --git a/tests/unit/src/sqlbuild/compiler/references/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/references/main/_test_types.py
index 2795314a33..e6d6a3d1e2 100644
--- a/tests/unit/src/sqlbuild/compiler/references/main/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/references/main/_test_types.py
@@ -2,6 +2,9 @@
from dataclasses import dataclass
+from sqlbuild.compiler.references.models import RelationName
+from sqlbuild.python_nodes.models import SqlResourceRef
+
@dataclass(frozen=True)
class AssertNoUnresolvedSqlMarkersTestCase:
@@ -10,3 +13,33 @@ class AssertNoUnresolvedSqlMarkersTestCase:
context: str
expected_error_fragment: str
expected_code: str | None = None
+
+
+@dataclass(frozen=True)
+class ExtractRelationNamesTestCase:
+ """SQL text and the relations and temporary tables it names."""
+
+ description: str
+ sql: str
+ dialect: str
+ expected_relations: tuple[RelationName, ...]
+ expected_temporary: frozenset[str] = frozenset()
+
+
+@dataclass(frozen=True)
+class MatchProjectRelationTestCase:
+ """A relation named in SQL and the project relation it resolves to, if any."""
+
+ description: str
+ relation: RelationName
+ default_schema: str | None
+ expected_ref: SqlResourceRef | None
+
+
+@dataclass(frozen=True)
+class UnparseableRelationNamesTestCase:
+ """SQL that cannot be parsed, which the hard-coded name checks must never block."""
+
+ description: str
+ sql: str
+ expected_result: None
diff --git a/tests/unit/src/sqlbuild/compiler/references/main/test_relation_names.py b/tests/unit/src/sqlbuild/compiler/references/main/test_relation_names.py
new file mode 100644
index 0000000000..f84b36da6d
--- /dev/null
+++ b/tests/unit/src/sqlbuild/compiler/references/main/test_relation_names.py
@@ -0,0 +1,135 @@
+"""Relation-name extraction and project-relation matching for hard-coded name checks."""
+
+from __future__ import annotations
+
+import pytest
+
+from sqlbuild.compiler.references.main.extract_relation_names import extract_relation_names
+from sqlbuild.compiler.references.main.match_project_relation import match_project_relation
+from sqlbuild.compiler.references.models import (
+ ProjectRelation,
+ ProjectRelationIndex,
+ RelationName,
+)
+from sqlbuild.refs import model, source
+from tests.unit.src.sqlbuild.compiler.references.main._test_types import (
+ ExtractRelationNamesTestCase,
+ MatchProjectRelationTestCase,
+ UnparseableRelationNamesTestCase,
+)
+
+_INDEX: ProjectRelationIndex = ProjectRelationIndex(
+ relations=(
+ ProjectRelation(
+ ref=model("customers"),
+ relation=RelationName(name="customers", schema="analytics", database="warehouse"),
+ ),
+ ProjectRelation(
+ ref=source("raw_orders"), relation=RelationName(name="orders", schema="raw")
+ ),
+ )
+)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ ExtractRelationNamesTestCase(
+ description="select, join, and CTE names",
+ sql="WITH recent AS (SELECT 1) SELECT * FROM recent JOIN analytics.customers c ON 1=1",
+ dialect="duckdb",
+ expected_relations=(RelationName(name="customers", schema="analytics"),),
+ ),
+ ExtractRelationNamesTestCase(
+ description="DML targets across statements and temporary tables",
+ sql=(
+ "CREATE TEMP TABLE staged AS SELECT 1 AS id; "
+ "DELETE FROM warehouse.analytics.customers WHERE id IN (SELECT id FROM staged)"
+ ),
+ dialect="duckdb",
+ expected_relations=(
+ RelationName(name="staged"),
+ RelationName(name="customers", schema="analytics", database="warehouse"),
+ ),
+ expected_temporary=frozenset({"staged"}),
+ ),
+ ExtractRelationNamesTestCase(
+ description="quoted dotted path splits into qualifier parts",
+ sql="SELECT * FROM `warehouse.analytics.customers`",
+ dialect="bigquery",
+ expected_relations=(
+ RelationName(name="customers", schema="analytics", database="warehouse"),
+ ),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_sql_when_extracting_relation_names_then_returns_named_relations(
+ test_case: ExtractRelationNamesTestCase,
+) -> None:
+ extracted: tuple[tuple[RelationName, ...], frozenset[str]] | None = extract_relation_names(
+ sql=test_case.sql, dialect=test_case.dialect
+ )
+
+ assert extracted is not None
+ assert set(extracted[0]) == set(test_case.expected_relations)
+ assert extracted[1] == test_case.expected_temporary
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ UnparseableRelationNamesTestCase(
+ description="malformed select", sql="SELEC FROM ((", expected_result=None
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_unparseable_sql_when_extracting_relation_names_then_returns_none(
+ test_case: UnparseableRelationNamesTestCase,
+) -> None:
+ assert extract_relation_names(sql=test_case.sql, dialect="duckdb") is test_case.expected_result
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ MatchProjectRelationTestCase(
+ description="unqualified name resolved against the matching current schema",
+ relation=RelationName(name="CUSTOMERS"),
+ default_schema="analytics",
+ expected_ref=model("customers"),
+ ),
+ MatchProjectRelationTestCase(
+ description="unqualified name in another current schema",
+ relation=RelationName(name="customers"),
+ default_schema="staging",
+ expected_ref=None,
+ ),
+ MatchProjectRelationTestCase(
+ description="qualified source table",
+ relation=RelationName(name="orders", schema="raw"),
+ default_schema="analytics",
+ expected_ref=source("raw_orders"),
+ ),
+ MatchProjectRelationTestCase(
+ description="non-project relation",
+ relation=RelationName(name="tables", schema="information_schema"),
+ default_schema="analytics",
+ expected_ref=None,
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_relation_when_matching_project_relations_then_resolves_missing_qualifiers(
+ test_case: MatchProjectRelationTestCase,
+) -> None:
+ match: ProjectRelation | None = match_project_relation(
+ index=_INDEX, relation=test_case.relation, default_schema=test_case.default_schema
+ )
+
+ assert getattr(match, "ref", None) == test_case.expected_ref
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-vv"])
diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/__init__.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py
new file mode 100644
index 0000000000..01660ee603
--- /dev/null
+++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py
@@ -0,0 +1,16 @@
+"""Test case types for the run-time relation guard."""
+
+from dataclasses import dataclass, field
+
+from sqlbuild.python_nodes.models import SqlResourceRef
+
+
+@dataclass(frozen=True)
+class RuntimeRelationGuardTestCase:
+ """SQL a Python node sends, and the hard-coded-name warnings it produces."""
+
+ description: str
+ statements: tuple[str, ...]
+ expected_warning_fragments: tuple[str, ...]
+ resolved_refs: tuple[SqlResourceRef, ...] = field(default_factory=tuple)
+ own_refs: frozenset[SqlResourceRef] = frozenset()
diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py
new file mode 100644
index 0000000000..78b0ad7e14
--- /dev/null
+++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py
@@ -0,0 +1,92 @@
+"""Run-time warnings for project relation names hard-coded in Python SQL."""
+
+from __future__ import annotations
+
+import pytest
+
+from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.refs import model, source
+from tests.unit.src.sqlbuild.executor.python_nodes.classes._test_types import (
+ RuntimeRelationGuardTestCase,
+)
+
+_PROJECT_RELATIONS: dict[SqlResourceRef, str] = {
+ model("customers"): "main.customers",
+ model("orders"): "main.orders",
+ source("raw_payments"): "raw.payments",
+}
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ RuntimeRelationGuardTestCase(
+ description="unqualified model name resolved against the current schema warns once",
+ statements=("SELECT * FROM customers", "SELECT count(*) FROM main.customers"),
+ expected_warning_fragments=(
+ "[P008] task 'export_orders' named model:customers as 'customers'",
+ ),
+ ),
+ RuntimeRelationGuardTestCase(
+ description="qualified source table warns",
+ statements=("DELETE FROM raw.payments WHERE 1 = 0",),
+ expected_warning_fragments=("named source:raw_payments as 'raw.payments'",),
+ ),
+ RuntimeRelationGuardTestCase(
+ description="relation resolved through ctx.relation does not warn",
+ statements=("SELECT * FROM main.customers",),
+ expected_warning_fragments=(),
+ resolved_refs=(model("customers"),),
+ ),
+ RuntimeRelationGuardTestCase(
+ description="own destination does not warn",
+ statements=("INSERT INTO orders SELECT 1",),
+ expected_warning_fragments=(),
+ own_refs=frozenset({model("orders")}),
+ ),
+ RuntimeRelationGuardTestCase(
+ description="temporary tables and non-project relations do not warn",
+ statements=(
+ "CREATE TEMP TABLE customers AS SELECT 1 AS id",
+ "SELECT * FROM customers JOIN information_schema.tables ON 1 = 0",
+ ),
+ expected_warning_fragments=(),
+ ),
+ RuntimeRelationGuardTestCase(
+ description="unparseable SQL is not blocked and does not warn",
+ statements=("VACUUM ((( customers",),
+ expected_warning_fragments=(),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_python_sql_when_checking_then_warns_for_hard_coded_project_relations(
+ test_case: RuntimeRelationGuardTestCase,
+) -> None:
+ warnings: list[str] = []
+ guard: RuntimeRelationGuard = RuntimeRelationGuard(
+ owner_label="task 'export_orders'",
+ declare_help="declare it with depends_on={typed}",
+ project_relations=_PROJECT_RELATIONS,
+ dialect="duckdb",
+ default_database=None,
+ default_schema="main",
+ warnings=warnings,
+ own_refs=test_case.own_refs,
+ )
+ for ref in test_case.resolved_refs:
+ guard.record_resolved(ref)
+
+ for statement in test_case.statements:
+ guard.check(statement)
+
+ assert len(warnings) == len(test_case.expected_warning_fragments)
+ assert all(
+ fragment in warning
+ for warning, fragment in zip(warnings, test_case.expected_warning_fragments, strict=True)
+ )
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-vv"])
diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py
index 7546e169d8..00f2284dce 100644
--- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py
+++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py
@@ -2,6 +2,7 @@
from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent
from sqlbuild.executor.run.types import AuditGateReuseReason, AuditGateStatus, ExecutionPhase
+from sqlbuild.python_nodes.models import SqlResourceRef
from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily
@@ -440,3 +441,32 @@ class ReusedFinalAuditResultsTestCase:
description: str
reads_outside_target_lineage: bool
expected_reused: bool
+
+
+@dataclass(frozen=True)
+class PythonHookReadsTestCase:
+ """A Python hook resolving relations, and the relation or error it gets."""
+
+ description: str
+ reads: tuple[SqlResourceRef, ...]
+ requested: SqlResourceRef
+ expected_relation: str
+
+
+@dataclass(frozen=True)
+class PythonHookUndeclaredReadTestCase:
+ """A Python hook resolving a relation it did not declare."""
+
+ description: str
+ reads: tuple[SqlResourceRef, ...]
+ requested: SqlResourceRef
+ expected_error_fragment: str
+
+
+@dataclass(frozen=True)
+class PythonHookHardCodedRelationTestCase:
+ """A Python hook sending SQL that names a project relation directly."""
+
+ description: str
+ enforce_explicit_references: bool
+ expected_warning_fragments: tuple[str, ...]
diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py
index e2ce01e86b..4c6844f2fe 100644
--- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py
+++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py
@@ -1,3 +1,4 @@
+from collections.abc import Callable
from pathlib import Path
from typing import Any
@@ -19,11 +20,15 @@
CompiledRelationLocation,
)
from sqlbuild.compiler.compile.types import CompiledResourceType
+from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry
from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry
from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason
from sqlbuild.executor.auditing.models import AuditExecutionResult
-from sqlbuild.executor.run.models import HookContext
-from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily
+from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks
+from sqlbuild.executor.run.models import HookContext, HookRelationLookup, HookRunContext
+from sqlbuild.executor.run.types import HookPhase
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily, SourceEntry
def insert_snapshot_hook_log(ctx: HookContext, phase: str) -> None:
@@ -298,3 +303,56 @@ def build_recording_adapter(
"postgres": RecordingPostgresAdapter,
"duckdb": RecordingDuckDbAdapter,
}[adapter_name]()
+
+
+def run_python_hook_with_relations(
+ *,
+ function: Callable[..., object],
+ reads: tuple[SqlResourceRef, ...],
+ enforce_explicit_references: bool = True,
+ warnings: list[str] | None = None,
+) -> None:
+ """Run one post-hook on ``orders`` with ``customers`` and ``raw_orders`` planned."""
+
+ adapter: DuckDbAdapter = DuckDbAdapter()
+ connection: Any = adapter.connect({"database": ":memory:"})
+ connection.execute("CREATE TABLE customers AS SELECT 1 AS customer_id")
+ connection.execute("CREATE SCHEMA raw")
+ connection.execute("CREATE TABLE raw.orders AS SELECT 1 AS order_id")
+ execute_hooks(
+ connection=connection,
+ adapter=adapter,
+ hooks=[PythonHookEntry(name="refresh_lookup", kwargs={})],
+ phase=HookPhase.POST_HOOKS,
+ hook_functions=(
+ DiscoveredHookFunction(
+ file_path=Path(__file__),
+ relative_path=Path("hooks/python/lookups.py"),
+ name="refresh_lookup",
+ function=function,
+ reads=reads,
+ ),
+ ),
+ hook_run=HookRunContext(
+ model_name="orders",
+ destination=CompiledRelationLocation(
+ database=None, schema="main", name="orders", qualified_name=None
+ ),
+ run_id="run-1",
+ relation_lookup=HookRelationLookup(
+ model_locations={
+ "customers": CompiledRelationLocation(
+ database=None, schema="main", name="customers", qualified_name=None
+ ),
+ "orders": CompiledRelationLocation(
+ database=None, schema="main", name="orders", qualified_name=None
+ ),
+ },
+ source_map={
+ "raw_orders": SourceEntry(name="raw_orders", schema="raw", table="orders")
+ },
+ ),
+ warnings=warnings,
+ enforce_explicit_references=enforce_explicit_references,
+ ),
+ )
diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py
index a8bd19dc7b..97cd0f69f1 100644
--- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py
+++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import re
from dataclasses import replace
from pathlib import Path
from typing import Any
@@ -32,6 +33,7 @@
dispatcher_scope,
invocation_scope,
)
+from sqlbuild.refs import model, source
from tests.unit.src.sqlbuild.executor.python_nodes._helpers.helpers import (
python_operation_events,
)
@@ -40,13 +42,19 @@
PublicHookContextExportTestCase,
PythonHookContextParameterTestCase,
PythonHookExecutionTestCase,
+ PythonHookHardCodedRelationTestCase,
PythonHookInvalidReturnTestCase,
PythonHookInvocationTestCase,
+ PythonHookReadsTestCase,
PythonHookRuntimeErrorTestCase,
PythonHookSkipTestCase,
+ PythonHookUndeclaredReadTestCase,
RenderHooksTestCase,
)
-from tests.unit.src.sqlbuild.executor.run._helpers.helpers import build_result_model_plan_entry
+from tests.unit.src.sqlbuild.executor.run._helpers.helpers import (
+ build_result_model_plan_entry,
+ run_python_hook_with_relations,
+)
@pytest.mark.parametrize(
@@ -669,3 +677,101 @@ def test_given_invalid_hook_entry_when_executing_then_it_reports_hook_index(
hooks=test_case.hooks,
phase=HookPhase.POST_HOOKS,
)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ PythonHookReadsTestCase(
+ description="declared model read resolves to the planned relation",
+ reads=(model("customers"),),
+ requested=model("customers"),
+ expected_relation="main.customers",
+ ),
+ PythonHookReadsTestCase(
+ description="declared source read resolves to the planned source relation",
+ reads=(source("raw_orders"),),
+ requested=source("raw_orders"),
+ expected_relation="raw.orders",
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_python_hook_reads_when_resolving_relation_then_returns_declared_relations_only(
+ test_case: PythonHookReadsTestCase,
+) -> None:
+ resolved: list[str] = []
+
+ def refresh_lookup(ctx: HookContext) -> None:
+ resolved.append(ctx.relation(test_case.requested))
+
+ run_python_hook_with_relations(function=refresh_lookup, reads=test_case.reads)
+
+ assert resolved == [test_case.expected_relation]
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ PythonHookUndeclaredReadTestCase(
+ description="undeclared read raises",
+ reads=(model("customers"),),
+ requested=model("orders"),
+ expected_error_fragment=(
+ "SQL relation ref 'orders' must be declared in @hook(reads=...) before use"
+ ),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_python_hook_undeclared_read_when_resolving_relation_then_raises(
+ test_case: PythonHookUndeclaredReadTestCase,
+) -> None:
+ def refresh_lookup(ctx: HookContext) -> None:
+ ctx.relation(test_case.requested)
+
+ with pytest.raises(ExecutorInputError, match=re.escape(test_case.expected_error_fragment)):
+ run_python_hook_with_relations(function=refresh_lookup, reads=test_case.reads)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ PythonHookHardCodedRelationTestCase(
+ description="hard-coded model name warns and the query still runs",
+ enforce_explicit_references=True,
+ expected_warning_fragments=(
+ "[P008] hook 'refresh_lookup' on model 'orders' named model:customers",
+ ),
+ ),
+ PythonHookHardCodedRelationTestCase(
+ description="enforcement disabled does not warn",
+ enforce_explicit_references=False,
+ expected_warning_fragments=(),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_python_hook_hard_coding_a_project_relation_when_running_then_warns_only(
+ test_case: PythonHookHardCodedRelationTestCase,
+) -> None:
+ rows: list[list[tuple[object, ...]]] = []
+ warnings: list[str] = []
+
+ def refresh_lookup(ctx: HookContext) -> None:
+ name: str = "customers"
+ rows.append(ctx.query(f"SELECT customer_id FROM {name}"))
+
+ run_python_hook_with_relations(
+ function=refresh_lookup,
+ reads=(),
+ enforce_explicit_references=test_case.enforce_explicit_references,
+ warnings=warnings,
+ )
+
+ assert rows == [[(1,)]]
+ assert len(warnings) == len(test_case.expected_warning_fragments)
+ assert all(
+ fragment in warning
+ for warning, fragment in zip(warnings, test_case.expected_warning_fragments, strict=True)
+ )
From 15fa6e27f0e6d83d927e744493f5b3a42f59ff49 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Sun, 27 Sep 2026 21:02:03 +0100
Subject: [PATCH 03/13] docs: document typed macro references, hook reads and
[references] enforce_explicit
---
.../references/docs/concepts/macros.md | 45 ++++++++++++++++++
.../docs/concepts/models/hooks/python.md | 47 ++++++++++++++++++-
.../docs/concepts/project-configuration.md | 23 +++++++++
.../concepts/python-nodes/sql-references.md | 41 +++++++++++++++-
.../skills/sqlbuild/references/docs/index.md | 2 +-
.../references/macros-and-declarations.md | 7 +++
.../src/content/docs/docs/concepts/macros.mdx | 45 ++++++++++++++++++
.../docs/concepts/models/hooks/python.mdx | 37 ++++++++++++++-
.../docs/concepts/project-configuration.mdx | 22 +++++++++
.../concepts/python-nodes/sql-references.mdx | 41 +++++++++++++++-
10 files changed, 305 insertions(+), 5 deletions(-)
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md
index 5135a22a7e..978b7a6d77 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md
@@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/concepts/macros/
- Create a macro
- Call a macro from SQL
- Arguments
+- Typed reference arguments
- Keep implementation details private
- Use macros in tests
- Use macros in hooks
@@ -77,6 +78,8 @@ Macro calls accept Python literal values:
- Dictionaries: `{"key": "value"}`
- `None`
- The result of another macro call
+- `__ref("model")`, `__source("source")`, and `__seed("seed")` references, alone or inside lists
+ and dictionaries
Positional and keyword arguments are supported:
@@ -87,6 +90,48 @@ Positional and keyword arguments are supported:
Use quoted strings when passing SQL expressions such as column names. The macro decides how to
place that text into its returned SQL.
+## Typed reference arguments
+
+Pass the models, sources, and seeds a macro reads as reference arguments:
+
+```sql
+SELECT * FROM @union_all([__ref("orders_eu"), __ref("orders_us")])
+```
+
+The macro receives each reference as a typed `SqlResourceRef` from `sqlbuild.refs`, the same type
+Python nodes use. It has `kind` and `name` attributes, and formatting it into SQL renders the
+relation with the adapter's quoting:
+
+```python
+# models/marts/_sqlbuild/_macros/unions.py
+from sqlbuild.refs import SqlResourceRef
+
+def union_all(relations: list[SqlResourceRef]) -> str:
+ return " UNION ALL ".join(f"SELECT order_id FROM {relation}" for relation in relations)
+```
+
+The dependency belongs to the model that makes the call. The graph, column lineage, and
+declaration scopes treat `@union_all([__ref("orders_eu"), __ref("orders_us")])` exactly like a
+model that wrote `__ref("orders_eu")` and `__ref("orders_us")` itself.
+
+### References in macro output
+
+A macro must not produce references on its own. Any `__ref()`, `__source()`, or `__seed()` in a
+macro's expanded output fails to compile with `P006`, whether the macro's Python code wrote it or
+it arrived as a quoted string argument such as `@wrap('__ref("stg_orders")')`:
+
+```
+error[P006]: model:order_summary depends on model:stg_orders through macro orders_base()
+ --> models/marts/_sqlbuild/_macros/orders.py
+ = help: write the reference in the model, or pass it in: @orders_base(__ref("stg_orders"))
+ = help: while migrating a project, allow macro-generated references with [references] enforce_explicit = false in sqlbuild_project.toml
+```
+
+A dependency hidden inside a macro is invisible to reviewers of the model file and bypasses
+declaration scopes. For projects migrating from other tools,
+[`[references] enforce_explicit = false`](project-configuration.md#references) allows
+macro-generated references project-wide.
+
## Keep implementation details private
Only public functions owned by the file are exported as macros. Prefix helpers, constants, classes,
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
index 94ef5e69d6..4c7698e9b3 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
@@ -6,6 +6,17 @@
Online: https://sqlbuild.com/docs/concepts/models/hooks/python/
+## Contents
+
+- Define a Python hook
+- Declared reads
+- Invoke a Python hook
+- Signature validation
+- Hook context
+- Providers
+- Skips and failures
+- Identity and diagnostics
+
Python hooks run model lifecycle logic that needs Python control flow, providers, warehouse queries, logging, or explicit skip decisions.
For shared lifecycle ordering, failure timing, naming, and identity rules, see the [Hooks overview](../hooks.md).
@@ -29,7 +40,40 @@ def notify_complete(ctx, channel="#data-builds"):
)
```
-By default, the hook name is the function name. The decorator accepts optional `name` and `description` arguments. A Python file may define multiple decorated hooks.
+By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A Python file may define multiple decorated hooks.
+
+## Declared reads
+
+A hook that queries other models, sources, or seeds declares them with `reads` and resolves each
+one with `ctx.relation(...)`:
+
+```python
+from sqlbuild.hooks import hook
+from sqlbuild.refs import model
+
+@hook(reads=model("dim_customers"))
+def refresh_lookup(ctx):
+ customers = ctx.relation(model("dim_customers"))
+ ctx.execute_sql(f"CREATE OR REPLACE TABLE customer_lookup AS SELECT * FROM {customers}")
+```
+
+`reads` accepts one reference or a list of `model(...)`, `source(...)`, and `seed(...)`
+references from `sqlbuild.refs`. `ctx.relation(...)` returns the adapter-qualified relation for
+the current run, following deferral like SQL references do. Resolving a reference that is not
+declared raises, as it does for [Python nodes](../../python-nodes/sql-references.md).
+
+Declared reads order the build: a model that runs the hook waits for the resources the hook reads
+when both are in the build. In a partial build, unselected reads are not built, and the hook reads
+their existing relations. A hook cannot read a resource built from the model that runs it, because
+that model would have to wait for its own dependants; that fails to compile with `P007`. Reading
+an unknown model, source, or seed fails to compile. Use `ctx.destination` for the hook's own model.
+
+Declaring reads does not change who may call the hook: the model must still see the hook under the
+usual [scope rules](../hooks.md#project-layout).
+
+Literal project relation names in `ctx.query()` and `ctx.execute_sql()` SQL fail to compile, and
+dynamically built names produce run-time warnings; see
+[Hard-coded relation names](../../python-nodes/sql-references.md#hard-coded-relation-names).
Files named `__init__.py` or beginning with `_` are skipped. Imported decorated functions are not registered again from the importing module.
@@ -73,6 +117,7 @@ Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, o
| `ctx.target` | Active target |
| `ctx.vars` | Effective project variables |
| `ctx.destination` | Destination relation metadata |
+| `ctx.relation(ref)` | Adapter-qualified relation for a [declared read](#declared-reads) |
| `ctx.adapter_name` | Active adapter name |
| `ctx.adapter` | Adapter instance |
| `ctx.connection` | Live connection |
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
index 181bac33fd..6665d856f8 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
@@ -16,6 +16,7 @@ Online: https://sqlbuild.com/docs/concepts/project-configuration/
- Settings
- Rules
- Scopes
+- References
- Project variables
- Janitor
- Scenario
@@ -430,6 +431,28 @@ enforce_placement = false
Visibility errors are unaffected. See
[Where to Put Declarations](declaration-scopes/placement.md).
+## References
+
+Every reference to a model, source, or seed must be explicit and typed. By default:
+
+- A macro whose expanded output contains `__ref()`, `__source()`, or `__seed()` fails to compile
+ with `P006`. Pass the reference to the macro as an argument instead; see
+ [Typed reference arguments](macros.md#typed-reference-arguments).
+- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in a task, asset, or Python hook that
+ names a project relation fails to compile with `P008`.
+- The same check runs on SQL sent at run time and reports a `P008` warning; the query still runs.
+ See [Hard-coded relation names](python-nodes/sql-references.md#hard-coded-relation-names).
+
+While migrating a project from another tool, turn these checks off for the whole project:
+
+```toml
+[references]
+enforce_explicit = false # default true
+```
+
+The switch has no per-model or per-folder form. Undeclared `ctx.relation(...)` calls still fail,
+and declared hook reads still order the build.
+
## Project variables
Variables are simple string substitutions available in model SQL via the `@@name` syntax:
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md
index ec849b5955..1f60ff1316 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md
@@ -46,7 +46,44 @@ def export_orders(ctx):
`ctx.relation(...)` returns the adapter-qualified relation name for the current run. Passing a reference that was not declared in `depends_on` raises an error.
-Do not hardcode model or source names in raw SQL. A model's relation depends on the target, its schema and any deferral, so a query like `ctx.query("SELECT * FROM fact_orders")` can fail or read the wrong relation. Always use `ctx.relation(model("fact_orders"))`.
+Do not hardcode model or source names in raw SQL. A model's relation depends on the target, its schema and any deferral, so a query like `ctx.query("SELECT * FROM fact_orders")` can fail or read the wrong relation. Always use `ctx.relation(model("fact_orders"))`. SQLBuild checks for this; see [Hard-coded relation names](#hard-coded-relation-names).
+
+## Hard-coded relation names
+
+SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in tasks, assets, and
+[Python hooks](../models/hooks/python.md) for names of project relations: models, sources,
+and, for hooks, seeds.
+
+**At compile time**, SQL written as a string literal, or as an f-string whose table names are
+literal text, is parsed with the project's SQL dialect. Naming a project relation fails to compile
+with `P008`:
+
+```
+error[P008]: task:export_orders names model:fact_orders as 'fact_orders' in SQL passed to ctx.query()
+ --> python/tasks/export.py:9
+ = help: declare it with depends_on=model("fact_orders") and use ctx.relation(model("fact_orders")) instead of the relation name
+ = help: while migrating a project, allow hard-coded relation names with [references] enforce_explicit = false in sqlbuild_project.toml
+```
+
+**At run time**, SQL built dynamically cannot be checked earlier. Before sending it,
+`ctx.query()` and `ctx.execute_sql()` parse it in the adapter's dialect, resolve unqualified names
+against the current schema, and compare them with the project relations resolved for the run. A
+project relation that the node did not obtain through `ctx.relation(...)` produces a `P008`
+**warning**. The query still runs. The warning appears in the build summary, counts the node as
+`WARN`, and is stored under the node's `warnings` in the `--json` run results.
+
+Both checks ignore:
+
+- Relations resolved through `ctx.relation(...)`, and a hook's own model (`ctx.destination`)
+- Relations that are not project relations, such as `information_schema` tables, external tables,
+ and temporary tables the node creates
+- CTE names
+- SQL that cannot be parsed; it is never blocked, only logged at debug level
+
+This is a guard against accidental hard-coding, not a sandbox. SQL sent directly through
+`ctx.connection` or `ctx.adapter` is not checked. Checks and loaders cannot declare SQL
+references, so their SQL is not checked either. To turn both checks off while migrating a project,
+set [`[references] enforce_explicit = false`](../project-configuration.md#references).
## model vs source
@@ -60,5 +97,7 @@ Do not hardcode model or source names in raw SQL. A model's relation depends on
## Where references are allowed
- **Tasks** and **assets** may declare `model()` and `source()` references and read them with `ctx.relation(...)`.
+- **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](../models/hooks/python.md#declared-reads).
+- `seed()` references are accepted by hook reads only; tasks and assets cannot depend on seeds.
- **Checks** may not reference SQL resources. Validate SQL with [audits](../audits.md).
- A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies.
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/index.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/index.md
index eef8d5fb4b..525ca6e3bf 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/index.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/index.md
@@ -7,7 +7,7 @@
Online: https://sqlbuild.com/docs/
SQLBuild is a free, open-source framework for SQL and Python data pipelines. It brings compile-time
-checks, tests and diffs to your warehouse, so you can change it as often as you change your code.
+checks, tests and safe renames to your warehouse, so you can change it as often as you change your code.
Valid SQL isn't the same as correct SQL. A query can compile, run and return rows, and still produce
a wrong number that people already trust. SQLBuild is built to catch that before it reaches
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md
index b13b5282bd..95ad3d134d 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md
@@ -28,9 +28,16 @@ FROM __ref("stg_orders")
- Arguments are Python literals: strings, numbers, `True`/`False`, lists, dicts, `None`, or another
macro call. Pass SQL expressions and column names as quoted strings. Keyword arguments work:
`@mock_orders(count=5, status="completed")`.
+- Pass the models, sources and seeds a macro reads as typed arguments:
+ `@union_all([__ref("orders_eu"), __ref("orders_us")])`. The macro receives `SqlResourceRef`
+ values (`sqlbuild.refs`); formatting one into SQL renders the relation, and the dependency
+ belongs to the calling model.
- A macro used in SQL must return a string, and its output is final SQL. Never return text
containing another `@macro()` call; compose by calling the Python functions directly, including
functions imported from other visible macro files (`from macros.currency import add_tax`).
+- Never return `__ref()`, `__source()` or `__seed()` from a macro, including through a quoted
+ string argument: it fails compile with `P006`. Only `[references] enforce_explicit = false` in
+ `sqlbuild_project.toml` allows it, as a project-wide migration switch.
- Macros work in model SQL, SQL hooks, unit tests, scenarios, audits, SQL functions and inline
source expressions. They are not allowed in ordinary `MODEL()` configuration fields.
- Imported functions keep their original identity; they are not re-exported by the importing
diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx
index b18be33927..57ee084ffc 100644
--- a/website/src/content/docs/docs/concepts/macros.mdx
+++ b/website/src/content/docs/docs/concepts/macros.mdx
@@ -66,6 +66,8 @@ Macro calls accept Python literal values:
- Dictionaries: `{"key": "value"}`
- `None`
- The result of another macro call
+- `__ref("model")`, `__source("source")`, and `__seed("seed")` references, alone or inside lists
+ and dictionaries
Positional and keyword arguments are supported:
@@ -76,6 +78,49 @@ Positional and keyword arguments are supported:
Use quoted strings when passing SQL expressions such as column names. The macro decides how to
place that text into its returned SQL.
+## Typed reference arguments
+
+Pass the models, sources, and seeds a macro reads as reference arguments:
+
+```sql
+SELECT * FROM @union_all([__ref("orders_eu"), __ref("orders_us")])
+```
+
+The macro receives each reference as a typed `SqlResourceRef` from `sqlbuild.refs`, the same type
+Python nodes use. It has `kind` and `name` attributes, and formatting it into SQL renders the
+relation with the adapter's quoting:
+
+```python
+# models/marts/_sqlbuild/_macros/unions.py
+from sqlbuild.refs import SqlResourceRef
+
+
+def union_all(relations: list[SqlResourceRef]) -> str:
+ return " UNION ALL ".join(f"SELECT order_id FROM {relation}" for relation in relations)
+```
+
+The dependency belongs to the model that makes the call. The graph, column lineage, and
+declaration scopes treat `@union_all([__ref("orders_eu"), __ref("orders_us")])` exactly like a
+model that wrote `__ref("orders_eu")` and `__ref("orders_us")` itself.
+
+### References in macro output
+
+A macro must not produce references on its own. Any `__ref()`, `__source()`, or `__seed()` in a
+macro's expanded output fails to compile with `P006`, whether the macro's Python code wrote it or
+it arrived as a quoted string argument such as `@wrap('__ref("stg_orders")')`:
+
+```
+error[P006]: model:order_summary depends on model:stg_orders through macro orders_base()
+ --> models/marts/_sqlbuild/_macros/orders.py
+ = help: write the reference in the model, or pass it in: @orders_base(__ref("stg_orders"))
+ = help: while migrating a project, allow macro-generated references with [references] enforce_explicit = false in sqlbuild_project.toml
+```
+
+A dependency hidden inside a macro is invisible to reviewers of the model file and bypasses
+declaration scopes. For projects migrating from other tools,
+[`[references] enforce_explicit = false`](/docs/concepts/project-configuration#references) allows
+macro-generated references project-wide.
+
## Keep implementation details private
Only public functions owned by the file are exported as macros. Prefix helpers, constants, classes,
diff --git a/website/src/content/docs/docs/concepts/models/hooks/python.mdx b/website/src/content/docs/docs/concepts/models/hooks/python.mdx
index 62dde7d6ec..e408d84972 100644
--- a/website/src/content/docs/docs/concepts/models/hooks/python.mdx
+++ b/website/src/content/docs/docs/concepts/models/hooks/python.mdx
@@ -27,7 +27,41 @@ def notify_complete(ctx, channel="#data-builds"):
)
```
-By default, the hook name is the function name. The decorator accepts optional `name` and `description` arguments. A Python file may define multiple decorated hooks.
+By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A Python file may define multiple decorated hooks.
+
+## Declared reads
+
+A hook that queries other models, sources, or seeds declares them with `reads` and resolves each
+one with `ctx.relation(...)`:
+
+```python
+from sqlbuild.hooks import hook
+from sqlbuild.refs import model
+
+
+@hook(reads=model("dim_customers"))
+def refresh_lookup(ctx):
+ customers = ctx.relation(model("dim_customers"))
+ ctx.execute_sql(f"CREATE OR REPLACE TABLE customer_lookup AS SELECT * FROM {customers}")
+```
+
+`reads` accepts one reference or a list of `model(...)`, `source(...)`, and `seed(...)`
+references from `sqlbuild.refs`. `ctx.relation(...)` returns the adapter-qualified relation for
+the current run, following deferral like SQL references do. Resolving a reference that is not
+declared raises, as it does for [Python nodes](/docs/concepts/python-nodes/sql-references).
+
+Declared reads order the build: a model that runs the hook waits for the resources the hook reads
+when both are in the build. In a partial build, unselected reads are not built, and the hook reads
+their existing relations. A hook cannot read a resource built from the model that runs it, because
+that model would have to wait for its own dependants; that fails to compile with `P007`. Reading
+an unknown model, source, or seed fails to compile. Use `ctx.destination` for the hook's own model.
+
+Declaring reads does not change who may call the hook: the model must still see the hook under the
+usual [scope rules](/docs/concepts/models/hooks#project-layout).
+
+Literal project relation names in `ctx.query()` and `ctx.execute_sql()` SQL fail to compile, and
+dynamically built names produce run-time warnings; see
+[Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names).
Files named `__init__.py` or beginning with `_` are skipped. Imported decorated functions are not registered again from the importing module.
@@ -71,6 +105,7 @@ Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, o
| `ctx.target` | Active target |
| `ctx.vars` | Effective project variables |
| `ctx.destination` | Destination relation metadata |
+| `ctx.relation(ref)` | Adapter-qualified relation for a [declared read](#declared-reads) |
| `ctx.adapter_name` | Active adapter name |
| `ctx.adapter` | Adapter instance |
| `ctx.connection` | Live connection |
diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx
index f78ed2b28f..6adecf25a7 100644
--- a/website/src/content/docs/docs/concepts/project-configuration.mdx
+++ b/website/src/content/docs/docs/concepts/project-configuration.mdx
@@ -409,6 +409,28 @@ enforce_placement = false
Visibility errors are unaffected. See
[Where to Put Declarations](/docs/concepts/declaration-scopes/placement).
+## References
+
+Every reference to a model, source, or seed must be explicit and typed. By default:
+
+- A macro whose expanded output contains `__ref()`, `__source()`, or `__seed()` fails to compile
+ with `P006`. Pass the reference to the macro as an argument instead; see
+ [Typed reference arguments](/docs/concepts/macros#typed-reference-arguments).
+- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in a task, asset, or Python hook that
+ names a project relation fails to compile with `P008`.
+- The same check runs on SQL sent at run time and reports a `P008` warning; the query still runs.
+ See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names).
+
+While migrating a project from another tool, turn these checks off for the whole project:
+
+```toml
+[references]
+enforce_explicit = false # default true
+```
+
+The switch has no per-model or per-folder form. Undeclared `ctx.relation(...)` calls still fail,
+and declared hook reads still order the build.
+
## Project variables
Variables are simple string substitutions available in model SQL via the `@@name` syntax:
diff --git a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx
index 761ea84051..f4f11e8170 100644
--- a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx
+++ b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx
@@ -48,9 +48,46 @@ def export_orders(ctx):
`ctx.relation(...)` returns the adapter-qualified relation name for the current run. Passing a reference that was not declared in `depends_on` raises an error.
-Do not hardcode model or source names in raw SQL. A model's relation depends on the target, its schema and any deferral, so a query like `ctx.query("SELECT * FROM fact_orders")` can fail or read the wrong relation. Always use `ctx.relation(model("fact_orders"))`.
+Do not hardcode model or source names in raw SQL. A model's relation depends on the target, its schema and any deferral, so a query like `ctx.query("SELECT * FROM fact_orders")` can fail or read the wrong relation. Always use `ctx.relation(model("fact_orders"))`. SQLBuild checks for this; see [Hard-coded relation names](#hard-coded-relation-names).
+## Hard-coded relation names
+
+SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in tasks, assets, and
+[Python hooks](/docs/concepts/models/hooks/python) for names of project relations: models, sources,
+and, for hooks, seeds.
+
+**At compile time**, SQL written as a string literal, or as an f-string whose table names are
+literal text, is parsed with the project's SQL dialect. Naming a project relation fails to compile
+with `P008`:
+
+```
+error[P008]: task:export_orders names model:fact_orders as 'fact_orders' in SQL passed to ctx.query()
+ --> python/tasks/export.py:9
+ = help: declare it with depends_on=model("fact_orders") and use ctx.relation(model("fact_orders")) instead of the relation name
+ = help: while migrating a project, allow hard-coded relation names with [references] enforce_explicit = false in sqlbuild_project.toml
+```
+
+**At run time**, SQL built dynamically cannot be checked earlier. Before sending it,
+`ctx.query()` and `ctx.execute_sql()` parse it in the adapter's dialect, resolve unqualified names
+against the current schema, and compare them with the project relations resolved for the run. A
+project relation that the node did not obtain through `ctx.relation(...)` produces a `P008`
+**warning**. The query still runs. The warning appears in the build summary, counts the node as
+`WARN`, and is stored under the node's `warnings` in the `--json` run results.
+
+Both checks ignore:
+
+- Relations resolved through `ctx.relation(...)`, and a hook's own model (`ctx.destination`)
+- Relations that are not project relations, such as `information_schema` tables, external tables,
+ and temporary tables the node creates
+- CTE names
+- SQL that cannot be parsed; it is never blocked, only logged at debug level
+
+This is a guard against accidental hard-coding, not a sandbox. SQL sent directly through
+`ctx.connection` or `ctx.adapter` is not checked. Checks and loaders cannot declare SQL
+references, so their SQL is not checked either. To turn both checks off while migrating a project,
+set [`[references] enforce_explicit = false`](/docs/concepts/project-configuration#references).
+
## model vs source
| Reference | Resolves to |
@@ -63,5 +100,7 @@ Do not hardcode model or source names in raw SQL. A model's relation depends on
## Where references are allowed
- **Tasks** and **assets** may declare `model()` and `source()` references and read them with `ctx.relation(...)`.
+- **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](/docs/concepts/models/hooks/python#declared-reads).
+- `seed()` references are accepted by hook reads only; tasks and assets cannot depend on seeds.
- **Checks** may not reference SQL resources. Validate SQL with [audits](/docs/concepts/audits).
- A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies.
From c5196a54a63877bf774df2d7cb89ce5ab0024301 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Sun, 27 Sep 2026 22:12:56 +0100
Subject: [PATCH 04/13] feat!: accept seed() and SQL references in checks, and
check loader and check SQL
Tasks, assets and checks may declare model(), source() and seed() in
depends_on and resolve them with ctx.relation(...). Seed dependencies
order read-side tasks after the seed, appear as edges in sqb dag --json
and Dagster, and are validated by sqb check. A check with SQL
dependencies runs at the end of sqb build when those resources ran in the
build, and its checked assets include them.
Selector expansion now crosses the SQL/Python boundary: +task:x also
selects the models, sources and seeds x declares and their upstream, and
x+ on a SQL resource also selects the tasks and assets that read it.
Literal SQL in checks and loaders that names a project relation now fails
compile with P008, and dynamic SQL produces a P008 run-time warning, as
for tasks and assets. Loader help points to ctx.source(...) or
ctx.loader(...), and a model or seed named in loader SQL is reported as
unreadable because loaders run before models and seeds. ctx.source() and
ctx.loader() count as resolved. Unqualified matches add a help to
qualify an external table that shares the name. Loader warnings appear in
the build summary and in the JSON run results.
BREAKING CHANGE: Python checks and loaders that hard-code project relation
names in literal SQL now fail compile; use ctx.relation(...) in checks and
ctx.source(...) or ctx.loader(...) in loaders.
BREAKING CHANGE: + selectors now include the SQL resources
the node declares, and + selectors include the tasks and
assets that declare it.
---
.../sqlbuild/references/docs/CONTENTS.md | 4 +-
.../sqlbuild/references/docs/cli/check.md | 2 +-
.../docs/concepts/models/hooks/python.md | 4 +
.../docs/concepts/project-configuration.md | 4 +-
.../docs/concepts/python-nodes/assets.md | 2 +-
.../docs/concepts/python-nodes/checks.md | 29 ++-
.../docs/concepts/python-nodes/loaders.md | 2 +
.../docs/concepts/python-nodes/overview.md | 6 +-
.../concepts/python-nodes/sql-references.md | 53 +++--
.../docs/concepts/python-nodes/tasks.md | 2 +-
.../_helpers/build_execution/checks.py | 19 ++
.../_helpers/build_execution/execution.py | 4 +
.../build_python_nodes/python_lifecycle.py | 11 +-
.../cli/commands/_helpers/check/core.py | 53 ++++-
.../cli/commands/_helpers/check/execution.py | 6 +
.../cli/commands/_helpers/check/planning.py | 13 +-
.../classes/build_progress_callbacks.py | 47 +++--
src/sqlbuild/cli/commands/models.py | 1 +
.../_helpers/execution_result_document.py | 7 +-
.../explicit_references/python_sql.py | 88 ++++++--
.../compiler/dag/_helpers/artifact.py | 11 +-
.../_helpers/validation/discovery.py | 13 +-
.../compiler/graph/_helpers/hook_reads.py | 16 +-
.../compiler/graph/_helpers/sql_refs.py | 20 ++
.../compiler/graph/main/sql_ref_key.py | 13 ++
.../pipeline/_helpers/relation_targets.py | 125 +++++++++++
.../pipeline/main/project_relation_targets.py | 18 ++
.../pipeline/main/relation_targets.py | 68 +-----
.../_helpers/unified_selectors.py | 96 +++++++--
.../compiler/references/_helpers/remedies.py | 38 ++++
.../main/hard_coded_relation_remedy.py | 26 +++
src/sqlbuild/compiler/references/types.py | 8 +
.../executor/build/_helpers/source_node.py | 1 +
src/sqlbuild/executor/build/models.py | 2 +
.../load/_helpers/loader_invocation.py | 38 +++-
src/sqlbuild/executor/load/main/_execute.py | 48 ++++-
src/sqlbuild/executor/load/models.py | 21 +-
.../python_nodes/_helpers/execution.py | 26 +--
.../_helpers/ingress_execution.py | 1 +
.../python_nodes/_helpers/python_checks.py | 18 +-
.../_helpers/read_side_execution.py | 5 +-
.../python_nodes/_helpers/relation_guard.py | 25 +++
.../classes/runtime_relation_guard.py | 22 +-
src/sqlbuild/executor/python_nodes/models.py | 1 +
.../executor/run/_helpers/execution/hooks.py | 3 +-
.../cli/commands/main/build/_test_types.py | 13 ++
.../cli/commands/main/build/helpers.py | 15 ++
.../test_build_no_tests_no_audits.py | 21 +-
.../main/build/test_explicit_references.py | 197 ++++++++++++++++++
.../commands/_helpers/check/test_outputs.py | 1 +
.../compiler/compile/_helpers/_test_types.py | 3 +-
.../compiler/compile/_helpers/helpers.py | 28 +++
.../_helpers/test_python_sql_references.py | 140 ++++++++++++-
.../sqlbuild/compiler/discovery/test_main.py | 84 +++-----
.../_helpers/test_unified_selectors.py | 50 +++++
.../python_nodes/classes/_test_types.py | 5 +
.../classes/test_runtime_relation_guard.py | 70 ++++++-
57 files changed, 1323 insertions(+), 324 deletions(-)
create mode 100644 src/sqlbuild/compiler/graph/_helpers/sql_refs.py
create mode 100644 src/sqlbuild/compiler/graph/main/sql_ref_key.py
create mode 100644 src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py
create mode 100644 src/sqlbuild/compiler/pipeline/main/project_relation_targets.py
create mode 100644 src/sqlbuild/compiler/references/_helpers/remedies.py
create mode 100644 src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py
create mode 100644 src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md
index 7769bd733b..b9111d6836 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md
@@ -88,10 +88,10 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in
- [Loaders](concepts/python-nodes/loaders.md) (`concepts/python-nodes/loaders`) - Load external data into source tables with Python functions.
- [Tasks](concepts/python-nodes/tasks.md) (`concepts/python-nodes/tasks`) - Run Python computation and side effects as nodes in the SQLBuild graph.
- [Assets](concepts/python-nodes/assets.md) (`concepts/python-nodes/assets`) - Produce or observe external artifacts as nodes in the SQLBuild graph.
-- [Checks](concepts/python-nodes/checks.md) (`concepts/python-nodes/checks`) - Validate tasks, assets, and loaders with Python checks.
+- [Checks](concepts/python-nodes/checks.md) (`concepts/python-nodes/checks`) - Validate tasks, assets, loaders, and SQL relations with Python checks.
- [Factories](concepts/python-nodes/factories.md) (`concepts/python-nodes/factories`) - Generate Python nodes programmatically with @factory.
- [Providers](concepts/python-nodes/providers.md) (`concepts/python-nodes/providers`) - Shared runtime services for Python nodes and hooks.
-- [SQL References](concepts/python-nodes/sql-references.md) (`concepts/python-nodes/sql-references`) - Read SQL models and sources from Python nodes without creating SQL dependencies.
+- [SQL References](concepts/python-nodes/sql-references.md) (`concepts/python-nodes/sql-references`) - Read SQL models, sources, and seeds from Python nodes without creating SQL dependencies.
## Integrations
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md
index 986f891925..8fef3284f9 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/check.md
@@ -51,4 +51,4 @@ Results are written to `target/run/checks/python_checks.json`.
## Relationship to build
-`sqb build` runs relevant Python checks by default after the tasks, assets, and loaders they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand.
+`sqb build` runs relevant Python checks by default after the tasks, assets, loaders, models, sources, and seeds they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand.
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
index 4c7698e9b3..ff0240a5a1 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
@@ -68,6 +68,10 @@ their existing relations. A hook cannot read a resource built from the model tha
that model would have to wait for its own dependants; that fails to compile with `P007`. Reading
an unknown model, source, or seed fails to compile. Use `ctx.destination` for the hook's own model.
+The Dagster integration does not add declared reads to its asset graph, the same as reads of
+attached audits. Within one `sqb build` the ordering holds; when Dagster runs the hooked model in a
+separate build from the resources it reads, the hook reads their existing relations.
+
Declaring reads does not change who may call the hook: the model must still see the hook under the
usual [scope rules](../hooks.md#project-layout).
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
index 6665d856f8..6607720f9d 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
@@ -438,8 +438,8 @@ Every reference to a model, source, or seed must be explicit and typed. By defau
- A macro whose expanded output contains `__ref()`, `__source()`, or `__seed()` fails to compile
with `P006`. Pass the reference to the macro as an argument instead; see
[Typed reference arguments](macros.md#typed-reference-arguments).
-- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in a task, asset, or Python hook that
- names a project relation fails to compile with `P008`.
+- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in a task, asset, check, loader, or
+ Python hook that names a project relation fails to compile with `P008`.
- The same check runs on SQL sent at run time and reports a `P008` warning; the query still runs.
See [Hard-coded relation names](python-nodes/sql-references.md#hard-coded-relation-names).
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md
index a78487b7b4..8140ad5614 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md
@@ -124,7 +124,7 @@ See [Tasks](tasks.md#retries) for the full retry policy fields.
| Parameter | Description |
|-----------|-------------|
| `name` | Override the node name (defaults to the function name) |
-| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()` for read-only SQL refs |
+| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()`/`seed()` for read-only SQL refs |
| `columns` | Column declarations for the produced artifact |
| `column_lineage` | Map of asset column to upstream `{node, column}` references |
| `tags` | Labels for selection and grouping |
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md
index b81c8232ad..85a07e2094 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md
@@ -2,7 +2,7 @@
# Checks
-> Validate tasks, assets, and loaders with Python checks.
+> Validate tasks, assets, loaders, and SQL relations with Python checks.
Online: https://sqlbuild.com/docs/concepts/python-nodes/checks/
@@ -16,7 +16,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/checks/
- Running checks
- Checks vs audits
-Checks are Python nodes that validate other Python nodes. They are the Python analog of SQL [audits](../audits.md): audits validate SQL relations, checks validate the output of tasks, assets, and loaders. See [Python Nodes](overview.md) for the shared model.
+Checks are Python nodes that validate the output of tasks, assets, and loaders, or the data in SQL models, sources, and seeds. They are the Python analog of SQL [audits](../audits.md). See [Python Nodes](overview.md) for the shared model.
Checks are separate graph nodes, not callbacks embedded in a task or asset. A check declares what it validates through `depends_on`.
@@ -39,6 +39,21 @@ def check_orders_exported(ctx: CheckContext):
The check receives a `CheckContext` and reads its dependencies' persisted results with `ctx.result_of(...)`.
+A check can also validate SQL relations. Declare them with `model()`, `source()`, or `seed()` from `sqlbuild.refs`, and resolve them with `ctx.relation(...)`:
+
+```python
+from sqlbuild.checks import check
+from sqlbuild.refs import model
+
+@check(depends_on=model("fact_orders"))
+def orders_present(ctx):
+ orders = ctx.relation(model("fact_orders"))
+ count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0]
+ return ctx.pass_() if count else ctx.fail("no orders")
+```
+
+Do not hard-code relation names in check SQL; see [Hard-coded relation names](sql-references.md#hard-coded-relation-names).
+
## Results
Return a result through the context helpers, or a bool shorthand:
@@ -79,7 +94,7 @@ def orders_freshness(ctx):
## What checks can depend on
- Checks may depend on **tasks, assets, and loaders**.
-- Checks may **not** depend on SQL models, sources, seeds, or functions. Use SQL [audits](../audits.md) to validate SQL relations.
+- Checks may depend on SQL **models, sources, and seeds** through `model()`, `source()`, and `seed()` references. They may **not** depend on functions.
- Checks may **not** depend on other checks.
- Checks may **not** depend on a terminal source loader directly. Validate loaded source data with a source audit instead.
@@ -89,7 +104,7 @@ A check that depends on a single node is displayed grouped under that node. Mult
| Parameter | Description |
|-----------|-------------|
-| `depends_on` | Required. Tasks/assets/loaders to validate (function, tuple, or list) |
+| `depends_on` | Required. Tasks/assets/loaders to validate, and `model()`/`source()`/`seed()` references (function, reference, tuple, or list) |
| `name` | Override the node name (defaults to the function name) |
| `severity` | `error` (default) or `warn` |
| `tags` | Labels for selection and grouping |
@@ -101,7 +116,7 @@ Checks do not support `columns`, `column_lineage`, or `retry`.
## Running checks
-Checks run automatically during `sqb build` when their Python dependencies run. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](../../cli/check.md):
+Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](../../cli/check.md):
```bash
# Run all checks
@@ -114,13 +129,13 @@ sqb check --select +check_orders_exported
sqb check --select tag:exports
```
-`sqb check` rejects selecting non-check nodes; use `sqb build` to run tasks and assets. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout.
+`sqb check` rejects selecting non-check nodes; use `sqb build` to run tasks and assets. The models, sources, and seeds a check depends on must already exist; `sqb check` does not build them. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout.
## Checks vs audits
| | Checks | Audits |
|---|--------|--------|
-| Validates | Python tasks, assets, loaders | SQL relations |
+| Validates | Python tasks, assets, loaders, and SQL relations | SQL relations |
| Authored in | `python/` (Python) | `MODEL()` headers / `audits/` (SQL) |
| Run by | `sqb build`, `sqb check` | `sqb build`, `sqb audit` |
| Severity | `error`, `warn` | `error`, `warn` |
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md
index eed742a5e3..d3a6d8ac74 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md
@@ -268,6 +268,8 @@ def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]:
]
```
+Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders run before models and seeds and cannot read them. See [Hard-coded relation names](sql-references.md#hard-coded-relation-names).
+
Dependencies form a DAG. SQLBuild schedules loaders in topological order and executes independent loaders concurrently when `--concurrency` is set.
Intermediate loaders (those referenced only via `depends_on`, with no managed source of the same name) are given synthetic source entries and write to `__loader__` tables by default. Only the terminal loader - the one whose name matches a managed source - populates that source; intermediate loaders feed it. Use the `destination` parameter on the decorator to override the intermediate relation:
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md
index 7433ce2673..b0642c0854 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md
@@ -68,7 +68,7 @@ The most important rule: **SQL models never depend on Python nodes.** The depend
- SQL models depend only on other SQL resources (models, sources, seeds, functions).
- Python nodes may depend on other Python nodes.
- The only way Python data reaches SQL is through a **loader populating a source**: `loader -> source -> model`.
-- Python nodes may **read** SQL models and sources at runtime through typed references (see [SQL references](sql-references.md)), but reading a model does not make it a SQL dependency.
+- Tasks, assets, and checks may **read** SQL models, sources, and seeds at runtime through typed references (see [SQL references](sql-references.md)), but reading a model does not make it a SQL dependency.
This keeps the SQL graph fully analyzable and testable on its own, while letting Python participate around the edges.
@@ -79,7 +79,7 @@ This keeps the SQL graph fully analyzable and testable on its own, while letting
| (read only) v
| asset
v |
- source <-- loader | check (validates tasks/assets/loaders)
+ source <-- loader | check (validates tasks/assets/loaders/SQL)
|
v
model (SQL) ----> model (SQL)
@@ -91,7 +91,7 @@ Every decorator accepts the same organizational metadata:
| Argument | Description |
|----------|-------------|
-| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()` references) |
+| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) |
| `tags` | Labels for selection, filtering, and catalog grouping |
| `group` | A display/catalog grouping string |
| `description` | Human-readable docs (defaults to the function docstring) |
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md
index 1f60ff1316..96ab6b2873 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md
@@ -2,18 +2,26 @@
# SQL References
-> Read SQL models and sources from Python nodes without creating SQL dependencies.
+> Read SQL models, sources, and seeds from Python nodes without creating SQL dependencies.
Online: https://sqlbuild.com/docs/concepts/python-nodes/sql-references/
-Python nodes can **read** SQL models and sources at runtime, but they cannot depend on them as graph edges (see the [SQL boundary](overview.md#the-sql-boundary)). Typed references make this read-only access explicit and safe.
+## Contents
+
+- Declaring a reference
+- Resolving with ctx.relation
+- Hard-coded relation names
+- model vs source
+- Where references are allowed
+
+Python nodes can **read** SQL models, sources, and seeds at runtime, but SQL cannot depend on Python nodes (see the [SQL boundary](overview.md#the-sql-boundary)). Typed references make this read-only access explicit and safe.
## Declaring a reference
-Use `model()` and `source()` from `sqlbuild.refs` in a node's `depends_on`:
+Use `model()`, `source()`, and `seed()` from `sqlbuild.refs` in the `depends_on` of a task, asset, or check:
```python
-from sqlbuild.refs import model, source
+from sqlbuild.refs import model, seed, source
from sqlbuild.tasks import task
@task(depends_on=model("fact_orders"))
@@ -23,6 +31,10 @@ def export_orders(ctx):
@task(depends_on=source("raw_orders"))
def inspect_raw(ctx):
...
+
+@task(depends_on=seed("country_codes"))
+def export_countries(ctx):
+ ...
```
Declaring the reference does two things:
@@ -32,6 +44,8 @@ Declaring the reference does two things:
It does **not** make the SQL resource depend on the Python node. The dependency is one-way: Python reads SQL, never the reverse.
+The reference is a graph edge. It appears in `sqb dag --json`, and selector expansion follows it: `+task:export_countries` also selects the `country_codes` seed and its upstream, and `country_codes+` also selects the tasks and assets that read it. Checks are never added by expansion; they run when their dependencies run.
+
## Resolving with ctx.relation
Always resolve a reference to its concrete relation with `ctx.relation(...)` instead of hardcoding the table name:
@@ -50,9 +64,9 @@ Do not hardcode model or source names in raw SQL. A model's relation depends on
## Hard-coded relation names
-SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in tasks, assets, and
-[Python hooks](../models/hooks/python.md) for names of project relations: models, sources,
-and, for hooks, seeds.
+SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in tasks, assets, checks,
+loaders, and [Python hooks](../models/hooks/python.md) for names of project relations:
+models, sources, and seeds.
**At compile time**, SQL written as a string literal, or as an f-string whose table names are
literal text, is parsed with the project's SQL dialect. Naming a project relation fails to compile
@@ -62,27 +76,36 @@ with `P008`:
error[P008]: task:export_orders names model:fact_orders as 'fact_orders' in SQL passed to ctx.query()
--> python/tasks/export.py:9
= help: declare it with depends_on=model("fact_orders") and use ctx.relation(model("fact_orders")) instead of the relation name
+ = help: if 'fact_orders' is an external table that shares the name, qualify it with its schema
= help: while migrating a project, allow hard-coded relation names with [references] enforce_explicit = false in sqlbuild_project.toml
```
+An unqualified name that matches a project relation is treated as that relation. If you mean an
+external table that shares the name, qualify it with its schema.
+
+Loaders run before models and seeds, so a loader naming a model or seed fails with a help saying it
+cannot read them. A loader naming a source is told to use `ctx.source("name")`, or
+`ctx.loader(upstream_loader)` when the source is populated by a loader it depends on.
+
**At run time**, SQL built dynamically cannot be checked earlier. Before sending it,
`ctx.query()` and `ctx.execute_sql()` parse it in the adapter's dialect, resolve unqualified names
against the current schema, and compare them with the project relations resolved for the run. A
-project relation that the node did not obtain through `ctx.relation(...)` produces a `P008`
-**warning**. The query still runs. The warning appears in the build summary, counts the node as
+project relation that the node did not obtain through `ctx.relation(...)` (or, in a loader,
+`ctx.source(...)` or `ctx.loader(...)`) produces a `P008` **warning**. The query still runs. The warning appears in the build summary, counts the node as
`WARN`, and is stored under the node's `warnings` in the `--json` run results.
Both checks ignore:
-- Relations resolved through `ctx.relation(...)`, and a hook's own model (`ctx.destination`)
+- Relations resolved through `ctx.relation(...)`, `ctx.source(...)`, or `ctx.loader(...)`, a hook's
+ own model (`ctx.destination`), and a loader's own source
- Relations that are not project relations, such as `information_schema` tables, external tables,
and temporary tables the node creates
- CTE names
- SQL that cannot be parsed; it is never blocked, only logged at debug level
This is a guard against accidental hard-coding, not a sandbox. SQL sent directly through
-`ctx.connection` or `ctx.adapter` is not checked. Checks and loaders cannot declare SQL
-references, so their SQL is not checked either. To turn both checks off while migrating a project,
+`ctx.connection` or `ctx.adapter` is not checked. Run-time checks apply to `sqb build` and
+`sqb check`. To turn both checks off while migrating a project,
set [`[references] enforce_explicit = false`](../project-configuration.md#references).
## model vs source
@@ -91,13 +114,13 @@ set [`[references] enforce_explicit = false`](../project-configuration.md#refere
|-----------|-------------|
| `model("name")` | The built model relation |
| `source("name")` | The source read relation, following deferred-source semantics |
+| `seed("name")` | The loaded seed relation |
`source(...)` respects `defer_sources_to`, so a node reading a source in `dev` can read the deferred target's data just like SQL models do.
## Where references are allowed
-- **Tasks** and **assets** may declare `model()` and `source()` references and read them with `ctx.relation(...)`.
+- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`.
- **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](../models/hooks/python.md#declared-reads).
-- `seed()` references are accepted by hook reads only; tasks and assets cannot depend on seeds.
-- **Checks** may not reference SQL resources. Validate SQL with [audits](../audits.md).
+- **Loaders** may not reference SQL resources; they read sources with `ctx.source(...)` and upstream loaders with `ctx.loader(...)`.
- A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies.
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md
index 4af571d7de..032ca4c17b 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md
@@ -123,7 +123,7 @@ The default is no retry. Set `retry_on` explicitly rather than relying on the br
| Parameter | Description |
|-----------|-------------|
| `name` | Override the node name (defaults to the function name) |
-| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()` for read-only SQL refs |
+| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()`/`seed()` for read-only SQL refs |
| `tags` | Labels for selection and grouping |
| `group` | Display/catalog grouping |
| `description` | Docs (defaults to docstring) |
diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py
index 31c6858abe..df113307de 100644
--- a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py
+++ b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py
@@ -5,6 +5,7 @@
from typing import Any
from sqlbuild.cli.commands._helpers.check.core import (
+ build_check_relation_targets,
load_results_by_loader_name,
record_python_run_state_results,
relevant_check_functions,
@@ -17,6 +18,7 @@
BuildRunOutcome,
)
from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction
+from sqlbuild.compiler.pipeline.main.project_relation_targets import build_project_relation_targets
from sqlbuild.compiler.pipeline.models import CompilePipelineResult
from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph
from sqlbuild.compiler.python_nodes.models import PythonNodeGraph
@@ -57,6 +59,13 @@ def run_post_build_python_checks(
load_results=outcome.result.load_results,
)
),
+ completed_sql_names=frozenset(
+ (
+ *(result.model_name for result in outcome.result.model_results),
+ *(result.seed_name for result in outcome.result.seed_results),
+ *(result.source_name for result in outcome.result.load_results),
+ )
+ ),
)
if not check_functions:
return ()
@@ -90,6 +99,16 @@ def run_post_build_python_checks(
is_reload=request.reload_sources,
default_database=invocation.adapter.default_database(),
default_schema=invocation.adapter.default_schema(),
+ relation_targets=build_check_relation_targets(
+ adapter=invocation.adapter,
+ pipeline_result=pipeline_result,
+ python_graph=python_graph,
+ selected_python_names=frozenset(check.name for check in check_functions),
+ ),
+ project_relations=build_project_relation_targets(
+ adapter=invocation.adapter,
+ plan_output=pipeline_result.plan_output,
+ ),
providers=providers,
),
run_state=check_run_state,
diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py
index 74857928e6..f0f64cda94 100644
--- a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py
+++ b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py
@@ -30,6 +30,7 @@
ConnectionProgressReporter,
)
from sqlbuild.cli.progress.main._write_execution_header import write_execution_header
+from sqlbuild.compiler.pipeline.main.project_relation_targets import build_project_relation_targets
from sqlbuild.compiler.pipeline.models import CompilePipelineResult
from sqlbuild.compiler.planner.models import CursorOverrides
from sqlbuild.executor.build.models import (
@@ -164,6 +165,9 @@ def execute_build_plan(
end_cursor_int=preparation.end_cursor_int,
use_color=invocation.use_color,
providers=providers,
+ project_relations=build_project_relation_targets(
+ adapter=invocation.adapter, plan_output=pipeline_result.plan_output
+ ),
),
callbacks=BuildCallbacks(
on_migration_progress=lambda message: _write_progress_line(
diff --git a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py
index ca24146c94..3794f15de0 100644
--- a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py
+++ b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py
@@ -21,6 +21,9 @@
from sqlbuild.compiler.compile.models import CompiledObjectKey
from sqlbuild.compiler.compile.types import CompiledResourceType
from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs
+from sqlbuild.compiler.pipeline.main.project_relation_targets import (
+ build_project_relation_targets,
+)
from sqlbuild.compiler.pipeline.main.relation_targets import build_python_relation_targets
from sqlbuild.compiler.pipeline.models import CompilePipelineResult
from sqlbuild.compiler.planner.main.selection.loader_dag import build_intermediate_source_map
@@ -100,12 +103,8 @@ def prepare_direct_python_lifecycle(
),
),
)
- project_relations: dict[SqlResourceRef, str] | None = (
- build_python_relation_targets(
- adapter=adapter, project=pipeline_result.project, plan_output=plan_output
- )
- if pipeline_result.project.enforce_explicit_references
- else None
+ project_relations: dict[SqlResourceRef, str] | None = build_project_relation_targets(
+ adapter=adapter, plan_output=plan_output
)
default_database: str | None = pipeline_result.project.effective_target_database
if default_database is None:
diff --git a/src/sqlbuild/cli/commands/_helpers/check/core.py b/src/sqlbuild/cli/commands/_helpers/check/core.py
index 5b09357aab..4790a9723e 100644
--- a/src/sqlbuild/cli/commands/_helpers/check/core.py
+++ b/src/sqlbuild/cli/commands/_helpers/check/core.py
@@ -30,8 +30,13 @@
from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector
from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector
from sqlbuild.compiler.python_nodes.main.selectors import resolve_python_nodes_from_selectors
-from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan
+from sqlbuild.compiler.python_nodes.models import (
+ DiscoveredPythonNode,
+ PythonNodeGraph,
+ PythonSqlRunLifecyclePlan,
+)
from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus
+from sqlbuild.executor.build.models import SeedExecutionResult
from sqlbuild.executor.build.types import ExecutionStatus
from sqlbuild.executor.load.models import LoadExecutionResult
from sqlbuild.executor.python_nodes.main.read_side import create_read_side_python_execution_tracker
@@ -187,6 +192,7 @@ def run_check_read_side_dependencies(
lifecycle_plan: PythonSqlRunLifecyclePlan,
relation_targets: dict[SqlResourceRef, str],
validation_refs: frozenset[SqlResourceRef] | None = None,
+ project_relations: dict[SqlResourceRef, str] | None = None,
providers: ProviderContainer | None = None,
) -> tuple[PythonNodeExecutionResult, ...]:
"""Run read-side Python dependencies for checks against existing SQL relations."""
@@ -239,6 +245,7 @@ def run_check_read_side_dependencies(
default_database=adapter.default_database(),
default_schema=adapter.default_schema(),
relation_targets=relation_targets,
+ project_relations=project_relations,
providers=providers,
),
)
@@ -247,6 +254,10 @@ def run_check_read_side_dependencies(
tracker.record_sql_result(
ModelExecutionResult(model_name=sql_ref.name, status=ExecutionStatus.SUCCESS)
)
+ elif sql_ref.kind == SqlResourceRefKind.SEED:
+ tracker.record_sql_result(
+ SeedExecutionResult(seed_name=sql_ref.name, status=ExecutionStatus.SUCCESS)
+ )
elif sql_ref.kind == SqlResourceRefKind.SOURCE:
tracker.record_sql_result(
LoadExecutionResult(
@@ -362,6 +373,7 @@ def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str
"message": result.message,
"error_message": result.error_message,
"metadata": result.metadata,
+ "warnings": list(result.warning_messages) or None,
}
for result in results
],
@@ -381,8 +393,9 @@ def relevant_check_functions(
python_graph: PythonNodeGraph,
exclude: tuple[str, ...],
selected_dependency_names: frozenset[str],
+ completed_sql_names: frozenset[str] = frozenset(),
) -> tuple[DiscoveredCheckFunction, ...]:
- """Return checks whose Python dependencies all ran and were not excluded."""
+ """Return non-excluded checks whose Python and SQL dependencies all ran in this build."""
excluded_names: frozenset[str] = _resolve_python_check_excludes(
exclude=exclude, python_graph=python_graph
@@ -394,7 +407,12 @@ def relevant_check_functions(
if node.name in excluded_names:
continue
upstream_names: tuple[str, ...] = python_graph.upstream_deps.get(node.name, ())
- if upstream_names and all(name in selected_dependency_names for name in upstream_names):
+ sql_names: tuple[str, ...] = tuple(sql_ref.name for sql_ref in node.sql_deps)
+ if (
+ (upstream_names or sql_names)
+ and all(name in selected_dependency_names for name in upstream_names)
+ and all(name in completed_sql_names for name in sql_names)
+ ):
check_names.add(node.name)
return tuple(check for check in discovered_inputs.check_functions if check.name in check_names)
@@ -470,6 +488,19 @@ def _validate_check_sql_ref_exists(
database=target.database, schema=target.schema, name=target.name
)
relation: str = resolve_relation_location_qualified_name(adapter=adapter, location=target)
+ elif ref.kind == SqlResourceRefKind.SEED:
+ seed_target: CompiledRelationLocation | None = (
+ pipeline_result.plan_output.seed_locations.get(ref.name)
+ )
+ if seed_target is None:
+ raise CliUserError(
+ f"Python check dependency requires unknown seed '{ref.name}'",
+ code="C682",
+ )
+ exists = relation_lookup.exists(
+ database=seed_target.database, schema=seed_target.schema, name=seed_target.name
+ )
+ relation = resolve_relation_location_qualified_name(adapter=adapter, location=seed_target)
elif ref.kind == SqlResourceRefKind.SOURCE:
source: SourceEntry | None = (
pipeline_result.plan_output.source_read_map or pipeline_result.plan_output.source_map
@@ -501,6 +532,13 @@ def _check_sql_ref_location(
if target is None:
return None
return (target.database, target.schema, target.name)
+ if ref.kind == SqlResourceRefKind.SEED:
+ seed_target: CompiledRelationLocation | None = (
+ pipeline_result.plan_output.seed_locations.get(ref.name)
+ )
+ if seed_target is None:
+ return None
+ return (seed_target.database, seed_target.schema, seed_target.name)
if ref.kind == SqlResourceRefKind.SOURCE:
source: SourceEntry | None = (
pipeline_result.plan_output.source_read_map or pipeline_result.plan_output.source_map
@@ -543,7 +581,11 @@ def _check_group_label(
python_graph: PythonNodeGraph | None,
) -> str:
if python_graph is not None:
- upstream_names: tuple[str, ...] = python_graph.upstream_deps.get(result_name, ())
+ node: DiscoveredPythonNode | None = python_graph.nodes_by_name.get(result_name)
+ upstream_names: tuple[str, ...] = (
+ *python_graph.upstream_deps.get(result_name, ()),
+ *(() if node is None else (sql_ref.name for sql_ref in node.sql_deps)),
+ )
if len(upstream_names) == 1:
return upstream_names[0]
if len(upstream_names) > 1:
@@ -570,3 +612,6 @@ def _write_check_result_row(
if detail:
stream.write(f" {detail}")
stream.write("\n")
+ warning_message: str
+ for warning_message in result.warning_messages:
+ stream.write(f" {style.warning(warning_message)}\n")
diff --git a/src/sqlbuild/cli/commands/_helpers/check/execution.py b/src/sqlbuild/cli/commands/_helpers/check/execution.py
index bf26ceef55..4011a33ad8 100644
--- a/src/sqlbuild/cli/commands/_helpers/check/execution.py
+++ b/src/sqlbuild/cli/commands/_helpers/check/execution.py
@@ -23,6 +23,7 @@
current_native_progress_projector,
)
from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction
+from sqlbuild.compiler.pipeline.main.project_relation_targets import build_project_relation_targets
from sqlbuild.compiler.pipeline.models import CompilePipelineResult
from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph
from sqlbuild.compiler.python_nodes.main.run_lifecycle import build_python_sql_run_lifecycle
@@ -88,6 +89,9 @@ def prepare_check_execution(
relation_refs=relation_refs,
default_database=_default_database(invocation=invocation, pipeline_result=pipeline_result),
default_schema=_default_schema(invocation=invocation, pipeline_result=pipeline_result),
+ project_relations=build_project_relation_targets(
+ adapter=invocation.adapter, plan_output=pipeline_result.plan_output
+ ),
)
@@ -151,6 +155,7 @@ def execute_check_plan(
default_database=preparation.default_database,
default_schema=preparation.default_schema,
relation_targets=preparation.relation_targets,
+ project_relations=preparation.project_relations,
providers=providers,
),
run_state=ingress_result.run_state,
@@ -233,6 +238,7 @@ def _execute_check_read_side(
lifecycle_plan=preparation.lifecycle_plan,
relation_targets=preparation.relation_targets,
validation_refs=preparation.relation_refs,
+ project_relations=preparation.project_relations,
providers=providers,
)
record_python_run_state_results(
diff --git a/src/sqlbuild/cli/commands/_helpers/check/planning.py b/src/sqlbuild/cli/commands/_helpers/check/planning.py
index 0c27e54c95..81981b5c35 100644
--- a/src/sqlbuild/cli/commands/_helpers/check/planning.py
+++ b/src/sqlbuild/cli/commands/_helpers/check/planning.py
@@ -12,6 +12,7 @@
from sqlbuild.cli.commands.models import CheckCommandRequest, CheckInvocation
from sqlbuild.compiler.compile.models import CompiledObjectKey
from sqlbuild.compiler.compile.types import CompiledResourceType
+from sqlbuild.compiler.graph.main.sql_ref_key import sql_ref_key
from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context
from sqlbuild.compiler.pipeline.main.static_result import build_static_pipeline_result
from sqlbuild.compiler.pipeline.models import (
@@ -24,7 +25,6 @@
from sqlbuild.compiler.python_nodes.models import PythonNodeGraph
from sqlbuild.compiler.python_nodes.types import PythonNodeKind
from sqlbuild.python_nodes.models import SqlResourceRef
-from sqlbuild.python_nodes.types import SqlResourceRefKind
def compile_check_plan(
@@ -56,16 +56,7 @@ def compile_check_plan(
required_loader_names=required_loader_names,
)
selected_keys: frozenset[CompiledObjectKey] = frozenset(
- CompiledObjectKey(
- resource_type=(
- CompiledResourceType.MODEL
- if ref.kind == SqlResourceRefKind.MODEL
- else CompiledResourceType.SOURCE
- ),
- name=ref.name,
- )
- for ref in required_refs
- if ref.kind in {SqlResourceRefKind.MODEL, SqlResourceRefKind.SOURCE}
+ sql_ref_key(ref) for ref in required_refs
) | frozenset(
CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=name)
for name in required_source_names
diff --git a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py
index 4b90820806..3a3ff52e7f 100644
--- a/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py
+++ b/src/sqlbuild/cli/commands/classes/build_progress_callbacks.py
@@ -854,7 +854,9 @@ def _count_build_footer_results(
load_result: LoadExecutionResult
for load_result in result.load_results:
- if load_result.status == ExecutionStatus.SUCCESS:
+ if load_result.status == ExecutionStatus.SUCCESS and load_result.warning_messages:
+ warn_count += 1
+ elif load_result.status == ExecutionStatus.SUCCESS:
pass_count += 1
elif load_result.status == ExecutionStatus.FAILED:
fail_count += 1
@@ -1215,32 +1217,33 @@ def _format_warning_details(
lines.append(line)
lines.append("")
- function_result: FunctionExecutionResult
- for function_result in result.function_results:
- if not function_result.warning_messages:
- continue
- if not has_warnings:
- lines.append("")
- lines.append(style.warning_strong("Warnings:"))
- lines.append("")
- has_warnings = True
- lines.append(f" {function_result.function_name} ({function_result.function_kind})")
- warning_msg: str
- for warning_msg in function_result.warning_messages:
- lines.append(f" {warning_msg}")
- lines.append("")
-
- python_result: PythonNodeExecutionResult
- for python_result in python_node_results:
- if not python_result.warning_messages:
- continue
+ labelled_warnings: tuple[tuple[str, tuple[str, ...]], ...] = (
+ *(
+ (f"{function_result.function_name} ({function_result.function_kind})", messages)
+ for function_result in result.function_results
+ if (messages := function_result.warning_messages)
+ ),
+ *(
+ (f"{load_result.source_name} (loader {load_result.loader_name})", messages)
+ for load_result in result.load_results
+ if (messages := load_result.warning_messages)
+ ),
+ *(
+ (f"{python_result.node_name} ({python_result.kind.value})", messages)
+ for python_result in python_node_results
+ if (messages := python_result.warning_messages)
+ ),
+ )
+ label: str
+ messages: tuple[str, ...]
+ for label, messages in labelled_warnings:
if not has_warnings:
lines.append("")
lines.append(style.warning_strong("Warnings:"))
lines.append("")
has_warnings = True
- lines.append(f" {python_result.node_name} ({python_result.kind.value})")
- for warning_msg in python_result.warning_messages:
+ lines.append(f" {label}")
+ for warning_msg in messages:
lines.append(f" {warning_msg}")
lines.append("")
diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py
index 945f2fa93a..e22960ed9d 100644
--- a/src/sqlbuild/cli/commands/models.py
+++ b/src/sqlbuild/cli/commands/models.py
@@ -320,6 +320,7 @@ class CheckExecutionPreparation:
relation_refs: frozenset[SqlResourceRef]
default_database: str | None
default_schema: str | None
+ project_relations: dict[SqlResourceRef, str] | None = None
@dataclass(frozen=True)
diff --git a/src/sqlbuild/cli/output/_helpers/execution_result_document.py b/src/sqlbuild/cli/output/_helpers/execution_result_document.py
index feb08cfeef..960550078a 100644
--- a/src/sqlbuild/cli/output/_helpers/execution_result_document.py
+++ b/src/sqlbuild/cli/output/_helpers/execution_result_document.py
@@ -156,7 +156,10 @@ def format_build_execution_json(
)
python_check_fail_count: int = sum(1 for result in python_check_results if result.failed)
python_check_warn_count: int = sum(1 for result in python_check_results if result.warned)
- python_warning_count: int = sum(len(result.warning_messages) for result in python_node_results)
+ python_warning_count: int = sum(
+ len(result.warning_messages)
+ for result in (*python_node_results, *python_check_results, *load_results)
+ )
python_check_pass_count: int = sum(1 for result in python_check_results if result.passed)
status: BuildStatus = (
BuildStatus.FAILED
@@ -742,6 +745,7 @@ def _format_load_assets(
"skip_mode": result.skip_mode.value if result.skip_mode else None,
"skip_reason": result.skip_reason,
"error_message": result.error_message,
+ "warnings": result.warning_messages or None,
}
)
for result in results
@@ -898,6 +902,7 @@ def _format_python_check_results(
"message": result.message,
"error_message": result.error_message,
"metadata": result.metadata,
+ "warnings": result.warning_messages or None,
}
)
for result in results
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
index e006aaaecd..3c95b4fc0e 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
@@ -5,8 +5,8 @@
import ast
import inspect
import textwrap
-from collections.abc import Callable, Iterator
-from dataclasses import dataclass
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass, field
from pathlib import Path
from sqlbuild.compiler.compile.constants import HARD_CODED_PROJECT_RELATION_CODE
@@ -18,13 +18,18 @@
compiled_project_relations,
)
from sqlbuild.compiler.references.main.extract_relation_names import extract_relation_names
+from sqlbuild.compiler.references.main.hard_coded_relation_remedy import (
+ hard_coded_relation_remedy,
+)
from sqlbuild.compiler.references.main.match_project_relation import match_project_relation
from sqlbuild.compiler.references.models import (
ProjectRelation,
ProjectRelationIndex,
RelationName,
)
-from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
+from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition
+from sqlbuild.python_nodes.models import LoaderDefinition, SqlResourceRef
from sqlbuild.python_nodes.types import SqlResourceRefKind
_SQL_METHOD_NAMES: frozenset[str] = frozenset({"query", "execute_sql"})
@@ -35,11 +40,11 @@
@dataclass(frozen=True)
class _PythonSqlOwner:
label: str
+ kind: HardCodedRelationOwnerKind
function: Callable[..., object]
relative_path: Path
- declare_help: str
- own_refs: frozenset[SqlResourceRef]
- relation_kinds: frozenset[SqlResourceRefKind]
+ own_refs: frozenset[SqlResourceRef] = frozenset()
+ upstream_loader_by_source: Mapping[str, str] = field(default_factory=dict)
@dataclass(frozen=True)
@@ -52,7 +57,7 @@ class _LiteralSql:
def validate_python_sql_references(
*, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs
) -> None:
- """Reject literal SQL in tasks, assets, and hooks that names a project relation."""
+ """Reject literal SQL in Python nodes and hooks that names a project relation."""
owners: tuple[_PythonSqlOwner, ...] = _owners(
project=project, discovered_inputs=discovered_inputs
@@ -77,25 +82,22 @@ def validate_python_sql_references(
def _owners(
*, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs
) -> tuple[_PythonSqlOwner, ...]:
- node_kinds: frozenset[SqlResourceRefKind] = frozenset(
- {SqlResourceRefKind.MODEL, SqlResourceRefKind.SOURCE}
- )
owners: list[_PythonSqlOwner] = []
for label, nodes in (
("task", discovered_inputs.task_functions),
("asset", discovered_inputs.asset_functions),
+ ("check", discovered_inputs.check_functions),
):
owners.extend(
_PythonSqlOwner(
label=f"{label}:{node.name}",
+ kind=HardCodedRelationOwnerKind.NODE,
function=node.function,
relative_path=node.relative_path,
- declare_help="declare it with depends_on={typed}",
- own_refs=frozenset(),
- relation_kinds=node_kinds,
)
for node in nodes
)
+ owners.extend(_loader_owners(project=project, discovered_inputs=discovered_inputs))
attached_models: dict[str, set[SqlResourceRef]] = {}
for model in project.models:
for hook_name in model_python_hook_names(model=model):
@@ -105,17 +107,52 @@ def _owners(
owners.extend(
_PythonSqlOwner(
label=f"hook:{hook.name}",
+ kind=HardCodedRelationOwnerKind.HOOK,
function=hook.function,
relative_path=hook.relative_path,
- declare_help="declare it with @hook(reads={typed})",
own_refs=frozenset(attached_models.get(hook.name, ())),
- relation_kinds=frozenset(SqlResourceRefKind),
)
for hook in project.hook_functions
)
return tuple(owners)
+def _loader_owners(
+ *, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs
+) -> tuple[_PythonSqlOwner, ...]:
+ sources_by_loader: dict[str, set[str]] = {}
+ for source in project.sources:
+ if source.source_entry.loader is not None:
+ sources_by_loader.setdefault(source.source_entry.loader, set()).add(source.name)
+ owners: list[_PythonSqlOwner] = []
+ for loader in discovered_inputs.loader_functions:
+ upstream_loader_by_source: dict[str, str] = {}
+ for dependency in loader.depends_on:
+ definition: LoaderDefinition | None = (
+ read_loader_definition(dependency) if callable(dependency) else None
+ )
+ if definition is None:
+ continue
+ for source_name in sources_by_loader.get(definition.name, ()):
+ upstream_loader_by_source[source_name] = getattr(
+ dependency, "__name__", definition.name
+ )
+ owners.append(
+ _PythonSqlOwner(
+ label=f"loader:{loader.name}",
+ kind=HardCodedRelationOwnerKind.LOADER,
+ function=loader.function,
+ relative_path=loader.relative_path,
+ own_refs=frozenset(
+ SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source_name)
+ for source_name in sources_by_loader.get(loader.name, ())
+ ),
+ upstream_loader_by_source=upstream_loader_by_source,
+ )
+ )
+ return tuple(owners)
+
+
def _literal_sql_calls(function: Callable[..., object]) -> Iterator[_LiteralSql]:
try:
lines: list[str]
@@ -193,24 +230,29 @@ def _reject_hard_coded_relation(
if relation.schema is None and relation.name.casefold() in temporary:
continue
match: ProjectRelation | None = match_project_relation(index=index, relation=relation)
- if (
- match is None
- or match.ref in owner.own_refs
- or match.ref.kind not in owner.relation_kinds
- ):
+ if match is None or match.ref in owner.own_refs:
continue
- typed: str = f'{match.ref.kind.value}("{match.ref.name}")'
written: str = ".".join(
part for part in (relation.database, relation.schema, relation.name) if part
)
+ remedy: str = hard_coded_relation_remedy(
+ owner_kind=owner.kind,
+ ref=match.ref,
+ upstream_loader_by_source=owner.upstream_loader_by_source,
+ )
+ qualify_help: str = (
+ f"\n = help: if '{written}' is an external table that shares the name, qualify it "
+ "with its schema"
+ if relation.schema is None
+ else ""
+ )
raise CompileInputError(
f"{owner.label} names {match.ref.kind.value}:{match.ref.name} as '{written}' in SQL "
f"passed to ctx.{literal.method}()\n --> {owner.relative_path.as_posix()}:"
f"{literal.line}",
code=HARD_CODED_PROJECT_RELATION_CODE,
help=(
- f"{owner.declare_help.format(typed=typed)} and use ctx.relation({typed}) instead "
- "of the relation name\n"
+ f"{remedy}{qualify_help}\n"
" = help: while migrating a project, allow hard-coded relation names with "
"[references] enforce_explicit = false in sqlbuild_project.toml"
),
diff --git a/src/sqlbuild/compiler/dag/_helpers/artifact.py b/src/sqlbuild/compiler/dag/_helpers/artifact.py
index 94d63dc650..bd768286ac 100644
--- a/src/sqlbuild/compiler/dag/_helpers/artifact.py
+++ b/src/sqlbuild/compiler/dag/_helpers/artifact.py
@@ -33,12 +33,12 @@
)
from sqlbuild.compiler.dag.types import NodeKind
from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction
+from sqlbuild.compiler.graph.main.sql_ref_key import sql_ref_key
from sqlbuild.compiler.pipeline.models import ProjectGraph
from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph
from sqlbuild.compiler.python_nodes.types import PythonNodeKind
from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type
from sqlbuild.python_nodes.models import ColumnLineageRef, SqlResourceRef
-from sqlbuild.python_nodes.types import SqlResourceRefKind
from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts
from sqlbuild.spec.contracts.models import (
LoaderDestinationParts,
@@ -437,7 +437,7 @@ def _build_python_check(*, node: DiscoveredPythonNode, python_graph: PythonNodeG
)
for edge in python_graph.dependency_edges
if edge.downstream_name == node.name
- )
+ ) + tuple(_sql_ref_node_id(sql_dep) for sql_dep in node.sql_deps)
return DagCheck(
id=_python_node_id(kind=node.kind, node_name=node.name),
kind=NodeKind.PYTHON_CHECK.value,
@@ -495,12 +495,7 @@ def _python_node_id(*, kind: PythonNodeKind, node_name: str) -> str:
def _sql_ref_node_id(ref: SqlResourceRef) -> str:
- resource_type: CompiledResourceType = (
- CompiledResourceType.MODEL
- if ref.kind == SqlResourceRefKind.MODEL
- else CompiledResourceType.SOURCE
- )
- return _node_id(CompiledObjectKey(resource_type, ref.name))
+ return _node_id(sql_ref_key(ref))
def _source_by_loader(graph: ProjectGraph) -> dict[str, SourceEntry]:
diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py
index bb2800cac5..e124b9cf7a 100644
--- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py
+++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py
@@ -40,7 +40,6 @@
SqlResourceRef,
TaskDefinition,
)
-from sqlbuild.python_nodes.types import SqlResourceRefKind
from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry, SourceEntry
@@ -356,11 +355,6 @@ def _validate_python_node_dependencies(
f"'{dependency.name}'; loaders "
"may depend on Python loader/task/asset functions only"
)
- if dependency.kind is SqlResourceRefKind.SEED:
- raise DiscoveryConflictError(
- f"Python node '{node.name}' depends on seed '{dependency.name}'; "
- "Python nodes may depend on models and sources"
- )
continue
if _python_node_dependency_key(dependency) not in node_by_dependency_key:
if isinstance(node, DiscoveredLoaderFunction):
@@ -429,15 +423,12 @@ def _validate_check_dependencies(
dependency: object
for dependency in check_function.depends_on:
if isinstance(dependency, SqlResourceRef):
- raise DiscoveryConflictError(
- f"Check '{check_function.name}' depends on SQL resource '{dependency.name}'; "
- "checks may depend on loaders, tasks, and assets only"
- )
+ continue
dependency_key: object | tuple[str, str] = _python_node_dependency_key(dependency)
if dependency_key in check_dependency_keys:
raise DiscoveryConflictError(
f"Check '{check_function.name}' depends on another check; checks may "
- "depend on loaders, tasks, and assets only"
+ "depend on loaders, tasks, assets, models, sources, and seeds only"
)
if dependency_key not in allowed_dependency_keys:
raise DiscoveryConflictError(
diff --git a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
index baec15d9ac..d77c7846ae 100644
--- a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
+++ b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
@@ -3,27 +3,15 @@
from __future__ import annotations
from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, CompiledProject
-from sqlbuild.compiler.compile.types import CompiledResourceType
from sqlbuild.compiler.discovery.models import PythonHookEntry
from sqlbuild.compiler.graph._helpers.algorithms import transitive_closure_many_impl
from sqlbuild.compiler.graph._helpers.audit_gates import attached_audit_gate_edges_impl
from sqlbuild.compiler.graph._helpers.lineage import build_lineage_upstream_deps_impl
+from sqlbuild.compiler.graph._helpers.sql_refs import sql_ref_key_impl
from sqlbuild.compiler.graph.models import HookReadEdge
from sqlbuild.python_nodes.models import SqlResourceRef
-from sqlbuild.python_nodes.types import SqlResourceRefKind
_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks")
-_RESOURCE_TYPE_BY_REF_KIND: dict[SqlResourceRefKind, CompiledResourceType] = {
- SqlResourceRefKind.MODEL: CompiledResourceType.MODEL,
- SqlResourceRefKind.SOURCE: CompiledResourceType.SOURCE,
- SqlResourceRefKind.SEED: CompiledResourceType.SEED,
-}
-
-
-def hook_read_key(ref: SqlResourceRef) -> CompiledObjectKey:
- """Return the compiled graph key a declared hook read names."""
-
- return CompiledObjectKey(resource_type=_RESOURCE_TYPE_BY_REF_KIND[ref.kind], name=ref.name)
def model_python_hook_names_impl(model: CompiledModel) -> tuple[str, ...]:
@@ -51,7 +39,7 @@ def hook_read_edges_impl(*, project: CompiledProject) -> tuple[HookReadEdge, ...
for model in project.models:
for hook_name in model_python_hook_names_impl(model):
for ref in reads_by_hook.get(hook_name, ()):
- read: CompiledObjectKey = hook_read_key(ref)
+ read: CompiledObjectKey = sql_ref_key_impl(ref)
if read != model.key:
edges[HookReadEdge(hook_name=hook_name, gated=model.key, read=read)] = None
return tuple(edges)
diff --git a/src/sqlbuild/compiler/graph/_helpers/sql_refs.py b/src/sqlbuild/compiler/graph/_helpers/sql_refs.py
new file mode 100644
index 0000000000..9caa97ade6
--- /dev/null
+++ b/src/sqlbuild/compiler/graph/_helpers/sql_refs.py
@@ -0,0 +1,20 @@
+"""Graph keys for typed SQL resource references."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.models import CompiledObjectKey
+from sqlbuild.compiler.compile.types import CompiledResourceType
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
+
+_RESOURCE_TYPE_BY_REF_KIND: dict[SqlResourceRefKind, CompiledResourceType] = {
+ SqlResourceRefKind.MODEL: CompiledResourceType.MODEL,
+ SqlResourceRefKind.SOURCE: CompiledResourceType.SOURCE,
+ SqlResourceRefKind.SEED: CompiledResourceType.SEED,
+}
+
+
+def sql_ref_key_impl(ref: SqlResourceRef) -> CompiledObjectKey:
+ """Return the compiled graph key a typed SQL reference names."""
+
+ return CompiledObjectKey(resource_type=_RESOURCE_TYPE_BY_REF_KIND[ref.kind], name=ref.name)
diff --git a/src/sqlbuild/compiler/graph/main/sql_ref_key.py b/src/sqlbuild/compiler/graph/main/sql_ref_key.py
new file mode 100644
index 0000000000..2a7ed40d58
--- /dev/null
+++ b/src/sqlbuild/compiler/graph/main/sql_ref_key.py
@@ -0,0 +1,13 @@
+"""Typed SQL reference graph key entrypoint."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.models import CompiledObjectKey
+from sqlbuild.compiler.graph._helpers.sql_refs import sql_ref_key_impl
+from sqlbuild.python_nodes.models import SqlResourceRef
+
+
+def sql_ref_key(ref: SqlResourceRef) -> CompiledObjectKey:
+ """Return the compiled graph key a typed model, source, or seed reference names."""
+
+ return sql_ref_key_impl(ref)
diff --git a/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py b/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py
new file mode 100644
index 0000000000..e53c375664
--- /dev/null
+++ b/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py
@@ -0,0 +1,125 @@
+"""Runtime SQL relation maps for Python node contexts and hard-coded name guards."""
+
+from __future__ import annotations
+
+from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
+from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import (
+ resolve_relation_location_qualified_name,
+)
+from sqlbuild.compiler.compile.models import (
+ CompiledProject,
+ CompiledRelationLocation,
+)
+from sqlbuild.compiler.planner.models import PlanOutput
+from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
+from sqlbuild.errors.contracts.exceptions import SharedInputError
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
+from sqlbuild.spec.contracts.models import SourceEntry
+
+
+def build_python_relation_targets_impl(
+ *,
+ adapter: BaseAdapter,
+ project: CompiledProject,
+ plan_output: PlanOutput,
+ required_refs: frozenset[SqlResourceRef] | None = None,
+) -> dict[SqlResourceRef, str]:
+ """Return adapter-qualified runtime relations required by selected Python nodes."""
+
+ targets: dict[SqlResourceRef, str] = {}
+ refs: frozenset[SqlResourceRef] = (
+ required_refs
+ if required_refs is not None
+ else _planned_relation_refs(plan_output=plan_output)
+ )
+ ref: SqlResourceRef
+ for ref in refs:
+ targets[ref] = _resolve_relation(
+ adapter=adapter,
+ project=project,
+ plan_output=plan_output,
+ ref=ref,
+ )
+ return targets
+
+
+def build_project_relation_targets_impl(
+ *, adapter: BaseAdapter, plan_output: PlanOutput
+) -> dict[SqlResourceRef, str] | None:
+ """Return every planned project relation for hard-coded name warnings, or None when off."""
+
+ if not plan_output.enforce_explicit_references:
+ return None
+ return {
+ ref: relation
+ for ref in _planned_relation_refs(plan_output=plan_output)
+ if (relation := _planned_relation(adapter=adapter, plan_output=plan_output, ref=ref))
+ is not None
+ }
+
+
+def _planned_relation_refs(*, plan_output: PlanOutput) -> frozenset[SqlResourceRef]:
+ refs: set[SqlResourceRef] = {
+ SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=name)
+ for name in plan_output.model_locations
+ }
+ refs.update(
+ SqlResourceRef(kind=SqlResourceRefKind.SEED, name=name)
+ for name in plan_output.seed_locations
+ )
+ source_name: str
+ for source_name in plan_output.source_read_map or plan_output.source_map:
+ refs.add(SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source_name))
+ return frozenset(refs)
+
+
+def _resolve_relation(
+ *,
+ adapter: BaseAdapter,
+ project: CompiledProject,
+ plan_output: PlanOutput,
+ ref: SqlResourceRef,
+) -> str:
+ planned: str | None = _planned_relation(adapter=adapter, plan_output=plan_output, ref=ref)
+ if planned is not None:
+ return planned
+ if ref.kind == SqlResourceRefKind.SOURCE:
+ raise SharedInputError(
+ f"Python node source '{ref.name}' is missing from the resolved plan source map"
+ )
+ location: CompiledRelationLocation | None = next(
+ (
+ resource.destination
+ for resource in (
+ project.models if ref.kind == SqlResourceRefKind.MODEL else project.seeds
+ )
+ if resource.name == ref.name
+ ),
+ None,
+ )
+ if location is None:
+ raise SharedInputError(f"Python node references unknown {ref.kind.value} '{ref.name}'")
+ return resolve_relation_location_qualified_name(adapter=adapter, location=location)
+
+
+def _planned_relation(
+ *, adapter: BaseAdapter, plan_output: PlanOutput, ref: SqlResourceRef
+) -> str | None:
+ if ref.kind == SqlResourceRefKind.SOURCE:
+ planned_sources: dict[str, SourceEntry] = (
+ plan_output.source_read_map or plan_output.source_map
+ )
+ planned_source: SourceEntry | None = planned_sources.get(ref.name)
+ if planned_source is None:
+ return None
+ return render_source_relation(entry=planned_source, adapter=adapter)
+ locations: dict[str, CompiledRelationLocation] = (
+ plan_output.model_locations
+ if ref.kind == SqlResourceRefKind.MODEL
+ else plan_output.seed_locations
+ )
+ location: CompiledRelationLocation | None = locations.get(ref.name)
+ if location is None:
+ return None
+ return resolve_relation_location_qualified_name(adapter=adapter, location=location)
diff --git a/src/sqlbuild/compiler/pipeline/main/project_relation_targets.py b/src/sqlbuild/compiler/pipeline/main/project_relation_targets.py
new file mode 100644
index 0000000000..d517110bfe
--- /dev/null
+++ b/src/sqlbuild/compiler/pipeline/main/project_relation_targets.py
@@ -0,0 +1,18 @@
+"""Planned project relations for hard-coded relation name guards."""
+
+from __future__ import annotations
+
+from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
+from sqlbuild.compiler.pipeline._helpers.relation_targets import (
+ build_project_relation_targets_impl,
+)
+from sqlbuild.compiler.planner.models import PlanOutput
+from sqlbuild.python_nodes.models import SqlResourceRef
+
+
+def build_project_relation_targets(
+ *, adapter: BaseAdapter, plan_output: PlanOutput
+) -> dict[SqlResourceRef, str] | None:
+ """Return every planned project relation for hard-coded name warnings, or None when off."""
+
+ return build_project_relation_targets_impl(adapter=adapter, plan_output=plan_output)
diff --git a/src/sqlbuild/compiler/pipeline/main/relation_targets.py b/src/sqlbuild/compiler/pipeline/main/relation_targets.py
index 77f01a2d72..cada81cd53 100644
--- a/src/sqlbuild/compiler/pipeline/main/relation_targets.py
+++ b/src/sqlbuild/compiler/pipeline/main/relation_targets.py
@@ -3,20 +3,12 @@
from __future__ import annotations
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
-from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import (
- resolve_relation_location_qualified_name,
-)
-from sqlbuild.compiler.compile.models import (
- CompiledModel,
- CompiledProject,
- CompiledRelationLocation,
+from sqlbuild.compiler.compile.models import CompiledProject
+from sqlbuild.compiler.pipeline._helpers.relation_targets import (
+ build_python_relation_targets_impl,
)
from sqlbuild.compiler.planner.models import PlanOutput
-from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
-from sqlbuild.errors.contracts.exceptions import SharedInputError
from sqlbuild.python_nodes.models import SqlResourceRef
-from sqlbuild.python_nodes.types import SqlResourceRefKind
-from sqlbuild.spec.contracts.models import SourceEntry
def build_python_relation_targets(
@@ -28,56 +20,6 @@ def build_python_relation_targets(
) -> dict[SqlResourceRef, str]:
"""Return adapter-qualified runtime relations required by selected Python nodes."""
- targets: dict[SqlResourceRef, str] = {}
- refs: frozenset[SqlResourceRef] = (
- required_refs
- if required_refs is not None
- else _planned_relation_refs(plan_output=plan_output)
- )
- ref: SqlResourceRef
- for ref in refs:
- targets[ref] = _resolve_relation(
- adapter=adapter,
- project=project,
- plan_output=plan_output,
- ref=ref,
- )
- return targets
-
-
-def _planned_relation_refs(*, plan_output: PlanOutput) -> frozenset[SqlResourceRef]:
- refs: set[SqlResourceRef] = {
- SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=name)
- for name in plan_output.model_locations
- }
- source_name: str
- for source_name in plan_output.source_read_map or plan_output.source_map:
- refs.add(SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source_name))
- return frozenset(refs)
-
-
-def _resolve_relation(
- *,
- adapter: BaseAdapter,
- project: CompiledProject,
- plan_output: PlanOutput,
- ref: SqlResourceRef,
-) -> str:
- if ref.kind == SqlResourceRefKind.MODEL:
- planned_model: CompiledRelationLocation | None = plan_output.model_locations.get(ref.name)
- if planned_model is not None:
- return resolve_relation_location_qualified_name(adapter=adapter, location=planned_model)
- model: CompiledModel
- for model in project.models:
- if model.name == ref.name:
- return resolve_relation_location_qualified_name(
- adapter=adapter, location=model.destination
- )
- raise SharedInputError(f"Python node references unknown model '{ref.name}'")
- planned_sources: dict[str, SourceEntry] = plan_output.source_read_map or plan_output.source_map
- planned_source: SourceEntry | None = planned_sources.get(ref.name)
- if planned_source is not None:
- return render_source_relation(entry=planned_source, adapter=adapter)
- raise SharedInputError(
- f"Python node source '{ref.name}' is missing from the resolved plan source map"
+ return build_python_relation_targets_impl(
+ adapter=adapter, project=project, plan_output=plan_output, required_refs=required_refs
)
diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py
index eda7d19cca..c6eb8f6022 100644
--- a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py
+++ b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py
@@ -7,7 +7,9 @@
from sqlbuild.compiler.compile.models import CompiledObjectKey
from sqlbuild.compiler.compile.types import CompiledResourceType
+from sqlbuild.compiler.graph.main.sql_ref_key import sql_ref_key
from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure
+from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many
from sqlbuild.compiler.pipeline.models import ProjectGraph
from sqlbuild.compiler.planner.constants import PATH_SELECTOR_EXPLICIT_ROOT_ERROR
from sqlbuild.compiler.planner.exceptions import PlannerInputError
@@ -30,7 +32,6 @@
PythonSqlSelection,
)
from sqlbuild.compiler.python_nodes.types import PythonNodeKind
-from sqlbuild.python_nodes.types import SqlResourceRefKind
_PYTHON_SELECTOR_KINDS: frozenset[SelectorKind] = frozenset(
{
@@ -181,21 +182,10 @@ def _validate_python_sql_refs(
raise PlannerInputError(
f"Python node '{node.name}' depends on unknown SQL resource '{sql_ref.name}'"
)
- if (
- sql_ref.kind == SqlResourceRefKind.MODEL
- and sql_key.resource_type != CompiledResourceType.MODEL
- ):
- raise PlannerInputError(
- f"Python node '{node.name}' declares model('{sql_ref.name}') but "
- f"'{sql_ref.name}' is a {sql_key.resource_type}"
- )
- if (
- sql_ref.kind == SqlResourceRefKind.SOURCE
- and sql_key.resource_type != CompiledResourceType.SOURCE
- ):
+ if sql_key.resource_type != sql_ref_key(sql_ref).resource_type:
raise PlannerInputError(
- f"Python node '{node.name}' declares source('{sql_ref.name}') but "
- f"'{sql_ref.name}' is a {sql_key.resource_type}"
+ f"Python node '{node.name}' declares {sql_ref.kind.value}('{sql_ref.name}') "
+ f"but '{sql_ref.name}' is a {sql_key.resource_type}"
)
@@ -267,6 +257,82 @@ def _resolve_token(
def _resolve_single(
*, raw: str, project_graph: ProjectGraph, python_graph: PythonNodeGraph
+) -> frozenset[_SelectionAtom]:
+ atoms: frozenset[_SelectionAtom] = _resolve_single_side(
+ raw=raw, project_graph=project_graph, python_graph=python_graph
+ )
+ parsed: ParsedSelector | PathSelector = parse_project_selector(raw)
+ if isinstance(parsed, PathSelector):
+ return atoms
+ if parsed.upstream:
+ atoms = atoms | _sql_dependency_upstream_atoms(
+ atoms=atoms, project_graph=project_graph, python_graph=python_graph
+ )
+ if parsed.downstream:
+ atoms = atoms | _sql_dependent_python_atoms(atoms=atoms, python_graph=python_graph)
+ return atoms
+
+
+def _sql_dependency_upstream_atoms(
+ *,
+ atoms: frozenset[_SelectionAtom],
+ project_graph: ProjectGraph,
+ python_graph: PythonNodeGraph,
+) -> frozenset[_SelectionAtom]:
+ """Return the SQL resources selected Python nodes declare, with their SQL upstream."""
+
+ dependency_keys: set[CompiledObjectKey] = set()
+ atom: _SelectionAtom
+ for atom in atoms:
+ if isinstance(atom.value, str):
+ dependency_keys.update(_sql_dependency_keys(python_graph.nodes_by_name[atom.value]))
+ if not dependency_keys:
+ return frozenset()
+ sql_atoms: set[_SelectionAtom] = {
+ _sql_atom(key)
+ for key in transitive_closure_many(
+ starts=dependency_keys, edges=project_graph.upstream_deps, include_starts=True
+ )
+ }
+ return frozenset(
+ sql_atoms
+ | _required_terminal_loader_atoms(
+ selected_atoms=sql_atoms, project_graph=project_graph, python_graph=python_graph
+ )
+ )
+
+
+def _sql_dependent_python_atoms(
+ *, atoms: frozenset[_SelectionAtom], python_graph: PythonNodeGraph
+) -> frozenset[_SelectionAtom]:
+ """Return runnable Python nodes that declare a selected SQL resource, with their downstream."""
+
+ selected_keys: frozenset[CompiledObjectKey] = frozenset(
+ atom.value for atom in atoms if isinstance(atom.value, CompiledObjectKey)
+ )
+ dependent_names: frozenset[str] = frozenset(
+ node.name
+ for node in python_graph.nodes
+ if node.kind != PythonNodeKind.CHECK
+ and not _sql_dependency_keys(node).isdisjoint(selected_keys)
+ )
+ if not dependent_names:
+ return frozenset()
+ return frozenset(
+ _python_atom(name)
+ for name in transitive_closure_many(
+ starts=dependent_names, edges=python_graph.downstream_deps, include_starts=True
+ )
+ if python_graph.nodes_by_name[name].kind != PythonNodeKind.CHECK
+ )
+
+
+def _sql_dependency_keys(node: DiscoveredPythonNode) -> frozenset[CompiledObjectKey]:
+ return frozenset(sql_ref_key(sql_ref) for sql_ref in node.sql_deps)
+
+
+def _resolve_single_side(
+ *, raw: str, project_graph: ProjectGraph, python_graph: PythonNodeGraph
) -> frozenset[_SelectionAtom]:
parsed: ParsedSelector | PathSelector = parse_project_selector(raw)
atoms: frozenset[_SelectionAtom]
diff --git a/src/sqlbuild/compiler/references/_helpers/remedies.py b/src/sqlbuild/compiler/references/_helpers/remedies.py
new file mode 100644
index 0000000000..3e5944d18e
--- /dev/null
+++ b/src/sqlbuild/compiler/references/_helpers/remedies.py
@@ -0,0 +1,38 @@
+"""Remedies for project relation names hard-coded in Python SQL."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
+
+
+def hard_coded_relation_remedy_impl(
+ *,
+ owner_kind: HardCodedRelationOwnerKind,
+ ref: SqlResourceRef,
+ upstream_loader_by_source: Mapping[str, str],
+) -> str:
+ """Return how the owner should read ``ref`` instead of naming it in SQL."""
+
+ typed: str = f'{ref.kind.value}("{ref.name}")'
+ if owner_kind is HardCodedRelationOwnerKind.LOADER:
+ if ref.kind is not SqlResourceRefKind.SOURCE:
+ return (
+ "loaders run before models and seeds and cannot read them; read "
+ f"{ref.kind.value}:{ref.name} from a task or asset instead"
+ )
+ loader_name: str | None = upstream_loader_by_source.get(ref.name)
+ if loader_name is not None:
+ return f"use ctx.loader({loader_name}) instead of the relation name"
+ return f'use ctx.source("{ref.name}") instead of the relation name'
+ declaration: str = (
+ f"@hook(reads={typed})"
+ if owner_kind is HardCodedRelationOwnerKind.HOOK
+ else f"depends_on={typed}"
+ )
+ return (
+ f"declare it with {declaration} and use ctx.relation({typed}) instead of the relation name"
+ )
diff --git a/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py b/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py
new file mode 100644
index 0000000000..94ee894819
--- /dev/null
+++ b/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py
@@ -0,0 +1,26 @@
+"""Hard-coded project relation remedy entrypoint."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from sqlbuild.compiler.references._helpers.remedies import hard_coded_relation_remedy_impl
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
+from sqlbuild.python_nodes.models import SqlResourceRef
+
+
+def hard_coded_relation_remedy(
+ *,
+ owner_kind: HardCodedRelationOwnerKind,
+ ref: SqlResourceRef,
+ upstream_loader_by_source: Mapping[str, str] | None = None,
+) -> str:
+ """Return how Python code should read ``ref`` instead of naming it in SQL."""
+
+ return hard_coded_relation_remedy_impl(
+ owner_kind=owner_kind,
+ ref=ref,
+ upstream_loader_by_source={}
+ if upstream_loader_by_source is None
+ else upstream_loader_by_source,
+ )
diff --git a/src/sqlbuild/compiler/references/types.py b/src/sqlbuild/compiler/references/types.py
index 855de928c0..468a5b7108 100644
--- a/src/sqlbuild/compiler/references/types.py
+++ b/src/sqlbuild/compiler/references/types.py
@@ -66,3 +66,11 @@ def resolve_reference(
ref_package: str | None,
) -> str | None:
"""Return the physical relation for an external reference, if resolvable."""
+
+
+class HardCodedRelationOwnerKind(StrEnum):
+ """The kind of Python code that names a project relation in SQL."""
+
+ NODE = "node"
+ HOOK = "hook"
+ LOADER = "loader"
diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py
index f90a1d44af..ce74485e38 100644
--- a/src/sqlbuild/executor/build/_helpers/source_node.py
+++ b/src/sqlbuild/executor/build/_helpers/source_node.py
@@ -80,6 +80,7 @@ def execute_build_source_node(
providers=runtime.providers,
result_store=result_store,
schema_prepared=schema_prepared,
+ project_relations=runtime.project_relations,
),
statement_recorder=StatementRecorder(),
loader_ref_entries=loader_ref_entries,
diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py
index 078781a16c..15bd6e6c8b 100644
--- a/src/sqlbuild/executor/build/models.py
+++ b/src/sqlbuild/executor/build/models.py
@@ -31,6 +31,7 @@
from sqlbuild.microbatches.models import MicrobatchScope
from sqlbuild.microbatches.types import MicrobatchEventStore
from sqlbuild.provider.main.runtime import ProviderContainer
+from sqlbuild.python_nodes.models import SqlResourceRef
from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, NodeStartCallback
from sqlbuild.spec.contracts.models import SnapshotsConfig
@@ -91,6 +92,7 @@ class BuildRuntimeParams:
| None
) = None
microbatch_lease_check: Callable[[], None] | None = None
+ project_relations: Mapping[SqlResourceRef, str] | None = None
@dataclass(frozen=True)
diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py
index 4d3964fc91..21db821933 100644
--- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py
+++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py
@@ -4,12 +4,13 @@
import logging
import time
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
from typing import Any
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder
from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
from sqlbuild.errors.contracts.exceptions import ExecutorInputError
from sqlbuild.executor.load._helpers.execution import (
is_untargeted_self_managed_intermediate,
@@ -28,8 +29,11 @@
LoadRuntimeParams,
)
from sqlbuild.executor.node_results.models import NodeResultEnvelope
+from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
from sqlbuild.executor.scheduling.types import ExecutionStatus
from sqlbuild.provider.main.runtime import _empty_provider_container
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
from sqlbuild.runtime.contracts.types import ExecutionResourceKind
from sqlbuild.spec.contracts.models import SourceEntry
from sqlbuild.spec.contracts.types import SourceWriteStrategy
@@ -121,6 +125,38 @@ def build_loader_context(
),
result_store=runtime.result_store,
on_progress=on_progress,
+ relation_guard=ref_bindings.relation_guard,
+ )
+
+
+def build_loader_relation_guard(
+ *,
+ source_entry: SourceEntry,
+ loader_function: DiscoveredLoaderFunction,
+ adapter: BaseAdapter,
+ runtime: LoadRuntimeParams,
+ loader_ref_entries: Mapping[Callable[..., object], SourceEntry] | None,
+ warnings: list[str],
+) -> RuntimeRelationGuard | None:
+ """Return the hard-coded relation guard for one loader run, or None when enforcement is off."""
+
+ if runtime.project_relations is None:
+ return None
+ return RuntimeRelationGuard(
+ owner_label=f"loader '{loader_function.name}'",
+ owner_kind=HardCodedRelationOwnerKind.LOADER,
+ project_relations=runtime.project_relations,
+ dialect=adapter.sql_analysis_dialect(),
+ default_database=adapter.default_database(),
+ default_schema=adapter.default_schema(),
+ warnings=warnings,
+ own_refs=frozenset(
+ {SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source_entry.name)}
+ ),
+ upstream_loader_by_source={
+ entry.name: getattr(function, "__name__", entry.name)
+ for function, entry in (loader_ref_entries or {}).items()
+ },
)
diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py
index 79e2b59d9a..9d6537fcc1 100644
--- a/src/sqlbuild/executor/load/main/_execute.py
+++ b/src/sqlbuild/executor/load/main/_execute.py
@@ -4,6 +4,7 @@
import time
from collections.abc import Callable, Mapping
+from dataclasses import replace
from typing import Any
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
@@ -28,6 +29,7 @@
from sqlbuild.executor.load._helpers.external import execute_external_source_load
from sqlbuild.executor.load._helpers.loader_invocation import (
build_loader_context,
+ build_loader_relation_guard,
interpret_loader_return,
validate_source_write_strategy,
)
@@ -63,6 +65,47 @@ def execute_source_load(
) -> LoadExecutionResult:
"""Run one source loader and write returned rows using the table strategy."""
+ warnings: list[str] = []
+ result: LoadExecutionResult = _execute_source_load(
+ source_entry=source_entry,
+ loader_function=loader_function,
+ adapter=adapter,
+ connection_config=connection_config,
+ connection=connection,
+ runtime=runtime,
+ statement_recorder=statement_recorder,
+ ref_bindings=LoaderRefBindings(
+ loader_ref_entries=loader_ref_entries,
+ source_ref_entries=source_ref_entries,
+ relation_guard=build_loader_relation_guard(
+ source_entry=source_entry,
+ loader_function=loader_function,
+ adapter=adapter,
+ runtime=runtime,
+ loader_ref_entries=loader_ref_entries,
+ warnings=warnings,
+ ),
+ ),
+ on_progress=on_progress,
+ )
+ if not warnings:
+ return result
+ return replace(result, warning_messages=tuple(warnings))
+
+
+def _execute_source_load(
+ *,
+ source_entry: SourceEntry,
+ loader_function: DiscoveredLoaderFunction,
+ adapter: BaseAdapter,
+ connection_config: dict[str, object],
+ connection: Any,
+ runtime: LoadRuntimeParams,
+ statement_recorder: StatementRecorder,
+ ref_bindings: LoaderRefBindings,
+ on_progress: Callable[[str], None] | None = None,
+) -> LoadExecutionResult:
+
destination_name: str = (
source_entry.table if source_entry.table is not None else source_entry.name
)
@@ -119,10 +162,7 @@ def execute_source_load(
destination=destination,
runtime=runtime,
statement_recorder=statement_recorder,
- ref_bindings=LoaderRefBindings(
- loader_ref_entries=loader_ref_entries,
- source_ref_entries=source_ref_entries,
- ),
+ ref_bindings=ref_bindings,
on_progress=on_progress,
)
with OperationLifecycle(operation_kind="loader", operation_name="managed_source_load"):
diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py
index f260b5e26c..7bf948b434 100644
--- a/src/sqlbuild/executor/load/models.py
+++ b/src/sqlbuild/executor/load/models.py
@@ -22,9 +22,12 @@
from sqlbuild.executor.load.constants import LOADER_RELATION_QUALIFIER_SEPARATOR
from sqlbuild.executor.load.types import LoadProgressCallback
from sqlbuild.executor.node_results.models import NodeResultEnvelope
+from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT
from sqlbuild.executor.scheduling.types import ExecutionStatus
from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container
+from sqlbuild.python_nodes.models import SqlResourceRef
+from sqlbuild.python_nodes.types import SqlResourceRefKind
from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, ExecutionResourceKind
from sqlbuild.spec.contracts.models import SourceEntry
@@ -121,12 +124,17 @@ class LoaderContext:
result_store: Any | None = None
runtime_dir: Path = Path("target")
on_progress: Callable[[str], None] | None = None
+ relation_guard: RuntimeRelationGuard | None = field(default=None, repr=False)
def execute_sql(self, sql: str) -> Any:
+ if self.relation_guard is not None:
+ self.relation_guard.check(sql)
self.statement_recorder.record(sql)
return self.adapter.execute(connection=self.connection, sql=sql)
def query(self, sql: str) -> Any:
+ if self.relation_guard is not None:
+ self.relation_guard.check(sql)
self.statement_recorder.record(sql)
return self.adapter.execute(connection=self.connection, sql=sql)
@@ -224,6 +232,7 @@ def loader(self, loader_fn: Callable[..., object]) -> LoaderRelationRef:
relation_ref: LoaderRelationRef | None = self.loader_refs.get(loader_fn)
if relation_ref is None:
raise ExecutorInputError("Unknown loader reference passed to ctx.loader(...)")
+ self._record_resolved(relation_ref)
return relation_ref
def source(self, source_name: str) -> LoaderRelationRef:
@@ -234,8 +243,15 @@ def source(self, source_name: str) -> LoaderRelationRef:
raise ExecutorInputError(
f"Unknown source reference passed to ctx.source(...): {source_name}"
)
+ self._record_resolved(relation_ref)
return relation_ref
+ def _record_resolved(self, relation_ref: LoaderRelationRef) -> None:
+ if self.relation_guard is not None:
+ self.relation_guard.record_resolved(
+ SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=relation_ref.name)
+ )
+
def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]:
task_definition: object = getattr(node_function, "__sqlbuild_task__", None)
if task_definition is not None:
@@ -274,6 +290,7 @@ class LoadExecutionResult:
result_payload: object | None = None
result_metadata: dict[str, object] = field(default_factory=dict)
result_materialized: bool | None = None
+ warning_messages: tuple[str, ...] = field(default_factory=tuple)
@dataclass(frozen=True)
@@ -305,6 +322,7 @@ class LoadRuntimeParams:
providers: ProviderContainer | None = None
result_store: Any | None = None
schema_prepared: bool = False
+ project_relations: Mapping[SqlResourceRef, str] | None = None
@dataclass(frozen=True)
@@ -327,10 +345,11 @@ class LoaderDestination:
@dataclass(frozen=True)
class LoaderRefBindings:
- """Loader and source relation-ref entry maps for one loader run."""
+ """Loader and source relation bindings, and the hard-coded name guard, for one loader run."""
loader_ref_entries: Mapping[Callable[..., object], SourceEntry] | None = None
source_ref_entries: Mapping[str, SourceEntry] | None = None
+ relation_guard: RuntimeRelationGuard | None = None
@dataclass(frozen=True)
diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py
index ea63510a98..0a291eb538 100644
--- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py
+++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py
@@ -20,12 +20,12 @@
from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store
from sqlbuild.executor.node_results.models import NodeResultRecord
from sqlbuild.executor.node_results.types import NodeResultStatus
+from sqlbuild.executor.python_nodes._helpers.relation_guard import build_python_node_relation_guard
from sqlbuild.executor.python_nodes._helpers.results import (
build_python_node_failure_result,
evaluate_python_node_fan_in,
normalize_python_node_return,
)
-from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
from sqlbuild.executor.python_nodes.models import (
AssetContext,
PythonNodeExecutionResult,
@@ -384,8 +384,8 @@ def _build_context(
default_schema=runtime.default_schema,
relation_targets=runtime.resolved_relation_targets,
allowed_sql_refs=allowed_sql_refs,
- relation_guard=_node_relation_guard(
- node=node, node_kind=node_kind, runtime=runtime, warnings=warnings
+ relation_guard=build_python_node_relation_guard(
+ owner_label=f"{node_kind.value} '{node.name}'", runtime=runtime, warnings=warnings
),
providers=providers,
start_cursor_ts=runtime.start_cursor_ts,
@@ -395,26 +395,6 @@ def _build_context(
)
-def _node_relation_guard(
- *,
- node: ExecutablePythonNode,
- node_kind: PythonNodeKind,
- runtime: PythonNodeRuntime,
- warnings: list[str],
-) -> RuntimeRelationGuard | None:
- if runtime.project_relations is None:
- return None
- return RuntimeRelationGuard(
- owner_label=f"{node_kind.value} '{node.name}'",
- declare_help="declare it with depends_on={typed}",
- project_relations=runtime.project_relations,
- dialect=runtime.adapter.sql_analysis_dialect(),
- default_database=runtime.default_database,
- default_schema=runtime.default_schema,
- warnings=warnings,
- )
-
-
def _node_kind(node: ExecutablePythonNode) -> PythonNodeKind:
if isinstance(node, DiscoveredAssetFunction):
return PythonNodeKind.ASSET
diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py
index 23d14ecfad..64d67e6450 100644
--- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py
+++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py
@@ -294,6 +294,7 @@ def _execute_ingress_loader(
use_color=callbacks.use_color,
providers=runtime.providers,
result_store=runtime.result_store,
+ project_relations=runtime.project_relations,
),
statement_recorder=StatementRecorder(),
loader_ref_entries=_loader_ref_entries(
diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py
index 21b0e046c2..86f89689f4 100644
--- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py
+++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py
@@ -9,7 +9,11 @@
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder
from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction
-from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonNodeIdentity
+from sqlbuild.compiler.python_nodes.models import (
+ DiscoveredPythonNode,
+ PythonNodeGraph,
+ PythonNodeIdentity,
+)
from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus
from sqlbuild.cost.classes.cost_context import CostContext
from sqlbuild.errors.contracts.exceptions import ExecutorInputError
@@ -20,6 +24,7 @@
from sqlbuild.executor.python_nodes._helpers.fingerprinting import (
try_write_python_node_identity_fingerprint,
)
+from sqlbuild.executor.python_nodes._helpers.relation_guard import build_python_node_relation_guard
from sqlbuild.executor.python_nodes._helpers.results import normalize_python_check_return
from sqlbuild.executor.python_nodes.models import (
CheckContext,
@@ -143,6 +148,8 @@ def _execute_one_python_check(
) as lifecycle,
):
if blocked is None:
+ node: DiscoveredPythonNode = python_graph.nodes_by_name[check_function.name]
+ warnings: list[str] = []
context: CheckContext = CheckContext(
adapter=adapter,
connection_config=runtime.connection_config,
@@ -159,7 +166,12 @@ def _execute_one_python_check(
default_database=runtime.default_database,
default_schema=runtime.default_schema,
relation_targets=runtime.resolved_relation_targets,
- allowed_sql_refs=frozenset(),
+ allowed_sql_refs=frozenset(node.sql_deps),
+ relation_guard=build_python_node_relation_guard(
+ owner_label=f"check '{check_function.name}'",
+ runtime=runtime,
+ warnings=warnings,
+ ),
providers=providers if providers is not None else _empty_provider_container(),
start_cursor_ts=runtime.start_cursor_ts,
end_cursor_ts=runtime.end_cursor_ts,
@@ -188,6 +200,7 @@ def _execute_one_python_check(
severity=severity,
message=check_result.message,
metadata=check_result.metadata,
+ warning_messages=tuple(warnings),
)
except Exception as error:
result = PythonCheckExecutionResult(
@@ -195,6 +208,7 @@ def _execute_one_python_check(
passed=False,
severity=PythonCheckSeverity.ERROR,
error_message=str(error),
+ warning_messages=tuple(warnings),
)
else:
result = blocked
diff --git a/src/sqlbuild/executor/python_nodes/_helpers/read_side_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/read_side_execution.py
index f7abaff092..020e3a63a8 100644
--- a/src/sqlbuild/executor/python_nodes/_helpers/read_side_execution.py
+++ b/src/sqlbuild/executor/python_nodes/_helpers/read_side_execution.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from sqlbuild.executor.build.models import SeedExecutionResult
from sqlbuild.executor.load.models import LoadExecutionResult
from sqlbuild.executor.run.models import ModelExecutionResult
from sqlbuild.executor.scheduling.types import ExecutionStatus
@@ -12,10 +13,12 @@ def _sql_result_name(result: object) -> str | None:
return result.model_name
if isinstance(result, LoadExecutionResult):
return result.source_name
+ if isinstance(result, SeedExecutionResult):
+ return result.seed_name
return None
def _sql_result_failed(result: object) -> bool:
- if isinstance(result, ModelExecutionResult | LoadExecutionResult):
+ if isinstance(result, ModelExecutionResult | LoadExecutionResult | SeedExecutionResult):
return result.status == ExecutionStatus.FAILED
return False
diff --git a/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py b/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py
new file mode 100644
index 0000000000..5e6eead8b3
--- /dev/null
+++ b/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py
@@ -0,0 +1,25 @@
+"""Runtime relation guards for task, asset, and check contexts."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
+from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
+from sqlbuild.executor.python_nodes.models import PythonNodeRuntime
+
+
+def build_python_node_relation_guard(
+ *, owner_label: str, runtime: PythonNodeRuntime, warnings: list[str]
+) -> RuntimeRelationGuard | None:
+ """Return the hard-coded relation guard for one node, or None when enforcement is off."""
+
+ if runtime.project_relations is None:
+ return None
+ return RuntimeRelationGuard(
+ owner_label=owner_label,
+ owner_kind=HardCodedRelationOwnerKind.NODE,
+ project_relations=runtime.project_relations,
+ dialect=runtime.adapter.sql_analysis_dialect(),
+ default_database=runtime.default_database,
+ default_schema=runtime.default_schema,
+ warnings=warnings,
+ )
diff --git a/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py b/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py
index a1e68af7ea..9b2ed69015 100644
--- a/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py
+++ b/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py
@@ -6,6 +6,9 @@
from collections.abc import Mapping
from sqlbuild.compiler.references.main.extract_relation_names import extract_relation_names
+from sqlbuild.compiler.references.main.hard_coded_relation_remedy import (
+ hard_coded_relation_remedy,
+)
from sqlbuild.compiler.references.main.match_project_relation import match_project_relation
from sqlbuild.compiler.references.main.runtime_project_relations import (
runtime_project_relations,
@@ -15,6 +18,7 @@
ProjectRelationIndex,
RelationName,
)
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
from sqlbuild.python_nodes.models import SqlResourceRef
_LOGGER: logging.Logger = logging.getLogger(__name__)
@@ -27,16 +31,18 @@ def __init__( # noqa: PLR0913
self,
*,
owner_label: str,
- declare_help: str,
+ owner_kind: HardCodedRelationOwnerKind,
project_relations: Mapping[SqlResourceRef, str],
dialect: str | None,
default_database: str | None,
default_schema: str | None,
warnings: list[str],
own_refs: frozenset[SqlResourceRef] = frozenset(),
+ upstream_loader_by_source: Mapping[str, str] | None = None,
) -> None:
self._owner_label: str = owner_label
- self._declare_help: str = declare_help
+ self._owner_kind: HardCodedRelationOwnerKind = owner_kind
+ self._upstream_loader_by_source: Mapping[str, str] | None = upstream_loader_by_source
self._project_relations: Mapping[SqlResourceRef, str] = project_relations
self._dialect: str | None = dialect
self._default_database: str | None = default_database
@@ -48,7 +54,7 @@ def __init__( # noqa: PLR0913
self._index: ProjectRelationIndex | None = None
def record_resolved(self, ref: SqlResourceRef) -> None:
- """Record a relation the node obtained through ctx.relation()."""
+ """Record a relation the node resolved through its context instead of by name."""
self._resolved_refs.add(ref)
@@ -92,13 +98,15 @@ def _relation_index(self) -> ProjectRelationIndex:
return self._index
def _warning(self, *, match: ProjectRelation, relation: RelationName) -> str:
- typed: str = f'{match.ref.kind.value}("{match.ref.name}")'
written: str = ".".join(
part for part in (relation.database, relation.schema, relation.name) if part
)
+ remedy: str = hard_coded_relation_remedy(
+ owner_kind=self._owner_kind,
+ ref=match.ref,
+ upstream_loader_by_source=self._upstream_loader_by_source,
+ )
return (
f"[P008] {self._owner_label} named {match.ref.kind.value}:{match.ref.name} as "
- f"'{written}' in SQL without ctx.relation(); "
- f"{self._declare_help.format(typed=typed)} and use ctx.relation({typed}), or set "
- "[references] enforce_explicit = false"
+ f"'{written}' in SQL; {remedy}, or set [references] enforce_explicit = false"
)
diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py
index 71644dfd30..1ff25a8a67 100644
--- a/src/sqlbuild/executor/python_nodes/models.py
+++ b/src/sqlbuild/executor/python_nodes/models.py
@@ -72,6 +72,7 @@ class PythonCheckExecutionResult:
message: str | None = None
metadata: dict[str, object] = field(default_factory=dict)
error_message: str | None = None
+ warning_messages: tuple[str, ...] = field(default_factory=tuple)
@property
def warned(self) -> bool:
diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py
index 5810b046fa..1395e7b70d 100644
--- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py
+++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py
@@ -21,6 +21,7 @@
from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity
from sqlbuild.compiler.python_nodes.types import SkipMode
from sqlbuild.compiler.references.main.render_source_relation import render_source_relation
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
from sqlbuild.cost.classes.cost_context import CostContext
from sqlbuild.cost.models import CostResourceContext
from sqlbuild.errors.contracts.exceptions import ExecutorInputError
@@ -507,7 +508,7 @@ def _hook_relation_guard(
)
return RuntimeRelationGuard(
owner_label=f"hook '{hook_name}' on model '{model_name}'",
- declare_help="declare it with @hook(reads={typed})",
+ owner_kind=HardCodedRelationOwnerKind.HOOK,
project_relations=_resolve_hook_relations(adapter=adapter, refs=all_refs, lookup=lookup),
dialect=adapter.sql_analysis_dialect(),
default_database=adapter.default_database(),
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
index d90d49cb7a..499e898ba9 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
@@ -1023,6 +1023,19 @@ class ExplicitReferenceBuildE2ETestCase:
expected_order_before: tuple[str, str]
+@dataclass(frozen=True)
+class ExplicitReferencePythonDependencyE2ETestCase:
+ """Python nodes declaring seed and model dependencies, built and checked end to end."""
+
+ description: str
+ select: tuple[str, ...]
+ expected_dag_edges: tuple[tuple[str, str], ...]
+ expected_checked_asset_ids: tuple[str, ...]
+ expected_order_before: tuple[str, str]
+ expected_country_counts: tuple[tuple[object, ...], ...]
+ expected_check_row_pattern: str
+
+
@dataclass(frozen=True)
class ExplicitReferenceFailureE2ETestCase:
"""A project whose references are not explicit, and the failure it reports."""
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py
index 4d3d22b2f5..33e16d2f86 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py
@@ -736,3 +736,18 @@ def explicit_reference_project_files(
EXPLICIT_REFERENCE_MACRO_PATH: EXPLICIT_REFERENCE_MACROS,
EXPLICIT_REFERENCE_HOOK_PATH: EXPLICIT_REFERENCE_HOOKS,
} | overrides
+
+
+def explicit_reference_literal_loader(*, table: str) -> str:
+ """Return loaders ``raw_regions`` and ``raw_customers``; the latter queries ``table``."""
+
+ return (
+ "from sqlbuild.loaders import loader\n\n\n"
+ "@loader\n"
+ "def raw_regions(ctx):\n"
+ " return [{'id': 1}]\n\n\n"
+ "@loader\n"
+ "def raw_customers(ctx):\n"
+ f' ctx.query("SELECT count(*) FROM {table}")\n'
+ " return [{'id': 1}]\n"
+ )
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py
index e63afac281..d9235b499f 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py
@@ -768,7 +768,8 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built
"from sqlbuild.tasks import task\n\n"
"@task(depends_on=model('fact_orders'))\n"
"def summarize_orders(ctx):\n"
- " rows = ctx.query('SELECT COUNT(*) FROM fact_orders').fetchall()[0][0]\n"
+ " orders = ctx.relation(model('fact_orders'))\n"
+ " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n"
" Path(__file__).parents[2].joinpath('summary.txt').write_text(str(rows))\n"
" return ctx.result(metadata={'rows': rows})\n"
),
@@ -839,7 +840,8 @@ def test_given_asset_depends_on_terminal_model_when_running_run_then_asset_reads
"from sqlbuild.refs import model\n\n"
"@asset(depends_on=model('fact_orders'))\n"
"def export_fact_orders(ctx):\n"
- " rows = ctx.query('SELECT COUNT(*) FROM fact_orders').fetchall()[0][0]\n"
+ " orders = ctx.relation(model('fact_orders'))\n"
+ " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n"
" Path(__file__).parents[2].joinpath('export.txt').write_text(str(rows))\n"
" return ctx.result(metadata={'rows': rows}, materialized=True)\n"
),
@@ -976,10 +978,12 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load
" type: INTEGER\n"
),
"python/tasks/orders.py": (
+ "from sqlbuild.refs import source\n"
"from sqlbuild.tasks import task\n\n"
- "@task\n"
+ "@task(depends_on=source('raw_orders'))\n"
"def summarize_loaded_orders(ctx):\n"
- " rows = ctx.query('SELECT COUNT(*) FROM raw_orders').fetchall()[0][0]\n"
+ " orders = ctx.relation(source('raw_orders'))\n"
+ " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n"
" return ctx.result(payload={'rows': rows}, metadata={'rows': rows})\n"
),
"python/assets/orders.py": (
@@ -1213,7 +1217,8 @@ def test_given_task_depends_on_model_when_running_run_then_task_runs_before_down
"from sqlbuild.tasks import task\n\n"
"@task(depends_on=model('stg_orders'))\n"
"def profile_stg_orders(ctx):\n"
- " rows = ctx.query('SELECT COUNT(*) FROM stg_orders').fetchall()[0][0]\n"
+ " orders = ctx.relation(model('stg_orders'))\n"
+ " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n"
" return ctx.result(payload={'rows': rows})\n"
),
},
@@ -1288,7 +1293,8 @@ def test_given_task_depends_on_source_when_running_run_then_task_runs_after_sour
"from sqlbuild.tasks import task\n\n"
"@task(depends_on=source('raw_orders'))\n"
"def profile_raw_orders(ctx):\n"
- " rows = ctx.query('SELECT COUNT(*) FROM raw_orders').fetchall()[0][0]\n"
+ " orders = ctx.relation(source('raw_orders'))\n"
+ " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n"
" return ctx.result(payload={'rows': rows})\n"
),
},
@@ -1953,7 +1959,8 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running
"from sqlbuild.tasks import task\n\n"
"@task(depends_on=model('fact_orders'))\n"
"def profile_fact_orders(ctx):\n"
- " order_id = ctx.query('SELECT order_id FROM fact_orders').fetchall()[0][0]\n"
+ " orders = ctx.relation(model('fact_orders'))\n"
+ " order_id = ctx.query(f'SELECT order_id FROM {orders}').fetchall()[0][0]\n"
" return ctx.result(payload={'order_id': order_id}, metadata={'rows': 1})\n"
),
"python/assets/export.py": (
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
index 6e808adb18..5ada83e129 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
+import re
import subprocess
from itertools import chain
from pathlib import Path
@@ -12,6 +13,7 @@
from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
ExplicitReferenceBuildE2ETestCase,
ExplicitReferenceFailureE2ETestCase,
+ ExplicitReferencePythonDependencyE2ETestCase,
ExplicitReferenceRuntimeWarningE2ETestCase,
)
from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import (
@@ -20,6 +22,7 @@
EXPLICIT_REFERENCE_MACRO_PATH,
EXPLICIT_REFERENCE_MACROS,
EXPLICIT_REFERENCE_MODELS,
+ explicit_reference_literal_loader,
explicit_reference_project_files,
)
from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import (
@@ -53,6 +56,117 @@
)
+_COUNTRY_SEED_FILES: dict[str, str] = {
+ "seeds/countries.yml": (
+ "seeds:\n - name: countries\n columns:\n - name: code\n type: VARCHAR\n"
+ ),
+ "seeds/countries.csv": "code\nGB\nFR\n",
+}
+_SEED_TASK: str = (
+ "from sqlbuild.refs import seed\n"
+ "from sqlbuild.tasks import task\n\n\n"
+ '@task(depends_on=seed("countries"))\n'
+ "def count_countries(ctx):\n"
+ ' countries = ctx.relation(seed("countries"))\n'
+ " ctx.execute_sql(\n"
+ ' f"CREATE OR REPLACE TABLE country_counts AS SELECT count(*) AS n FROM {countries}"\n'
+ " )\n"
+)
+_MODEL_CHECK: str = (
+ "from sqlbuild.checks import check\n"
+ "from sqlbuild.refs import model\n\n\n"
+ '@check(depends_on=model("all_orders"))\n'
+ "def orders_present(ctx):\n"
+ ' orders = ctx.relation(model("all_orders"))\n'
+ ' count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0]\n'
+ ' return ctx.pass_() if count else ctx.fail(message="no orders")\n'
+)
+_LITERAL_CHECK: str = _MODEL_CHECK.replace(
+ 'ctx.query(f"SELECT count(*) FROM {orders}")', 'ctx.query("SELECT count(*) FROM stg_customers")'
+)
+_RUNTIME_CHECK: str = _MODEL_CHECK.replace(
+ ' return ctx.pass_() if count else ctx.fail(message="no orders")\n',
+ ' customers = "stg_" + "customers"\n'
+ ' ctx.query(f"SELECT count(*) FROM {customers}")\n'
+ ' return ctx.pass_() if count else ctx.fail(message="no orders")\n',
+)
+_RAW_SOURCES: str = (
+ "sources:\n"
+ " - name: raw_regions\n managed: true\n write_strategy: table\n"
+ " columns:\n - name: id\n type: INTEGER\n"
+ " - name: raw_customers\n managed: true\n write_strategy: table\n"
+ " columns:\n - name: id\n type: INTEGER\n"
+)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ ExplicitReferencePythonDependencyE2ETestCase(
+ description="task on a seed and check on a model build, order, select and check",
+ select=("+task:count_countries",),
+ expected_dag_edges=(
+ ("seed:countries", "task:count_countries"),
+ ("model:all_orders", "check:orders_present"),
+ ),
+ expected_checked_asset_ids=("model:all_orders",),
+ expected_order_before=("countries", "count_countries"),
+ expected_country_counts=((2,),),
+ expected_check_row_pattern=r"check\s+orders_present\s+PASS",
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_python_seed_and_model_dependencies_when_building_then_graph_selection_and_checks_hold(
+ test_case: ExplicitReferencePythonDependencyE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="explicit_refs",
+ repo_files=explicit_reference_project_files(
+ overrides={
+ **_COUNTRY_SEED_FILES,
+ "python/tasks/countries.py": _SEED_TASK,
+ "python/checks/orders.py": _MODEL_CHECK,
+ }
+ ),
+ )
+
+ dag: subprocess.CompletedProcess[str] = run_sqb(
+ command=("dag", "--json"), project_dir=project_dir
+ )
+ selected: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build", "--select", *test_case.select), project_dir=project_dir
+ )
+ full: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build"), project_dir=project_dir
+ )
+ checked: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "check"), project_dir=project_dir
+ )
+
+ assert dag.returncode == 0, dag.stdout + dag.stderr
+ payload: dict[str, object] = json.loads(dag.stdout)
+ edges: set[tuple[str, str]] = {(edge["from_id"], edge["to_id"]) for edge in payload["edges"]}
+ assert set(test_case.expected_dag_edges) <= edges
+ check_assets: dict[str, tuple[str, ...]] = {
+ check["id"]: tuple(check["checked_asset_ids"]) for check in payload["checks"]
+ }
+ assert check_assets["check:orders_present"] == test_case.expected_checked_asset_ids
+ assert selected.returncode == 0, selected.stdout + selected.stderr
+ first, second = test_case.expected_order_before
+ assert selected.stdout.index(f" {first} ") < selected.stdout.index(f" {second} ")
+ assert " all_orders " not in selected.stdout
+ assert query_duckdb(
+ db_path=project_dir / "warehouse.duckdb", sql="SELECT n FROM country_counts"
+ ) == list(test_case.expected_country_counts)
+ assert full.returncode == 0, full.stdout + full.stderr
+ assert checked.returncode == 0, checked.stdout + checked.stderr
+ assert re.search(test_case.expected_check_row_pattern, full.stdout), full.stdout
+ assert re.search(test_case.expected_check_row_pattern, checked.stdout), checked.stdout
+
+
@pytest.mark.parametrize(
"test_case",
[
@@ -139,6 +253,43 @@ def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_an
"[references] enforce_explicit = false",
),
),
+ ExplicitReferenceFailureE2ETestCase(
+ description="check literal SQL naming an undeclared model fails compile",
+ overrides={"python/checks/orders.py": _LITERAL_CHECK},
+ command=("--no-color", "compile"),
+ expected_exit_code=1,
+ expected_output_fragments=(
+ "error[P008]: check:orders_present names model:stg_customers as 'stg_customers'",
+ "--> python/checks/orders.py:8",
+ 'declare it with depends_on=model("stg_customers")',
+ ),
+ ),
+ ExplicitReferenceFailureE2ETestCase(
+ description="loader literal SQL naming a source fails compile with ctx.source help",
+ overrides={
+ "sources/raw.yml": _RAW_SOURCES,
+ "python/loaders/raw.py": explicit_reference_literal_loader(table="raw_regions"),
+ },
+ command=("--no-color", "compile"),
+ expected_exit_code=1,
+ expected_output_fragments=(
+ "error[P008]: loader:raw_customers names source:raw_regions as 'raw_regions'",
+ 'use ctx.source("raw_regions") instead of the relation name',
+ ),
+ ),
+ ExplicitReferenceFailureE2ETestCase(
+ description="loader literal SQL naming a model fails compile",
+ overrides={
+ "sources/raw.yml": _RAW_SOURCES,
+ "python/loaders/raw.py": explicit_reference_literal_loader(table="stg_customers"),
+ },
+ command=("--no-color", "compile"),
+ expected_exit_code=1,
+ expected_output_fragments=(
+ "error[P008]: loader:raw_customers names model:stg_customers as 'stg_customers'",
+ "loaders run before models and seeds and cannot read them",
+ ),
+ ),
],
ids=lambda case: case.description,
)
@@ -225,3 +376,49 @@ def test_given_runtime_hard_coded_relation_when_building_then_it_warns_without_f
fragment in warning
for warning, fragment in zip(stored, test_case.expected_warning_fragments, strict=True)
)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ ExplicitReferenceRuntimeWarningE2ETestCase(
+ description="check naming an undeclared model at run time warns and passes",
+ overrides={"python/checks/orders.py": _RUNTIME_CHECK},
+ enforce_explicit=True,
+ expected_exit_code=0,
+ expected_warning_fragments=(
+ "[P008] check 'orders_present' named model:stg_customers as 'stg_customers'",
+ ),
+ expected_final_line_prefix="",
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_without_failing(
+ test_case: ExplicitReferenceRuntimeWarningE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="explicit_refs",
+ repo_files=explicit_reference_project_files(
+ overrides=test_case.overrides, enforce_explicit=test_case.enforce_explicit
+ ),
+ )
+
+ machine: subprocess.CompletedProcess[str] = run_sqb(
+ command=("build", "--json"), project_dir=project_dir
+ )
+
+ assert machine.returncode == test_case.expected_exit_code, machine.stdout + machine.stderr
+ assert all(fragment in machine.stderr for fragment in test_case.expected_warning_fragments)
+ stored: list[str] = list(
+ chain.from_iterable(
+ check.get("warnings") or () for check in json.loads(machine.stdout)["checks"]
+ )
+ )
+ assert len(stored) == len(test_case.expected_warning_fragments)
+ assert all(
+ fragment in warning
+ for warning, fragment in zip(stored, test_case.expected_warning_fragments, strict=True)
+ )
diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py
index d5da47abff..149e1f5fa0 100644
--- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py
+++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_outputs.py
@@ -181,6 +181,7 @@ def execute_checks(**_: Any) -> tuple[PythonCheckExecutionResult, ...]:
relation_targets={},
default_database=None,
default_schema=None,
+ project_relations=None,
),
)
pipeline_result: CompilePipelineResult = cast(
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
index 8f774763e1..442fbd67b4 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
@@ -1007,11 +1007,12 @@ class HookReadErrorTestCase:
@dataclass(frozen=True)
class PythonSqlReferenceErrorTestCase:
- """A task, asset, or hook whose literal SQL hard-codes a project relation name."""
+ """A Python node or hook whose literal SQL hard-codes a project relation name."""
description: str
files: dict[str, str]
expected_error_fragments: tuple[str, ...]
+ unexpected_error_fragments: tuple[str, ...] = ()
@dataclass(frozen=True)
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
index 3682e528bf..6cdf857817 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
@@ -371,3 +371,31 @@ def python_task_source(*, body: str) -> str:
"def export_orders(ctx):\n"
f"{body}"
)
+
+
+def python_check_source(*, depends_on: str, body: str) -> str:
+ """Return a Python check module defining ``orders_present`` with the given dependencies."""
+
+ return (
+ "from sqlbuild.checks import check\n"
+ "from sqlbuild.refs import model\n\n\n"
+ f"@check(depends_on={depends_on})\n"
+ "def orders_present(ctx):\n"
+ f"{body}"
+ " return ctx.pass_()\n"
+ )
+
+
+def python_loader_source(*, depends_on: str, body: str) -> str:
+ """Return loaders ``raw_regions`` and ``raw_customers``; the latter runs ``body``."""
+
+ return (
+ "from sqlbuild.loaders import loader\n\n\n"
+ "@loader\n"
+ "def raw_regions(ctx):\n"
+ " return [{'id': 1}]\n\n\n"
+ f"@loader(depends_on=[{depends_on}])\n"
+ "def raw_customers(ctx):\n"
+ f"{body}"
+ " return [{'id': 1}]\n"
+ )
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
index a6d1925b59..da2a01a3c4 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
@@ -1,4 +1,4 @@
-"""Literal SQL in tasks, assets, and hooks must not hard-code project relation names."""
+"""Literal SQL in Python nodes and hooks must not hard-code project relation names."""
from __future__ import annotations
@@ -15,7 +15,9 @@
from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import (
compile_and_assemble,
gate_model,
+ python_check_source,
python_hook_source,
+ python_loader_source,
python_task_source,
)
@@ -43,6 +45,19 @@
}
+_CHECK_PATH: str = "python/checks/orders.py"
+_LOADER_PATH: str = "python/loaders/raw.py"
+_RAW_SOURCES: dict[str, str] = {
+ "sources/raw.yml": (
+ "sources:\n"
+ " - name: raw_regions\n managed: true\n write_strategy: table\n"
+ " columns:\n - name: id\n type: INTEGER\n"
+ " - name: raw_customers\n managed: true\n write_strategy: table\n"
+ " columns:\n - name: id\n type: INTEGER\n"
+ ),
+}
+
+
@pytest.mark.parametrize(
"test_case",
(
@@ -61,9 +76,88 @@
"ctx.query()",
"python/tasks/export.py:7",
'depends_on=model("customers")',
+ "if 'customers' is an external table that shares the name, qualify it with its "
+ "schema",
"[references] enforce_explicit = false",
),
),
+ PythonSqlReferenceErrorTestCase(
+ description="task literal SQL naming a seed",
+ files={
+ **_MODELS,
+ **_COUNTRY_SEED,
+ _HOOK_PATH: python_hook_source(),
+ _TASK_PATH: python_task_source(
+ body=' ctx.query("SELECT * FROM country_codes")\n'
+ ),
+ },
+ expected_error_fragments=(
+ "task:export_orders names seed:country_codes",
+ 'declare it with depends_on=seed("country_codes") and use '
+ 'ctx.relation(seed("country_codes"))',
+ ),
+ ),
+ PythonSqlReferenceErrorTestCase(
+ description="check literal SQL naming an undeclared model",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(),
+ _CHECK_PATH: python_check_source(
+ depends_on='model("orders")',
+ body=' ctx.query("SELECT count(*) FROM customers")\n',
+ ),
+ },
+ expected_error_fragments=(
+ "[P008]",
+ "check:orders_present names model:customers as 'customers' in SQL passed to "
+ "ctx.query()",
+ "python/checks/orders.py:7",
+ 'declare it with depends_on=model("customers")',
+ ),
+ ),
+ PythonSqlReferenceErrorTestCase(
+ description="loader literal SQL naming a source it does not depend on",
+ files={
+ **_MODELS,
+ **_RAW_SOURCES,
+ _HOOK_PATH: python_hook_source(),
+ _LOADER_PATH: python_loader_source(
+ depends_on="", body=' ctx.query("SELECT * FROM raw_regions")\n'
+ ),
+ },
+ expected_error_fragments=(
+ "loader:raw_customers names source:raw_regions as 'raw_regions'",
+ 'use ctx.source("raw_regions") instead of the relation name',
+ ),
+ ),
+ PythonSqlReferenceErrorTestCase(
+ description="loader literal SQL naming an upstream loader's source",
+ files={
+ **_MODELS,
+ **_RAW_SOURCES,
+ _HOOK_PATH: python_hook_source(),
+ _LOADER_PATH: python_loader_source(
+ depends_on="raw_regions", body=' ctx.query("SELECT * FROM raw_regions")\n'
+ ),
+ },
+ expected_error_fragments=("use ctx.loader(raw_regions) instead of the relation name",),
+ ),
+ PythonSqlReferenceErrorTestCase(
+ description="loader literal SQL naming a model",
+ files={
+ **_MODELS,
+ **_RAW_SOURCES,
+ _HOOK_PATH: python_hook_source(),
+ _LOADER_PATH: python_loader_source(
+ depends_on="", body=' ctx.query("SELECT * FROM customers")\n'
+ ),
+ },
+ expected_error_fragments=(
+ "loader:raw_customers names model:customers as 'customers'",
+ "loaders run before models and seeds and cannot read them; read model:customers "
+ "from a task or asset instead",
+ ),
+ ),
PythonSqlReferenceErrorTestCase(
description="hook f-string with a literal seed table part",
files={
@@ -92,6 +186,17 @@
},
expected_error_fragments=("names model:customers as 'main.customers'",),
),
+ PythonSqlReferenceErrorTestCase(
+ description="qualified name omits the external table help",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(
+ body=' ctx.query("SELECT * FROM main.customers")\n'
+ ),
+ },
+ expected_error_fragments=("names model:customers",),
+ unexpected_error_fragments=("external table",),
+ ),
),
ids=lambda case: case.description,
)
@@ -109,6 +214,7 @@ def test_given_hard_coded_project_relation_when_compiling_then_it_fails(
f"[{getattr(error.value, 'code', '')}] {error.value} {getattr(error.value, 'help', '')}"
)
assert all(fragment in rendered for fragment in test_case.expected_error_fragments)
+ assert not any(fragment in rendered for fragment in test_case.unexpected_error_fragments)
@pytest.mark.parametrize(
@@ -151,6 +257,38 @@ def test_given_hard_coded_project_relation_when_compiling_then_it_fails(
},
expected_model_names=("customers", "orders"),
),
+ PythonSqlReferenceAllowedTestCase(
+ description="check reading its declared model through ctx.relation",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(),
+ _CHECK_PATH: python_check_source(
+ depends_on='model("orders")',
+ body=(
+ ' orders = ctx.relation(model("orders"))\n'
+ ' ctx.query(f"SELECT count(*) FROM {orders}")\n'
+ ),
+ ),
+ },
+ expected_model_names=("customers", "orders"),
+ ),
+ PythonSqlReferenceAllowedTestCase(
+ description="loader naming its own source and reading upstream through ctx.loader",
+ files={
+ **_MODELS,
+ **_RAW_SOURCES,
+ _HOOK_PATH: python_hook_source(),
+ _LOADER_PATH: python_loader_source(
+ depends_on="raw_regions",
+ body=(
+ " regions = ctx.loader(raw_regions)\n"
+ ' ctx.query(f"SELECT * FROM {regions.destination}")\n'
+ ' ctx.query("SELECT count(*) FROM raw_customers")\n'
+ ),
+ ),
+ },
+ expected_model_names=("customers", "orders"),
+ ),
),
ids=lambda case: case.description,
)
diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py
index 450abc6e1d..e3292c7636 100644
--- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py
+++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py
@@ -559,6 +559,33 @@ def prepare(ctx):
},
expected_task_names=("prepare_orders",),
),
+ DiscoverFactoryValidationTestCase(
+ description="generated check and task may depend on a model and a seed",
+ repo_files=base_repo_files()
+ | {
+ "python/factories/generated.py": """
+from sqlbuild.checks import check
+from sqlbuild.factories import factory
+from sqlbuild.refs import model, seed
+from sqlbuild.tasks import task
+
+
+@factory
+def generated_nodes():
+ @task(name="export_countries", depends_on=seed("country_codes"))
+ def export_countries(ctx):
+ return None
+
+ @check(name="check_orders", depends_on=[model("orders"), seed("country_codes")])
+ def check_orders(ctx):
+ return True
+
+ return [export_countries, check_orders]
+""",
+ },
+ expected_task_names=("export_countries",),
+ expected_check_names=("check_orders",),
+ ),
),
ids=lambda case: case.description,
)
@@ -816,48 +843,6 @@ def raw_orders(ctx):
},
expected_error_fragment="Loader 'raw_orders' depends on SQL resource 'orders'",
),
- DiscoverFactoryValidationTestCase(
- description="generated check SQL dependency fails validation",
- repo_files=base_repo_files()
- | {
- "python/checks/generated.py": """
-from sqlbuild.checks import check
-from sqlbuild.factories import factory
-from sqlbuild.refs import model
-
-
-@factory
-def generated_checks():
- @check(name="check_orders", depends_on=model("orders"))
- def check_orders(ctx):
- return True
- return check_orders
-""",
- },
- expected_error_fragment="Check 'check_orders' depends on SQL resource 'orders'",
- ),
- DiscoverFactoryValidationTestCase(
- description="generated task seed dependency fails validation",
- repo_files=base_repo_files()
- | {
- "python/tasks/generated.py": """
-from sqlbuild.factories import factory
-from sqlbuild.refs import seed
-from sqlbuild.tasks import task
-
-
-@factory
-def generated_tasks():
- @task(name="export_countries", depends_on=seed("country_codes"))
- def export_countries(ctx):
- return None
- return export_countries
-""",
- },
- expected_error_fragment=(
- "Python node 'export_countries' depends on seed 'country_codes'"
- ),
- ),
DiscoverFactoryValidationTestCase(
description="generated managed source loader mismatch fails validation",
repo_files=base_repo_files()
@@ -1450,21 +1435,6 @@ def load_orders(ctx):
},
expected_error_fragment="Loader 'load_orders' depends on SQL resource 'stg_orders'",
),
- DiscoverProjectInputsErrorTestCase(
- description="raises when check declares SQL model dependency",
- repo_files=base_repo_files()
- | {
- "python/checks/orders.py": """
-from sqlbuild.checks import check
-from sqlbuild.refs import model
-
-@check(depends_on=model('stg_orders'))
-def check_orders(ctx):
- return True
-""",
- },
- expected_error_fragment="Check 'check_orders' depends on SQL resource 'stg_orders'",
- ),
DiscoverProjectInputsErrorTestCase(
description="raises when check depends on check",
repo_files=base_repo_files()
diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py
index 2430ea467c..8e64225233 100644
--- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py
+++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py
@@ -10,6 +10,7 @@
validate_python_sql_boundaries,
)
from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlSelection
+from sqlbuild.refs import seed
from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import (
PythonSqlSelectorErrorTestCase,
PythonSqlSelectorTestCase,
@@ -211,6 +212,47 @@ def test_given_managed_source_loader_glob_when_resolving_then_requires_source_se
)
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ PythonSqlSelectorTestCase(
+ description="upstream expansion of a Python node includes its SQL dependencies",
+ select=("+profile_orders",),
+ exclude=(),
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ expected_python_node_names=frozenset({"profile_orders"}),
+ ),
+ PythonSqlSelectorTestCase(
+ description="downstream expansion of a SQL resource includes dependent Python nodes",
+ select=("raw_orders+",),
+ exclude=(),
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ expected_python_node_names=frozenset({"profile_orders"}),
+ ),
+ PythonSqlSelectorTestCase(
+ description="unexpanded Python node selects no SQL dependency",
+ select=("profile_orders",),
+ exclude=(),
+ expected_sql_names=frozenset(),
+ expected_python_node_names=frozenset({"profile_orders"}),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_python_sql_dependency_when_expanding_selectors_then_crosses_the_boundary(
+ test_case: PythonSqlSelectorTestCase,
+) -> None:
+ result: PythonSqlSelection = resolve_python_sql_selectors(
+ select=test_case.select,
+ exclude=test_case.exclude,
+ project_graph=build_orders_project_graph(),
+ python_graph=build_sql_ref_python_node_graph(dependency=model_ref("orders")),
+ )
+
+ assert frozenset(key.name for key in result.sql_keys) == test_case.expected_sql_names
+ assert result.python_node_names == test_case.expected_python_node_names
+
+
@pytest.mark.parametrize(
"test_case",
[
@@ -238,6 +280,14 @@ def test_given_managed_source_loader_glob_when_resolving_then_requires_source_se
expected_error_fragment="declares model.*raw_orders.*but.*raw_orders.*is a source",
sql_ref_dependency=model_ref("raw_orders"),
),
+ PythonSqlSelectorErrorTestCase(
+ description="raises when seed ref names a model",
+ select=("profile_orders",),
+ exclude=(),
+ expected_error_type=ValueError,
+ expected_error_fragment="declares seed.*orders.*but.*orders.*is a model",
+ sql_ref_dependency=seed("orders"),
+ ),
],
ids=lambda case: case.description,
)
diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py
index 01660ee603..89b16ada8b 100644
--- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py
+++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py
@@ -2,6 +2,7 @@
from dataclasses import dataclass, field
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
from sqlbuild.python_nodes.models import SqlResourceRef
@@ -12,5 +13,9 @@ class RuntimeRelationGuardTestCase:
description: str
statements: tuple[str, ...]
expected_warning_fragments: tuple[str, ...]
+ expected_warning_count: int
resolved_refs: tuple[SqlResourceRef, ...] = field(default_factory=tuple)
own_refs: frozenset[SqlResourceRef] = frozenset()
+ owner_label: str = "task 'export_orders'"
+ owner_kind: HardCodedRelationOwnerKind = HardCodedRelationOwnerKind.NODE
+ upstream_loader_by_source: dict[str, str] = field(default_factory=dict)
diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py
index 78b0ad7e14..fb6847b0d3 100644
--- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py
+++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py
@@ -4,6 +4,7 @@
import pytest
+from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind
from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard
from sqlbuild.python_nodes.models import SqlResourceRef
from sqlbuild.refs import model, source
@@ -25,24 +26,77 @@
description="unqualified model name resolved against the current schema warns once",
statements=("SELECT * FROM customers", "SELECT count(*) FROM main.customers"),
expected_warning_fragments=(
- "[P008] task 'export_orders' named model:customers as 'customers'",
+ "[P008] task 'export_orders' named model:customers as 'customers' in SQL; "
+ 'declare it with depends_on=model("customers") and use '
+ 'ctx.relation(model("customers")) instead of the relation name',
),
+ expected_warning_count=1,
+ ),
+ RuntimeRelationGuardTestCase(
+ description="check naming an undeclared model warns with the depends_on remedy",
+ statements=("SELECT count(*) FROM orders",),
+ expected_warning_fragments=(
+ "[P008] check 'orders_present' named model:orders as 'orders'",
+ 'declare it with depends_on=model("orders")',
+ ),
+ expected_warning_count=1,
+ owner_label="check 'orders_present'",
+ ),
+ RuntimeRelationGuardTestCase(
+ description="loader naming a model warns that loaders run before models",
+ statements=("SELECT * FROM customers",),
+ expected_warning_fragments=(
+ "[P008] loader 'raw_orders' named model:customers as 'customers'",
+ "loaders run before models and seeds and cannot read them",
+ ),
+ expected_warning_count=1,
+ owner_label="loader 'raw_orders'",
+ owner_kind=HardCodedRelationOwnerKind.LOADER,
+ ),
+ RuntimeRelationGuardTestCase(
+ description="loader naming a source warns with the ctx.source remedy",
+ statements=("SELECT * FROM raw.payments",),
+ expected_warning_fragments=('use ctx.source("raw_payments") instead',),
+ expected_warning_count=1,
+ owner_label="loader 'raw_orders'",
+ owner_kind=HardCodedRelationOwnerKind.LOADER,
+ ),
+ RuntimeRelationGuardTestCase(
+ description="loader naming an upstream loader's source warns with the ctx.loader remedy",
+ statements=("SELECT * FROM raw.payments",),
+ expected_warning_fragments=("use ctx.loader(raw_payments) instead",),
+ expected_warning_count=1,
+ owner_label="loader 'raw_orders'",
+ owner_kind=HardCodedRelationOwnerKind.LOADER,
+ upstream_loader_by_source={"raw_payments": "raw_payments"},
+ ),
+ RuntimeRelationGuardTestCase(
+ description="loader source resolved through ctx.source does not warn",
+ statements=("SELECT * FROM raw.payments",),
+ expected_warning_fragments=(),
+ expected_warning_count=0,
+ resolved_refs=(source("raw_payments"),),
+ owner_label="loader 'raw_orders'",
+ owner_kind=HardCodedRelationOwnerKind.LOADER,
),
RuntimeRelationGuardTestCase(
description="qualified source table warns",
statements=("DELETE FROM raw.payments WHERE 1 = 0",),
expected_warning_fragments=("named source:raw_payments as 'raw.payments'",),
+ expected_warning_count=1,
),
RuntimeRelationGuardTestCase(
description="relation resolved through ctx.relation does not warn",
statements=("SELECT * FROM main.customers",),
expected_warning_fragments=(),
+ expected_warning_count=0,
resolved_refs=(model("customers"),),
),
RuntimeRelationGuardTestCase(
description="own destination does not warn",
statements=("INSERT INTO orders SELECT 1",),
expected_warning_fragments=(),
+ expected_warning_count=0,
own_refs=frozenset({model("orders")}),
),
RuntimeRelationGuardTestCase(
@@ -52,11 +106,13 @@
"SELECT * FROM customers JOIN information_schema.tables ON 1 = 0",
),
expected_warning_fragments=(),
+ expected_warning_count=0,
),
RuntimeRelationGuardTestCase(
description="unparseable SQL is not blocked and does not warn",
statements=("VACUUM ((( customers",),
expected_warning_fragments=(),
+ expected_warning_count=0,
),
),
ids=lambda case: case.description,
@@ -66,14 +122,15 @@ def test_given_python_sql_when_checking_then_warns_for_hard_coded_project_relati
) -> None:
warnings: list[str] = []
guard: RuntimeRelationGuard = RuntimeRelationGuard(
- owner_label="task 'export_orders'",
- declare_help="declare it with depends_on={typed}",
+ owner_label=test_case.owner_label,
+ owner_kind=test_case.owner_kind,
project_relations=_PROJECT_RELATIONS,
dialect="duckdb",
default_database=None,
default_schema="main",
warnings=warnings,
own_refs=test_case.own_refs,
+ upstream_loader_by_source=test_case.upstream_loader_by_source,
)
for ref in test_case.resolved_refs:
guard.record_resolved(ref)
@@ -81,11 +138,8 @@ def test_given_python_sql_when_checking_then_warns_for_hard_coded_project_relati
for statement in test_case.statements:
guard.check(statement)
- assert len(warnings) == len(test_case.expected_warning_fragments)
- assert all(
- fragment in warning
- for warning, fragment in zip(warnings, test_case.expected_warning_fragments, strict=True)
- )
+ assert len(warnings) == test_case.expected_warning_count
+ assert all(fragment in "\n".join(warnings) for fragment in test_case.expected_warning_fragments)
if __name__ == "__main__":
From bff926332d42d755f30c9521a3d5363fb7667cf8 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Sun, 27 Sep 2026 22:12:57 +0100
Subject: [PATCH 05/13] docs: document seed and SQL references in checks, and
loader relation rules
Checks may depend on models, sources and seeds; tasks and assets may depend
on seeds; selector expansion follows declared SQL references; loaders read
upstream relations through ctx.source() and ctx.loader(); Dagster does not
add hook reads to its asset graph.
---
website/src/content/docs/docs/cli/check.mdx | 2 +-
.../docs/concepts/models/hooks/python.mdx | 4 ++
.../docs/concepts/project-configuration.mdx | 4 +-
.../docs/concepts/python-nodes/assets.mdx | 2 +-
.../docs/concepts/python-nodes/checks.mdx | 30 +++++++++---
.../docs/concepts/python-nodes/loaders.mdx | 2 +
.../docs/concepts/python-nodes/overview.mdx | 6 +--
.../concepts/python-nodes/sql-references.mdx | 46 +++++++++++++------
.../docs/docs/concepts/python-nodes/tasks.mdx | 2 +-
9 files changed, 68 insertions(+), 30 deletions(-)
diff --git a/website/src/content/docs/docs/cli/check.mdx b/website/src/content/docs/docs/cli/check.mdx
index f712d13f59..3707cbe444 100644
--- a/website/src/content/docs/docs/cli/check.mdx
+++ b/website/src/content/docs/docs/cli/check.mdx
@@ -50,4 +50,4 @@ Results are written to `target/run/checks/python_checks.json`.
## Relationship to build
-`sqb build` runs relevant Python checks by default after the tasks, assets, and loaders they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand.
+`sqb build` runs relevant Python checks by default after the tasks, assets, loaders, models, sources, and seeds they depend on have run. `sqb build --no-audits` skips checks alongside audits. `sqb check` runs only checks, on demand.
diff --git a/website/src/content/docs/docs/concepts/models/hooks/python.mdx b/website/src/content/docs/docs/concepts/models/hooks/python.mdx
index e408d84972..048dc7eb7f 100644
--- a/website/src/content/docs/docs/concepts/models/hooks/python.mdx
+++ b/website/src/content/docs/docs/concepts/models/hooks/python.mdx
@@ -56,6 +56,10 @@ their existing relations. A hook cannot read a resource built from the model tha
that model would have to wait for its own dependants; that fails to compile with `P007`. Reading
an unknown model, source, or seed fails to compile. Use `ctx.destination` for the hook's own model.
+The Dagster integration does not add declared reads to its asset graph, the same as reads of
+attached audits. Within one `sqb build` the ordering holds; when Dagster runs the hooked model in a
+separate build from the resources it reads, the hook reads their existing relations.
+
Declaring reads does not change who may call the hook: the model must still see the hook under the
usual [scope rules](/docs/concepts/models/hooks#project-layout).
diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx
index 6adecf25a7..70dc7e46bb 100644
--- a/website/src/content/docs/docs/concepts/project-configuration.mdx
+++ b/website/src/content/docs/docs/concepts/project-configuration.mdx
@@ -416,8 +416,8 @@ Every reference to a model, source, or seed must be explicit and typed. By defau
- A macro whose expanded output contains `__ref()`, `__source()`, or `__seed()` fails to compile
with `P006`. Pass the reference to the macro as an argument instead; see
[Typed reference arguments](/docs/concepts/macros#typed-reference-arguments).
-- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in a task, asset, or Python hook that
- names a project relation fails to compile with `P008`.
+- Literal SQL passed to `ctx.query()` or `ctx.execute_sql()` in a task, asset, check, loader, or
+ Python hook that names a project relation fails to compile with `P008`.
- The same check runs on SQL sent at run time and reports a `P008` warning; the query still runs.
See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names).
diff --git a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx
index 19a06bc589..31f0c61a10 100644
--- a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx
+++ b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx
@@ -115,7 +115,7 @@ See [Tasks](/docs/concepts/python-nodes/tasks#retries) for the full retry policy
| Parameter | Description |
|-----------|-------------|
| `name` | Override the node name (defaults to the function name) |
-| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()` for read-only SQL refs |
+| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()`/`seed()` for read-only SQL refs |
| `columns` | Column declarations for the produced artifact |
| `column_lineage` | Map of asset column to upstream `{node, column}` references |
| `tags` | Labels for selection and grouping |
diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx
index 6b8112dff7..668acfd5f3 100644
--- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx
+++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx
@@ -1,9 +1,9 @@
---
title: "Checks"
-description: "Validate tasks, assets, and loaders with Python checks."
+description: "Validate tasks, assets, loaders, and SQL relations with Python checks."
---
-Checks are Python nodes that validate other Python nodes. They are the Python analog of SQL [audits](/docs/concepts/audits): audits validate SQL relations, checks validate the output of tasks, assets, and loaders. See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model.
+Checks are Python nodes that validate the output of tasks, assets, and loaders, or the data in SQL models, sources, and seeds. They are the Python analog of SQL [audits](/docs/concepts/audits). See [Python Nodes](/docs/concepts/python-nodes/overview) for the shared model.
Checks are separate graph nodes, not callbacks embedded in a task or asset. A check declares what it validates through `depends_on`.
@@ -27,6 +27,22 @@ def check_orders_exported(ctx: CheckContext):
The check receives a `CheckContext` and reads its dependencies' persisted results with `ctx.result_of(...)`.
+A check can also validate SQL relations. Declare them with `model()`, `source()`, or `seed()` from `sqlbuild.refs`, and resolve them with `ctx.relation(...)`:
+
+```python
+from sqlbuild.checks import check
+from sqlbuild.refs import model
+
+
+@check(depends_on=model("fact_orders"))
+def orders_present(ctx):
+ orders = ctx.relation(model("fact_orders"))
+ count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0]
+ return ctx.pass_() if count else ctx.fail("no orders")
+```
+
+Do not hard-code relation names in check SQL; see [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names).
+
## Results
Return a result through the context helpers, or a bool shorthand:
@@ -67,7 +83,7 @@ def orders_freshness(ctx):
## What checks can depend on
- Checks may depend on **tasks, assets, and loaders**.
-- Checks may **not** depend on SQL models, sources, seeds, or functions. Use SQL [audits](/docs/concepts/audits) to validate SQL relations.
+- Checks may depend on SQL **models, sources, and seeds** through `model()`, `source()`, and `seed()` references. They may **not** depend on functions.
- Checks may **not** depend on other checks.
- Checks may **not** depend on a terminal source loader directly. Validate loaded source data with a source audit instead.
@@ -77,7 +93,7 @@ A check that depends on a single node is displayed grouped under that node. Mult
| Parameter | Description |
|-----------|-------------|
-| `depends_on` | Required. Tasks/assets/loaders to validate (function, tuple, or list) |
+| `depends_on` | Required. Tasks/assets/loaders to validate, and `model()`/`source()`/`seed()` references (function, reference, tuple, or list) |
| `name` | Override the node name (defaults to the function name) |
| `severity` | `error` (default) or `warn` |
| `tags` | Labels for selection and grouping |
@@ -89,7 +105,7 @@ Checks do not support `columns`, `column_lineage`, or `retry`.
## Running checks
-Checks run automatically during `sqb build` when their Python dependencies run. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](/docs/cli/check):
+Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](/docs/cli/check):
```bash
# Run all checks
@@ -102,13 +118,13 @@ sqb check --select +check_orders_exported
sqb check --select tag:exports
```
-`sqb check` rejects selecting non-check nodes; use `sqb build` to run tasks and assets. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout.
+`sqb check` rejects selecting non-check nodes; use `sqb build` to run tasks and assets. The models, sources, and seeds a check depends on must already exist; `sqb check` does not build them. Check results are written to `target/run/checks/python_checks.json`, and `sqb check --json` prints them to stdout.
## Checks vs audits
| | Checks | Audits |
|---|--------|--------|
-| Validates | Python tasks, assets, loaders | SQL relations |
+| Validates | Python tasks, assets, loaders, and SQL relations | SQL relations |
| Authored in | `python/` (Python) | `MODEL()` headers / `audits/` (SQL) |
| Run by | `sqb build`, `sqb check` | `sqb build`, `sqb audit` |
| Severity | `error`, `warn` | `error`, `warn` |
diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx
index cd8cdc46e4..2c1e44c548 100644
--- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx
+++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx
@@ -249,6 +249,8 @@ def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]:
]
```
+Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders run before models and seeds and cannot read them. See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names).
+
Dependencies form a DAG. SQLBuild schedules loaders in topological order and executes independent loaders concurrently when `--concurrency` is set.
Intermediate loaders (those referenced only via `depends_on`, with no managed source of the same name) are given synthetic source entries and write to `__loader__` tables by default. Only the terminal loader - the one whose name matches a managed source - populates that source; intermediate loaders feed it. Use the `destination` parameter on the decorator to override the intermediate relation:
diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx
index 0fdb61d35d..732da0adec 100644
--- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx
+++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx
@@ -52,7 +52,7 @@ The most important rule: **SQL models never depend on Python nodes.** The depend
- SQL models depend only on other SQL resources (models, sources, seeds, functions).
- Python nodes may depend on other Python nodes.
- The only way Python data reaches SQL is through a **loader populating a source**: `loader -> source -> model`.
-- Python nodes may **read** SQL models and sources at runtime through typed references (see [SQL references](/docs/concepts/python-nodes/sql-references)), but reading a model does not make it a SQL dependency.
+- Tasks, assets, and checks may **read** SQL models, sources, and seeds at runtime through typed references (see [SQL references](/docs/concepts/python-nodes/sql-references)), but reading a model does not make it a SQL dependency.
This keeps the SQL graph fully analyzable and testable on its own, while letting Python participate around the edges.
@@ -63,7 +63,7 @@ This keeps the SQL graph fully analyzable and testable on its own, while letting
| (read only) v
| asset
v |
- source <-- loader | check (validates tasks/assets/loaders)
+ source <-- loader | check (validates tasks/assets/loaders/SQL)
|
v
model (SQL) ----> model (SQL)
@@ -75,7 +75,7 @@ Every decorator accepts the same organizational metadata:
| Argument | Description |
|----------|-------------|
-| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()` references) |
+| `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) |
| `tags` | Labels for selection, filtering, and catalog grouping |
| `group` | A display/catalog grouping string |
| `description` | Human-readable docs (defaults to the function docstring) |
diff --git a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx
index f4f11e8170..bf7391dfd1 100644
--- a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx
+++ b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx
@@ -1,18 +1,18 @@
---
title: "SQL References"
-description: "Read SQL models and sources from Python nodes without creating SQL dependencies."
+description: "Read SQL models, sources, and seeds from Python nodes without creating SQL dependencies."
---
import Warning from '../../../../../components/mintlify/Warning.astro';
-Python nodes can **read** SQL models and sources at runtime, but they cannot depend on them as graph edges (see the [SQL boundary](/docs/concepts/python-nodes/overview#the-sql-boundary)). Typed references make this read-only access explicit and safe.
+Python nodes can **read** SQL models, sources, and seeds at runtime, but SQL cannot depend on Python nodes (see the [SQL boundary](/docs/concepts/python-nodes/overview#the-sql-boundary)). Typed references make this read-only access explicit and safe.
## Declaring a reference
-Use `model()` and `source()` from `sqlbuild.refs` in a node's `depends_on`:
+Use `model()`, `source()`, and `seed()` from `sqlbuild.refs` in the `depends_on` of a task, asset, or check:
```python
-from sqlbuild.refs import model, source
+from sqlbuild.refs import model, seed, source
from sqlbuild.tasks import task
@@ -24,6 +24,11 @@ def export_orders(ctx):
@task(depends_on=source("raw_orders"))
def inspect_raw(ctx):
...
+
+
+@task(depends_on=seed("country_codes"))
+def export_countries(ctx):
+ ...
```
Declaring the reference does two things:
@@ -33,6 +38,8 @@ Declaring the reference does two things:
It does **not** make the SQL resource depend on the Python node. The dependency is one-way: Python reads SQL, never the reverse.
+The reference is a graph edge. It appears in `sqb dag --json`, and selector expansion follows it: `+task:export_countries` also selects the `country_codes` seed and its upstream, and `country_codes+` also selects the tasks and assets that read it. Checks are never added by expansion; they run when their dependencies run.
+
## Resolving with ctx.relation
Always resolve a reference to its concrete relation with `ctx.relation(...)` instead of hardcoding the table name:
@@ -53,9 +60,9 @@ Do not hardcode model or source names in raw SQL. A model's relation depends on
## Hard-coded relation names
-SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in tasks, assets, and
-[Python hooks](/docs/concepts/models/hooks/python) for names of project relations: models, sources,
-and, for hooks, seeds.
+SQLBuild checks SQL passed to `ctx.query()` and `ctx.execute_sql()` in tasks, assets, checks,
+loaders, and [Python hooks](/docs/concepts/models/hooks/python) for names of project relations:
+models, sources, and seeds.
**At compile time**, SQL written as a string literal, or as an f-string whose table names are
literal text, is parsed with the project's SQL dialect. Naming a project relation fails to compile
@@ -65,27 +72,36 @@ with `P008`:
error[P008]: task:export_orders names model:fact_orders as 'fact_orders' in SQL passed to ctx.query()
--> python/tasks/export.py:9
= help: declare it with depends_on=model("fact_orders") and use ctx.relation(model("fact_orders")) instead of the relation name
+ = help: if 'fact_orders' is an external table that shares the name, qualify it with its schema
= help: while migrating a project, allow hard-coded relation names with [references] enforce_explicit = false in sqlbuild_project.toml
```
+An unqualified name that matches a project relation is treated as that relation. If you mean an
+external table that shares the name, qualify it with its schema.
+
+Loaders run before models and seeds, so a loader naming a model or seed fails with a help saying it
+cannot read them. A loader naming a source is told to use `ctx.source("name")`, or
+`ctx.loader(upstream_loader)` when the source is populated by a loader it depends on.
+
**At run time**, SQL built dynamically cannot be checked earlier. Before sending it,
`ctx.query()` and `ctx.execute_sql()` parse it in the adapter's dialect, resolve unqualified names
against the current schema, and compare them with the project relations resolved for the run. A
-project relation that the node did not obtain through `ctx.relation(...)` produces a `P008`
-**warning**. The query still runs. The warning appears in the build summary, counts the node as
+project relation that the node did not obtain through `ctx.relation(...)` (or, in a loader,
+`ctx.source(...)` or `ctx.loader(...)`) produces a `P008` **warning**. The query still runs. The warning appears in the build summary, counts the node as
`WARN`, and is stored under the node's `warnings` in the `--json` run results.
Both checks ignore:
-- Relations resolved through `ctx.relation(...)`, and a hook's own model (`ctx.destination`)
+- Relations resolved through `ctx.relation(...)`, `ctx.source(...)`, or `ctx.loader(...)`, a hook's
+ own model (`ctx.destination`), and a loader's own source
- Relations that are not project relations, such as `information_schema` tables, external tables,
and temporary tables the node creates
- CTE names
- SQL that cannot be parsed; it is never blocked, only logged at debug level
This is a guard against accidental hard-coding, not a sandbox. SQL sent directly through
-`ctx.connection` or `ctx.adapter` is not checked. Checks and loaders cannot declare SQL
-references, so their SQL is not checked either. To turn both checks off while migrating a project,
+`ctx.connection` or `ctx.adapter` is not checked. Run-time checks apply to `sqb build` and
+`sqb check`. To turn both checks off while migrating a project,
set [`[references] enforce_explicit = false`](/docs/concepts/project-configuration#references).
## model vs source
@@ -94,13 +110,13 @@ set [`[references] enforce_explicit = false`](/docs/concepts/project-configurati
|-----------|-------------|
| `model("name")` | The built model relation |
| `source("name")` | The source read relation, following deferred-source semantics |
+| `seed("name")` | The loaded seed relation |
`source(...)` respects `defer_sources_to`, so a node reading a source in `dev` can read the deferred target's data just like SQL models do.
## Where references are allowed
-- **Tasks** and **assets** may declare `model()` and `source()` references and read them with `ctx.relation(...)`.
+- **Tasks**, **assets**, and **checks** may declare `model()`, `source()`, and `seed()` references and read them with `ctx.relation(...)`.
- **Python hooks** declare the models, sources, and seeds they read with `@hook(reads=...)`; see [Python Hooks](/docs/concepts/models/hooks/python#declared-reads).
-- `seed()` references are accepted by hook reads only; tasks and assets cannot depend on seeds.
-- **Checks** may not reference SQL resources. Validate SQL with [audits](/docs/concepts/audits).
+- **Loaders** may not reference SQL resources; they read sources with `ctx.source(...)` and upstream loaders with `ctx.loader(...)`.
- A Python node referencing a SQL resource never turns into a SQL dependency; selector expansion will not pull Python outputs into SQL model dependencies.
diff --git a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx
index a33e5eeca4..836879bdfc 100644
--- a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx
+++ b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx
@@ -114,7 +114,7 @@ The default is no retry. Set `retry_on` explicitly rather than relying on the br
| Parameter | Description |
|-----------|-------------|
| `name` | Override the node name (defaults to the function name) |
-| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()` for read-only SQL refs |
+| `depends_on` | Upstream nodes (function, tuple, or list); `model()`/`source()`/`seed()` for read-only SQL refs |
| `tags` | Labels for selection and grouping |
| `group` | Display/catalog grouping |
| `description` | Docs (defaults to docstring) |
From c57c4744088132c8e45bbdbd506cd0d287060c2d Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Mon, 28 Sep 2026 03:46:38 +0100
Subject: [PATCH 06/13] fix: report every P006, P007 and P008 violation in one
compile
Macro-generated references, hook reads of a hook's own dependant, and
hard-coded relation names in Python SQL now become compile diagnostics
instead of stopping at the first one. One compile reports each violation
once (per consumer, macro and reference for P006; per hook read for P007;
per node, line and relation for P008) in deterministic order, in the
terminal and in sqb compile --json with has_errors. Commands that plan or
build still fail before planning and list every error.
Re-expanding model SQL with mocked macros for SQL tests no longer checks
macro-generated references again; the model's own expansion reports them.
---
.../docs/concepts/project-configuration.md | 3 +
.../compile/_helpers/assembly/project.py | 4 +-
.../_helpers/explicit_references/collector.py | 68 +++++++++++
.../explicit_references/hook_reads.py | 46 ++++++--
.../explicit_references/macro_arguments.py | 67 ++++++++---
.../explicit_references/python_sql.py | 106 ++++++++++++------
.../classes/explicit_reference_violations.py | 23 ++++
.../compile/main/_assemble_project.py | 23 +++-
.../compile/main/_build_compile_inputs.py | 4 +
.../compiler/graph/_helpers/hook_reads.py | 20 ++--
...ook_read_cycle.py => _hook_read_cycles.py} | 8 +-
.../cli/commands/main/build/_test_types.py | 10 ++
.../main/build/test_explicit_references.py | 72 ++++++++++++
.../compiler/compile/_helpers/_test_types.py | 10 ++
.../compiler/compile/_helpers/helpers.py | 24 ++++
.../compile/_helpers/test_hook_reads.py | 40 +++++++
.../_helpers/test_python_sql_references.py | 36 +++++-
.../_helpers/test_typed_macro_references.py | 35 ++++++
.../docs/concepts/project-configuration.mdx | 3 +
19 files changed, 511 insertions(+), 91 deletions(-)
create mode 100644 src/sqlbuild/compiler/compile/_helpers/explicit_references/collector.py
create mode 100644 src/sqlbuild/compiler/compile/classes/explicit_reference_violations.py
rename src/sqlbuild/compiler/graph/main/{_hook_read_cycle.py => _hook_read_cycles.py} (54%)
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
index 6607720f9d..5f8ca1c9cf 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md
@@ -443,6 +443,9 @@ Every reference to a model, source, or seed must be explicit and typed. By defau
- The same check runs on SQL sent at run time and reports a `P008` warning; the query still runs.
See [Hard-coded relation names](python-nodes/sql-references.md#hard-coded-relation-names).
+One compile reports every `P006`, `P007`, and `P008` violation in the project, in the terminal and in
+`sqb compile --json`.
+
While migrating a project from another tool, turn these checks off for the whole project:
```toml
diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
index aaca60446f..03bd9bb7c1 100644
--- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
+++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
@@ -1921,7 +1921,7 @@ def _build_test_model_query_overrides(
return {}
if not test_input.payload.macro_mocks:
return {}
- macro_context: MacroContext = inputs.macro_context or MacroContext(
+ model_macro_context: MacroContext = inputs.macro_context or MacroContext(
adapter_name=resolve_effective_adapter_name(
project_config=inputs.project_config,
local_config=inputs.local_config,
@@ -1929,8 +1929,8 @@ def _build_test_model_query_overrides(
sql_analysis_enabled=inputs.effective_settings.sql_analysis,
target_name=inputs.effective_target_name,
vars=inputs.effective_vars,
- _enforce_explicit_references=inputs.project_config.references.enforce_explicit,
)
+ macro_context: MacroContext = replace(model_macro_context, _enforce_explicit_references=False)
overrides: dict[str, str] = {}
model_input: CompileModelInput
for model_input in model_inputs:
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/collector.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/collector.py
new file mode 100644
index 0000000000..5eac95dde3
--- /dev/null
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/collector.py
@@ -0,0 +1,68 @@
+"""Collect explicit-reference violations so one compile reports all of them."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar, Token
+from dataclasses import replace
+from functools import wraps
+
+from sqlbuild.compiler.compile.classes.explicit_reference_violations import (
+ ExplicitReferenceViolations,
+)
+from sqlbuild.compiler.compile.exceptions import CompileInputError
+from sqlbuild.compiler.compile.models import CompileProjectInputs, CompilerDiagnostic
+
+_ACTIVE_COLLECTOR: ContextVar[ExplicitReferenceViolations | None] = ContextVar(
+ "sqlbuild_explicit_reference_violations", default=None
+)
+
+
+@contextmanager
+def collect_explicit_reference_violations() -> Iterator[ExplicitReferenceViolations]:
+ """Collect violations reported inside the block instead of raising the first one."""
+
+ collector: ExplicitReferenceViolations = ExplicitReferenceViolations()
+ token: Token[ExplicitReferenceViolations | None] = _ACTIVE_COLLECTOR.set(collector)
+ try:
+ yield collector
+ finally:
+ _ACTIVE_COLLECTOR.reset(token)
+
+
+def with_explicit_reference_diagnostics[**P](
+ build: Callable[P, CompileProjectInputs],
+) -> Callable[P, CompileProjectInputs]:
+ """Append every violation reported while building compile inputs to their diagnostics."""
+
+ @wraps(build)
+ def collecting(*args: P.args, **kwargs: P.kwargs) -> CompileProjectInputs:
+ with collect_explicit_reference_violations() as violations:
+ inputs: CompileProjectInputs = build(*args, **kwargs)
+ if not violations.diagnostics:
+ return inputs
+ return replace(inputs, diagnostics=(*inputs.diagnostics, *violations.diagnostics))
+
+ return collecting
+
+
+def report_explicit_reference_violation(
+ *, key: tuple[str, ...], diagnostic: CompilerDiagnostic
+) -> None:
+ """Record a violation in the active collector, or raise it when none is active."""
+
+ collector: ExplicitReferenceViolations | None = _ACTIVE_COLLECTOR.get()
+ if collector is not None:
+ collector.add(key=key, diagnostic=diagnostic)
+ return
+ location: str = (
+ f"\n --> {diagnostic.location.path.as_posix()}:{diagnostic.location.line}"
+ if diagnostic.location is not None
+ else f"\n --> {diagnostic.path.as_posix()}"
+ if diagnostic.path is not None
+ else ""
+ )
+ raise CompileInputError(
+ f"{diagnostic.message}{location}", code=diagnostic.code, help=diagnostic.help
+ )
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
index 726fbd2c3c..3aebf71c3d 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
@@ -2,17 +2,28 @@
from __future__ import annotations
+from pathlib import Path
+
from sqlbuild.compiler.compile.constants import HOOK_READS_OWN_DEPENDANT_CODE
from sqlbuild.compiler.compile.exceptions import CompileInputError
-from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject
-from sqlbuild.compiler.graph.main._hook_read_cycle import hook_read_cycle
+from sqlbuild.compiler.compile.models import (
+ CompiledObjectKey,
+ CompiledProject,
+ CompilerDiagnostic,
+)
+from sqlbuild.compiler.compile.types import (
+ CompiledResourceType,
+ DiagnosticPhase,
+ DiagnosticSeverity,
+)
+from sqlbuild.compiler.graph.main._hook_read_cycles import hook_read_cycles
from sqlbuild.compiler.graph.models import HookReadEdge
from sqlbuild.python_nodes.models import SqlResourceRef
from sqlbuild.python_nodes.types import SqlResourceRefKind
-def validate_hook_reads(*, project: CompiledProject) -> None:
- """Reject unknown declared hook reads and reads that would wait for the hook's own model."""
+def hook_read_diagnostics(*, project: CompiledProject) -> tuple[CompilerDiagnostic, ...]:
+ """Reject unknown hook reads; return a P007 error for each read of a hook's own dependant."""
known: dict[SqlResourceRefKind, frozenset[str]] = {
SqlResourceRefKind.MODEL: frozenset(model.name for model in project.models),
@@ -27,15 +38,26 @@ def validate_hook_reads(*, project: CompiledProject) -> None:
f"Python hook '{hook.name}' in '{hook.relative_path.as_posix()}' reads unknown "
f"{ref.kind.value} '{ref.name}'"
)
- cycle: HookReadEdge | None = hook_read_cycle(project=project)
- if cycle is None:
- return
- raise CompileInputError(
- f"Python hook '{cycle.hook_name}' on {_describe(cycle.gated)} reads "
- f"{_describe(cycle.read)}, "
- f"which depends on '{cycle.gated.name}'; a hook runs as part of its model, so it cannot "
- "wait for a resource built from that model",
+ hook_paths: dict[str, Path] = {hook.name: hook.relative_path for hook in project.hook_functions}
+ return tuple(
+ _cycle_diagnostic(cycle=cycle, hook_path=hook_paths.get(cycle.hook_name))
+ for cycle in hook_read_cycles(project=project)
+ )
+
+
+def _cycle_diagnostic(*, cycle: HookReadEdge, hook_path: Path | None) -> CompilerDiagnostic:
+ return CompilerDiagnostic(
+ phase=DiagnosticPhase.COMPILE,
+ severity=DiagnosticSeverity.ERROR,
code=HOOK_READS_OWN_DEPENDANT_CODE,
+ message=(
+ f"Python hook '{cycle.hook_name}' on {_describe(cycle.gated)} reads "
+ f"{_describe(cycle.read)}, which depends on '{cycle.gated.name}'; a hook runs as "
+ "part of its model, so it cannot wait for a resource built from that model"
+ ),
+ resource_type=CompiledResourceType.MODEL,
+ resource_name=cycle.gated.name,
+ path=hook_path,
help=(
f"attach the hook to a model built after '{cycle.read.name}', or drop the read and "
f"use ctx.destination for '{cycle.gated.name}' itself"
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
index 183261fb3d..d4be2f8130 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
@@ -6,10 +6,18 @@
import re
from pathlib import Path
+from sqlbuild.compiler.compile._helpers.explicit_references.collector import (
+ report_explicit_reference_violation,
+)
from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references
from sqlbuild.compiler.compile.constants import MACRO_GENERATED_REFERENCE_CODE
from sqlbuild.compiler.compile.exceptions import CompileInputError
-from sqlbuild.compiler.compile.models import CompileSqlReference, LoadedMacro
+from sqlbuild.compiler.compile.models import CompilerDiagnostic, CompileSqlReference, LoadedMacro
+from sqlbuild.compiler.compile.types import (
+ CompiledResourceType,
+ DiagnosticPhase,
+ DiagnosticSeverity,
+)
from sqlbuild.compiler.references.types import SqlReferenceKind
from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity
from sqlbuild.compiler.scopes.types import ResourceKind
@@ -29,6 +37,11 @@
SqlReferenceKind.SOURCE.value: SqlResourceRefKind.SOURCE,
SqlReferenceKind.SEED.value: SqlResourceRefKind.SEED,
}
+_RESOURCE_TYPE_BY_CONSUMER_KIND: dict[ResourceKind, CompiledResourceType] = {
+ ResourceKind.MODEL: CompiledResourceType.MODEL,
+ ResourceKind.SOURCE: CompiledResourceType.SOURCE,
+ ResourceKind.SEED: CompiledResourceType.SEED,
+}
_REFERENCE_CALL_MARKERS: tuple[str, ...] = tuple(f"{name}(" for name in _TYPED_REFERENCE_KINDS)
_RELATION_PLACEHOLDER_PREFIX: str = "__sqlbuild_relation_"
_RELATION_PLACEHOLDER_PATTERN: re.Pattern[str] = re.compile(
@@ -64,11 +77,10 @@ def reject_macro_generated_references(
file_path: Path,
consumer: ResourceIdentity | DeclarationIdentity | None,
) -> None:
+ """Report each typed reference a macro emitted instead of receiving it as an argument."""
+
if not any(marker in macro_result for marker in _REFERENCE_CALL_MARKERS):
return
- generated: SqlResourceRef | None = _first_generated_reference(macro_result)
- if generated is None:
- return
consumer_label: str = (
f"{consumer.kind.value}:{consumer.name}" if consumer is not None else f"'{file_path}'"
)
@@ -77,29 +89,48 @@ def reject_macro_generated_references(
if isinstance(consumer, ResourceIdentity) and consumer.kind is ResourceKind.MODEL
else "the calling SQL"
)
- raise CompileInputError(
- f"{consumer_label} depends on {generated.kind.value}:{generated.name} through macro "
- f"{loaded_macro.name}()\n --> {loaded_macro.relative_path.as_posix()}",
- code=MACRO_GENERATED_REFERENCE_CODE,
- help=(
- f"write the reference in {location}, or pass it in: "
- f"@{loaded_macro.name}({_reference_call_text(generated)})\n"
- " = help: while migrating a project, allow macro-generated references with "
- "[references] enforce_explicit = false in sqlbuild_project.toml"
- ),
+ resource: ResourceIdentity | None = consumer if isinstance(consumer, ResourceIdentity) else None
+ resource_type: CompiledResourceType | None = (
+ _RESOURCE_TYPE_BY_CONSUMER_KIND.get(resource.kind) if resource is not None else None
)
+ generated: SqlResourceRef
+ for generated in _generated_references(macro_result):
+ report_explicit_reference_violation(
+ key=(consumer_label, loaded_macro.name, generated.kind.value, generated.name),
+ diagnostic=CompilerDiagnostic(
+ phase=DiagnosticPhase.COMPILE,
+ severity=DiagnosticSeverity.ERROR,
+ code=MACRO_GENERATED_REFERENCE_CODE,
+ message=(
+ f"{consumer_label} depends on {generated.kind.value}:{generated.name} "
+ f"through macro {loaded_macro.name}()"
+ ),
+ resource_type=resource_type,
+ resource_name=(
+ resource.name if resource is not None and resource_type is not None else None
+ ),
+ path=loaded_macro.relative_path,
+ help=(
+ f"write the reference in {location}, or pass it in: "
+ f"@{loaded_macro.name}({_reference_call_text(generated)})\n"
+ " = help: while migrating a project, allow macro-generated references with "
+ "[references] enforce_explicit = false in sqlbuild_project.toml"
+ ),
+ ),
+ )
-def _first_generated_reference(sql: str) -> SqlResourceRef | None:
+def _generated_references(sql: str) -> tuple[SqlResourceRef, ...]:
try:
references: tuple[CompileSqlReference, ...] = extract_sql_references(sql)
except CompileInputError:
- return None
+ return ()
+ generated: dict[SqlResourceRef, None] = {}
for reference in references:
kind: SqlResourceRefKind | None = _EXPLICIT_REFERENCE_KINDS.get(str(reference.ref_kind))
if kind is not None:
- return SqlResourceRef(kind=kind, name=reference.ref_name)
- return None
+ generated[SqlResourceRef(kind=kind, name=reference.ref_name)] = None
+ return tuple(generated)
def evaluate_typed_reference(*, node: ast.Call, file_path: Path) -> SqlResourceRef:
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
index 3c95b4fc0e..0c88fe1c42 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
@@ -10,8 +10,8 @@
from pathlib import Path
from sqlbuild.compiler.compile.constants import HARD_CODED_PROJECT_RELATION_CODE
-from sqlbuild.compiler.compile.exceptions import CompileInputError
-from sqlbuild.compiler.compile.models import CompiledProject
+from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic
+from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity
from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs
from sqlbuild.compiler.graph.main._model_python_hook_names import model_python_hook_names
from sqlbuild.compiler.references.main._compiled_project_relations import (
@@ -31,6 +31,7 @@
from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition
from sqlbuild.python_nodes.models import LoaderDefinition, SqlResourceRef
from sqlbuild.python_nodes.types import SqlResourceRefKind
+from sqlbuild.spec.contracts.models import SourceLocation
_SQL_METHOD_NAMES: frozenset[str] = frozenset({"query", "execute_sql"})
_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset({"ctx", "context", "_ctx", "hook_context"})
@@ -54,29 +55,40 @@ class _LiteralSql:
line: int
-def validate_python_sql_references(
+def python_sql_reference_diagnostics(
*, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs
-) -> None:
- """Reject literal SQL in Python nodes and hooks that names a project relation."""
+) -> tuple[CompilerDiagnostic, ...]:
+ """Return a P008 error for each project relation literal SQL in Python code names."""
owners: tuple[_PythonSqlOwner, ...] = _owners(
project=project, discovered_inputs=discovered_inputs
)
if not owners:
- return
+ return ()
index: ProjectRelationIndex = compiled_project_relations(project=project)
if not index.relations:
- return
+ return ()
+ diagnostics: dict[tuple[str, int, str, str], CompilerDiagnostic] = {}
for owner in owners:
temporary: frozenset[str] = frozenset()
for literal in sorted(_literal_sql_calls(owner.function), key=lambda item: item.line):
- temporary = _reject_hard_coded_relation(
+ found: dict[tuple[str, int, str, str], CompilerDiagnostic]
+ temporary, found = _hard_coded_relations(
owner=owner,
literal=literal,
index=index,
dialect=project.sql_analysis_dialect,
temporary=temporary,
)
+ for key, diagnostic in found.items():
+ diagnostics.setdefault(key, diagnostic)
+ return tuple(
+ diagnostics[key]
+ for key in sorted(
+ diagnostics,
+ key=lambda key: (diagnostics[key].path or Path(), key[1], key[0], key[2], key[3]),
+ )
+ )
def _owners(
@@ -207,19 +219,20 @@ def _literal_text(node: ast.expr) -> str | None:
return "".join(parts)
-def _reject_hard_coded_relation(
+def _hard_coded_relations(
*,
owner: _PythonSqlOwner,
literal: _LiteralSql,
index: ProjectRelationIndex,
dialect: str | None,
temporary: frozenset[str],
-) -> frozenset[str]:
+) -> tuple[frozenset[str], dict[tuple[str, int, str, str], CompilerDiagnostic]]:
extracted: tuple[tuple[RelationName, ...], frozenset[str]] | None = extract_relation_names(
sql=literal.sql, dialect=dialect
)
+ found: dict[tuple[str, int, str, str], CompilerDiagnostic] = {}
if extracted is None:
- return temporary
+ return temporary, found
relations, created = extracted
temporary = temporary | created
for relation in relations:
@@ -232,29 +245,54 @@ def _reject_hard_coded_relation(
match: ProjectRelation | None = match_project_relation(index=index, relation=relation)
if match is None or match.ref in owner.own_refs:
continue
- written: str = ".".join(
- part for part in (relation.database, relation.schema, relation.name) if part
+ key: tuple[str, int, str, str] = (
+ owner.label,
+ literal.line,
+ match.ref.kind.value,
+ match.ref.name,
)
- remedy: str = hard_coded_relation_remedy(
- owner_kind=owner.kind,
- ref=match.ref,
- upstream_loader_by_source=owner.upstream_loader_by_source,
- )
- qualify_help: str = (
- f"\n = help: if '{written}' is an external table that shares the name, qualify it "
- "with its schema"
- if relation.schema is None
- else ""
- )
- raise CompileInputError(
- f"{owner.label} names {match.ref.kind.value}:{match.ref.name} as '{written}' in SQL "
- f"passed to ctx.{literal.method}()\n --> {owner.relative_path.as_posix()}:"
- f"{literal.line}",
- code=HARD_CODED_PROJECT_RELATION_CODE,
- help=(
- f"{remedy}{qualify_help}\n"
- " = help: while migrating a project, allow hard-coded relation names with "
- "[references] enforce_explicit = false in sqlbuild_project.toml"
+ found.setdefault(
+ key,
+ _hard_coded_relation_diagnostic(
+ owner=owner, literal=literal, relation=relation, match=match
),
)
- return temporary
+ return temporary, found
+
+
+def _hard_coded_relation_diagnostic(
+ *,
+ owner: _PythonSqlOwner,
+ literal: _LiteralSql,
+ relation: RelationName,
+ match: ProjectRelation,
+) -> CompilerDiagnostic:
+ written: str = ".".join(
+ part for part in (relation.database, relation.schema, relation.name) if part
+ )
+ remedy: str = hard_coded_relation_remedy(
+ owner_kind=owner.kind,
+ ref=match.ref,
+ upstream_loader_by_source=owner.upstream_loader_by_source,
+ )
+ qualify_help: str = (
+ f"\n = help: if '{written}' is an external table that shares the name, qualify it "
+ "with its schema"
+ if relation.schema is None
+ else ""
+ )
+ return CompilerDiagnostic(
+ phase=DiagnosticPhase.COMPILE,
+ severity=DiagnosticSeverity.ERROR,
+ code=HARD_CODED_PROJECT_RELATION_CODE,
+ message=(
+ f"{owner.label} names {match.ref.kind.value}:{match.ref.name} as '{written}' in SQL "
+ f"passed to ctx.{literal.method}()"
+ ),
+ location=SourceLocation(path=owner.relative_path, line=literal.line, column=1),
+ help=(
+ f"{remedy}{qualify_help}\n"
+ " = help: while migrating a project, allow hard-coded relation names with "
+ "[references] enforce_explicit = false in sqlbuild_project.toml"
+ ),
+ )
diff --git a/src/sqlbuild/compiler/compile/classes/explicit_reference_violations.py b/src/sqlbuild/compiler/compile/classes/explicit_reference_violations.py
new file mode 100644
index 0000000000..816dd2f3b7
--- /dev/null
+++ b/src/sqlbuild/compiler/compile/classes/explicit_reference_violations.py
@@ -0,0 +1,23 @@
+"""Explicit-reference violations gathered during one compile phase."""
+
+from __future__ import annotations
+
+from sqlbuild.compiler.compile.models import CompilerDiagnostic
+
+
+class ExplicitReferenceViolations:
+ """Deduplicated explicit-reference diagnostics gathered during one compile phase."""
+
+ def __init__(self) -> None:
+ self._by_key: dict[tuple[str, ...], CompilerDiagnostic] = {}
+
+ def add(self, *, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None:
+ """Record one violation; a repeated key keeps the first diagnostic."""
+
+ self._by_key.setdefault(key, diagnostic)
+
+ @property
+ def diagnostics(self) -> tuple[CompilerDiagnostic, ...]:
+ """Return the recorded diagnostics in deterministic key order."""
+
+ return tuple(self._by_key[key] for key in sorted(self._by_key))
diff --git a/src/sqlbuild/compiler/compile/main/_assemble_project.py b/src/sqlbuild/compiler/compile/main/_assemble_project.py
index a7e8b7ba23..e23d8984d1 100644
--- a/src/sqlbuild/compiler/compile/main/_assemble_project.py
+++ b/src/sqlbuild/compiler/compile/main/_assemble_project.py
@@ -2,18 +2,20 @@
from __future__ import annotations
+from dataclasses import replace
from pathlib import Path
from sqlbuild.adapter.contract.models import ExpressionInferenceProfile
from sqlbuild.compiler.compile._helpers.assembly.audit_gates import validate_attached_audit_gates
from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project
-from sqlbuild.compiler.compile._helpers.explicit_references.hook_reads import validate_hook_reads
+from sqlbuild.compiler.compile._helpers.explicit_references.hook_reads import hook_read_diagnostics
from sqlbuild.compiler.compile._helpers.explicit_references.python_sql import (
- validate_python_sql_references,
+ python_sql_reference_diagnostics,
)
from sqlbuild.compiler.compile.models import (
CompiledProject,
CompileProjectInputs,
+ CompilerDiagnostic,
)
from sqlbuild.compiler.lineage.types import ColumnLineageMode
@@ -38,7 +40,16 @@ def assemble_project(
analysis_model_names=analysis_model_names,
)
validate_attached_audit_gates(project=project)
- validate_hook_reads(project=project)
- if inputs.project_config.references.enforce_explicit:
- validate_python_sql_references(project=project, discovered_inputs=inputs.discovered_inputs)
- return project
+ reference_diagnostics: tuple[CompilerDiagnostic, ...] = (
+ *hook_read_diagnostics(project=project),
+ *(
+ python_sql_reference_diagnostics(
+ project=project, discovered_inputs=inputs.discovered_inputs
+ )
+ if inputs.project_config.references.enforce_explicit
+ else ()
+ ),
+ )
+ if not reference_diagnostics:
+ return project
+ return replace(project, diagnostics=(*project.diagnostics, *reference_diagnostics))
diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
index 60453da560..7fbd0870e8 100644
--- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
+++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
@@ -31,6 +31,9 @@
from sqlbuild.compiler.compile._helpers.audit_factories.core import (
build_audit_factory_orphan_diagnostics,
)
+from sqlbuild.compiler.compile._helpers.explicit_references.collector import (
+ with_explicit_reference_diagnostics,
+)
from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id
from sqlbuild.compiler.compile._helpers.render.declarations import (
build_public_declaration_indexes,
@@ -71,6 +74,7 @@
from sqlbuild.spec.contracts.models import SettingsConfig, TargetConfig
+@with_explicit_reference_diagnostics
def build_compile_inputs(
*,
discovered_inputs: DiscoveredProjectInputs,
diff --git a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
index d77c7846ae..dbf9e52344 100644
--- a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
+++ b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
@@ -45,12 +45,12 @@ def hook_read_edges_impl(*, project: CompiledProject) -> tuple[HookReadEdge, ...
return tuple(edges)
-def hook_read_cycle_impl(*, project: CompiledProject) -> HookReadEdge | None:
- """Return the first declared hook read that depends on the model running the hook."""
+def hook_read_cycles_impl(*, project: CompiledProject) -> tuple[HookReadEdge, ...]:
+ """Return every declared hook read that depends on the model running the hook."""
edges: tuple[HookReadEdge, ...] = hook_read_edges_impl(project=project)
if not edges:
- return None
+ return ()
upstream: dict[CompiledObjectKey, list[CompiledObjectKey]] = {
key: list(deps) for key, deps in build_lineage_upstream_deps_impl(project).items()
}
@@ -58,9 +58,11 @@ def hook_read_cycle_impl(*, project: CompiledProject) -> HookReadEdge | None:
upstream.setdefault(gate.gated, []).append(gate.read)
for edge in edges:
upstream.setdefault(edge.gated, []).append(edge.read)
- for edge in sorted(edges, key=lambda item: (item.gated.name, item.hook_name, item.read.name)):
- if edge.gated in transitive_closure_many_impl(
- starts=(edge.read,), edges=upstream, include_starts=False
- ):
- return edge
- return None
+ return tuple(
+ edge
+ for edge in sorted(
+ edges, key=lambda item: (item.gated.name, item.hook_name, item.read.name)
+ )
+ if edge.gated
+ in transitive_closure_many_impl(starts=(edge.read,), edges=upstream, include_starts=False)
+ )
diff --git a/src/sqlbuild/compiler/graph/main/_hook_read_cycle.py b/src/sqlbuild/compiler/graph/main/_hook_read_cycles.py
similarity index 54%
rename from src/sqlbuild/compiler/graph/main/_hook_read_cycle.py
rename to src/sqlbuild/compiler/graph/main/_hook_read_cycles.py
index 7e57d62226..69c6e541a4 100644
--- a/src/sqlbuild/compiler/graph/main/_hook_read_cycle.py
+++ b/src/sqlbuild/compiler/graph/main/_hook_read_cycles.py
@@ -3,11 +3,11 @@
from __future__ import annotations
from sqlbuild.compiler.compile.models import CompiledProject
-from sqlbuild.compiler.graph._helpers.hook_reads import hook_read_cycle_impl
+from sqlbuild.compiler.graph._helpers.hook_reads import hook_read_cycles_impl
from sqlbuild.compiler.graph.models import HookReadEdge
-def hook_read_cycle(*, project: CompiledProject) -> HookReadEdge | None:
- """Return the first declared hook read that depends on the model running the hook."""
+def hook_read_cycles(*, project: CompiledProject) -> tuple[HookReadEdge, ...]:
+ """Return every declared hook read that depends on the model running the hook."""
- return hook_read_cycle_impl(project=project)
+ return hook_read_cycles_impl(project=project)
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
index 499e898ba9..6398e12b1a 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
@@ -1036,6 +1036,16 @@ class ExplicitReferencePythonDependencyE2ETestCase:
expected_check_row_pattern: str
+@dataclass(frozen=True)
+class ExplicitReferenceAllDiagnosticsE2ETestCase:
+ """A project with several explicit-reference violations, all reported by one compile."""
+
+ description: str
+ overrides: dict[str, str]
+ expected_output_fragments: tuple[str, ...]
+ expected_json_diagnostics: tuple[tuple[str, str], ...]
+
+
@dataclass(frozen=True)
class ExplicitReferenceFailureE2ETestCase:
"""A project whose references are not explicit, and the failure it reports."""
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
index 5ada83e129..a686e21069 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
@@ -11,6 +11,7 @@
import pytest
from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
+ ExplicitReferenceAllDiagnosticsE2ETestCase,
ExplicitReferenceBuildE2ETestCase,
ExplicitReferenceFailureE2ETestCase,
ExplicitReferencePythonDependencyE2ETestCase,
@@ -36,6 +37,9 @@
+ "\n\ndef orders_base():\n return 'SELECT * FROM __ref(\"stg_orders_eu\")'\n"
)
_EMITTING_MODEL: str = "MODEL (materialized table);\n@orders_base()\n"
+_SECOND_EMITTING_MACROS: str = (
+ "def customers_base():\n return 'SELECT * FROM __ref(\"stg_customers\")'\n"
+)
_UNDECLARED_HOOKS: str = EXPLICIT_REFERENCE_HOOKS + ' ctx.relation(model("all_orders"))\n'
_LITERAL_TASK: str = (
"from sqlbuild.refs import model\n"
@@ -422,3 +426,71 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit
fragment in warning
for warning, fragment in zip(stored, test_case.expected_warning_fragments, strict=True)
)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ ExplicitReferenceAllDiagnosticsE2ETestCase(
+ description="two ref-emitting macros and a hard-coded task name all report at once",
+ overrides={
+ EXPLICIT_REFERENCE_MACRO_PATH: _EMITTING_MACROS,
+ "models/sales/_sqlbuild/_macros/customers.py": _SECOND_EMITTING_MACROS,
+ f"{EXPLICIT_REFERENCE_MODELS}/hidden.sql": _EMITTING_MODEL,
+ f"{EXPLICIT_REFERENCE_MODELS}/hidden_customers.sql": (
+ "MODEL (materialized table);\n@customers_base()\n"
+ ),
+ "python/tasks/export.py": _LITERAL_TASK,
+ },
+ expected_output_fragments=(
+ "error[P006]: model:hidden depends on model:stg_orders_eu through macro "
+ "orders_base()",
+ "error[P006]: model:hidden_customers depends on model:stg_customers through macro "
+ "customers_base()",
+ "error[P008]: task:export_orders names model:stg_customers as 'stg_customers'",
+ "3 errors",
+ ),
+ expected_json_diagnostics=(
+ ("P006", "model:hidden depends on model:stg_orders_eu through macro orders_base()"),
+ (
+ "P006",
+ "model:hidden_customers depends on model:stg_customers through macro "
+ "customers_base()",
+ ),
+ (
+ "P008",
+ "task:export_orders names model:stg_customers as 'stg_customers' in SQL "
+ "passed to ctx.query()",
+ ),
+ ),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_several_implicit_references_when_compiling_then_one_compile_reports_all(
+ test_case: ExplicitReferenceAllDiagnosticsE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="explicit_refs",
+ repo_files=explicit_reference_project_files(overrides=test_case.overrides),
+ )
+
+ text: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "compile"), project_dir=project_dir
+ )
+ machine: subprocess.CompletedProcess[str] = run_sqb(
+ command=("compile", "--json", "--no-cache"), project_dir=project_dir
+ )
+
+ assert text.returncode == 1, text.stdout + text.stderr
+ output: str = text.stdout + text.stderr
+ assert all(fragment in output for fragment in test_case.expected_output_fragments), output
+ assert machine.returncode == 1, machine.stdout + machine.stderr
+ payload: dict[str, object] = json.loads(machine.stdout)
+ assert payload["has_errors"] is True
+ assert (
+ tuple((item["code"], item["message"]) for item in payload["diagnostics"])
+ == test_case.expected_json_diagnostics
+ )
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
index 442fbd67b4..62b4bc3b77 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
@@ -1022,3 +1022,13 @@ class PythonSqlReferenceAllowedTestCase:
description: str
files: dict[str, str]
expected_model_names: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class MacroGeneratedReferenceCollectionTestCase:
+ """Model SQL whose macros emit several references, all reported by one expansion."""
+
+ description: str
+ macro_file_contents: str
+ sql: str
+ expected_messages: tuple[str, ...]
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
index 6cdf857817..34924ba7e1 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
@@ -11,6 +11,9 @@
from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter
from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter
from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core
+from sqlbuild.compiler.compile._helpers.explicit_references.collector import (
+ collect_explicit_reference_violations,
+)
from sqlbuild.compiler.compile._helpers.render.macros import (
expand_sql_macros,
load_project_macros,
@@ -30,6 +33,7 @@
CompiledRelationLocation,
CompiledSqlTest,
CompileProjectInputs,
+ CompilerDiagnostic,
DeclarationExpansionContext,
DeclarationResolutionContext,
DeclarationRuntimeProjection,
@@ -399,3 +403,23 @@ def python_loader_source(*, depends_on: str, body: str) -> str:
f"{body}"
" return [{'id': 1}]\n"
)
+
+
+def collect_typed_macro_violations(
+ *, tmp_path: Path, macro_file_contents: str, sql: str
+) -> tuple[CompilerDiagnostic, ...]:
+ """Expand ``order_summary`` SQL and return the explicit-reference violations it reports."""
+
+ with collect_explicit_reference_violations() as violations:
+ expand_typed_macro_sql(tmp_path=tmp_path, macro_file_contents=macro_file_contents, sql=sql)
+ return violations.diagnostics
+
+
+def render_compile_diagnostics(*, project: CompiledProject) -> str:
+ """Render every project diagnostic with its code, message, location, and help."""
+
+ return "\n".join(
+ f"[{diagnostic.code}] {diagnostic.message} --> {diagnostic.path}:{diagnostic.line} "
+ f"= help: {diagnostic.help}"
+ for diagnostic in project.diagnostics
+ )
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
index 2d80815ede..a062ed7483 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
@@ -17,6 +17,7 @@
execution_edge_names,
gate_model,
python_hook_source,
+ render_compile_diagnostics,
)
_PROJECT_FILE: str = """
@@ -99,6 +100,45 @@ def test_given_hook_reads_when_compiling_then_hooked_model_runs_after_reads(
"which depends on 'orders'",
),
),
+ HookReadErrorTestCase(
+ description="every hook read of its own dependant is reported",
+ files={
+ _HOOK_PATH: python_hook_source(
+ reads='[model("order_rollup"), model("order_count")]'
+ ),
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id", header=_HOOKED_HEADER
+ ),
+ "models/marts/order_rollup.sql": gate_model(
+ sql='SELECT count(*) AS n FROM __ref("orders")'
+ ),
+ "models/marts/order_count.sql": gate_model(
+ sql='SELECT count(*) AS n FROM __ref("orders")'
+ ),
+ },
+ expected_error_fragments=(
+ "[P007] Python hook 'refresh_lookup' on model 'orders' reads model 'order_count'",
+ "[P007] Python hook 'refresh_lookup' on model 'orders' reads model 'order_rollup'",
+ ),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_hook_reading_its_own_dependant_when_compiling_then_it_reports_p007(
+ test_case: HookReadErrorTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
+
+ rendered: str = render_compile_diagnostics(project=compile_and_assemble(project_dir=tmp_path))
+
+ assert all(fragment in rendered for fragment in test_case.expected_error_fragments)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ (
HookReadErrorTestCase(
description="hook reading an unknown model",
files={
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
index da2a01a3c4..a63e658bac 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
@@ -19,6 +19,7 @@
python_hook_source,
python_loader_source,
python_task_source,
+ render_compile_diagnostics,
)
_PROJECT_FILE: str = """
@@ -81,6 +82,33 @@
"[references] enforce_explicit = false",
),
),
+ PythonSqlReferenceErrorTestCase(
+ description="every hard-coded relation across Python nodes is reported",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(),
+ _TASK_PATH: python_task_source(
+ body=(
+ ' ctx.query("SELECT count(*) FROM customers")\n'
+ ' ctx.query("SELECT * FROM customers JOIN orders ON true")\n'
+ )
+ ),
+ _CHECK_PATH: python_check_source(
+ depends_on='model("orders")',
+ body=' ctx.query("SELECT count(*) FROM customers")\n',
+ ),
+ },
+ expected_error_fragments=(
+ "[P008] check:orders_present names model:customers as 'customers' in SQL passed "
+ "to ctx.query() --> python/checks/orders.py:7",
+ "[P008] task:export_orders names model:customers as 'customers' in SQL passed to "
+ "ctx.query() --> python/tasks/export.py:7",
+ "[P008] task:export_orders names model:customers as 'customers' in SQL passed to "
+ "ctx.query() --> python/tasks/export.py:8",
+ "[P008] task:export_orders names model:orders as 'orders' in SQL passed to "
+ "ctx.query() --> python/tasks/export.py:8",
+ ),
+ ),
PythonSqlReferenceErrorTestCase(
description="task literal SQL naming a seed",
files={
@@ -200,19 +228,15 @@
),
ids=lambda case: case.description,
)
-def test_given_hard_coded_project_relation_when_compiling_then_it_fails(
+def test_given_hard_coded_project_relation_when_compiling_then_it_reports_p008(
test_case: PythonSqlReferenceErrorTestCase,
tmp_path: Path,
write_repo_files: Callable[[Path, dict[str, str]], None],
) -> None:
write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
- with pytest.raises(ValueError) as error:
- compile_and_assemble(project_dir=tmp_path)
+ rendered: str = render_compile_diagnostics(project=compile_and_assemble(project_dir=tmp_path))
- rendered: str = (
- f"[{getattr(error.value, 'code', '')}] {error.value} {getattr(error.value, 'help', '')}"
- )
assert all(fragment in rendered for fragment in test_case.expected_error_fragments)
assert not any(fragment in rendered for fragment in test_case.unexpected_error_fragments)
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
index 1c5126b885..a121a4497a 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
@@ -6,7 +6,9 @@
import pytest
from sqlbuild.compiler.compile.exceptions import CompileInputError
+from sqlbuild.compiler.compile.models import CompilerDiagnostic
from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import (
+ MacroGeneratedReferenceCollectionTestCase,
MacroGeneratedReferenceErrorTestCase,
MacroGeneratedReferenceSwitchTestCase,
MalformedTypedMacroReferenceTestCase,
@@ -14,6 +16,7 @@
TypedReferenceAdapterRenderingTestCase,
)
from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import (
+ collect_typed_macro_violations,
expand_typed_macro_sql,
resolve_model_references_for_adapter,
)
@@ -154,6 +157,38 @@ def test_given_macro_output_reference_when_expanding_then_raises_explicit_refere
assert all(fragment in rendered for fragment in test_case.expected_error_fragments)
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ MacroGeneratedReferenceCollectionTestCase(
+ description="every emitted reference is reported once, in order",
+ macro_file_contents=(
+ "def pair():\n"
+ ' return \'SELECT * FROM __ref("stg_orders") JOIN __seed("country_codes") ON 1\'\n\n'
+ "def raw_orders():\n return 'SELECT * FROM __source(\"raw_orders\")'\n"
+ ),
+ sql="@pair() UNION ALL @pair() UNION ALL @raw_orders()",
+ expected_messages=(
+ "model:order_summary depends on model:stg_orders through macro pair()",
+ "model:order_summary depends on seed:country_codes through macro pair()",
+ "model:order_summary depends on source:raw_orders through macro raw_orders()",
+ ),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_several_macro_output_references_when_collecting_then_reports_each_once(
+ test_case: MacroGeneratedReferenceCollectionTestCase, tmp_path: Path
+) -> None:
+ diagnostics: tuple[CompilerDiagnostic, ...] = collect_typed_macro_violations(
+ tmp_path=tmp_path, macro_file_contents=test_case.macro_file_contents, sql=test_case.sql
+ )
+
+ assert tuple(diagnostic.message for diagnostic in diagnostics) == test_case.expected_messages
+ assert {diagnostic.code for diagnostic in diagnostics} == {"P006"}
+ assert {diagnostic.resource_name for diagnostic in diagnostics} == {"order_summary"}
+
+
@pytest.mark.parametrize(
"test_case",
[
diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx
index 70dc7e46bb..5e33c5bbc1 100644
--- a/website/src/content/docs/docs/concepts/project-configuration.mdx
+++ b/website/src/content/docs/docs/concepts/project-configuration.mdx
@@ -421,6 +421,9 @@ Every reference to a model, source, or seed must be explicit and typed. By defau
- The same check runs on SQL sent at run time and reports a `P008` warning; the query still runs.
See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names).
+One compile reports every `P006`, `P007`, and `P008` violation in the project, in the terminal and in
+`sqb compile --json`.
+
While migrating a project from another tool, turn these checks off for the whole project:
```toml
From b440bc14b404b683ce8522c4b0651cf7a9d3dd2b Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Mon, 28 Sep 2026 04:28:01 +0100
Subject: [PATCH 07/13] fix: report every P004, P005 and scope placement
violation in one compile
Single-model singular audits (P004), attached audits reading a resource
built from their target (P005), declarations not visible from their
consumer (S006), and declaration placement errors (S007, S008, S009,
S010, S024) now become individual compile diagnostics instead of stopping
at the first one or folding every scope finding into one P001. Each
carries its file path and, where it has one, the model or audit name, so
the terminal marks the right rows FAIL and sqb compile --json reports
every error with has_errors. Commands that plan or build still fail
before planning and list every error.
Project-loading failures stay fatal: audits/ and _sqlbuild/ layout
errors, unknown seed YAML keys, and declaration-index parse, name and
duplicate errors.
---
.../compile/_helpers/assembly/audit_gates.py | 51 ++++--
.../compile/_helpers/assembly/project.py | 9 +-
.../compile/_helpers/attachment/audits.py | 78 ++++++----
.../compile/_helpers/diagnostics/collector.py | 70 +++++++++
.../compile/_helpers/diagnostics/scope.py | 75 +++++++++
.../_helpers/explicit_references/collector.py | 68 --------
.../explicit_references/macro_arguments.py | 14 +-
.../_helpers/named_declarations/core.py | 47 ++++--
.../named_declarations/model_schemas.py | 38 +++--
...ns.py => collected_compile_diagnostics.py} | 8 +-
.../compile/main/_assemble_project.py | 10 +-
.../compile/main/_build_compile_inputs.py | 6 +-
.../compiler/graph/_helpers/audit_gates.py | 19 ++-
...ycle.py => _attached_audit_gate_cycles.py} | 8 +-
.../cli/commands/main/build/_test_types.py | 4 +-
.../main/build/test_explicit_references.py | 114 +++++++++++++-
.../compiler/compile/_helpers/helpers.py | 6 +-
.../_helpers/test_attached_audit_gates.py | 9 +-
.../_helpers/test_declaration_scopes.py | 12 +-
.../test_scoped_named_declarations.py | 147 +++++++++++-------
20 files changed, 552 insertions(+), 241 deletions(-)
create mode 100644 src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py
create mode 100644 src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py
delete mode 100644 src/sqlbuild/compiler/compile/_helpers/explicit_references/collector.py
rename src/sqlbuild/compiler/compile/classes/{explicit_reference_violations.py => collected_compile_diagnostics.py} (67%)
rename src/sqlbuild/compiler/graph/main/{_attached_audit_gate_cycle.py => _attached_audit_gate_cycles.py} (51%)
diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/audit_gates.py b/src/sqlbuild/compiler/compile/_helpers/assembly/audit_gates.py
index c525b90db6..abce2fc5ab 100644
--- a/src/sqlbuild/compiler/compile/_helpers/assembly/audit_gates.py
+++ b/src/sqlbuild/compiler/compile/_helpers/assembly/audit_gates.py
@@ -2,11 +2,20 @@
from __future__ import annotations
+from pathlib import Path
+
from sqlbuild.compiler.compile.constants import ATTACHED_AUDIT_READS_OWN_DEPENDANT_CODE
-from sqlbuild.compiler.compile.exceptions import CompileInputError
-from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject
-from sqlbuild.compiler.compile.types import CompiledResourceType
-from sqlbuild.compiler.graph.main._attached_audit_gate_cycle import attached_audit_gate_cycle
+from sqlbuild.compiler.compile.models import (
+ CompiledObjectKey,
+ CompiledProject,
+ CompilerDiagnostic,
+)
+from sqlbuild.compiler.compile.types import (
+ CompiledResourceType,
+ DiagnosticPhase,
+ DiagnosticSeverity,
+)
+from sqlbuild.compiler.graph.main._attached_audit_gate_cycles import attached_audit_gate_cycles
from sqlbuild.compiler.graph.models import AttachedAuditGateEdge
_RESOURCE_LABELS: dict[CompiledResourceType, str] = {
@@ -18,19 +27,35 @@
}
-def validate_attached_audit_gates(*, project: CompiledProject) -> None:
- """Reject attached audits that would wait for a resource built from their own target."""
+def attached_audit_gate_diagnostics(*, project: CompiledProject) -> tuple[CompilerDiagnostic, ...]:
+ """Return a P005 error for each attached audit that reads a resource built from its target."""
+
+ audit_paths: dict[str, Path] = {
+ audit.name: audit.audit_file.relative_path for audit in project.audits
+ }
+ return tuple(
+ _cycle_diagnostic(cycle=cycle, audit_path=audit_paths.get(cycle.audit_name))
+ for cycle in attached_audit_gate_cycles(project=project)
+ )
- cycle: AttachedAuditGateEdge | None = attached_audit_gate_cycle(project=project)
- if cycle is None:
- return
+
+def _cycle_diagnostic(
+ *, cycle: AttachedAuditGateEdge, audit_path: Path | None
+) -> CompilerDiagnostic:
target: str = _describe(cycle.target)
read: str = _describe(cycle.read)
- raise CompileInputError(
- f"Audit '{cycle.audit_name}' on {target} reads {read}, which depends on "
- f"'{cycle.target.name}'; an attached audit gates its target, so it cannot wait for a "
- "resource built from that target",
+ return CompilerDiagnostic(
+ phase=DiagnosticPhase.COMPILE,
+ severity=DiagnosticSeverity.ERROR,
code=ATTACHED_AUDIT_READS_OWN_DEPENDANT_CODE,
+ message=(
+ f"Audit '{cycle.audit_name}' on {target} reads {read}, which depends on "
+ f"'{cycle.target.name}'; an attached audit gates its target, so it cannot wait for a "
+ "resource built from that target"
+ ),
+ resource_type=cycle.target.resource_type,
+ resource_name=cycle.target.name,
+ path=audit_path,
help=(
f"write a singular audit under audits/singular/ that references both "
f"'{cycle.target.name}' and '{cycle.read.name}'; it runs once both are built"
diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
index 03bd9bb7c1..1c0d270c76 100644
--- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
+++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py
@@ -80,6 +80,7 @@
sql_test_scope_deps,
)
from sqlbuild.compiler.compile._helpers.diagnostics.recovery import complete_semantic_diagnostics
+from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors
from sqlbuild.compiler.compile._helpers.render.context_templates import (
resolve_early_model_templates,
)
@@ -101,7 +102,6 @@
build_sql_test_case_fingerprint,
)
from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME
-from sqlbuild.compiler.compile.exceptions import CompileInputError
from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import (
scope_index_with_compile_usages,
)
@@ -161,8 +161,6 @@
from sqlbuild.compiler.profiling.main.record import record_compile_timing
from sqlbuild.compiler.references.types import SqlReferenceKind
from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type
-from sqlbuild.compiler.scopes.exceptions import ScopeValidationError
-from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index
from sqlbuild.compiler.scopes.models import ScopeIndex
from sqlbuild.compiler.sql_analysis.constants import (
BINDING_UNKNOWN_TABLE_INTERNAL_CODE,
@@ -347,10 +345,7 @@ def assemble_compiled_project(
complete_binding_schemas=complete_binding_schemas,
)
scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs)
- try:
- validate_scope_index(index=scope_index)
- except ScopeValidationError as error:
- raise CompileInputError(str(error)) from error
+ report_scope_index_errors(index=scope_index)
effective_target_values: dict[str, object] = resolve_early_model_templates(
values={
"database": (
diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py
index 2e272d4f3b..4610ae6471 100644
--- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py
+++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py
@@ -21,6 +21,7 @@
build_known_source_names,
validate_audit_references,
)
+from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic
from sqlbuild.compiler.compile._helpers.named_declarations.core import (
declaration_file_expansion,
named_declaration_usages,
@@ -39,6 +40,7 @@
AuthoredSqlExpansionResult,
CompileAuditInput,
CompileModelInput,
+ CompilerDiagnostic,
CompileSeedInput,
CompileSourceInput,
CompileSqlReference,
@@ -48,6 +50,9 @@
)
from sqlbuild.compiler.compile.types import (
AttachedAuditTargetKind,
+ CompiledResourceType,
+ DiagnosticPhase,
+ DiagnosticSeverity,
)
from sqlbuild.compiler.discovery.models import (
DiscoveredAuditBlock,
@@ -725,7 +730,7 @@ def singular_audit_resources(
audit_file: DiscoveredAuditFile,
audit_name: str,
) -> tuple[ResourceIdentity, ...]:
- """Require a singular audit to check more than one resource and return what it references."""
+ """Return the resources a singular audit references, reporting P004 if not cross-resource."""
model_names: frozenset[str] = frozenset(
reference.ref_name for reference in references if reference.ref_kind == SqlReferenceKind.REF
@@ -740,44 +745,55 @@ def singular_audit_resources(
for reference in references
if reference.ref_kind in companion_kinds
)
- label: str = f"Singular audit '{audit_name}' in {audit_file.relative_path}"
- if len(model_names) >= _CROSS_MODEL_MINIMUM or (model_names and companions):
- return tuple(
- sorted(
- {
- *(ResourceIdentity(ResourceKind.MODEL, name) for name in model_names),
- *companions,
- }
- )
+ resources: tuple[ResourceIdentity, ...] = tuple(
+ sorted({*(ResourceIdentity(ResourceKind.MODEL, name) for name in model_names), *companions})
+ )
+ if len(model_names) < _CROSS_MODEL_MINIMUM and not (model_names and companions):
+ message, help_text = _single_resource_problem(
+ model_names=model_names, companions=companions
+ )
+ report_compile_diagnostic(
+ key=(
+ SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE,
+ audit_file.relative_path.as_posix(),
+ audit_name,
+ ),
+ diagnostic=CompilerDiagnostic(
+ phase=DiagnosticPhase.COMPILE,
+ severity=DiagnosticSeverity.ERROR,
+ code=SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE,
+ message=f"Singular audit '{audit_name}' in {audit_file.relative_path} {message}",
+ resource_type=CompiledResourceType.AUDIT,
+ resource_name=audit_name,
+ path=audit_file.relative_path,
+ help=help_text,
+ ),
)
+ return resources
+
+
+def _single_resource_problem(
+ *, model_names: frozenset[str], companions: frozenset[ResourceIdentity]
+) -> tuple[str, str]:
if len(model_names) == 1:
model_name: str = next(iter(model_names))
- raise CompileInputError(
- f"{label} checks only model '{model_name}'; singular audits must be cross-resource",
- code=SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE,
- help=(
- f"attach a generic audit to '{model_name}' instead: write it in an "
- "audits/generic/ role and select FROM @relation, joining @relation to itself for "
- "self-join checks"
- ),
+ return (
+ f"checks only model '{model_name}'; singular audits must be cross-resource",
+ f"attach a generic audit to '{model_name}' instead: write it in an audits/generic/ "
+ "role and select FROM @relation, joining @relation to itself for self-join checks",
)
if companions:
- raise CompileInputError(
- f"{label} checks only sources or seeds; singular audits must reference at least one "
- "model",
- code=SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE,
- help="attach YAML audits to the source or seed instead",
+ return (
+ "checks only sources or seeds; singular audits must reference at least one model",
+ "attach YAML audits to the source or seed instead",
)
- raise CompileInputError(
- f"{label} references no SQLBuild resource; singular audits must reference two or more "
- f"models, or a model plus a source, seed, or table function via "
+ return (
+ "references no SQLBuild resource; singular audits must reference two or more models, or "
+ "a model plus a source, seed, or table function via "
f"{SqlReferenceKind.REF.placeholder_call()}, {SqlReferenceKind.SOURCE.placeholder_call()}, "
f"or {SqlReferenceKind.SEED.placeholder_call()}",
- code=SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE,
- help=(
- "reference resources through SQLBuild calls instead of hard-coded relation names, or "
- "attach a generic audit to the resource being checked"
- ),
+ "reference resources through SQLBuild calls instead of hard-coded relation names, or "
+ "attach a generic audit to the resource being checked",
)
diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py
new file mode 100644
index 0000000000..ae6080acb7
--- /dev/null
+++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py
@@ -0,0 +1,70 @@
+"""Collect compile diagnostics so one compile reports every violation, not only the first."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar, Token
+from dataclasses import replace
+from functools import wraps
+
+from sqlbuild.compiler.compile.classes.collected_compile_diagnostics import (
+ CollectedCompileDiagnostics,
+)
+from sqlbuild.compiler.compile.exceptions import CompileInputError
+from sqlbuild.compiler.compile.models import (
+ CompiledProject,
+ CompileProjectInputs,
+ CompilerDiagnostic,
+)
+
+_ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar(
+ "sqlbuild_collected_compile_diagnostics", default=None
+)
+
+
+@contextmanager
+def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]:
+ """Collect diagnostics reported inside the block instead of raising the first one."""
+
+ collector: CollectedCompileDiagnostics = CollectedCompileDiagnostics()
+ token: Token[CollectedCompileDiagnostics | None] = _ACTIVE_COLLECTOR.set(collector)
+ try:
+ yield collector
+ finally:
+ _ACTIVE_COLLECTOR.reset(token)
+
+
+def with_collected_compile_diagnostics[**P, R: (CompileProjectInputs, CompiledProject)](
+ build: Callable[P, R],
+) -> Callable[P, R]:
+ """Append every diagnostic reported while ``build`` runs to the diagnostics it returns."""
+
+ @wraps(build)
+ def collecting(*args: P.args, **kwargs: P.kwargs) -> R:
+ with collect_compile_diagnostics() as violations:
+ built: R = build(*args, **kwargs)
+ if not violations.diagnostics:
+ return built
+ return replace(built, diagnostics=(*built.diagnostics, *violations.diagnostics))
+
+ return collecting
+
+
+def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None:
+ """Record a diagnostic in the active collector, or raise it when none is active."""
+
+ collector: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get()
+ if collector is not None:
+ collector.add(key=key, diagnostic=diagnostic)
+ return
+ location: str = (
+ f"\n --> {diagnostic.location.path.as_posix()}:{diagnostic.location.line}"
+ if diagnostic.location is not None
+ else f"\n --> {diagnostic.path.as_posix()}"
+ if diagnostic.path is not None
+ else ""
+ )
+ raise CompileInputError(
+ f"{diagnostic.message}{location}", code=diagnostic.code, help=diagnostic.help
+ )
diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py
new file mode 100644
index 0000000000..b78cd9b79c
--- /dev/null
+++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py
@@ -0,0 +1,75 @@
+"""Report declaration-scope index errors as individual compile diagnostics."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic
+from sqlbuild.compiler.compile.models import CompilerDiagnostic
+from sqlbuild.compiler.compile.types import (
+ CompiledResourceType,
+ DiagnosticPhase,
+ DiagnosticSeverity,
+)
+from sqlbuild.compiler.scopes.models import ScopeDiagnostic, ScopeIndex
+from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind
+from sqlbuild.compiler.scopes.types import DiagnosticSeverity as ScopeSeverity
+
+_RESOURCE_TYPES: dict[ResourceKind, CompiledResourceType] = {
+ ResourceKind.MODEL: CompiledResourceType.MODEL,
+ ResourceKind.SOURCE: CompiledResourceType.SOURCE,
+ ResourceKind.SEED: CompiledResourceType.SEED,
+ ResourceKind.FUNCTION: CompiledResourceType.UDF,
+ ResourceKind.TEST: CompiledResourceType.SQL_TEST,
+ ResourceKind.SCENARIO: CompiledResourceType.SQL_SCENARIO,
+}
+_DECLARATION_RESOURCE_TYPES: dict[DeclarationKind, CompiledResourceType] = {
+ DeclarationKind.AUDIT: CompiledResourceType.AUDIT,
+ DeclarationKind.SINGULAR_AUDIT: CompiledResourceType.AUDIT,
+}
+
+
+def compiled_resource_type(kind: ResourceKind) -> CompiledResourceType | None:
+ """Return the compiled resource type diagnostics use for a scope resource kind."""
+
+ return _RESOURCE_TYPES.get(kind)
+
+
+def report_scope_index_errors(*, index: ScopeIndex) -> None:
+ """Report each error-severity scope finding as its own compile diagnostic."""
+
+ for item in index.diagnostics:
+ if item.severity is not ScopeSeverity.ERROR:
+ continue
+ report_compile_diagnostic(
+ key=(
+ item.code.value,
+ item.path or "",
+ str(item.declaration or ""),
+ str(item.resource or ""),
+ item.message,
+ ),
+ diagnostic=_compile_diagnostic(item),
+ )
+
+
+def _compile_diagnostic(item: ScopeDiagnostic) -> CompilerDiagnostic:
+ resource_type: CompiledResourceType | None = None
+ resource_name: str | None = None
+ if item.resource is not None and item.resource.kind in _RESOURCE_TYPES:
+ resource_type = _RESOURCE_TYPES[item.resource.kind]
+ resource_name = item.resource.name
+ elif item.declaration is not None and item.declaration.kind in _DECLARATION_RESOURCE_TYPES:
+ resource_type = _DECLARATION_RESOURCE_TYPES[item.declaration.kind]
+ resource_name = item.declaration.name
+ return CompilerDiagnostic(
+ phase=DiagnosticPhase.COMPILE,
+ severity=DiagnosticSeverity.ERROR,
+ code=item.code.value,
+ message=item.message,
+ resource_type=resource_type,
+ resource_name=resource_name,
+ path=Path(item.path) if item.path is not None else None,
+ line=item.line,
+ column=item.column,
+ )
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/collector.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/collector.py
deleted file mode 100644
index 5eac95dde3..0000000000
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/collector.py
+++ /dev/null
@@ -1,68 +0,0 @@
-"""Collect explicit-reference violations so one compile reports all of them."""
-
-from __future__ import annotations
-
-from collections.abc import Callable, Iterator
-from contextlib import contextmanager
-from contextvars import ContextVar, Token
-from dataclasses import replace
-from functools import wraps
-
-from sqlbuild.compiler.compile.classes.explicit_reference_violations import (
- ExplicitReferenceViolations,
-)
-from sqlbuild.compiler.compile.exceptions import CompileInputError
-from sqlbuild.compiler.compile.models import CompileProjectInputs, CompilerDiagnostic
-
-_ACTIVE_COLLECTOR: ContextVar[ExplicitReferenceViolations | None] = ContextVar(
- "sqlbuild_explicit_reference_violations", default=None
-)
-
-
-@contextmanager
-def collect_explicit_reference_violations() -> Iterator[ExplicitReferenceViolations]:
- """Collect violations reported inside the block instead of raising the first one."""
-
- collector: ExplicitReferenceViolations = ExplicitReferenceViolations()
- token: Token[ExplicitReferenceViolations | None] = _ACTIVE_COLLECTOR.set(collector)
- try:
- yield collector
- finally:
- _ACTIVE_COLLECTOR.reset(token)
-
-
-def with_explicit_reference_diagnostics[**P](
- build: Callable[P, CompileProjectInputs],
-) -> Callable[P, CompileProjectInputs]:
- """Append every violation reported while building compile inputs to their diagnostics."""
-
- @wraps(build)
- def collecting(*args: P.args, **kwargs: P.kwargs) -> CompileProjectInputs:
- with collect_explicit_reference_violations() as violations:
- inputs: CompileProjectInputs = build(*args, **kwargs)
- if not violations.diagnostics:
- return inputs
- return replace(inputs, diagnostics=(*inputs.diagnostics, *violations.diagnostics))
-
- return collecting
-
-
-def report_explicit_reference_violation(
- *, key: tuple[str, ...], diagnostic: CompilerDiagnostic
-) -> None:
- """Record a violation in the active collector, or raise it when none is active."""
-
- collector: ExplicitReferenceViolations | None = _ACTIVE_COLLECTOR.get()
- if collector is not None:
- collector.add(key=key, diagnostic=diagnostic)
- return
- location: str = (
- f"\n --> {diagnostic.location.path.as_posix()}:{diagnostic.location.line}"
- if diagnostic.location is not None
- else f"\n --> {diagnostic.path.as_posix()}"
- if diagnostic.path is not None
- else ""
- )
- raise CompileInputError(
- f"{diagnostic.message}{location}", code=diagnostic.code, help=diagnostic.help
- )
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
index d4be2f8130..a54ac68f06 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
@@ -6,8 +6,8 @@
import re
from pathlib import Path
-from sqlbuild.compiler.compile._helpers.explicit_references.collector import (
- report_explicit_reference_violation,
+from sqlbuild.compiler.compile._helpers.diagnostics.collector import (
+ report_compile_diagnostic,
)
from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references
from sqlbuild.compiler.compile.constants import MACRO_GENERATED_REFERENCE_CODE
@@ -95,8 +95,14 @@ def reject_macro_generated_references(
)
generated: SqlResourceRef
for generated in _generated_references(macro_result):
- report_explicit_reference_violation(
- key=(consumer_label, loaded_macro.name, generated.kind.value, generated.name),
+ report_compile_diagnostic(
+ key=(
+ MACRO_GENERATED_REFERENCE_CODE,
+ consumer_label,
+ loaded_macro.name,
+ generated.kind.value,
+ generated.name,
+ ),
diagnostic=CompilerDiagnostic(
phase=DiagnosticPhase.COMPILE,
severity=DiagnosticSeverity.ERROR,
diff --git a/src/sqlbuild/compiler/compile/_helpers/named_declarations/core.py b/src/sqlbuild/compiler/compile/_helpers/named_declarations/core.py
index 6be4b0399c..62604564bc 100644
--- a/src/sqlbuild/compiler/compile/_helpers/named_declarations/core.py
+++ b/src/sqlbuild/compiler/compile/_helpers/named_declarations/core.py
@@ -5,9 +5,15 @@
from dataclasses import replace
from pathlib import Path
+from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic
+from sqlbuild.compiler.compile._helpers.diagnostics.scope import compiled_resource_type
from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion
-from sqlbuild.compiler.compile.exceptions import CompileInputError
-from sqlbuild.compiler.compile.models import DeclarationExpansionContext, DeclarationScopeResolver
+from sqlbuild.compiler.compile.models import (
+ CompilerDiagnostic,
+ DeclarationExpansionContext,
+ DeclarationScopeResolver,
+)
+from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity
from sqlbuild.compiler.scopes.constants import DECLARATION_ROLE_PARTS
from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path
from sqlbuild.compiler.scopes.main._declaration_visibility import declaration_visibility
@@ -79,14 +85,35 @@ def named_declaration_usages(
)
owner: str = record.owning_path or record.ownership_root.path
role: str = "/".join(DECLARATION_ROLE_PARTS[kind])
- raise CompileInputError(
- f"{_KIND_LABELS[kind]} '{name}' is not visible from '{consumer_label}'. It is "
- f"defined at {record.path} with {record.scope.value} scope owned by '{owner}'",
- code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value,
- help=(
- f"move '{record.path}' to a {role}/ role owned by '{consumer_folder}' or one of "
- f"its parent folders, or to the project-wide {role}/ when consumers span "
- "resource trees"
+ resource: ResourceIdentity | None = (
+ consumer if isinstance(consumer, ResourceIdentity) else None
+ )
+ report_compile_diagnostic(
+ key=(
+ ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value,
+ consumer_label,
+ kind.value,
+ name,
+ ),
+ diagnostic=CompilerDiagnostic(
+ phase=DiagnosticPhase.COMPILE,
+ severity=DiagnosticSeverity.ERROR,
+ code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value,
+ message=(
+ f"{_KIND_LABELS[kind]} '{name}' is not visible from '{consumer_label}'. It "
+ f"is defined at {record.path} with {record.scope.value} scope owned by "
+ f"'{owner}'"
+ ),
+ resource_type=(
+ compiled_resource_type(resource.kind) if resource is not None else None
+ ),
+ resource_name=resource.name if resource is not None else None,
+ path=Path(consumer_label),
+ help=(
+ f"move '{record.path}' to a {role}/ role owned by '{consumer_folder}' or one "
+ f"of its parent folders, or to the project-wide {role}/ when consumers span "
+ "resource trees"
+ ),
),
)
return (UsageRecord(consumer=consumer, declaration=record.identity),)
diff --git a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py b/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py
index ffa0fe710e..4e175ca41d 100644
--- a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py
+++ b/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py
@@ -2,12 +2,15 @@
from __future__ import annotations
+from pathlib import Path
+
+from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic
from sqlbuild.compiler.compile._helpers.named_declarations.core import (
named_declaration_record,
named_declaration_usages,
)
-from sqlbuild.compiler.compile.exceptions import CompileInputError
-from sqlbuild.compiler.compile.models import DeclarationScopeResolver
+from sqlbuild.compiler.compile.models import CompilerDiagnostic, DeclarationScopeResolver
+from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity
from sqlbuild.compiler.discovery.models import (
DiscoveredModelSchemaFile,
EnumDeclaration,
@@ -94,15 +97,28 @@ def _enum_type_records(
continue
if declaration_visibility(declaration=enum_record, consumer=record) is None:
owner: str = enum_record.owning_path or enum_record.ownership_root.path
- raise CompileInputError(
- f"Schema '{declaration.name}' in {record.path} column '{column.name}' uses enum "
- f"'{column.type}', which is not visible from the schema's location. The enum is "
- f"defined at {enum_record.path} with {enum_record.scope.value} scope owned by "
- f"'{owner}'",
- code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value,
- help=(
- "move the enum to a role visible from the schema's owning folder, or move "
- "the schema next to the enum"
+ report_compile_diagnostic(
+ key=(
+ ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value,
+ record.path,
+ declaration.name,
+ column.name,
+ ),
+ diagnostic=CompilerDiagnostic(
+ phase=DiagnosticPhase.COMPILE,
+ severity=DiagnosticSeverity.ERROR,
+ code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value,
+ message=(
+ f"Schema '{declaration.name}' in {record.path} column '{column.name}' "
+ f"uses enum '{column.type}', which is not visible from the schema's "
+ f"location. The enum is defined at {enum_record.path} with "
+ f"{enum_record.scope.value} scope owned by '{owner}'"
+ ),
+ path=Path(record.path),
+ help=(
+ "move the enum to a role visible from the schema's owning folder, or "
+ "move the schema next to the enum"
+ ),
),
)
records.append(enum_record)
diff --git a/src/sqlbuild/compiler/compile/classes/explicit_reference_violations.py b/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py
similarity index 67%
rename from src/sqlbuild/compiler/compile/classes/explicit_reference_violations.py
rename to src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py
index 816dd2f3b7..9a1a32e725 100644
--- a/src/sqlbuild/compiler/compile/classes/explicit_reference_violations.py
+++ b/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py
@@ -1,18 +1,18 @@
-"""Explicit-reference violations gathered during one compile phase."""
+"""Compile diagnostics collected during one compile phase instead of raised."""
from __future__ import annotations
from sqlbuild.compiler.compile.models import CompilerDiagnostic
-class ExplicitReferenceViolations:
- """Deduplicated explicit-reference diagnostics gathered during one compile phase."""
+class CollectedCompileDiagnostics:
+ """Deduplicated compile diagnostics gathered during one compile phase."""
def __init__(self) -> None:
self._by_key: dict[tuple[str, ...], CompilerDiagnostic] = {}
def add(self, *, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None:
- """Record one violation; a repeated key keeps the first diagnostic."""
+ """Record one diagnostic; a repeated key keeps the first diagnostic."""
self._by_key.setdefault(key, diagnostic)
diff --git a/src/sqlbuild/compiler/compile/main/_assemble_project.py b/src/sqlbuild/compiler/compile/main/_assemble_project.py
index e23d8984d1..b7a12c0f8b 100644
--- a/src/sqlbuild/compiler/compile/main/_assemble_project.py
+++ b/src/sqlbuild/compiler/compile/main/_assemble_project.py
@@ -6,8 +6,13 @@
from pathlib import Path
from sqlbuild.adapter.contract.models import ExpressionInferenceProfile
-from sqlbuild.compiler.compile._helpers.assembly.audit_gates import validate_attached_audit_gates
+from sqlbuild.compiler.compile._helpers.assembly.audit_gates import (
+ attached_audit_gate_diagnostics,
+)
from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project
+from sqlbuild.compiler.compile._helpers.diagnostics.collector import (
+ with_collected_compile_diagnostics,
+)
from sqlbuild.compiler.compile._helpers.explicit_references.hook_reads import hook_read_diagnostics
from sqlbuild.compiler.compile._helpers.explicit_references.python_sql import (
python_sql_reference_diagnostics,
@@ -20,6 +25,7 @@
from sqlbuild.compiler.lineage.types import ColumnLineageMode
+@with_collected_compile_diagnostics
def assemble_project(
*,
inputs: CompileProjectInputs,
@@ -39,8 +45,8 @@ def assemble_project(
analysis_cache_dir=analysis_cache_dir,
analysis_model_names=analysis_model_names,
)
- validate_attached_audit_gates(project=project)
reference_diagnostics: tuple[CompilerDiagnostic, ...] = (
+ *attached_audit_gate_diagnostics(project=project),
*hook_read_diagnostics(project=project),
*(
python_sql_reference_diagnostics(
diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
index 7fbd0870e8..c9b972a175 100644
--- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
+++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py
@@ -31,8 +31,8 @@
from sqlbuild.compiler.compile._helpers.audit_factories.core import (
build_audit_factory_orphan_diagnostics,
)
-from sqlbuild.compiler.compile._helpers.explicit_references.collector import (
- with_explicit_reference_diagnostics,
+from sqlbuild.compiler.compile._helpers.diagnostics.collector import (
+ with_collected_compile_diagnostics,
)
from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id
from sqlbuild.compiler.compile._helpers.render.declarations import (
@@ -74,7 +74,7 @@
from sqlbuild.spec.contracts.models import SettingsConfig, TargetConfig
-@with_explicit_reference_diagnostics
+@with_collected_compile_diagnostics
def build_compile_inputs(
*,
discovered_inputs: DiscoveredProjectInputs,
diff --git a/src/sqlbuild/compiler/graph/_helpers/audit_gates.py b/src/sqlbuild/compiler/graph/_helpers/audit_gates.py
index dd915ed19f..eb03e2d935 100644
--- a/src/sqlbuild/compiler/graph/_helpers/audit_gates.py
+++ b/src/sqlbuild/compiler/graph/_helpers/audit_gates.py
@@ -112,13 +112,16 @@ def _extra_reads(
)
-def attached_audit_gate_cycle_impl(*, project: CompiledProject) -> AttachedAuditGateEdge | None:
- """Return the first attached-audit read that depends on the audit's own target."""
+def attached_audit_gate_cycles_impl(
+ *, project: CompiledProject
+) -> tuple[AttachedAuditGateEdge, ...]:
+ """Return every attached-audit read that depends on the audit's own target."""
edges: tuple[AttachedAuditGateEdge, ...] = attached_audit_gate_edges_impl(project=project)
upstream: dict[CompiledObjectKey, list[CompiledObjectKey]] = {
key: list(deps) for key, deps in build_lineage_upstream_deps_impl(project).items()
}
+ cycles: dict[AttachedAuditGateEdge, None] = {}
edge: AttachedAuditGateEdge
for edge in edges:
upstream.setdefault(edge.gated, []).append(edge.read)
@@ -135,7 +138,11 @@ def attached_audit_gate_cycle_impl(*, project: CompiledProject) -> AttachedAudit
if target in transitive_closure_many_impl(
starts=(read,), edges=upstream, include_starts=False
):
- return AttachedAuditGateEdge(
- audit_name=audit.name, target=target, gated=target, read=read
- )
- return None
+ cycles[
+ AttachedAuditGateEdge(
+ audit_name=audit.name, target=target, gated=target, read=read
+ )
+ ] = None
+ return tuple(
+ sorted(cycles, key=lambda item: (item.audit_name, item.target.name, item.read.name))
+ )
diff --git a/src/sqlbuild/compiler/graph/main/_attached_audit_gate_cycle.py b/src/sqlbuild/compiler/graph/main/_attached_audit_gate_cycles.py
similarity index 51%
rename from src/sqlbuild/compiler/graph/main/_attached_audit_gate_cycle.py
rename to src/sqlbuild/compiler/graph/main/_attached_audit_gate_cycles.py
index 779ca509bf..2d117e43f3 100644
--- a/src/sqlbuild/compiler/graph/main/_attached_audit_gate_cycle.py
+++ b/src/sqlbuild/compiler/graph/main/_attached_audit_gate_cycles.py
@@ -3,11 +3,11 @@
from __future__ import annotations
from sqlbuild.compiler.compile.models import CompiledProject
-from sqlbuild.compiler.graph._helpers.audit_gates import attached_audit_gate_cycle_impl
+from sqlbuild.compiler.graph._helpers.audit_gates import attached_audit_gate_cycles_impl
from sqlbuild.compiler.graph.models import AttachedAuditGateEdge
-def attached_audit_gate_cycle(*, project: CompiledProject) -> AttachedAuditGateEdge | None:
- """Return the first attached-audit read that depends on the audit's own target."""
+def attached_audit_gate_cycles(*, project: CompiledProject) -> tuple[AttachedAuditGateEdge, ...]:
+ """Return every attached-audit read that depends on the audit's own target."""
- return attached_audit_gate_cycle_impl(project=project)
+ return attached_audit_gate_cycles_impl(project=project)
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
index 6398e12b1a..17bc1a6c12 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
@@ -1037,8 +1037,8 @@ class ExplicitReferencePythonDependencyE2ETestCase:
@dataclass(frozen=True)
-class ExplicitReferenceAllDiagnosticsE2ETestCase:
- """A project with several explicit-reference violations, all reported by one compile."""
+class CollectedCompileDiagnosticsE2ETestCase:
+ """A project with several compile violations, all reported by one compile."""
description: str
overrides: dict[str, str]
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
index a686e21069..33920bb2b5 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
@@ -7,11 +7,12 @@
import subprocess
from itertools import chain
from pathlib import Path
+from typing import cast
import pytest
from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
- ExplicitReferenceAllDiagnosticsE2ETestCase,
+ CollectedCompileDiagnosticsE2ETestCase,
ExplicitReferenceBuildE2ETestCase,
ExplicitReferenceFailureE2ETestCase,
ExplicitReferencePythonDependencyE2ETestCase,
@@ -41,6 +42,15 @@
"def customers_base():\n return 'SELECT * FROM __ref(\"stg_customers\")'\n"
)
_UNDECLARED_HOOKS: str = EXPLICIT_REFERENCE_HOOKS + ' ctx.relation(model("all_orders"))\n'
+_AUDITS: str = "models/sales/_sqlbuild"
+_AUDIT_READING: str = (
+ "AUDIT ();\nSELECT s.order_id FROM @relation s\n"
+ 'LEFT JOIN __ref("{read}") a USING (order_id) WHERE a.order_id IS NULL\n'
+)
+_AUDITED_MODEL: str = "MODEL (materialized table, audits [{audit}]);\nSELECT 1 AS order_id\n"
+_SUMMARY_MODEL: str = 'MODEL (materialized table);\nSELECT * FROM __ref("{read}")\n'
+_ONE_MODEL_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("{read}") WHERE order_id IS NULL\n'
+_NULL_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE order_id IS NULL\n'
_LITERAL_TASK: str = (
"from sqlbuild.refs import model\n"
"from sqlbuild.tasks import task\n\n\n"
@@ -431,7 +441,7 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit
@pytest.mark.parametrize(
"test_case",
[
- ExplicitReferenceAllDiagnosticsE2ETestCase(
+ CollectedCompileDiagnosticsE2ETestCase(
description="two ref-emitting macros and a hard-coded task name all report at once",
overrides={
EXPLICIT_REFERENCE_MACRO_PATH: _EMITTING_MACROS,
@@ -464,11 +474,85 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit
),
),
),
+ CollectedCompileDiagnosticsE2ETestCase(
+ description="two single-model singular audits and two gating audits all report",
+ overrides={
+ f"{_AUDITS}/audits/singular/one_orders.sql": _ONE_MODEL_AUDIT.format(
+ read="orders_a"
+ ),
+ f"{_AUDITS}/audits/singular/one_customers.sql": _ONE_MODEL_AUDIT.format(
+ read="customers_b"
+ ),
+ f"{_AUDITS}/_audits/generic/a_check.sql": _AUDIT_READING.format(
+ read="orders_a_summary"
+ ),
+ f"{_AUDITS}/_audits/generic/b_check.sql": _AUDIT_READING.format(
+ read="customers_b_summary"
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/orders_a.sql": _AUDITED_MODEL.format(audit="a_check"),
+ f"{EXPLICIT_REFERENCE_MODELS}/customers_b.sql": _AUDITED_MODEL.format(
+ audit="b_check"
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/orders_a_summary.sql": _SUMMARY_MODEL.format(
+ read="orders_a"
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/customers_b_summary.sql": _SUMMARY_MODEL.format(
+ read="customers_b"
+ ),
+ },
+ expected_output_fragments=(
+ "error[P004]: Singular audit 'one_customers'",
+ "error[P004]: Singular audit 'one_orders'",
+ "--> models/sales/_sqlbuild/audits/singular/one_orders.sql",
+ "error[P005]: Audit 'a_check' on model 'orders_a' reads model 'orders_a_summary'",
+ "error[P005]: Audit 'b_check' on model 'customers_b' reads model "
+ "'customers_b_summary'",
+ "--> models/sales/_sqlbuild/_audits/generic/a_check.sql",
+ "4 errors",
+ ),
+ expected_json_diagnostics=(
+ ("P004", "Singular audit 'one_customers'"),
+ ("P004", "Singular audit 'one_orders'"),
+ ("P005", "Audit 'a_check' on model 'orders_a' reads model 'orders_a_summary'"),
+ (
+ "P005",
+ "Audit 'b_check' on model 'customers_b' reads model 'customers_b_summary'",
+ ),
+ ),
+ ),
+ CollectedCompileDiagnosticsE2ETestCase(
+ description="every declaration placement error reports on its own",
+ overrides={
+ "audits/generic/eu_check.sql": _NULL_AUDIT,
+ "audits/generic/us_check.sql": _NULL_AUDIT,
+ f"{_AUDITS}/audits/generic/unused_check.sql": _NULL_AUDIT,
+ f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_eu.sql": (
+ "MODEL (materialized table, audits [eu_check]);\n"
+ "SELECT 1 AS order_id, 10 AS amount\n"
+ ),
+ f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_us.sql": (
+ "MODEL (materialized table, audits [us_check]);\n"
+ "SELECT 2 AS order_id, 20 AS amount\n"
+ ),
+ },
+ expected_output_fragments=(
+ "error[S010]: Unused inherited declaration 'audit:unused_check'",
+ "error[S024]: Declaration 'audit:eu_check'",
+ "error[S024]: Declaration 'audit:us_check'",
+ "--> audits/generic/eu_check.sql",
+ "3 errors",
+ ),
+ expected_json_diagnostics=(
+ ("S010", "Unused inherited declaration 'audit:unused_check'"),
+ ("S024", "Declaration 'audit:eu_check'"),
+ ("S024", "Declaration 'audit:us_check'"),
+ ),
+ ),
],
ids=lambda case: case.description,
)
-def test_given_several_implicit_references_when_compiling_then_one_compile_reports_all(
- test_case: ExplicitReferenceAllDiagnosticsE2ETestCase,
+def test_given_several_compile_violations_when_compiling_then_one_compile_reports_all(
+ test_case: CollectedCompileDiagnosticsE2ETestCase,
tmp_path: Path,
) -> None:
project_dir: Path = prepare_inline_project(
@@ -490,7 +574,23 @@ def test_given_several_implicit_references_when_compiling_then_one_compile_repor
assert machine.returncode == 1, machine.stdout + machine.stderr
payload: dict[str, object] = json.loads(machine.stdout)
assert payload["has_errors"] is True
- assert (
- tuple((item["code"], item["message"]) for item in payload["diagnostics"])
- == test_case.expected_json_diagnostics
+ diagnostics: list[dict[str, str]] = cast(list[dict[str, str]], payload["diagnostics"])
+ assert [item["code"] for item in diagnostics] == [
+ code for code, _ in test_case.expected_json_diagnostics
+ ]
+ assert all(
+ fragment in item["message"]
+ for item, (_, fragment) in zip(
+ diagnostics, test_case.expected_json_diagnostics, strict=True
+ )
+ )
+ build: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build"), project_dir=project_dir
+ )
+ assert build.returncode == 1, build.stdout + build.stderr
+ assert all(
+ f"[{code}]" in build.stdout + build.stderr and fragment in build.stdout + build.stderr
+ for code, fragment in test_case.expected_json_diagnostics
)
+ assert "Project compilation failed." in build.stdout + build.stderr
+ assert "Compiled project." not in build.stdout + build.stderr
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
index 34924ba7e1..e9a41cfaa1 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
@@ -11,8 +11,8 @@
from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter
from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter
from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core
-from sqlbuild.compiler.compile._helpers.explicit_references.collector import (
- collect_explicit_reference_violations,
+from sqlbuild.compiler.compile._helpers.diagnostics.collector import (
+ collect_compile_diagnostics,
)
from sqlbuild.compiler.compile._helpers.render.macros import (
expand_sql_macros,
@@ -410,7 +410,7 @@ def collect_typed_macro_violations(
) -> tuple[CompilerDiagnostic, ...]:
"""Expand ``order_summary`` SQL and return the explicit-reference violations it reports."""
- with collect_explicit_reference_violations() as violations:
+ with collect_compile_diagnostics() as violations:
expand_typed_macro_sql(tmp_path=tmp_path, macro_file_contents=macro_file_contents, sql=sql)
return violations.diagnostics
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py
index a37833b59f..861e1d6b35 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py
@@ -17,6 +17,7 @@
execution_edge_names,
gate_audit,
gate_model,
+ render_compile_diagnostics,
)
_PROJECT_FILE: str = """
@@ -178,19 +179,15 @@ def test_given_attached_audit_reading_another_resource_when_compiling_then_it_ad
),
ids=lambda case: case.description,
)
-def test_given_attached_audit_reading_its_targets_dependant_when_compiling_then_it_fails(
+def test_given_attached_audit_reading_its_targets_dependant_when_compiling_then_it_reports_p005(
test_case: AttachedAuditGateCycleTestCase,
tmp_path: Path,
write_repo_files: Callable[[Path, dict[str, str]], None],
) -> None:
write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
- with pytest.raises(ValueError) as error:
- compile_and_assemble(project_dir=tmp_path)
+ rendered: str = render_compile_diagnostics(project=compile_and_assemble(project_dir=tmp_path))
- rendered: str = (
- f"[{getattr(error.value, 'code', '')}] {error.value} {getattr(error.value, 'help', '')}"
- )
for fragment in test_case.expected_error_fragments:
assert fragment in rendered
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py
index 9168839eca..018008170d 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import re
from collections.abc import Callable, Iterator
from dataclasses import replace
from itertools import chain
@@ -49,6 +50,7 @@
from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import (
DUCKDB_COMPILE_ADAPTER_CONTEXT,
compile_project_inputs,
+ render_compile_diagnostics,
visible_declarations_without_runtime_values,
)
@@ -709,7 +711,7 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu
),
ids=lambda case: case.description,
)
-def test_given_invalid_declaration_placement_when_assembling_then_project_is_rejected(
+def test_given_invalid_declaration_placement_when_assembling_then_it_reports_an_error(
test_case: ScopePlacementCompileTestCase,
tmp_path: Path,
write_repo_files: Callable[[Path, dict[str, str]], None],
@@ -720,8 +722,12 @@ def test_given_invalid_declaration_placement_when_assembling_then_project_is_rej
)
inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path)
- with pytest.raises(CompileInputError, match=cast(str, test_case.expected_fragment)):
- _ = assemble_project(inputs=inputs, skip_column_inference=True)
+ project: CompiledProject = assemble_project(inputs=inputs, skip_column_inference=True)
+
+ assert re.search(
+ cast(str, test_case.expected_fragment),
+ render_compile_diagnostics(project=project),
+ )
@pytest.mark.parametrize(
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py
index a7bac3a013..587c5b3e45 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py
@@ -17,6 +17,7 @@
from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import (
compile_and_assemble,
model_header,
+ render_compile_diagnostics,
singular_audit_files,
)
@@ -207,6 +208,80 @@ def test_given_valid_named_declaration_layout_when_compiling_then_project_is_acc
)
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ NamedDeclarationErrorTestCase(
+ "audit file directly under audits/",
+ {"models/orders.sql": _ORDERS, "audits/check.sql": _GENERIC_AUDIT},
+ ("Unsupported entries in audits/", "audits/generic/ or audits/singular/"),
+ ),
+ NamedDeclarationErrorTestCase(
+ "audit folder other than generic or singular",
+ {"models/orders.sql": _ORDERS, "audits/checks/check.sql": _GENERIC_AUDIT},
+ ("Unsupported entries in audits/: audits/checks",),
+ ),
+ NamedDeclarationErrorTestCase(
+ "folder-only singular audit role",
+ {
+ "models/marts/orders.sql": _ORDERS,
+ "models/marts/_sqlbuild/_audits/singular/check.sql": _GENERIC_AUDIT,
+ },
+ ("models/marts/_sqlbuild/_audits/singular/ is invalid",),
+ ),
+ NamedDeclarationErrorTestCase(
+ "_sqlbuild directly under a resource tree root",
+ {
+ "models/orders.sql": model_header(key="audits", value="[order_check]"),
+ "models/_sqlbuild/audits/generic/order_check.sql": _GENERIC_AUDIT,
+ },
+ ("Grouped declaration root models/_sqlbuild/ must be below a concrete owner",),
+ ),
+ NamedDeclarationErrorTestCase(
+ "unsupported hook language folder",
+ {
+ "models/marts/orders.sql": _ORDERS,
+ "models/marts/_sqlbuild/hooks/shell/touch.sql": _SQL_HOOK,
+ },
+ ("hooks/ accepts only python/, sql/",),
+ ),
+ NamedDeclarationErrorTestCase(
+ "scoped audit uses a constant that is not visible from the audit's folder",
+ {
+ "models/staging/_sqlbuild/_constants/order_limit.sql": (
+ "CONSTANT (name order_limit, value 10);"
+ ),
+ "models/staging/orders_stage.sql": (
+ 'MODEL ();\nSELECT @const("order_limit") AS order_id'
+ ),
+ "models/marts/_sqlbuild/_audits/generic/order_check.sql": (
+ 'AUDIT ();\nSELECT * FROM __ref("@model") '
+ 'WHERE order_id > @const("order_limit")'
+ ),
+ "models/marts/orders.sql": model_header(key="audits", value="[order_check]"),
+ },
+ ("order_limit", "known but inaccessible"),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_invalid_named_declaration_layout_when_compiling_then_error_names_the_fix(
+ test_case: NamedDeclarationErrorTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
+
+ with pytest.raises(ValueError) as error:
+ compile_and_assemble(project_dir=tmp_path)
+
+ rendered: str = (
+ f"[{getattr(error.value, 'code', '')}] {error.value} {getattr(error.value, 'help', '')}"
+ )
+ for fragment in test_case.expected_error_fragments:
+ assert fragment in rendered
+
+
@pytest.mark.parametrize(
"test_case",
(
@@ -248,40 +323,6 @@ def test_given_valid_named_declaration_layout_when_compiling_then_project_is_acc
),
("[P004]", "references no SQLBuild resource"),
),
- NamedDeclarationErrorTestCase(
- "audit file directly under audits/",
- {"models/orders.sql": _ORDERS, "audits/check.sql": _GENERIC_AUDIT},
- ("Unsupported entries in audits/", "audits/generic/ or audits/singular/"),
- ),
- NamedDeclarationErrorTestCase(
- "audit folder other than generic or singular",
- {"models/orders.sql": _ORDERS, "audits/checks/check.sql": _GENERIC_AUDIT},
- ("Unsupported entries in audits/: audits/checks",),
- ),
- NamedDeclarationErrorTestCase(
- "folder-only singular audit role",
- {
- "models/marts/orders.sql": _ORDERS,
- "models/marts/_sqlbuild/_audits/singular/check.sql": _GENERIC_AUDIT,
- },
- ("models/marts/_sqlbuild/_audits/singular/ is invalid",),
- ),
- NamedDeclarationErrorTestCase(
- "_sqlbuild directly under a resource tree root",
- {
- "models/orders.sql": model_header(key="audits", value="[order_check]"),
- "models/_sqlbuild/audits/generic/order_check.sql": _GENERIC_AUDIT,
- },
- ("Grouped declaration root models/_sqlbuild/ must be below a concrete owner",),
- ),
- NamedDeclarationErrorTestCase(
- "unsupported hook language folder",
- {
- "models/marts/orders.sql": _ORDERS,
- "models/marts/_sqlbuild/hooks/shell/touch.sql": _SQL_HOOK,
- },
- ("hooks/ accepts only python/, sql/",),
- ),
NamedDeclarationErrorTestCase(
"top-level generic audit used under one folder",
{
@@ -374,6 +415,19 @@ def test_given_valid_named_declaration_layout_when_compiling_then_project_is_acc
"Generic audit 'order_check' is not visible from 'models/staging/orders.sql'",
),
),
+ NamedDeclarationErrorTestCase(
+ "every consumer of an invisible scoped audit is reported",
+ {
+ "models/marts/_sqlbuild/audits/generic/order_check.sql": _GENERIC_AUDIT,
+ "models/staging/orders.sql": model_header(key="audits", value="[order_check]"),
+ "models/staging/customers.sql": model_header(key="audits", value="[order_check]"),
+ },
+ (
+ "[S006] Generic audit 'order_check' is not visible from "
+ "'models/staging/customers.sql'",
+ "[S006] Generic audit 'order_check' is not visible from 'models/staging/orders.sql'",
+ ),
+ ),
NamedDeclarationErrorTestCase(
"scoped generic audit used from a parent folder",
{
@@ -465,39 +519,18 @@ def test_given_valid_named_declaration_layout_when_compiling_then_project_is_acc
},
("[S006]", "uses enum 'order_status', which is not visible from the schema's location"),
),
- NamedDeclarationErrorTestCase(
- "scoped audit uses a constant that is not visible from the audit's folder",
- {
- "models/staging/_sqlbuild/_constants/order_limit.sql": (
- "CONSTANT (name order_limit, value 10);"
- ),
- "models/staging/orders_stage.sql": (
- 'MODEL ();\nSELECT @const("order_limit") AS order_id'
- ),
- "models/marts/_sqlbuild/_audits/generic/order_check.sql": (
- 'AUDIT ();\nSELECT * FROM __ref("@model") '
- 'WHERE order_id > @const("order_limit")'
- ),
- "models/marts/orders.sql": model_header(key="audits", value="[order_check]"),
- },
- ("order_limit", "known but inaccessible"),
- ),
),
ids=lambda case: case.description,
)
-def test_given_invalid_named_declaration_layout_when_compiling_then_error_names_the_fix(
+def test_given_invalid_named_declaration_use_when_compiling_then_diagnostics_name_the_fix(
test_case: NamedDeclarationErrorTestCase,
tmp_path: Path,
write_repo_files: Callable[[Path, dict[str, str]], None],
) -> None:
write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
- with pytest.raises(ValueError) as error:
- compile_and_assemble(project_dir=tmp_path)
+ rendered: str = render_compile_diagnostics(project=compile_and_assemble(project_dir=tmp_path))
- rendered: str = (
- f"[{getattr(error.value, 'code', '')}] {error.value} {getattr(error.value, 'help', '')}"
- )
for fragment in test_case.expected_error_fragments:
assert fragment in rendered
From 08d9cebd9306fc45c46034c2d12feb1541020fdd Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Mon, 28 Sep 2026 04:55:27 +0100
Subject: [PATCH 08/13] fix: expand both-way selectors across the SQL and
Python boundary from the selected roots
+x+ selected the downstream of x's upstream, so a task or model selector
also ran sibling tasks that read x's upstream. Upstream and downstream
expansion now both start from the selected resources, so +x+ is x, its
ancestors and its descendants, and SQL-only selections are unchanged.
---
.../_helpers/unified_selectors.py | 39 ++++--
.../cli/commands/main/build/_test_types.py | 10 ++
.../main/build/test_explicit_references.py | 67 ++++++++++
.../python_nodes/_helpers/_test_types.py | 9 ++
.../compiler/python_nodes/_helpers/helpers.py | 27 ++++
.../_helpers/test_unified_selectors.py | 120 ++++++++++++++++++
6 files changed, 263 insertions(+), 9 deletions(-)
diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py
index c6eb8f6022..af24f270b2 100644
--- a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py
+++ b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py
@@ -11,12 +11,16 @@
from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure
from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many
from sqlbuild.compiler.pipeline.models import ProjectGraph
-from sqlbuild.compiler.planner.constants import PATH_SELECTOR_EXPLICIT_ROOT_ERROR
+from sqlbuild.compiler.planner.constants import (
+ PATH_SELECTOR_EXPLICIT_ROOT_ERROR,
+ SELECTOR_EXPANSION_MARKER,
+)
from sqlbuild.compiler.planner.exceptions import PlannerInputError
from sqlbuild.compiler.planner.main.selection._build_resources import (
expand_build_resource_selection,
)
from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors
+from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion
from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector
from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector
from sqlbuild.compiler.planner.types import SelectorKind
@@ -258,18 +262,35 @@ def _resolve_token(
def _resolve_single(
*, raw: str, project_graph: ProjectGraph, python_graph: PythonNodeGraph
) -> frozenset[_SelectionAtom]:
- atoms: frozenset[_SelectionAtom] = _resolve_single_side(
- raw=raw, project_graph=project_graph, python_graph=python_graph
- )
parsed: ParsedSelector | PathSelector = parse_project_selector(raw)
- if isinstance(parsed, PathSelector):
- return atoms
+ if isinstance(parsed, PathSelector) or not (parsed.upstream or parsed.downstream):
+ return _resolve_single_side(raw=raw, project_graph=project_graph, python_graph=python_graph)
+ core: str = split_selector_expansion(raw).core
+ atoms: frozenset[_SelectionAtom] = frozenset()
if parsed.upstream:
- atoms = atoms | _sql_dependency_upstream_atoms(
- atoms=atoms, project_graph=project_graph, python_graph=python_graph
+ upstream: frozenset[_SelectionAtom] = _resolve_single_side(
+ raw=f"{SELECTOR_EXPANSION_MARKER}{core}",
+ project_graph=project_graph,
+ python_graph=python_graph,
+ )
+ atoms = (
+ atoms
+ | upstream
+ | _sql_dependency_upstream_atoms(
+ atoms=upstream, project_graph=project_graph, python_graph=python_graph
+ )
)
if parsed.downstream:
- atoms = atoms | _sql_dependent_python_atoms(atoms=atoms, python_graph=python_graph)
+ downstream: frozenset[_SelectionAtom] = _resolve_single_side(
+ raw=f"{core}{SELECTOR_EXPANSION_MARKER}",
+ project_graph=project_graph,
+ python_graph=python_graph,
+ )
+ atoms = (
+ atoms
+ | downstream
+ | _sql_dependent_python_atoms(atoms=downstream, python_graph=python_graph)
+ )
return atoms
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
index 17bc1a6c12..f34eefb9e2 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
@@ -1067,3 +1067,13 @@ class ExplicitReferenceRuntimeWarningE2ETestCase:
expected_exit_code: int
expected_warning_fragments: tuple[str, ...]
expected_final_line_prefix: str
+
+
+@dataclass(frozen=True)
+class CrossBoundarySelectionE2ETestCase:
+ """A both-way selector over tasks that read a model and, separately, its upstream."""
+
+ description: str
+ select: tuple[str, ...]
+ expected_ran: tuple[str, ...]
+ expected_not_ran: tuple[str, ...]
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
index 33920bb2b5..2f3af35a29 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
@@ -13,6 +13,7 @@
from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
CollectedCompileDiagnosticsE2ETestCase,
+ CrossBoundarySelectionE2ETestCase,
ExplicitReferenceBuildE2ETestCase,
ExplicitReferenceFailureE2ETestCase,
ExplicitReferencePythonDependencyE2ETestCase,
@@ -31,6 +32,7 @@
prepare_inline_project,
query_duckdb,
run_sqb,
+ table_exists,
)
_EMITTING_MACROS: str = (
@@ -181,6 +183,71 @@ def test_given_python_seed_and_model_dependencies_when_building_then_graph_selec
assert re.search(test_case.expected_check_row_pattern, checked.stdout), checked.stdout
+_SIBLING_TASKS: str = (
+ "from sqlbuild.refs import model\n"
+ "from sqlbuild.tasks import task\n\n\n"
+ '@task(depends_on=model("all_orders"))\n'
+ "def export_summary(ctx):\n"
+ ' ctx.execute_sql("CREATE OR REPLACE TABLE export_summary_ran AS SELECT 1 AS n")\n\n\n'
+ '@task(depends_on=model("stg_orders_eu"))\n'
+ "def export_eu(ctx):\n"
+ ' ctx.execute_sql("CREATE OR REPLACE TABLE export_eu_ran AS SELECT 1 AS n")\n'
+)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ CrossBoundarySelectionE2ETestCase(
+ description="both-way task selection skips a task reading its upstream",
+ select=("+task:export_summary+",),
+ expected_ran=("export_summary_ran",),
+ expected_not_ran=("export_eu_ran",),
+ ),
+ CrossBoundarySelectionE2ETestCase(
+ description="both-way model selection skips a task reading its upstream",
+ select=("+all_orders+",),
+ expected_ran=("export_summary_ran",),
+ expected_not_ran=("export_eu_ran",),
+ ),
+ CrossBoundarySelectionE2ETestCase(
+ description="downstream selection of the upstream model runs both tasks",
+ select=("stg_orders_eu+",),
+ expected_ran=("export_summary_ran", "export_eu_ran"),
+ expected_not_ran=(),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_both_way_selector_when_building_then_it_expands_from_the_selected_roots(
+ test_case: CrossBoundarySelectionE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="explicit_refs",
+ repo_files=explicit_reference_project_files(
+ overrides={"python/tasks/exports.py": _SIBLING_TASKS}
+ ),
+ )
+
+ prepared: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build", "--select", "stg_orders_us", "stg_customers"),
+ project_dir=project_dir,
+ )
+ result: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build", "--select", *test_case.select), project_dir=project_dir
+ )
+
+ assert prepared.returncode == 0, prepared.stdout + prepared.stderr
+ assert result.returncode == 0, result.stdout + result.stderr
+ database: Path = project_dir / "warehouse.duckdb"
+ assert all(table_exists(db_path=database, table_name=table) for table in test_case.expected_ran)
+ assert not any(
+ table_exists(db_path=database, table_name=table) for table in test_case.expected_not_ran
+ )
+
+
@pytest.mark.parametrize(
"test_case",
[
diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py
index 384066c5bb..5221ff37ff 100644
--- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py
@@ -110,3 +110,12 @@ class PythonNodeIdentityChangeTestCase:
class PythonNodeIdentityConfigTestCase:
description: str
expected_decorator_config_keys_by_selector: dict[str, tuple[str, ...]]
+
+
+@dataclass(frozen=True)
+class SqlOnlySelectorParityTestCase:
+ """A SQL-only selector whose unified SQL keys must match the SQL-only selection."""
+
+ description: str
+ selector: str
+ expected_sql_names: frozenset[str]
diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py
index 778546978a..c15e5e59c6 100644
--- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py
+++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py
@@ -270,6 +270,33 @@ def build_sql_ref_python_node_graph(*, dependency: SqlResourceRef) -> PythonNode
)
+def build_sibling_sql_ref_python_node_graph() -> PythonNodeGraph:
+ """Return tasks reading orders and, separately, its upstream raw_orders source."""
+
+ return build_python_node_graph(
+ discovered_inputs=DiscoveredProjectInputs(
+ project_config=ProjectConfig(name="demo", adapter="duckdb"),
+ local_config=LocalConfig(),
+ task_functions=(
+ DiscoveredTaskFunction(
+ file_path=Path("/project/tasks/orders.py"),
+ relative_path=Path("python/tasks/orders.py"),
+ name="export_orders",
+ function=prepare_orders,
+ depends_on=(model_ref("orders"),),
+ ),
+ DiscoveredTaskFunction(
+ file_path=Path("/project/tasks/raw.py"),
+ relative_path=Path("python/tasks/raw.py"),
+ name="export_raw_orders",
+ function=prepare_orders,
+ depends_on=(source_ref("raw_orders"),),
+ ),
+ ),
+ )
+ )
+
+
def build_sql_downstream_task_to_loader_python_node_graph() -> PythonNodeGraph:
return build_python_node_graph(
discovered_inputs=DiscoveredProjectInputs(
diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py
index 8e64225233..33d4ac2e18 100644
--- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py
+++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py
@@ -4,7 +4,9 @@
import pytest
+from sqlbuild.compiler.compile.models import CompiledObjectKey
from sqlbuild.compiler.pipeline.models import ProjectGraph
+from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors
from sqlbuild.compiler.python_nodes._helpers.unified_selectors import (
resolve_python_sql_selectors,
validate_python_sql_boundaries,
@@ -14,12 +16,14 @@
from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import (
PythonSqlSelectorErrorTestCase,
PythonSqlSelectorTestCase,
+ SqlOnlySelectorParityTestCase,
)
from tests.unit.src.sqlbuild.compiler.python_nodes._helpers.helpers import (
build_model_depends_on_intermediate_loader_project_graph,
build_orders_project_graph,
build_orders_python_node_graph,
build_python_node_graph_for_case,
+ build_sibling_sql_ref_python_node_graph,
build_sql_downstream_task_to_loader_python_node_graph,
build_sql_ref_python_node_graph,
build_terminal_loader_python_node_graph,
@@ -253,6 +257,122 @@ def test_given_python_sql_dependency_when_expanding_selectors_then_crosses_the_b
assert result.python_node_names == test_case.expected_python_node_names
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ PythonSqlSelectorTestCase(
+ description="both-way expansion of a task skips siblings of its upstream",
+ select=("+export_orders+",),
+ exclude=(),
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ expected_python_node_names=frozenset({"export_orders"}),
+ ),
+ PythonSqlSelectorTestCase(
+ description="both-way expansion of a model skips tasks of its upstream",
+ select=("+orders+",),
+ exclude=(),
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ expected_python_node_names=frozenset({"export_orders"}),
+ ),
+ PythonSqlSelectorTestCase(
+ description="downstream expansion of the upstream source keeps both tasks",
+ select=("raw_orders+",),
+ exclude=(),
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ expected_python_node_names=frozenset({"export_orders", "export_raw_orders"}),
+ ),
+ PythonSqlSelectorTestCase(
+ description="excluding a both-way expansion leaves the sibling task",
+ select=(),
+ exclude=("+export_orders+",),
+ expected_sql_names=frozenset(),
+ expected_python_node_names=frozenset({"export_raw_orders"}),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_sibling_python_nodes_when_expanding_both_ways_then_starts_from_the_roots(
+ test_case: PythonSqlSelectorTestCase,
+) -> None:
+ result: PythonSqlSelection = resolve_python_sql_selectors(
+ select=test_case.select,
+ exclude=test_case.exclude,
+ project_graph=build_orders_project_graph(),
+ python_graph=build_sibling_sql_ref_python_node_graph(),
+ validate_dependencies=False,
+ )
+
+ assert frozenset(key.name for key in result.sql_keys) == test_case.expected_sql_names
+ assert result.python_node_names == test_case.expected_python_node_names
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ SqlOnlySelectorParityTestCase(
+ description="orders",
+ selector="orders",
+ expected_sql_names=frozenset({"orders"}),
+ ),
+ SqlOnlySelectorParityTestCase(
+ description="upstream of a model",
+ selector="+orders",
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ ),
+ SqlOnlySelectorParityTestCase(
+ description="downstream of a model",
+ selector="orders+",
+ expected_sql_names=frozenset({"orders"}),
+ ),
+ SqlOnlySelectorParityTestCase(
+ description="both ways from a model",
+ selector="+orders+",
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ ),
+ SqlOnlySelectorParityTestCase(
+ description="downstream of a source",
+ selector="raw_orders+",
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ ),
+ SqlOnlySelectorParityTestCase(
+ description="both ways from a source",
+ selector="+raw_orders+",
+ expected_sql_names=frozenset({"orders", "raw_orders"}),
+ ),
+ SqlOnlySelectorParityTestCase(
+ description="downstream of a tag",
+ selector="tag:daily+",
+ expected_sql_names=frozenset({"orders"}),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_sql_only_selector_when_resolving_unified_then_sql_keys_match_sql_selection(
+ test_case: SqlOnlySelectorParityTestCase,
+) -> None:
+ project_graph: ProjectGraph = build_orders_project_graph()
+
+ result: PythonSqlSelection = resolve_python_sql_selectors(
+ select=(test_case.selector,),
+ exclude=(),
+ project_graph=project_graph,
+ python_graph=build_sibling_sql_ref_python_node_graph(),
+ validate_dependencies=False,
+ )
+ sql_only: frozenset[CompiledObjectKey] = resolve_project_selectors(
+ select=(test_case.selector,),
+ exclude=(),
+ all_keys=project_graph.all_keys,
+ upstream_deps=project_graph.upstream_deps,
+ downstream_deps=project_graph.downstream_deps,
+ tag_index=project_graph.tag_index,
+ path_index=project_graph.path_index,
+ )
+
+ assert frozenset(key.name for key in result.sql_keys) == test_case.expected_sql_names
+ assert frozenset(key.name for key in sql_only) == test_case.expected_sql_names
+
+
@pytest.mark.parametrize(
"test_case",
[
From 8668bb9c0de1a7c648feffe343045e1558b00685 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Mon, 28 Sep 2026 05:05:28 +0100
Subject: [PATCH 09/13] fix: resolve Python reads of managed sources through
source deferral
Python hooks resolved ctx.relation(source(...)) from the plan's load map,
so under defer_sources_to a hook read the local source while the model it
ran on read the deferred one. Tasks, assets and checks only deferred a
source when a selected SQL model also read it, and sqb check never
deferred.
The plan now carries the source entries Python reads resolve, each as a
selected SQL read of that source sees it. Hooks, tasks, assets, checks,
the check existence guard and the hard-coded relation guard use it, and
sqb check plans with source deferral like sqb build.
---
Makefile | 1 +
.../cli/commands/_helpers/check/core.py | 12 +-
.../cli/commands/_helpers/check/planning.py | 1 -
.../pipeline/_helpers/relation_targets.py | 6 +-
.../_helpers/command_planning/static.py | 1 +
.../planner/_helpers/output/plan_entry.py | 14 ++
.../planner/_helpers/output/plan_output.py | 1 +
.../_helpers/warehouse/source_deferral.py | 15 ++-
src/sqlbuild/compiler/planner/models.py | 8 ++
.../executor/build/classes/build_scheduler.py | 1 +
.../run/_helpers/execution/hook_phases.py | 2 +-
src/sqlbuild/executor/run/models.py | 1 +
.../executor/scenario/main/_execute.py | 2 +
.../cli/commands/main/build/_test_types.py | 10 ++
.../main/build/test_python_source_deferral.py | 123 ++++++++++++++++++
.../sqlbuild/executor/build/custom/helpers.py | 1 +
.../src/sqlbuild/executor/run/helpers.py | 1 +
.../executor/run/incremental/helpers.py | 1 +
.../executor/run/microbatch/helpers.py | 1 +
.../test_microbatch_reconciliation.py | 6 +
.../src/sqlbuild/executor/run/view/helpers.py | 1 +
.../planner/_helpers/planning/helpers.py | 1 +
.../planner/_helpers/test_source_freshness.py | 1 +
.../executor/run/_helpers/test_hooks.py | 1 +
.../test_snapshot_adapter_boundary.py | 4 +
.../run/_helpers/test_snapshot_lifecycle.py | 3 +
26 files changed, 202 insertions(+), 17 deletions(-)
create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py
diff --git a/Makefile b/Makefile
index 9f50994d7a..8065fe4d26 100644
--- a/Makefile
+++ b/Makefile
@@ -76,6 +76,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py \
+ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py \
diff --git a/src/sqlbuild/cli/commands/_helpers/check/core.py b/src/sqlbuild/cli/commands/_helpers/check/core.py
index 4790a9723e..c37597e83a 100644
--- a/src/sqlbuild/cli/commands/_helpers/check/core.py
+++ b/src/sqlbuild/cli/commands/_helpers/check/core.py
@@ -502,9 +502,9 @@ def _validate_check_sql_ref_exists(
)
relation = resolve_relation_location_qualified_name(adapter=adapter, location=seed_target)
elif ref.kind == SqlResourceRefKind.SOURCE:
- source: SourceEntry | None = (
- pipeline_result.plan_output.source_read_map or pipeline_result.plan_output.source_map
- ).get(ref.name)
+ source: SourceEntry | None = (pipeline_result.plan_output.python_source_entries).get(
+ ref.name
+ )
if source is None or source.expression is not None or source.table is None:
return
exists = relation_lookup.exists(
@@ -540,9 +540,9 @@ def _check_sql_ref_location(
return None
return (seed_target.database, seed_target.schema, seed_target.name)
if ref.kind == SqlResourceRefKind.SOURCE:
- source: SourceEntry | None = (
- pipeline_result.plan_output.source_read_map or pipeline_result.plan_output.source_map
- ).get(ref.name)
+ source: SourceEntry | None = (pipeline_result.plan_output.python_source_entries).get(
+ ref.name
+ )
if source is None or source.expression is not None or source.table is None:
return None
return (source.database, source.schema, source.table)
diff --git a/src/sqlbuild/cli/commands/_helpers/check/planning.py b/src/sqlbuild/cli/commands/_helpers/check/planning.py
index 81981b5c35..2689787c33 100644
--- a/src/sqlbuild/cli/commands/_helpers/check/planning.py
+++ b/src/sqlbuild/cli/commands/_helpers/check/planning.py
@@ -67,7 +67,6 @@ def compile_check_plan(
options=CompilePipelineOptions(
selected_target=request.selected_target,
no_sql_validation=request.no_sql_validation,
- source_deferral_enabled=False,
connection_config=invocation.connection_config,
cli_vars=request.cli_vars,
external_sql_reference_resolver=resolve_external_sql_reference_resolver(
diff --git a/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py b/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py
index e53c375664..f6a639cc5d 100644
--- a/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py
+++ b/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py
@@ -69,7 +69,7 @@ def _planned_relation_refs(*, plan_output: PlanOutput) -> frozenset[SqlResourceR
for name in plan_output.seed_locations
)
source_name: str
- for source_name in plan_output.source_read_map or plan_output.source_map:
+ for source_name in plan_output.python_source_entries:
refs.add(SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source_name))
return frozenset(refs)
@@ -107,9 +107,7 @@ def _planned_relation(
*, adapter: BaseAdapter, plan_output: PlanOutput, ref: SqlResourceRef
) -> str | None:
if ref.kind == SqlResourceRefKind.SOURCE:
- planned_sources: dict[str, SourceEntry] = (
- plan_output.source_read_map or plan_output.source_map
- )
+ planned_sources: dict[str, SourceEntry] = plan_output.python_source_entries
planned_source: SourceEntry | None = planned_sources.get(ref.name)
if planned_source is None:
return None
diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py
index 1f4f79b2d6..d430ccc4b0 100644
--- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py
+++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py
@@ -247,6 +247,7 @@ def _base_plan_output(
function_locations=relations.function_locations,
source_map=relations.source_map,
source_read_map=relations.source_read_map,
+ python_source_read_map=relations.python_source_read_map,
hook_functions=project.hook_functions,
enforce_explicit_references=project.enforce_explicit_references,
)
diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
index 3034767498..af7fd0fcae 100644
--- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
+++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
@@ -246,12 +246,26 @@ def build_planner_relations_context(
if effective_deferral.source_deferral_enabled
else source_map
)
+ python_source_read_map: dict[str, SourceEntry] = (
+ build_source_read_map(
+ project=project,
+ source_map=source_map,
+ selected_keys=scope.selected_keys,
+ project_config=project_config,
+ local_config=local_config,
+ defer_sources_to=effective_deferral.defer_sources_to,
+ read_source_names=frozenset(source_map),
+ )
+ if effective_deferral.source_deferral_enabled
+ else source_map
+ )
return PlannerRelationsContext(
model_locations=model_locations,
seed_locations=seed_locations,
function_locations=function_locations,
source_map=source_map,
source_read_map=source_read_map,
+ python_source_read_map=python_source_read_map,
source_warehouse_columns=get_semantic_source_columns(
project=project,
adapter=adapter,
diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py
index 2304d0cc74..209a82aaa6 100644
--- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py
+++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py
@@ -165,6 +165,7 @@ def build_plan_output(
function_locations=relations.function_locations,
source_map=relations.source_map,
source_read_map=relations.source_read_map,
+ python_source_read_map=relations.python_source_read_map,
hook_functions=project.hook_functions,
enforce_explicit_references=project.enforce_explicit_references,
provider_usages=_build_provider_usages(
diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py
index c843f4b21f..7a0f35394b 100644
--- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py
+++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py
@@ -36,18 +36,23 @@ def build_source_read_map(
project_config: ProjectConfig | None,
local_config: LocalConfig | None,
defer_sources_to: str | None,
+ read_source_names: frozenset[str] | None = None,
) -> dict[str, SourceEntry]:
- """Resolve the source relation map used when selected SQL reads managed sources."""
+ """Resolve source reads for selected SQL, or for ``read_source_names`` when given."""
managed_source_names: frozenset[str] = frozenset(
source.source_entry.name
for source in project.sources
if source.source_entry.loader is not None
)
- selected_managed_sources: tuple[str, ...] = _selected_managed_source_refs(
- project=project,
- selected_keys=selected_keys,
- managed_source_names=managed_source_names,
+ selected_managed_sources: tuple[str, ...] = (
+ tuple(sorted(read_source_names & managed_source_names))
+ if read_source_names is not None
+ else _selected_managed_source_refs(
+ project=project,
+ selected_keys=selected_keys,
+ managed_source_names=managed_source_names,
+ )
)
if not selected_managed_sources:
return source_map
diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py
index bb1e2c493c..c51eed111a 100644
--- a/src/sqlbuild/compiler/planner/models.py
+++ b/src/sqlbuild/compiler/planner/models.py
@@ -739,6 +739,7 @@ class PlannerRelationsContext:
function_locations: dict[str, CompiledRelationLocation]
source_map: dict[str, SourceEntry]
source_read_map: dict[str, SourceEntry]
+ python_source_read_map: dict[str, SourceEntry]
source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]]
star_exclude_keyword: str
@@ -1355,6 +1356,7 @@ class PlanOutput:
function_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict)
source_map: dict[str, SourceEntry] = field(default_factory=dict)
source_read_map: dict[str, SourceEntry] = field(default_factory=dict)
+ python_source_read_map: dict[str, SourceEntry] = field(default_factory=dict)
hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple)
enforce_explicit_references: bool = True
provider_usages: tuple[PlanProviderUsage, ...] = field(default_factory=tuple)
@@ -1362,6 +1364,12 @@ class PlanOutput:
python_identity_fingerprints: dict[tuple[str, str], Fingerprint] = field(default_factory=dict)
metadata: dict[str, object] = field(default_factory=dict)
+ @property
+ def python_source_entries(self) -> dict[str, SourceEntry]:
+ """Return the source entries Python reads resolve, as a selected SQL read sees them."""
+
+ return self.python_source_read_map or self.source_read_map or self.source_map
+
@dataclass(frozen=True)
class PlannerSelection:
diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py
index 822653ded1..f4ed3cf3c4 100644
--- a/src/sqlbuild/executor/build/classes/build_scheduler.py
+++ b/src/sqlbuild/executor/build/classes/build_scheduler.py
@@ -1048,6 +1048,7 @@ def _execute_model_node(
model_locations=self._plan.model_locations,
seed_locations=self._plan.seed_locations,
source_map=self._plan.source_map,
+ python_source_read_map=self._plan.python_source_entries,
model_audits=model_audits,
run_id=self._run_id,
query_change_tracking=self._query_change_tracking,
diff --git a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py
index 0c2cbbaca8..ae1d8ae8a2 100644
--- a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py
+++ b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py
@@ -139,7 +139,7 @@ def build_model_hook_run(
relation_lookup=HookRelationLookup(
model_locations=context.model_locations,
seed_locations=context.seed_locations,
- source_map=context.source_map,
+ source_map=context.python_source_read_map,
),
warnings=warnings,
enforce_explicit_references=context.enforce_explicit_references,
diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py
index 3e1194b094..29e7fef1c6 100644
--- a/src/sqlbuild/executor/run/models.py
+++ b/src/sqlbuild/executor/run/models.py
@@ -276,6 +276,7 @@ class ModelMaterializationContext:
model_locations: dict[str, CompiledRelationLocation]
seed_locations: dict[str, CompiledRelationLocation]
source_map: dict[str, SourceEntry]
+ python_source_read_map: dict[str, SourceEntry]
model_audits: tuple[AuditPlanEntry, ...]
run_id: str
query_change_tracking: bool
diff --git a/src/sqlbuild/executor/scenario/main/_execute.py b/src/sqlbuild/executor/scenario/main/_execute.py
index 830e24609c..80162dead6 100644
--- a/src/sqlbuild/executor/scenario/main/_execute.py
+++ b/src/sqlbuild/executor/scenario/main/_execute.py
@@ -53,6 +53,7 @@ def execute_scenario_model(
model_locations=scenario_plan.relation_plan.model_locations,
seed_locations=scenario_plan.relation_plan.seed_locations,
source_map=scenario_plan.relation_plan.source_map,
+ python_source_read_map=scenario_plan.relation_plan.source_map,
model_audits=(),
run_id=run_id,
query_change_tracking=False,
@@ -74,6 +75,7 @@ def execute_scenario_model(
model_locations=scenario_plan.relation_plan.model_locations,
seed_locations=scenario_plan.relation_plan.seed_locations,
source_map=scenario_plan.relation_plan.source_map,
+ python_source_read_map=scenario_plan.relation_plan.source_map,
model_audits=(),
run_id=run_id,
query_change_tracking=False,
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
index f34eefb9e2..b5451dba46 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
@@ -1077,3 +1077,13 @@ class CrossBoundarySelectionE2ETestCase:
select: tuple[str, ...]
expected_ran: tuple[str, ...]
expected_not_ran: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class PythonSourceDeferralE2ETestCase:
+ """Python reads of a managed source under a target that defers source reads."""
+
+ description: str
+ select: tuple[str, ...]
+ expected_counts: dict[str, int]
+ expected_check_pattern: str
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py
new file mode 100644
index 0000000000..9f7f8020cb
--- /dev/null
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py
@@ -0,0 +1,123 @@
+"""Real-CLI coverage of Python reads of a managed source whose reads defer to another target."""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
+ PythonSourceDeferralE2ETestCase,
+)
+from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import (
+ execute_duckdb,
+ prepare_inline_project,
+ query_duckdb,
+ run_sqb,
+)
+
+_DATABASE: str = "warehouse.duckdb"
+_PROJECT_FILES: dict[str, str] = {
+ "sqlbuild_project.toml": (
+ 'name = "deferred_sources"\nadapter = "duckdb"\ndefault_target = "dev"\n\n'
+ f'[connection]\ndatabase = "{_DATABASE}"\n\n'
+ '[targets.dev]\nschema = "dev"\nloader_schema = "raw_dev"\ndefer_sources_to = "prod"\n\n'
+ '[targets.prod]\nschema = "prod"\nloader_schema = "raw_prod"\n'
+ ),
+ "sources/raw.yml": (
+ "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n"
+ " columns:\n - name: order_id\n type: INTEGER\n"
+ ),
+ "python/loaders/raw.py": (
+ "from sqlbuild.loaders import loader\n\n\n"
+ "@loader\ndef raw_orders(ctx):\n return [{'order_id': 9}]\n"
+ ),
+ "models/sales/_sqlbuild/_hooks/python/count.py": (
+ "from sqlbuild.hooks import hook\nfrom sqlbuild.refs import source\n\n\n"
+ '@hook(reads=source("raw_orders"))\n'
+ "def count_raw(ctx):\n"
+ ' raw = ctx.relation(source("raw_orders"))\n'
+ ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.hook_counts AS '
+ 'SELECT count(*) AS n FROM {raw}")\n'
+ ),
+ "models/sales/order_count.sql": (
+ 'MODEL (materialized table, post_hooks [python("count_raw")]);\n'
+ 'SELECT count(*) AS n FROM __source("raw_orders")\n'
+ ),
+ "models/sales/order_marker.sql": (
+ 'MODEL (materialized table, post_hooks [python("count_raw")]);\nSELECT 1 AS n\n'
+ ),
+ "python/tasks/count.py": (
+ "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n"
+ '@task(depends_on=source("raw_orders"))\n'
+ "def count_task(ctx):\n"
+ ' raw = ctx.relation(source("raw_orders"))\n'
+ ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.task_counts AS '
+ 'SELECT count(*) AS n FROM {raw}")\n'
+ ),
+ "python/checks/count.py": (
+ "from sqlbuild.checks import check\nfrom sqlbuild.refs import source\n\n\n"
+ '@check(depends_on=source("raw_orders"))\n'
+ "def two_raw_orders(ctx):\n"
+ ' raw = ctx.relation(source("raw_orders"))\n'
+ ' n = ctx.query(f"SELECT count(*) FROM {raw}").fetchone()[0]\n'
+ ' return ctx.pass_() if n == 2 else ctx.fail(message=f"read {n} rows")\n'
+ ),
+}
+_ENVIRONMENT_ROWS: str = (
+ "CREATE SCHEMA raw_dev; CREATE SCHEMA raw_prod;"
+ "CREATE TABLE raw_dev.raw_orders (order_id INTEGER); INSERT INTO raw_dev.raw_orders VALUES (1);"
+ "CREATE TABLE raw_prod.raw_orders (order_id INTEGER);"
+ "INSERT INTO raw_prod.raw_orders VALUES (1), (2);"
+)
+_CHECK_PASSED: str = r"check\s+two_raw_orders\s+PASS"
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ PythonSourceDeferralE2ETestCase(
+ description="hook and task read the deferred source a model reads",
+ select=("order_count", "task:count_task"),
+ expected_counts={"dev.order_count": 2, "main.hook_counts": 2, "main.task_counts": 2},
+ expected_check_pattern=_CHECK_PASSED,
+ ),
+ PythonSourceDeferralE2ETestCase(
+ description="hook on a model that does not read the source still reads it deferred",
+ select=("order_marker",),
+ expected_counts={"main.hook_counts": 2},
+ expected_check_pattern=_CHECK_PASSED,
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_deferred_source_reads_when_python_reads_the_source_then_it_reads_like_sql(
+ test_case: PythonSourceDeferralE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path, project_name="deferred_sources", repo_files=_PROJECT_FILES
+ )
+ database: Path = project_dir / _DATABASE
+ execute_duckdb(db_path=database, sql=_ENVIRONMENT_ROWS)
+
+ build: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build", "--select", *test_case.select), project_dir=project_dir
+ )
+ checked: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "check"), project_dir=project_dir
+ )
+
+ assert build.returncode == 0, build.stdout + build.stderr
+ assert {
+ table: query_duckdb(db_path=database, sql=f"SELECT n FROM {table}")[0][0]
+ for table in test_case.expected_counts
+ } == test_case.expected_counts
+ assert checked.returncode == 0, checked.stdout + checked.stderr
+ assert re.search(test_case.expected_check_pattern, checked.stdout), checked.stdout
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-vv"])
diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py
index 3a07e602bd..4447ccf2d0 100644
--- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py
+++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py
@@ -137,6 +137,7 @@ def run_custom_entry(
model_locations=model_locations or {},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=model_audits,
run_id="test_run",
query_change_tracking=True,
diff --git a/tests/integration/src/sqlbuild/executor/run/helpers.py b/tests/integration/src/sqlbuild/executor/run/helpers.py
index 8e850319ab..fc4c5bf5ea 100644
--- a/tests/integration/src/sqlbuild/executor/run/helpers.py
+++ b/tests/integration/src/sqlbuild/executor/run/helpers.py
@@ -264,6 +264,7 @@ def _execute_test(
model_locations=model_locations,
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=model_audits,
run_id="test_run",
query_change_tracking=getattr(test_case, "query_change_tracking", True),
diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py
index a21e8f1a6a..15d4924f49 100644
--- a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py
+++ b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py
@@ -346,6 +346,7 @@ def _execute_test(
model_locations=model_locations,
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=model_audits,
run_id="test_run",
query_change_tracking=getattr(test_case, "query_change_tracking", True),
diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py
index 599bce595a..85b5114203 100644
--- a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py
+++ b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py
@@ -415,6 +415,7 @@ def _execute_test(
model_locations=model_locations,
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=model_audits,
run_id="test_run",
query_change_tracking=getattr(test_case, "query_change_tracking", True),
diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py
index 44b24be70e..7455032d85 100644
--- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py
+++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py
@@ -251,6 +251,7 @@ def test_given_many_unaccounted_intervals_when_counting_then_queries_are_chunked
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="reconciliation-chunks",
query_change_tracking=False,
@@ -528,6 +529,7 @@ def test_given_replay_and_synthetic_facts_when_final_limit_fails_then_state_is_u
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="limited-replay",
query_change_tracking=False,
@@ -570,6 +572,7 @@ def test_given_legacy_recovery_outside_model_domain_when_clamping_then_only_over
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="domain-clamp",
query_change_tracking=False,
@@ -620,6 +623,7 @@ def test_given_full_refresh_wholly_after_cursor_end_when_planning_then_no_batch_
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="terminal-full-refresh",
query_change_tracking=False,
@@ -671,6 +675,7 @@ def test_given_no_work_full_refresh_with_absent_destination_when_executing_then_
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="absent-terminal-full-refresh",
query_change_tracking=False,
@@ -713,6 +718,7 @@ def test_given_no_work_first_run_with_absent_destination_when_executing_then_res
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="absent-first-run",
query_change_tracking=False,
diff --git a/tests/integration/src/sqlbuild/executor/run/view/helpers.py b/tests/integration/src/sqlbuild/executor/run/view/helpers.py
index 1de68a21f9..f182f88d24 100644
--- a/tests/integration/src/sqlbuild/executor/run/view/helpers.py
+++ b/tests/integration/src/sqlbuild/executor/run/view/helpers.py
@@ -236,6 +236,7 @@ def _execute_view_test(
model_locations=model_locations,
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=model_audits,
run_id="test_run",
query_change_tracking=getattr(test_case, "query_change_tracking", True),
diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py
index a47530c4a0..4696e862dc 100644
--- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py
+++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py
@@ -78,6 +78,7 @@ def build_retention_planner_inputs(
function_locations={},
source_map={},
source_read_map={},
+ python_source_read_map={},
source_warehouse_columns={},
star_exclude_keyword="EXCLUDE",
),
diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py
index 8952e47cd0..91898b9c3a 100644
--- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py
+++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py
@@ -75,6 +75,7 @@ def test_given_source_deferral_context_when_building_source_freshness_then_uses_
),
)
},
+ python_source_read_map={},
source_warehouse_columns={},
star_exclude_keyword="EXCLUDE",
),
diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py
index 97cd0f69f1..97043e06f8 100644
--- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py
+++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py
@@ -436,6 +436,7 @@ def maybe_skip(ctx: HookContext) -> object:
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="run-1",
query_change_tracking=False,
diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py
index fc12f374ea..165c3aa037 100644
--- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py
+++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py
@@ -199,6 +199,7 @@ def test_given_existing_snapshot_target_when_executing_then_uses_adapter_rendere
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="test_run",
query_change_tracking=False,
@@ -290,6 +291,7 @@ def test_given_existing_historical_timestamp_snapshot_when_executing_then_uses_a
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="test_run",
query_change_tracking=False,
@@ -372,6 +374,7 @@ def test_given_existing_historical_timestamp_changes_snapshot_when_executing_the
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="test_run",
query_change_tracking=False,
@@ -444,6 +447,7 @@ def test_given_new_historical_timestamp_changes_when_executing_then_uses_adapter
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="test_run",
query_change_tracking=False,
diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py
index 54cb5565cf..aee7401e98 100644
--- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py
+++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py
@@ -94,6 +94,7 @@ def test_given_successful_snapshot_when_executing_then_runs_lifecycle_side_effec
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id=test_case.run_id,
query_change_tracking=True,
@@ -176,6 +177,7 @@ def test_given_python_snapshot_hook_failure_when_executing_then_reports_hook_pha
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id="snapshot_hook_failure_run",
query_change_tracking=True,
@@ -310,6 +312,7 @@ def test_given_snapshot_contract_violation_when_executing_then_fails_before_targ
model_locations={},
seed_locations={},
source_map={},
+ python_source_read_map={},
model_audits=(),
run_id=test_case.run_id,
query_change_tracking=True,
From e9ff38a516415c1ddb2c2012890d98052b3828a3 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Mon, 28 Sep 2026 05:08:54 +0100
Subject: [PATCH 10/13] fix: report P006 for references a macro constructs
itself
Every SqlResourceRef formatted while a macro ran rendered as an approved
call-site placeholder, so a macro that built model("orders") itself
compiled without P006. Only references written as arguments at the call
site, including inside nested macro arguments, render as approved
placeholders now; any other reference renders as its reference call and
is reported as macro-generated, or becomes an ordinary reference with
[references] enforce_explicit = false.
---
.../explicit_references/macro_arguments.py | 18 ++++-
.../compile/_helpers/render/macros.py | 68 ++++++++++++++++---
.../_helpers/test_typed_macro_references.py | 40 +++++++++++
3 files changed, 113 insertions(+), 13 deletions(-)
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
index a54ac68f06..d1356d5f8f 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
@@ -62,14 +62,26 @@ def render_relation_placeholders(*, sql: str, relations: dict[SqlResourceRef, in
return sql
ordered: tuple[SqlResourceRef, ...] = tuple(relations)
return _RELATION_PLACEHOLDER_PATTERN.sub(
- lambda match: _reference_call_text(ordered[int(match.group(1))]), sql
+ lambda match: reference_call_text(ordered[int(match.group(1))]), sql
)
-def _reference_call_text(ref: SqlResourceRef) -> str:
+def reference_call_text(ref: SqlResourceRef) -> str:
+ """Return the SQL reference call that names ``ref``, such as ``__ref("orders")``."""
+
return f'{_REFERENCE_FUNCTION_BY_KIND[ref.kind]}("{ref.name}")'
+def call_site_sql_references(
+ refs: tuple[SqlResourceRef, ...],
+) -> tuple[CompileSqlReference, ...]:
+ """Return the SQL references written as typed macro arguments, as SQL extraction sees them."""
+
+ if not refs:
+ return ()
+ return extract_sql_references(" ".join(reference_call_text(ref) for ref in refs))
+
+
def reject_macro_generated_references(
*,
loaded_macro: LoadedMacro,
@@ -118,7 +130,7 @@ def reject_macro_generated_references(
path=loaded_macro.relative_path,
help=(
f"write the reference in {location}, or pass it in: "
- f"@{loaded_macro.name}({_reference_call_text(generated)})\n"
+ f"@{loaded_macro.name}({reference_call_text(generated)})\n"
" = help: while migrating a project, allow macro-generated references with "
"[references] enforce_explicit = false in sqlbuild_project.toml"
),
diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py
index e5b17b1d4d..eff717c29b 100644
--- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py
+++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py
@@ -18,6 +18,7 @@
from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import (
evaluate_typed_reference,
+ reference_call_text,
reject_macro_generated_references,
relation_placeholder_text,
render_relation_placeholders,
@@ -106,6 +107,9 @@ class _ExpansionFacts:
dependencies: list[DeclarationIdentity] = field(default_factory=list)
usages: list[UsageRecord] = field(default_factory=list)
relations: dict[SqlResourceRef, int] = field(default_factory=dict)
+ argument_references: dict[SqlResourceRef, None] = field(default_factory=dict)
+ call_site_refs: list[set[SqlResourceRef]] = field(default_factory=list)
+ rendering_approved: list[frozenset[SqlResourceRef]] = field(default_factory=list)
def add_dependency(self, identity: DeclarationIdentity) -> _ExpansionFacts:
"""Record a resolved dependency in encounter order."""
@@ -119,11 +123,43 @@ def add_usage(self, usage: UsageRecord) -> _ExpansionFacts:
self.usages.append(usage)
return self
- def relation_placeholder(self, ref: object) -> str:
- """Return the stable in-expansion placeholder for one typed reference argument."""
+ def record_call_site_refs(self, refs: tuple[SqlResourceRef, ...]) -> None:
+ """Record references written as arguments of every macro call being expanded."""
+
+ self.argument_references.update(dict.fromkeys(refs))
+ for approved in self.call_site_refs:
+ approved.update(refs)
+
+ def open_call_site(self) -> _ExpansionFacts:
+ """Start collecting the references written as one macro call's arguments."""
+
+ self.call_site_refs.append(set())
+ return self
+
+ def close_call_site(self) -> frozenset[SqlResourceRef]:
+ """Stop collecting for the innermost macro call and return its argument references."""
+
+ return frozenset(self.call_site_refs.pop())
+
+ def begin_rendering(self, approved: frozenset[SqlResourceRef]) -> _ExpansionFacts:
+ """Approve the call-site argument references of the macro about to run."""
+
+ self.rendering_approved.append(approved)
+ return self
+
+ def end_rendering(self) -> _ExpansionFacts:
+ """Restore the approvals of the enclosing macro after one macro returns."""
+
+ self.rendering_approved.pop()
+ return self
+
+ def render_relation(self, ref: object) -> str:
+ """Render a reference a macro formats; only call-site arguments become placeholders."""
if not isinstance(ref, SqlResourceRef):
raise CompileInputError("Only typed resource references render as relations")
+ if not self.rendering_approved or ref not in self.rendering_approved[-1]:
+ return reference_call_text(ref)
return relation_placeholder_text(self.relations.setdefault(ref, len(self.relations)))
def render_relation_placeholders(self, sql: str) -> str:
@@ -1439,13 +1475,17 @@ def _evaluate_macro_call(
args_source: str = sql[opening_paren_index + 1 : closing_paren_index]
args: tuple[object, ...]
kwargs: dict[str, object]
- args, kwargs = _parse_macro_arguments(
- args_source=args_source,
- file_path=file_path,
- state=state,
- declarations=declarations,
- stack=stack,
- )
+ _ = state.facts.open_call_site()
+ try:
+ args, kwargs = _parse_macro_arguments(
+ args_source=args_source,
+ file_path=file_path,
+ state=state,
+ declarations=declarations,
+ stack=stack,
+ )
+ finally:
+ approved: frozenset[SqlResourceRef] = state.facts.close_call_site()
try:
invocation_context: MacroContext = _build_macro_invocation_context(
macro_context=state.macro_context,
@@ -1454,7 +1494,7 @@ def _evaluate_macro_call(
state=state,
)
renderer_token: Token[Callable[[object], str] | None] = RELATION_RENDERER.set(
- state.facts.relation_placeholder
+ state.facts.begin_rendering(approved).render_relation
)
try:
macro_result: object = _call_loaded_macro(
@@ -1465,6 +1505,7 @@ def _evaluate_macro_call(
)
finally:
RELATION_RENDERER.reset(renderer_token)
+ _ = state.facts.end_rendering()
except CompileInputError:
raise
except TypeError as error:
@@ -1739,6 +1780,13 @@ def _parse_macro_arguments(
placeholder_values=placeholder_values,
file_path=file_path,
)
+ state.facts.record_call_site_refs(
+ tuple(
+ evaluate_typed_reference(node=node, file_path=file_path)
+ for node in ast.walk(call_expression)
+ if isinstance(node, ast.Call) and node is not call_expression
+ )
+ )
return args, kwargs
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
index a121a4497a..6b83be49c7 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
@@ -132,6 +132,37 @@ def test_given_typed_reference_arguments_when_expanding_then_renders_call_site_r
sql="@raw_orders()",
expected_error_fragments=("depends on source:raw_orders through macro raw_orders()",),
),
+ MacroGeneratedReferenceErrorTestCase(
+ description="macro formatting a reference it constructs itself",
+ macro_file_contents=(
+ "from sqlbuild.refs import model\n\n"
+ "def orders_base():\n return f'SELECT * FROM {model(\"stg_orders\")}'\n"
+ ),
+ sql="@orders_base()",
+ expected_error_fragments=(
+ "model:order_summary depends on model:stg_orders through macro orders_base()",
+ ),
+ ),
+ MacroGeneratedReferenceErrorTestCase(
+ description="macro formatting a constructed reference beside a passed one",
+ macro_file_contents=(
+ "from sqlbuild.refs import source\n\n"
+ "def enrich(relation):\n"
+ " return f'SELECT * FROM {relation} JOIN {source(\"raw_orders\")} ON true'\n"
+ ),
+ sql='@enrich(__ref("stg_orders"))',
+ expected_error_fragments=("depends on source:raw_orders through macro enrich()",),
+ ),
+ MacroGeneratedReferenceErrorTestCase(
+ description="nested macro returning a reference it constructs itself",
+ macro_file_contents=(
+ "from sqlbuild.refs import seed\n\n"
+ "def countries():\n return seed('country_codes')\n\n"
+ "def base(relation):\n return f'SELECT * FROM {relation}'\n"
+ ),
+ sql="@base(@countries())",
+ expected_error_fragments=("depends on seed:country_codes through macro base()",),
+ ),
MacroGeneratedReferenceErrorTestCase(
description="nested macro that emits a seed reference",
macro_file_contents=(
@@ -204,6 +235,15 @@ def test_given_several_macro_output_references_when_collecting_then_reports_each
sql="@wrap('__ref(\"stg_orders\")')",
expected_sql='SELECT * FROM __ref("stg_orders")',
),
+ MacroGeneratedReferenceSwitchTestCase(
+ description="macro formatting a reference it constructs itself",
+ macro_file_contents=(
+ "from sqlbuild.refs import model\n\n"
+ "def orders_base():\n return f'SELECT * FROM {model(\"stg_orders\")}'\n"
+ ),
+ sql="@orders_base()",
+ expected_sql='SELECT * FROM __ref("stg_orders")',
+ ),
],
ids=lambda case: case.description,
)
From f5e68626d8073a103c722f919a72a7117f16c648 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Mon, 28 Sep 2026 05:13:35 +0100
Subject: [PATCH 11/13] fix: record every typed macro argument as a dependency
of its caller
A reference passed to a macro became a dependency only if the macro
rendered it, so @unused(__ref("orders")) or an unchosen dictionary or list
entry left the caller without that dependency. Every __ref(), __source()
and __seed() written as a macro argument, including inside nested macro
arguments, is now a dependency of the calling model, audit or function,
so it orders, selects and appears in the DAG like a rendered reference.
---
.../compile/_helpers/attachment/audits.py | 23 +++++-
.../compile/_helpers/attachment/core.py | 10 ++-
.../compile/_helpers/attachment/functions.py | 8 +-
.../explicit_references/macro_arguments.py | 12 +++
.../compile/_helpers/render/macros.py | 2 +
.../compile/_helpers/render/sql_vars.py | 1 +
src/sqlbuild/compiler/compile/models.py | 2 +
.../cli/commands/main/build/_test_types.py | 10 +++
.../main/build/test_explicit_references.py | 73 ++++++++++++++++---
.../compiler/compile/_helpers/_test_types.py | 10 +++
.../_helpers/test_typed_macro_references.py | 64 +++++++++++++++-
11 files changed, 199 insertions(+), 16 deletions(-)
diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py
index 4610ae6471..bd184f1235 100644
--- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py
+++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py
@@ -22,6 +22,9 @@
validate_audit_references,
)
from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic
+from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import (
+ merge_call_site_references,
+)
from sqlbuild.compiler.compile._helpers.named_declarations.core import (
declaration_file_expansion,
named_declaration_usages,
@@ -200,8 +203,9 @@ def build_audit_inputs(
sql=expanded_evidence_sql,
context=f"Audit '{audit_block.name or audit_file.file_path.stem}' evidence",
)
- references: tuple[CompileSqlReference, ...] = _combined_references(
- expanded_sql_body, expanded_evidence_sql
+ references: tuple[CompileSqlReference, ...] = merge_call_site_references(
+ references=_combined_references(expanded_sql_body, expanded_evidence_sql),
+ argument_references=_argument_references(expansion, evidence_expansion),
)
validate_audit_references(
references=references,
@@ -624,8 +628,9 @@ def build_attached_audit_input(
sql=expanded_evidence_sql,
context=f"Audit '{audit_instance.definition_name}' evidence",
)
- references: tuple[CompileSqlReference, ...] = _combined_references(
- expanded_sql_body, expanded_evidence_sql
+ references: tuple[CompileSqlReference, ...] = merge_call_site_references(
+ references=_combined_references(expanded_sql_body, expanded_evidence_sql),
+ argument_references=_argument_references(expansion, evidence_expansion),
)
validate_audit_references(
references=references,
@@ -936,6 +941,16 @@ def _bool_from_dict(*, values: dict[str, object], key: str) -> bool:
return raw if isinstance(raw, bool) else False
+def _argument_references(
+ *expansions: AuthoredSqlExpansionResult | None,
+) -> tuple[CompileSqlReference, ...]:
+ references: list[CompileSqlReference] = []
+ for expansion in expansions:
+ if expansion is not None:
+ references.extend(expansion.argument_references)
+ return tuple(references)
+
+
def _combined_references(*sql_values: str | None) -> tuple[CompileSqlReference, ...]:
"""Extract references from independently compiled measurement/evidence queries."""
diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py
index 1c8a1920e1..a8e2903ee6 100644
--- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py
+++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py
@@ -48,6 +48,9 @@
validate_preserved_logical_namespace,
)
from sqlbuild.compiler.compile._helpers.config.table_type import resolve_storage_policies
+from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import (
+ merge_call_site_references,
+)
from sqlbuild.compiler.compile._helpers.named_declarations.core import (
named_declaration_usages,
)
@@ -583,6 +586,7 @@ def _build_model_inputs(
expanded_query_sql=expanded_query_sql,
sql_validation_placeholders=sql_validation_placeholders,
model_schema_columns=model_schema_columns,
+ argument_references=macro_expansion.argument_references,
)
hook_expansion: HookExpansionResult = expand_model_hook_macros_result(
values=effective_config.values,
@@ -744,6 +748,7 @@ def _validate_model_input(
expanded_query_sql: str,
sql_validation_placeholders: dict[str, str] | None,
model_schema_columns: tuple[SchemaColumn, ...] | None,
+ argument_references: tuple[CompileSqlReference, ...],
) -> tuple[bool, tuple[CompileSqlReference, ...]]:
model_name: str = model_file.file_path.stem
sql_validation_enabled: bool = _model_sql_validation_gate(
@@ -761,7 +766,10 @@ def _validate_model_input(
file_path=model_file.file_path,
placeholders=sql_validation_placeholders,
)
- references: tuple[CompileSqlReference, ...] = context.extract_references(expanded_query_sql)
+ references: tuple[CompileSqlReference, ...] = merge_call_site_references(
+ references=context.extract_references(expanded_query_sql),
+ argument_references=argument_references,
+ )
validate_model_references(
references=references,
model_file=model_file,
diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py
index 83d65cf6e3..c42fa581e4 100644
--- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py
+++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py
@@ -21,6 +21,9 @@
from sqlbuild.compiler.compile._helpers.config.namespace_validation import (
validate_preserved_logical_namespace,
)
+from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import (
+ merge_call_site_references,
+)
from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references
from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics
from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion
@@ -222,7 +225,10 @@ def build_sql_function_inputs(
return_columns=return_columns,
function_file=function_file,
)
- references: tuple[CompileSqlReference, ...] = extract_sql_references(expanded_body_sql)
+ references: tuple[CompileSqlReference, ...] = merge_call_site_references(
+ references=extract_sql_references(expanded_body_sql),
+ argument_references=expansion.argument_references,
+ )
validate_function_references(
references=references,
function_file=function_file,
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
index d1356d5f8f..55f31b944d 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
@@ -82,6 +82,18 @@ def call_site_sql_references(
return extract_sql_references(" ".join(reference_call_text(ref) for ref in refs))
+def merge_call_site_references(
+ *,
+ references: tuple[CompileSqlReference, ...],
+ argument_references: tuple[CompileSqlReference, ...],
+) -> tuple[CompileSqlReference, ...]:
+ """Add typed macro arguments to the references found in expanded SQL, once each."""
+
+ if not argument_references:
+ return references
+ return tuple(dict.fromkeys((*references, *argument_references)))
+
+
def reject_macro_generated_references(
*,
loaded_macro: LoadedMacro,
diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py
index eff717c29b..c7e1674f1a 100644
--- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py
+++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py
@@ -17,6 +17,7 @@
from typing import cast
from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import (
+ call_site_sql_references,
evaluate_typed_reference,
reference_call_text,
reject_macro_generated_references,
@@ -1273,6 +1274,7 @@ def expand_sql_macros_result(
return MacroExpansionResult(
sql=expanded_sql,
spans=spans,
+ argument_references=call_site_sql_references(tuple(facts.argument_references)),
dependencies=tuple(dict.fromkeys(facts.dependencies)),
usages=tuple(dict.fromkeys(facts.usages)),
)
diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py
index 788cca6555..f621c5c7cc 100644
--- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py
+++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py
@@ -97,6 +97,7 @@ def expand_authored_sql_result( # noqa: PLR0913
return AuthoredSqlExpansionResult(
sql=macro_result.sql,
usages=tuple(dict.fromkeys((*declaration_result.usages, *macro_result.usages))),
+ argument_references=macro_result.argument_references,
)
diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py
index 70d55d4cd5..9a208bd986 100644
--- a/src/sqlbuild/compiler/compile/models.py
+++ b/src/sqlbuild/compiler/compile/models.py
@@ -374,6 +374,7 @@ class MacroExpansionResult:
spans: tuple[ExpansionSpan, ...] = field(default_factory=tuple)
dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple)
usages: tuple[UsageRecord, ...] = field(default_factory=tuple)
+ argument_references: tuple[CompileSqlReference, ...] = field(default_factory=tuple)
@dataclass(frozen=True)
@@ -391,6 +392,7 @@ class AuthoredSqlExpansionResult:
sql: str
usages: tuple[UsageRecord, ...] = field(default_factory=tuple)
+ argument_references: tuple[CompileSqlReference, ...] = field(default_factory=tuple)
@dataclass(frozen=True)
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
index b5451dba46..6bd7de8f3a 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
@@ -1087,3 +1087,13 @@ class PythonSourceDeferralE2ETestCase:
select: tuple[str, ...]
expected_counts: dict[str, int]
expected_check_pattern: str
+
+
+@dataclass(frozen=True)
+class UnrenderedMacroArgumentE2ETestCase:
+ """A model passing typed references its macro does not render."""
+
+ description: str
+ model_sql: str
+ expected_dag_edges: tuple[tuple[str, str], ...]
+ expected_built: tuple[str, ...]
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
index 2f3af35a29..58e740eba6 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py
@@ -18,6 +18,7 @@
ExplicitReferenceFailureE2ETestCase,
ExplicitReferencePythonDependencyE2ETestCase,
ExplicitReferenceRuntimeWarningE2ETestCase,
+ UnrenderedMacroArgumentE2ETestCase,
)
from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import (
EXPLICIT_REFERENCE_HOOK_PATH,
@@ -115,6 +116,21 @@
)
+_PICK_MACROS: str = EXPLICIT_REFERENCE_MACROS + (
+ "\n\ndef pick(relations, key):\n return f'SELECT order_id FROM {relations[key]}'\n"
+)
+_SIBLING_TASKS: str = (
+ "from sqlbuild.refs import model\n"
+ "from sqlbuild.tasks import task\n\n\n"
+ '@task(depends_on=model("all_orders"))\n'
+ "def export_summary(ctx):\n"
+ ' ctx.execute_sql("CREATE OR REPLACE TABLE export_summary_ran AS SELECT 1 AS n")\n\n\n'
+ '@task(depends_on=model("stg_orders_eu"))\n'
+ "def export_eu(ctx):\n"
+ ' ctx.execute_sql("CREATE OR REPLACE TABLE export_eu_ran AS SELECT 1 AS n")\n'
+)
+
+
@pytest.mark.parametrize(
"test_case",
[
@@ -183,16 +199,55 @@ def test_given_python_seed_and_model_dependencies_when_building_then_graph_selec
assert re.search(test_case.expected_check_row_pattern, checked.stdout), checked.stdout
-_SIBLING_TASKS: str = (
- "from sqlbuild.refs import model\n"
- "from sqlbuild.tasks import task\n\n\n"
- '@task(depends_on=model("all_orders"))\n'
- "def export_summary(ctx):\n"
- ' ctx.execute_sql("CREATE OR REPLACE TABLE export_summary_ran AS SELECT 1 AS n")\n\n\n'
- '@task(depends_on=model("stg_orders_eu"))\n'
- "def export_eu(ctx):\n"
- ' ctx.execute_sql("CREATE OR REPLACE TABLE export_eu_ran AS SELECT 1 AS n")\n'
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ UnrenderedMacroArgumentE2ETestCase(
+ description="unchosen dictionary entry is a dependency the build selects",
+ model_sql=(
+ "MODEL (materialized table);\n"
+ '@pick({"eu": __ref("stg_orders_eu"), "us": __ref("stg_orders_us")}, key="eu")\n'
+ ),
+ expected_dag_edges=(
+ ("model:stg_orders_eu", "model:order_pick"),
+ ("model:stg_orders_us", "model:order_pick"),
+ ),
+ expected_built=("stg_orders_eu", "stg_orders_us", "order_pick"),
+ ),
+ ],
+ ids=lambda case: case.description,
)
+def test_given_unrendered_macro_argument_when_building_then_it_is_still_a_dependency(
+ test_case: UnrenderedMacroArgumentE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="explicit_refs",
+ repo_files=explicit_reference_project_files(
+ overrides={
+ EXPLICIT_REFERENCE_MACRO_PATH: _PICK_MACROS,
+ f"{EXPLICIT_REFERENCE_MODELS}/order_pick.sql": test_case.model_sql,
+ }
+ ),
+ )
+
+ dag: subprocess.CompletedProcess[str] = run_sqb(
+ command=("dag", "--json"), project_dir=project_dir
+ )
+ build: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build", "--select", "+order_pick"), project_dir=project_dir
+ )
+
+ assert dag.returncode == 0, dag.stdout + dag.stderr
+ payload: dict[str, object] = json.loads(dag.stdout)
+ edges: set[tuple[str, str]] = {
+ (edge["from_id"], edge["to_id"]) for edge in cast(list[dict[str, str]], payload["edges"])
+ }
+ assert set(test_case.expected_dag_edges) <= edges
+ assert build.returncode == 0, build.stdout + build.stderr
+ database: Path = project_dir / "warehouse.duckdb"
+ assert all(table_exists(db_path=database, table_name=name) for name in test_case.expected_built)
@pytest.mark.parametrize(
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
index 62b4bc3b77..8f11a18d13 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py
@@ -1032,3 +1032,13 @@ class MacroGeneratedReferenceCollectionTestCase:
macro_file_contents: str
sql: str
expected_messages: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class TypedMacroArgumentDependencyTestCase:
+ """A model passing typed references to a macro that may not render all of them."""
+
+ description: str
+ macro_file_contents: str
+ sql: str
+ expected_dependency_names: frozenset[str]
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
index 6b83be49c7..f787e23df3 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py
@@ -1,22 +1,25 @@
from __future__ import annotations
import re
+from collections.abc import Callable
from pathlib import Path
import pytest
from sqlbuild.compiler.compile.exceptions import CompileInputError
-from sqlbuild.compiler.compile.models import CompilerDiagnostic
+from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic
from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import (
MacroGeneratedReferenceCollectionTestCase,
MacroGeneratedReferenceErrorTestCase,
MacroGeneratedReferenceSwitchTestCase,
MalformedTypedMacroReferenceTestCase,
+ TypedMacroArgumentDependencyTestCase,
TypedMacroReferenceTestCase,
TypedReferenceAdapterRenderingTestCase,
)
from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import (
collect_typed_macro_violations,
+ compile_and_assemble,
expand_typed_macro_sql,
resolve_model_references_for_adapter,
)
@@ -29,6 +32,16 @@
_EMITTING_MACRO: str = "def orders_base():\n return 'SELECT * FROM __ref(\"stg_orders\")'\n"
+_DEPENDENCY_PROJECT_FILES: dict[str, str] = {
+ "sqlbuild_project.toml": (
+ 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_analysis = false\n'
+ "sql_validation = false\n"
+ ),
+ "models/orders_eu.sql": "MODEL ();\nSELECT 1 AS order_id",
+ "models/orders_us.sql": "MODEL ();\nSELECT 2 AS order_id",
+}
+
+
@pytest.mark.parametrize(
"test_case",
[
@@ -338,3 +351,52 @@ def test_given_adapter_when_resolving_typed_macro_reference_then_uses_adapter_qu
)
assert resolved == test_case.expected_resolved_sql
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ TypedMacroArgumentDependencyTestCase(
+ description="argument the macro ignores",
+ macro_file_contents="def unused(relation):\n return 'SELECT 1 AS order_id'\n",
+ sql='@unused(__ref("orders_eu"))',
+ expected_dependency_names=frozenset({"orders_eu"}),
+ ),
+ TypedMacroArgumentDependencyTestCase(
+ description="unchosen dictionary entry",
+ macro_file_contents=(
+ "def pick(relations, key):\n return f'SELECT * FROM {relations[key]}'\n"
+ ),
+ sql='@pick({"eu": __ref("orders_eu"), "us": __ref("orders_us")}, key="eu")',
+ expected_dependency_names=frozenset({"orders_eu", "orders_us"}),
+ ),
+ TypedMacroArgumentDependencyTestCase(
+ description="unchosen list entry inside a nested macro argument",
+ macro_file_contents=(
+ "def first(relations):\n return str(relations[0])\n\n"
+ "def base(relation):\n return f'SELECT * FROM {relation}'\n"
+ ),
+ sql='@base(@first([__ref("orders_eu"), __ref("orders_us")]))',
+ expected_dependency_names=frozenset({"orders_eu", "orders_us"}),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_typed_macro_arguments_when_compiling_then_every_argument_is_a_dependency(
+ test_case: TypedMacroArgumentDependencyTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(
+ tmp_path,
+ _DEPENDENCY_PROJECT_FILES
+ | {
+ "macros/common.py": test_case.macro_file_contents,
+ "models/order_summary.sql": f"MODEL ();\n{test_case.sql}",
+ },
+ )
+
+ project: CompiledProject = compile_and_assemble(project_dir=tmp_path)
+
+ summary: CompiledModel = {model.name: model for model in project.models}["order_summary"]
+ assert frozenset(dep.name for dep in summary.deps) == test_case.expected_dependency_names
From 3b64e5a0d606dba8e38fe144e245c2ffba83bd12 Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Mon, 28 Sep 2026 05:14:15 +0100
Subject: [PATCH 12/13] fix: check literal SQL passed to ctx.query and
ctx.execute_sql by keyword
Compile-time P008 only inspected the first positional argument, so
ctx.query(sql="...") and ctx.execute_sql(sql="...") skipped the
hard-coded project relation check. The sql keyword and a literal
**{"sql": ...} unpack are now checked like the positional form.
---
.../explicit_references/python_sql.py | 18 +++++++-
.../_helpers/test_python_sql_references.py | 41 +++++++++++++++++++
2 files changed, 57 insertions(+), 2 deletions(-)
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
index 0c88fe1c42..3ad9224695 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py
@@ -34,6 +34,7 @@
from sqlbuild.spec.contracts.models import SourceLocation
_SQL_METHOD_NAMES: frozenset[str] = frozenset({"query", "execute_sql"})
+_SQL_PARAMETER_NAME: str = "sql"
_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset({"ctx", "context", "_ctx", "hook_context"})
_VALUE_PLACEHOLDER: str = "__sqlbuild_python_value_"
@@ -197,14 +198,27 @@ def _literal_sql_calls(function: Callable[..., object]) -> Iterator[_LiteralSql]
and node.func.attr in _SQL_METHOD_NAMES
and isinstance(node.func.value, ast.Name)
and node.func.value.id == context_name
- and node.args
):
continue
- sql: str | None = _literal_text(node.args[0])
+ argument: ast.expr | None = _sql_argument(node)
+ sql: str | None = _literal_text(argument) if argument is not None else None
if sql is not None:
yield _LiteralSql(method=node.func.attr, sql=sql, line=start_line + node.lineno - 1)
+def _sql_argument(call: ast.Call) -> ast.expr | None:
+ if call.args:
+ return call.args[0]
+ for keyword in call.keywords:
+ if keyword.arg == _SQL_PARAMETER_NAME:
+ return keyword.value
+ if keyword.arg is None and isinstance(keyword.value, ast.Dict):
+ for key, value in zip(keyword.value.keys, keyword.value.values, strict=True):
+ if isinstance(key, ast.Constant) and key.value == _SQL_PARAMETER_NAME:
+ return value
+ return None
+
+
def _literal_text(node: ast.expr) -> str | None:
if isinstance(node, ast.Constant) and isinstance(node.value, str):
return node.value
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
index a63e658bac..56dbad3621 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py
@@ -109,6 +109,47 @@
"ctx.query() --> python/tasks/export.py:8",
),
),
+ PythonSqlReferenceErrorTestCase(
+ description="task literal SQL passed as the sql keyword",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(),
+ _TASK_PATH: python_task_source(
+ body=' ctx.query(sql="SELECT count(*) FROM customers")\n'
+ ),
+ },
+ expected_error_fragments=(
+ "[P008] task:export_orders names model:customers as 'customers' in SQL passed to "
+ "ctx.query() --> python/tasks/export.py:7",
+ ),
+ ),
+ PythonSqlReferenceErrorTestCase(
+ description="hook literal SQL passed to execute_sql as the sql keyword",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(
+ body=' ctx.execute_sql(sql="DELETE FROM customers")\n'
+ ),
+ },
+ expected_error_fragments=(
+ "[P008]",
+ "names model:customers as 'customers' in SQL passed to ctx.execute_sql()",
+ ),
+ ),
+ PythonSqlReferenceErrorTestCase(
+ description="task literal SQL unpacked from a keyword dictionary",
+ files={
+ **_MODELS,
+ _HOOK_PATH: python_hook_source(),
+ _TASK_PATH: python_task_source(
+ body=' ctx.query(**{"sql": "SELECT count(*) FROM customers"})\n'
+ ),
+ },
+ expected_error_fragments=(
+ "[P008] task:export_orders names model:customers as 'customers' in SQL passed to "
+ "ctx.query()",
+ ),
+ ),
PythonSqlReferenceErrorTestCase(
description="task literal SQL naming a seed",
files={
From 6d6fb35eae5e833679e117953c8b5f7871c0303c Mon Sep 17 00:00:00 2001
From: Kevin Longe <273747162+kvlonge@users.noreply.github.com>
Date: Mon, 28 Sep 2026 06:29:21 +0100
Subject: [PATCH 13/13] feat: resolve references in SQL hooks and treat them as
hook reads
__ref(), __source() and __seed() in inline and named SQL hook SQL, written
directly or passed as typed macro arguments, previously compiled without
being recorded and reached the warehouse unresolved. They now resolve at
plan time exactly like references in model SQL, including adapter quoting,
source deferral, --defer and target overrides.
Each reference is also a hook read with the same rules as Python
@hook(reads=...): the hooked model runs after the resources it reads when
both are in the build, reads never change selection, a read of a resource
built from the hooked model fails with P007, and an unknown reference fails
to compile. Macro-generated references in hook SQL still fail with P006.
---
Makefile | 1 +
.../docs/concepts/models/hooks/python.md | 3 +
.../docs/concepts/models/hooks/sql.md | 41 +++++
.../compile/_helpers/attachment/core.py | 69 ++++++--
.../explicit_references/hook_reads.py | 24 +--
.../explicit_references/macro_arguments.py | 13 ++
src/sqlbuild/compiler/compile/models.py | 1 +
src/sqlbuild/compiler/discovery/models.py | 1 +
.../compiler/graph/_helpers/hook_reads.py | 54 ++++--
src/sqlbuild/compiler/graph/models.py | 14 +-
src/sqlbuild/compiler/graph/types.py | 13 ++
.../compiler/planner/_helpers/graph/core.py | 3 +-
.../planner/_helpers/output/plan_entry.py | 13 +-
.../planner/_helpers/resolve/resolve.py | 151 ++++++++++++----
.../_helpers/warehouse/source_deferral.py | 22 ++-
.../cli/commands/main/build/_test_types.py | 19 ++
.../main/build/test_sql_hook_reads.py | 166 ++++++++++++++++++
.../compiler/compile/_helpers/helpers.py | 18 ++
.../compile/_helpers/test_hook_reads.py | 110 ++++++++++++
.../planner/_helpers/resolve/_test_types.py | 9 +
.../planner/_helpers/resolve/test_hooks.py | 85 +++++++++
.../docs/concepts/models/hooks/python.mdx | 3 +
.../docs/docs/concepts/models/hooks/sql.mdx | 40 +++++
23 files changed, 799 insertions(+), 74 deletions(-)
create mode 100644 src/sqlbuild/compiler/graph/types.py
create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py
create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py
diff --git a/Makefile b/Makefile
index 8065fe4d26..4f16add339 100644
--- a/Makefile
+++ b/Makefile
@@ -77,6 +77,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py \
+ tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py \
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
index ff0240a5a1..4aa0ad63fa 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md
@@ -75,6 +75,9 @@ separate build from the resources it reads, the hook reads their existing relati
Declaring reads does not change who may call the hook: the model must still see the hook under the
usual [scope rules](../hooks.md#project-layout).
+SQL hooks declare reads by referencing resources in their SQL; see
+[Read models, sources, and seeds](sql.md#read-models-sources-and-seeds).
+
Literal project relation names in `ctx.query()` and `ctx.execute_sql()` SQL fail to compile, and
dynamically built names produce run-time warnings; see
[Hard-coded relation names](../../python-nodes/sql-references.md#hard-coded-relation-names).
diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md
index 3c896355c3..753e1010d3 100644
--- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md
+++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md
@@ -13,6 +13,7 @@ Online: https://sqlbuild.com/docs/concepts/models/hooks/sql/
- SQL hook arguments
- Inline SQL hooks
- Compile-time context
+- Read models, sources, and seeds
- Validation and diagnostics
SQL hooks submit one rendered SQL payload to the adapter before or after model materialization. Use a named SQL hook for reusable behavior and `inline_sql(...)` for short model-specific payloads.
@@ -163,6 +164,46 @@ LANGUAGE SQL
AS 'SELECT 1'
```
+## Read models, sources, and seeds
+
+SQL hooks reference other resources the same way model SQL does, with `__ref("...")`,
+`__source("...")`, and `__seed("...")`, written directly or passed as typed macro arguments:
+
+```sql
+MODEL (
+ materialized table,
+ pre_hooks [
+ inline_sql('DELETE FROM @@CTX:destination.qualified WHERE order_id IN (SELECT order_id FROM __ref("cancelled_orders"))'),
+ ],
+ post_hooks [sql("record_counts")],
+);
+
+SELECT order_id FROM __ref("stg_orders")
+```
+
+**`models/marts/_sqlbuild/_hooks/sql/record_counts.sql`**
+
+```sql
+HOOK ();
+
+@count_rows(__ref("stg_orders"))
+```
+
+At plan time each reference resolves exactly like one in model SQL: adapter quoting, source
+deferral, `--defer`, and target overrides all apply. Every reference is also a hook read, with the
+same rules as [declared Python hook reads](python.md#declared-reads):
+
+- The model that runs the hook waits for the resources it reads when both are in the build. Reads
+ do not change selection; in a partial build the hook reads the existing relations.
+- A hook cannot read a resource built from its own model; that fails to compile with `P007`. Use
+ `@@CTX:destination.qualified` for the hook's own model.
+- Referencing an unknown model, source, or seed fails to compile.
+- A reference passed to a macro is a read even when the macro does not render it. A reference a
+ macro writes into its output instead fails with `P006`, as in model SQL.
+
+The model must still see a named hook under the usual
+[scope rules](../hooks.md#project-layout).
+
## Validation and diagnostics
SQLBuild strictly validates hook resource syntax, arguments, interpolation, macros, metadata, and non-empty payloads. It does not classify executable statement kinds or implement vendor SQL grammar. Each rendered hook payload is passed to the adapter in one `execute` call; whether a driver accepts multiple statements or client-side batch separators is adapter and warehouse behavior. Use separate hook entries when portable ordering between statements matters.
diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py
index a8e2903ee6..e0cc41a49f 100644
--- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py
+++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py
@@ -50,6 +50,7 @@
from sqlbuild.compiler.compile._helpers.config.table_type import resolve_storage_policies
from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import (
merge_call_site_references,
+ resource_references,
)
from sqlbuild.compiler.compile._helpers.named_declarations.core import (
named_declaration_usages,
@@ -58,6 +59,7 @@
model_schema_enum_declarations,
)
from sqlbuild.compiler.compile._helpers.refs.cache import cached_sql_reference_extractor
+from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references
from sqlbuild.compiler.compile._helpers.render.arguments import render_parameterized_sql
from sqlbuild.compiler.compile._helpers.render.context_templates import (
apply_environment_database_schema_overrides,
@@ -146,6 +148,7 @@
UsageKind,
VisibilityReason,
)
+from sqlbuild.python_nodes.models import SqlResourceRef
from sqlbuild.spec.contracts.models import (
DefaultsConfig,
LocalConfig,
@@ -257,10 +260,17 @@ class _HookExpansionContext:
@dataclass
class _HookExpansionFacts:
usages: list[UsageRecord]
+ references: list[CompileSqlReference] = field(default_factory=list)
def add(self, usages: tuple[UsageRecord, ...]) -> None:
self.usages.extend(usages)
+ def add_references(self, references: tuple[CompileSqlReference, ...]) -> None:
+ self.references.extend(references)
+
+ def reads_since(self, start: int) -> tuple[SqlResourceRef, ...]:
+ return resource_references(tuple(self.references[start:]))
+
@dataclass
class _ReusableModelConfigCache:
@@ -615,15 +625,11 @@ def _build_model_inputs(
sql_hook_definitions=sql_hook_definitions,
consumer=model_identity,
)
- expanded_config: CompileModelConfig = CompileModelConfig(
- values=hook_expansion.values,
- model_header_keys=effective_config.model_header_keys,
- matched_path_default=effective_config.matched_path_default,
- logical_schema=effective_config.logical_schema,
- layer_schema=effective_config.layer_schema,
- logical_database=effective_config.logical_database,
- time_travel_retention=effective_config.time_travel_retention,
- table_type=effective_config.table_type,
+ expanded_config: CompileModelConfig = _hook_expanded_config(
+ config=effective_config,
+ hook_expansion=hook_expansion,
+ model_file=model_file,
+ validation_context=validation_context,
)
hook_name: str
for hook_name in ("pre_hooks", "post_hooks"):
@@ -740,6 +746,35 @@ def _build_model_inputs(
return tuple(model_inputs)
+def _hook_expanded_config(
+ *,
+ config: CompileModelConfig,
+ hook_expansion: HookExpansionResult,
+ model_file: DiscoveredSqlModelFile,
+ validation_context: _ModelValidationContext,
+) -> CompileModelConfig:
+ validate_model_references(
+ references=hook_expansion.references,
+ model_file=model_file,
+ known_model_names=validation_context.known_model_names,
+ known_seed_names=validation_context.known_seed_names,
+ known_source_names=validation_context.known_source_names,
+ known_function_names=validation_context.known_function_names,
+ known_table_function_names=validation_context.known_table_function_names,
+ external_sql_reference_resolver=validation_context.external_sql_reference_resolver,
+ )
+ return CompileModelConfig(
+ values=hook_expansion.values,
+ model_header_keys=config.model_header_keys,
+ matched_path_default=config.matched_path_default,
+ logical_schema=config.logical_schema,
+ layer_schema=config.layer_schema,
+ logical_database=config.logical_database,
+ time_travel_retention=config.time_travel_retention,
+ table_type=config.table_type,
+ )
+
+
def _validate_model_input(
*,
context: _ModelValidationContext,
@@ -1198,7 +1233,11 @@ def expand_model_hook_macros_result(
),
hook_key=hook_key,
)
- return HookExpansionResult(values=expanded_values, usages=tuple(dict.fromkeys(facts.usages)))
+ return HookExpansionResult(
+ values=expanded_values,
+ usages=tuple(dict.fromkeys(facts.usages)),
+ references=tuple(dict.fromkeys(facts.references)),
+ )
def validate_model_hook_config(*, values: dict[str, object], model_name: str) -> None:
@@ -1436,8 +1475,15 @@ def expand_sql_macros_in_value(
)
facts: _HookExpansionFacts = context.facts
facts.add(expansion.usages)
+ facts.add_references(
+ merge_call_site_references(
+ references=extract_sql_references(expansion.sql),
+ argument_references=expansion.argument_references,
+ )
+ )
return expansion.sql
if isinstance(value, SqlHookEntry):
+ inline_start: int = len(context.facts.references)
expanded_statement: object = expand_sql_macros_in_value(
value=value.statement,
context=context,
@@ -1457,6 +1503,7 @@ def expand_sql_macros_in_value(
definition_sql=value.definition_sql,
kwargs=value.kwargs,
description=value.description,
+ reads=context.facts.reads_since(inline_start),
)
if isinstance(value, NamedSqlHookEntry):
hook_definition: DiscoveredSqlHookFile | None = context.sql_hook_definitions.get(value.name)
@@ -1498,6 +1545,7 @@ def expand_sql_macros_in_value(
definition_label=(f"SQL hook '{value.name}' in {hook_definition.relative_path}"),
reject_unused=True,
)
+ named_start: int = len(hook_facts.references)
expanded_statement = expand_sql_macros_in_value(
value=rendered_statement,
context=replace(
@@ -1525,6 +1573,7 @@ def expand_sql_macros_in_value(
definition_sql=hook_definition.sql_body,
kwargs=dict(value.kwargs),
description=hook_definition.description,
+ reads=hook_facts.reads_since(named_start),
)
if isinstance(value, PythonHookEntry):
return value
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
index 3aebf71c3d..3b4ffa6384 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/hook_reads.py
@@ -1,9 +1,7 @@
-"""Compile validation for resources Python hooks declare they read."""
+"""Compile validation for resources model hooks read."""
from __future__ import annotations
-from pathlib import Path
-
from sqlbuild.compiler.compile.constants import HOOK_READS_OWN_DEPENDANT_CODE
from sqlbuild.compiler.compile.exceptions import CompileInputError
from sqlbuild.compiler.compile.models import (
@@ -18,6 +16,7 @@
)
from sqlbuild.compiler.graph.main._hook_read_cycles import hook_read_cycles
from sqlbuild.compiler.graph.models import HookReadEdge
+from sqlbuild.compiler.graph.types import HookReadType
from sqlbuild.python_nodes.models import SqlResourceRef
from sqlbuild.python_nodes.types import SqlResourceRefKind
@@ -38,29 +37,30 @@ def hook_read_diagnostics(*, project: CompiledProject) -> tuple[CompilerDiagnost
f"Python hook '{hook.name}' in '{hook.relative_path.as_posix()}' reads unknown "
f"{ref.kind.value} '{ref.name}'"
)
- hook_paths: dict[str, Path] = {hook.name: hook.relative_path for hook in project.hook_functions}
- return tuple(
- _cycle_diagnostic(cycle=cycle, hook_path=hook_paths.get(cycle.hook_name))
- for cycle in hook_read_cycles(project=project)
- )
+ return tuple(_cycle_diagnostic(cycle) for cycle in hook_read_cycles(project=project))
-def _cycle_diagnostic(*, cycle: HookReadEdge, hook_path: Path | None) -> CompilerDiagnostic:
+def _cycle_diagnostic(cycle: HookReadEdge) -> CompilerDiagnostic:
+ destination: str = (
+ "ctx.destination"
+ if cycle.hook_type is HookReadType.PYTHON
+ else "@@CTX:destination.qualified"
+ )
return CompilerDiagnostic(
phase=DiagnosticPhase.COMPILE,
severity=DiagnosticSeverity.ERROR,
code=HOOK_READS_OWN_DEPENDANT_CODE,
message=(
- f"Python hook '{cycle.hook_name}' on {_describe(cycle.gated)} reads "
+ f"{cycle.label} on {_describe(cycle.gated)} reads "
f"{_describe(cycle.read)}, which depends on '{cycle.gated.name}'; a hook runs as "
"part of its model, so it cannot wait for a resource built from that model"
),
resource_type=CompiledResourceType.MODEL,
resource_name=cycle.gated.name,
- path=hook_path,
+ path=cycle.hook_path,
help=(
f"attach the hook to a model built after '{cycle.read.name}', or drop the read and "
- f"use ctx.destination for '{cycle.gated.name}' itself"
+ f"use {destination} for '{cycle.gated.name}' itself"
),
)
diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
index 55f31b944d..9ce894e4a0 100644
--- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
+++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py
@@ -82,6 +82,19 @@ def call_site_sql_references(
return extract_sql_references(" ".join(reference_call_text(ref) for ref in refs))
+def resource_references(
+ references: tuple[CompileSqlReference, ...],
+) -> tuple[SqlResourceRef, ...]:
+ """Return the model, source, and seed references among SQL references, once each."""
+
+ resources: dict[SqlResourceRef, None] = {}
+ for reference in references:
+ kind: SqlResourceRefKind | None = _EXPLICIT_REFERENCE_KINDS.get(str(reference.ref_kind))
+ if kind is not None:
+ resources[SqlResourceRef(kind=kind, name=reference.ref_name)] = None
+ return tuple(resources)
+
+
def merge_call_site_references(
*,
references: tuple[CompileSqlReference, ...],
diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py
index 9a208bd986..38e3e3236d 100644
--- a/src/sqlbuild/compiler/compile/models.py
+++ b/src/sqlbuild/compiler/compile/models.py
@@ -401,6 +401,7 @@ class HookExpansionResult:
values: dict[str, object]
usages: tuple[UsageRecord, ...] = field(default_factory=tuple)
+ references: tuple[CompileSqlReference, ...] = field(default_factory=tuple)
@dataclass(frozen=True)
diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py
index abf27116dc..7afe01b8b6 100644
--- a/src/sqlbuild/compiler/discovery/models.py
+++ b/src/sqlbuild/compiler/discovery/models.py
@@ -46,6 +46,7 @@ class SqlHookEntry:
definition_sql: str | None = None
kwargs: dict[str, object] | None = None
description: str | None = None
+ reads: tuple[SqlResourceRef, ...] = ()
@dataclass(frozen=True)
diff --git a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
index dbf9e52344..c072752a17 100644
--- a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
+++ b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py
@@ -1,14 +1,17 @@
-"""Ordering edges from the resources Python hooks declare they read."""
+"""Ordering edges from the resources model hooks read."""
from __future__ import annotations
+from pathlib import Path
+
from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, CompiledProject
-from sqlbuild.compiler.discovery.models import PythonHookEntry
+from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry
from sqlbuild.compiler.graph._helpers.algorithms import transitive_closure_many_impl
from sqlbuild.compiler.graph._helpers.audit_gates import attached_audit_gate_edges_impl
from sqlbuild.compiler.graph._helpers.lineage import build_lineage_upstream_deps_impl
from sqlbuild.compiler.graph._helpers.sql_refs import sql_ref_key_impl
from sqlbuild.compiler.graph.models import HookReadEdge
+from sqlbuild.compiler.graph.types import HookReadType
from sqlbuild.python_nodes.models import SqlResourceRef
_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks")
@@ -28,20 +31,51 @@ def model_python_hook_names_impl(model: CompiledModel) -> tuple[str, ...]:
def hook_read_edges_impl(*, project: CompiledProject) -> tuple[HookReadEdge, ...]:
- """Return edges from each declared hook read to the model that runs the hook."""
+ """Return edges from each hook read to the model that runs the hook."""
reads_by_hook: dict[str, tuple[SqlResourceRef, ...]] = {
hook.name: hook.reads for hook in project.hook_functions if hook.reads
}
- if not reads_by_hook:
- return ()
- edges: dict[HookReadEdge, None] = {}
+ paths_by_hook: dict[str, Path] = {
+ hook.name: hook.relative_path for hook in project.hook_functions
+ }
+ candidates: list[HookReadEdge] = []
for model in project.models:
for hook_name in model_python_hook_names_impl(model):
- for ref in reads_by_hook.get(hook_name, ()):
- read: CompiledObjectKey = sql_ref_key_impl(ref)
- if read != model.key:
- edges[HookReadEdge(hook_name=hook_name, gated=model.key, read=read)] = None
+ candidates.extend(
+ HookReadEdge(
+ hook_name=hook_name,
+ gated=model.key,
+ read=sql_ref_key_impl(ref),
+ hook_path=paths_by_hook.get(hook_name),
+ )
+ for ref in reads_by_hook.get(hook_name, ())
+ )
+ candidates.extend(_sql_hook_read_edges(model))
+ return tuple(dict.fromkeys(edge for edge in candidates if edge.read != edge.gated))
+
+
+def _sql_hook_read_edges(model: CompiledModel) -> tuple[HookReadEdge, ...]:
+ edges: list[HookReadEdge] = []
+ for hook_key in _MODEL_HOOK_KEYS:
+ entries: object = model.config.values.get(hook_key)
+ items: tuple[object, ...] = (
+ tuple(entries) if isinstance(entries, list | tuple) else (entries,)
+ )
+ for index, entry in enumerate(items):
+ if not isinstance(entry, SqlHookEntry) or not entry.reads:
+ continue
+ named: bool = entry.name is not None
+ edges.extend(
+ HookReadEdge(
+ hook_name=entry.name if entry.name is not None else f"{hook_key}[{index}]",
+ gated=model.key,
+ read=sql_ref_key_impl(ref),
+ hook_type=HookReadType.SQL if named else HookReadType.INLINE_SQL,
+ hook_path=entry.relative_path or model.relative_path,
+ )
+ for ref in entry.reads
+ )
return tuple(edges)
diff --git a/src/sqlbuild/compiler/graph/models.py b/src/sqlbuild/compiler/graph/models.py
index 021b31386e..b6e3e3661c 100644
--- a/src/sqlbuild/compiler/graph/models.py
+++ b/src/sqlbuild/compiler/graph/models.py
@@ -3,8 +3,10 @@
from __future__ import annotations
from dataclasses import dataclass
+from pathlib import Path
from sqlbuild.compiler.compile.models import CompiledObjectKey
+from sqlbuild.compiler.graph.types import HookReadType
@dataclass(frozen=True)
@@ -19,8 +21,18 @@ class AttachedAuditGateEdge:
@dataclass(frozen=True)
class HookReadEdge:
- """One ordering edge: a model waits for a resource its Python hook declares it reads."""
+ """One ordering edge: a model waits for a resource one of its hooks reads."""
hook_name: str
gated: CompiledObjectKey
read: CompiledObjectKey
+ hook_type: HookReadType = HookReadType.PYTHON
+ hook_path: Path | None = None
+
+ @property
+ def label(self) -> str:
+ """Return how diagnostics name the hook, such as ``SQL hook 'grant_access'``."""
+
+ if self.hook_type is HookReadType.INLINE_SQL:
+ return f"inline SQL hook {self.hook_name}"
+ return f"{self.hook_type.value} hook '{self.hook_name}'"
diff --git a/src/sqlbuild/compiler/graph/types.py b/src/sqlbuild/compiler/graph/types.py
new file mode 100644
index 0000000000..37940e899c
--- /dev/null
+++ b/src/sqlbuild/compiler/graph/types.py
@@ -0,0 +1,13 @@
+"""Enumerations shared by graph facts."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+
+class HookReadType(StrEnum):
+ """Kind of model hook that declares or writes a read."""
+
+ PYTHON = "Python"
+ SQL = "SQL"
+ INLINE_SQL = "inline SQL"
diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/core.py b/src/sqlbuild/compiler/planner/_helpers/graph/core.py
index fe2394afb3..55d67821ec 100644
--- a/src/sqlbuild/compiler/planner/_helpers/graph/core.py
+++ b/src/sqlbuild/compiler/planner/_helpers/graph/core.py
@@ -90,8 +90,7 @@ def build_execution_edge_origins(
continue
origins.setdefault(
(hook_edge.gated, hook_edge.read),
- f"hook '{hook_edge.hook_name}' on '{hook_edge.gated.name}' reads "
- f"'{hook_edge.read.name}'",
+ f"{hook_edge.label} on '{hook_edge.gated.name}' reads '{hook_edge.read.name}'",
)
return origins
diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
index af7fd0fcae..75a309a111 100644
--- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
+++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py
@@ -63,7 +63,10 @@
build_model_locations,
build_seed_locations,
)
-from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql
+from sqlbuild.compiler.planner._helpers.resolve.resolve import (
+ resolve_model_hook_entries,
+ resolve_model_sql,
+)
from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import (
get_semantic_source_columns,
)
@@ -746,9 +749,11 @@ def plan_model_from_change(
type_enforcement=type_enforcement,
)
- pre_hooks, post_hooks = (
- model.config.values.get("pre_hooks"),
- model.config.values.get("post_hooks"),
+ pre_hooks, post_hooks = resolve_model_hook_entries(
+ model=model,
+ adapter=adapter,
+ context=context,
+ external_sql_reference_resolver=external_sql_reference_resolver,
)
cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type")
cursor_grain: str | None = get_config_str(values=model.config.values, key="cursor_grain")
diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py
index c11d3b0ef8..39d5f40763 100644
--- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py
+++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from dataclasses import replace
from datetime import datetime
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
@@ -14,6 +15,7 @@
CompiledRelationLocation,
CursorInputRoles,
)
+from sqlbuild.compiler.discovery.models import SqlHookEntry
from sqlbuild.compiler.planner._helpers.output.inclusive_cursor_end import (
resolve_bounded_cursor_override,
)
@@ -40,6 +42,10 @@
from sqlbuild.compiler.planner._helpers.resolve.sources import (
resolve_source_references,
)
+from sqlbuild.compiler.planner.constants import (
+ MODEL_POST_HOOKS_CONFIG_KEY,
+ MODEL_PRE_HOOKS_CONFIG_KEY,
+)
from sqlbuild.compiler.planner.exceptions import PlannerInputError
from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety
from sqlbuild.compiler.planner.models import (
@@ -172,42 +178,21 @@ def resolve_function_sql(
) -> str:
"""Resolve relation and function references in a SQL function body."""
- query_sql: str = resolve_source_references(
- query_sql=function.body_sql,
- source_map=source_map,
- source_warehouse_columns=source_warehouse_columns,
- star_exclude_keyword=star_exclude_keyword,
- cursor_bounds=None,
- cursor_filter_inputs={},
- adapter=adapter,
- cursor_type=None,
- lower_bound_inclusive=True,
- )
- query_sql = resolve_ref_references(
- query_sql=query_sql,
- model_locations=model_locations,
- seed_locations=seed_locations,
- cursor_bounds=None,
- cursor_filter_inputs={},
- adapter=adapter,
- cursor_type=None,
- lower_bound_inclusive=True,
- )
- query_sql = resolve_dbt_ref_references(query_sql=query_sql)
- query_sql = resolve_udf_references(
- query_sql=query_sql,
- function_locations=function_locations,
- adapter=adapter,
- )
- query_sql = resolve_table_function_references(
- query_sql=query_sql,
- function_locations=function_locations,
+ return _resolve_unbounded_sql(
+ sql=function.body_sql,
+ context=ModelPlanContext(
+ model_locations=model_locations,
+ models_by_name={},
+ seed_locations=seed_locations,
+ function_locations=function_locations,
+ source_map=source_map,
+ source_warehouse_columns=source_warehouse_columns,
+ star_exclude_keyword=star_exclude_keyword,
+ ),
adapter=adapter,
+ external_sql_reference_resolver=None,
+ label=f"Function '{function.name}'",
)
- assert_no_unresolved_sql_markers(
- sql=query_sql, context=f"Function '{function.name}' planned SQL"
- )
- return query_sql
def _compute_model_cursor_bounds(
@@ -323,3 +308,101 @@ def _compute_model_cursor_bounds(
),
input_evidence=cursor_snapshot.input_evidence,
)
+
+
+def resolve_model_hook_entries(
+ *,
+ model: CompiledModel,
+ adapter: BaseAdapter,
+ context: ModelPlanContext,
+ external_sql_reference_resolver: ExternalSqlReferenceResolver | None,
+) -> tuple[object, object]:
+ """Return the model's pre and post hooks with SQL hook references resolved like model SQL."""
+
+ pre_hooks, post_hooks = (
+ _resolve_hook_value(
+ value=model.config.values.get(hook_key),
+ label=f"Model '{model.name}' {hook_key}",
+ adapter=adapter,
+ context=context,
+ external_sql_reference_resolver=external_sql_reference_resolver,
+ )
+ for hook_key in (MODEL_PRE_HOOKS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY)
+ )
+ return pre_hooks, post_hooks
+
+
+def _resolve_hook_value(
+ *,
+ value: object,
+ label: str,
+ adapter: BaseAdapter,
+ context: ModelPlanContext,
+ external_sql_reference_resolver: ExternalSqlReferenceResolver | None,
+) -> object:
+ if isinstance(value, SqlHookEntry):
+ return replace(
+ value,
+ statement=_resolve_unbounded_sql(
+ sql=value.statement,
+ label=label,
+ adapter=adapter,
+ context=context,
+ external_sql_reference_resolver=external_sql_reference_resolver,
+ ),
+ )
+ if isinstance(value, list | tuple):
+ resolved: list[object] = [
+ _resolve_hook_value(
+ value=item,
+ label=f"{label}[{index}]",
+ adapter=adapter,
+ context=context,
+ external_sql_reference_resolver=external_sql_reference_resolver,
+ )
+ for index, item in enumerate(value)
+ ]
+ return resolved if isinstance(value, list) else tuple(resolved)
+ return value
+
+
+def _resolve_unbounded_sql(
+ *,
+ sql: str,
+ label: str,
+ adapter: BaseAdapter,
+ context: ModelPlanContext,
+ external_sql_reference_resolver: ExternalSqlReferenceResolver | None,
+) -> str:
+ resolved: str = resolve_source_references(
+ query_sql=sql,
+ source_map=context.source_map,
+ source_warehouse_columns=context.source_warehouse_columns,
+ star_exclude_keyword=context.star_exclude_keyword,
+ cursor_bounds=None,
+ cursor_filter_inputs={},
+ adapter=adapter,
+ cursor_type=None,
+ lower_bound_inclusive=True,
+ )
+ resolved = resolve_ref_references(
+ query_sql=resolved,
+ model_locations=context.model_locations,
+ seed_locations=context.seed_locations,
+ cursor_bounds=None,
+ cursor_filter_inputs={},
+ adapter=adapter,
+ cursor_type=None,
+ lower_bound_inclusive=True,
+ )
+ resolved = resolve_dbt_ref_references(
+ query_sql=resolved, external_sql_reference_resolver=external_sql_reference_resolver
+ )
+ resolved = resolve_udf_references(
+ query_sql=resolved, function_locations=context.function_locations or {}, adapter=adapter
+ )
+ resolved = resolve_table_function_references(
+ query_sql=resolved, function_locations=context.function_locations or {}, adapter=adapter
+ )
+ assert_no_unresolved_sql_markers(sql=resolved, context=f"{label} planned SQL")
+ return resolved
diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py
index 7a0f35394b..e55dedb709 100644
--- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py
+++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py
@@ -14,12 +14,18 @@
CompiledProject,
CompiledSource,
)
-from sqlbuild.compiler.planner.constants import SOURCE_DEFERRAL_CONTEXT_FIELDS
+from sqlbuild.compiler.discovery.models import SqlHookEntry
+from sqlbuild.compiler.planner.constants import (
+ MODEL_POST_HOOKS_CONFIG_KEY,
+ MODEL_PRE_HOOKS_CONFIG_KEY,
+ SOURCE_DEFERRAL_CONTEXT_FIELDS,
+)
from sqlbuild.compiler.planner.exceptions import PlannerInputError
from sqlbuild.compiler.references.main._quoted_reference_call_pattern import (
quoted_reference_call_pattern,
)
from sqlbuild.compiler.references.types import SqlReferenceKind
+from sqlbuild.python_nodes.types import SqlResourceRefKind
from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config
from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry, TargetConfig
@@ -90,6 +96,19 @@ def build_source_read_map(
return result
+def _sql_hook_source_reads(model: CompiledModel) -> tuple[str, ...]:
+ names: list[str] = []
+ for hook_key in (MODEL_PRE_HOOKS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY):
+ value: object = model.config.values.get(hook_key)
+ entries: tuple[object, ...] = tuple(value) if isinstance(value, list | tuple) else (value,)
+ for entry in entries:
+ if isinstance(entry, SqlHookEntry):
+ names.extend(
+ ref.name for ref in entry.reads if ref.kind is SqlResourceRefKind.SOURCE
+ )
+ return tuple(names)
+
+
def _selected_managed_source_refs(
*,
project: CompiledProject,
@@ -106,6 +125,7 @@ def _selected_managed_source_refs(
sql=model.query_sql, managed_source_names=managed_source_names
)
)
+ names.update(name for name in _sql_hook_source_reads(model) if name in managed_source_names)
function: CompiledFunction
for function in project.functions:
if function.key not in selected_keys:
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
index 6bd7de8f3a..93c7d59100 100644
--- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
@@ -1097,3 +1097,22 @@ class UnrenderedMacroArgumentE2ETestCase:
model_sql: str
expected_dag_edges: tuple[tuple[str, str], ...]
expected_built: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class SqlHookReadBuildE2ETestCase:
+ """SQL hooks whose references a build must resolve and order like model SQL."""
+
+ description: str
+ select: tuple[str, ...]
+ expected_counts: dict[str, int]
+ expected_order_before: tuple[str, str]
+
+
+@dataclass(frozen=True)
+class SqlHookReadCycleE2ETestCase:
+ """A SQL hook that reads a model built from the model running the hook."""
+
+ description: str
+ overrides: dict[str, str]
+ expected_output_fragments: tuple[str, ...]
diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py
new file mode 100644
index 0000000000..6e005aff84
--- /dev/null
+++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py
@@ -0,0 +1,166 @@
+"""Real-CLI coverage of relation references written in model SQL hooks."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+from typing import cast
+
+import pytest
+
+from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
+ SqlHookReadBuildE2ETestCase,
+ SqlHookReadCycleE2ETestCase,
+)
+from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import (
+ execute_duckdb,
+ prepare_inline_project,
+ query_duckdb,
+ run_sqb,
+)
+
+_DATABASE: str = "warehouse.duckdb"
+_MODELS: str = "models/sales"
+_PROJECT_FILES: dict[str, str] = {
+ "sqlbuild_project.toml": (
+ 'name = "sql_hook_reads"\nadapter = "duckdb"\ndefault_target = "dev"\n\n'
+ f'[connection]\ndatabase = "{_DATABASE}"\n\n'
+ '[targets.dev]\nschema = "dev"\nloader_schema = "raw_dev"\ndefer_sources_to = "prod"\n\n'
+ '[targets.prod]\nschema = "prod"\nloader_schema = "raw_prod"\n'
+ ),
+ "sources/raw.yml": (
+ "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n"
+ " columns:\n - name: order_id\n type: INTEGER\n"
+ ),
+ "python/loaders/raw.py": (
+ "from sqlbuild.loaders import loader\n\n\n"
+ "@loader\ndef raw_orders(ctx):\n return [{'order_id': 9}]\n"
+ ),
+ f"{_MODELS}/z_orders.sql": (
+ "MODEL (materialized table);\nSELECT 1 AS order_id UNION ALL SELECT 2\n"
+ ),
+ f"{_MODELS}/_sqlbuild/_macros/counts.py": (
+ "def count_orders(relation):\n"
+ ' return f"CREATE OR REPLACE TABLE main.macro_counts AS '
+ 'SELECT count(*) AS n FROM {relation}"\n'
+ ),
+ f"{_MODELS}/_sqlbuild/_hooks/sql/record_counts.sql": (
+ 'HOOK ();\n@count_orders(__ref("z_orders"))\n'
+ ),
+ f"{_MODELS}/a_report.sql": (
+ "MODEL (\n materialized table,\n"
+ " pre_hooks [inline_sql('CREATE OR REPLACE TABLE main.inline_counts AS "
+ 'SELECT count(*) AS n FROM __ref("z_orders")\')],\n'
+ ' post_hooks [sql("record_counts")],\n);\nSELECT 1 AS n\n'
+ ),
+ f"{_MODELS}/raw_marker.sql": (
+ "MODEL (\n materialized table,\n"
+ " post_hooks [inline_sql('CREATE OR REPLACE TABLE main.raw_counts AS "
+ 'SELECT count(*) AS n FROM __source("raw_orders")\')],\n);\nSELECT 1 AS n\n'
+ ),
+}
+_ENVIRONMENT_ROWS: str = (
+ "CREATE SCHEMA raw_dev; CREATE SCHEMA raw_prod;"
+ "CREATE TABLE raw_dev.raw_orders (order_id INTEGER); INSERT INTO raw_dev.raw_orders VALUES (1);"
+ "CREATE TABLE raw_prod.raw_orders (order_id INTEGER);"
+ "INSERT INTO raw_prod.raw_orders VALUES (1), (2);"
+)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ SqlHookReadBuildE2ETestCase(
+ description="inline and named macro hook reads run after and read the model",
+ select=("a_report", "z_orders"),
+ expected_counts={"main.inline_counts": 2, "main.macro_counts": 2},
+ expected_order_before=("z_orders", "a_report"),
+ ),
+ SqlHookReadBuildE2ETestCase(
+ description="hook source read follows source deferral",
+ select=("raw_marker",),
+ expected_counts={"main.raw_counts": 2},
+ expected_order_before=("raw_orders", "raw_marker"),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_sql_hook_references_when_building_then_they_resolve_and_order_like_sql(
+ test_case: SqlHookReadBuildE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path, project_name="sql_hook_reads", repo_files=_PROJECT_FILES
+ )
+ database: Path = project_dir / _DATABASE
+ execute_duckdb(db_path=database, sql=_ENVIRONMENT_ROWS)
+
+ plan: subprocess.CompletedProcess[str] = run_sqb(
+ command=("plan", "--json", "--select", *test_case.select), project_dir=project_dir
+ )
+ build: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "build", "--select", *test_case.select), project_dir=project_dir
+ )
+
+ assert plan.returncode == 0, plan.stdout + plan.stderr
+ payload: dict[str, object] = json.loads(plan.stdout)
+ planned: list[str] = [
+ *(entry["name"] for entry in cast(list[dict[str, str]], payload["source_loads"])),
+ *(entry["name"] for entry in cast(list[dict[str, str]], payload["models"])),
+ ]
+ first, second = test_case.expected_order_before
+ assert planned.index(first) < planned.index(second), planned
+ assert build.returncode == 0, build.stdout + build.stderr
+ assert build.stdout.index(f" {first} ") < build.stdout.index(f" {second} ")
+ assert {
+ table: query_duckdb(db_path=database, sql=f"SELECT n FROM {table}")[0][0]
+ for table in test_case.expected_counts
+ } == test_case.expected_counts
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ SqlHookReadCycleE2ETestCase(
+ description="inline hook reading a model built from its own model",
+ overrides={
+ f"{_MODELS}/report_rollup.sql": (
+ 'MODEL (materialized table);\nSELECT * FROM __ref("a_report")\n'
+ ),
+ f"{_MODELS}/a_report.sql": (
+ "MODEL (\n materialized table,\n"
+ " post_hooks [inline_sql('SELECT * FROM __ref(\"report_rollup\")')],\n"
+ ");\nSELECT 1 AS n\n"
+ ),
+ },
+ expected_output_fragments=(
+ "error[P007]: inline SQL hook post_hooks[0] on model 'a_report' reads model "
+ "'report_rollup', which depends on 'a_report'",
+ "use @@CTX:destination.qualified for 'a_report' itself",
+ ),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_sql_hook_reading_its_own_dependant_when_compiling_then_it_reports_p007(
+ test_case: SqlHookReadCycleE2ETestCase,
+ tmp_path: Path,
+) -> None:
+ project_dir: Path = prepare_inline_project(
+ tmp_path=tmp_path,
+ project_name="sql_hook_reads",
+ repo_files=_PROJECT_FILES | test_case.overrides,
+ )
+
+ result: subprocess.CompletedProcess[str] = run_sqb(
+ command=("--no-color", "compile"), project_dir=project_dir
+ )
+
+ output: str = result.stdout + result.stderr
+ assert result.returncode == 1, output
+ assert all(fragment in output for fragment in test_case.expected_output_fragments), output
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-vv"])
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
index e9a41cfaa1..74fa229840 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py
@@ -44,6 +44,7 @@
from sqlbuild.compiler.compile.types import CompiledResourceType
from sqlbuild.compiler.discovery.main.discover import discover_project_inputs
from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs
+from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps
from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, ColumnTransformKind
from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project
from sqlbuild.compiler.planner._helpers.graph.core import build_execution_upstream_deps
@@ -307,6 +308,23 @@ def execution_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, st
return frozenset(edges)
+def inline_sql_hook_header(sql: str) -> str:
+ """Return a MODEL header running one inline SQL pre-hook."""
+
+ return f"MODEL (pre_hooks [inline_sql('{sql}')]);"
+
+
+def lineage_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, str]]:
+ """Return (node, upstream) name pairs of the project's lineage graph used for selection."""
+
+ edges: set[tuple[str, str]] = set()
+ key: CompiledObjectKey
+ deps: tuple[CompiledObjectKey, ...]
+ for key, deps in build_lineage_upstream_deps(project).items():
+ edges.update((key.name, dep.name) for dep in deps)
+ return frozenset(edges)
+
+
def expand_typed_macro_sql(
*, tmp_path: Path, macro_file_contents: str, sql: str, enforce_explicit: bool = True
) -> str:
diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
index a062ed7483..5e1598f99d 100644
--- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
+++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py
@@ -16,6 +16,8 @@
compile_and_assemble,
execution_edge_names,
gate_model,
+ inline_sql_hook_header,
+ lineage_edge_names,
python_hook_source,
render_compile_diagnostics,
)
@@ -34,6 +36,10 @@
"seeds:\n - name: country_codes\n columns:\n - name: code\n type: VARCHAR\n"
)
_SOURCES: str = "sources:\n - name: raw_orders\n table: orders\n"
+_SQL_HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/sql/record_rows.sql"
+_MACRO_PATH: str = "models/marts/_sqlbuild/_macros/counts.py"
+_IGNORING_MACRO: str = "def count_rows(relation):\n return 'SELECT 1'\n"
+_NAMED_SQL_HOOK_HEADER: str = 'MODEL (post_hooks [sql("record_rows")]);'
@pytest.mark.parametrize(
@@ -65,6 +71,44 @@
},
expected_edges=(("orders", "country_codes"), ("orders", "raw_orders")),
),
+ HookReadEdgeTestCase(
+ description="inline SQL hook reading a model directly",
+ files={
+ "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"),
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id",
+ header=inline_sql_hook_header('SELECT count(*) FROM __ref("customers")'),
+ ),
+ },
+ expected_edges=(("orders", "customers"),),
+ ),
+ HookReadEdgeTestCase(
+ description="named SQL hook passing a model to a macro that ignores it",
+ files={
+ _MACRO_PATH: _IGNORING_MACRO,
+ _SQL_HOOK_PATH: 'HOOK ();\n@count_rows(__ref("customers"))\n',
+ "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"),
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id", header=_NAMED_SQL_HOOK_HEADER
+ ),
+ },
+ expected_edges=(("orders", "customers"),),
+ ),
+ HookReadEdgeTestCase(
+ description="inline SQL hook reading a source and a seed",
+ files={
+ "seeds/country_codes.yml": _COUNTRY_SEED,
+ "seeds/country_codes.csv": "code\nGB\n",
+ "sources/raw.yml": _SOURCES,
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id",
+ header=inline_sql_hook_header(
+ 'SELECT * FROM __source("raw_orders"), __seed("country_codes")'
+ ),
+ ),
+ },
+ expected_edges=(("orders", "country_codes"), ("orders", "raw_orders")),
+ ),
),
ids=lambda case: case.description,
)
@@ -80,6 +124,36 @@ def test_given_hook_reads_when_compiling_then_hooked_model_runs_after_reads(
assert set(test_case.expected_edges) <= execution_edge_names(project=project)
+@pytest.mark.parametrize(
+ "test_case",
+ (
+ HookReadEdgeTestCase(
+ description="inline SQL hook read orders the build but is not lineage",
+ files={
+ "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"),
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id",
+ header=inline_sql_hook_header('SELECT count(*) FROM __ref("customers")'),
+ ),
+ },
+ expected_edges=(("orders", "customers"),),
+ ),
+ ),
+ ids=lambda case: case.description,
+)
+def test_given_sql_hook_read_when_compiling_then_lineage_and_selection_ignore_it(
+ test_case: HookReadEdgeTestCase,
+ tmp_path: Path,
+ write_repo_files: Callable[[Path, dict[str, str]], None],
+) -> None:
+ write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files)
+
+ project: CompiledProject = compile_and_assemble(project_dir=tmp_path)
+
+ assert set(test_case.expected_edges) <= execution_edge_names(project=project)
+ assert set(test_case.expected_edges).isdisjoint(lineage_edge_names(project=project))
+
+
@pytest.mark.parametrize(
"test_case",
(
@@ -121,6 +195,32 @@ def test_given_hook_reads_when_compiling_then_hooked_model_runs_after_reads(
"[P007] Python hook 'refresh_lookup' on model 'orders' reads model 'order_rollup'",
),
),
+ HookReadErrorTestCase(
+ description="inline and named SQL hooks reading a model built from their model",
+ files={
+ _MACRO_PATH: _IGNORING_MACRO,
+ _SQL_HOOK_PATH: 'HOOK ();\n@count_rows(__ref("order_count"))\n',
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id",
+ header=(
+ "MODEL (pre_hooks [inline_sql('SELECT * FROM __ref(\"order_rollup\")')], "
+ 'post_hooks [sql("record_rows")]);'
+ ),
+ ),
+ "models/marts/order_rollup.sql": gate_model(
+ sql='SELECT count(*) AS n FROM __ref("orders")'
+ ),
+ "models/marts/order_count.sql": gate_model(
+ sql='SELECT count(*) AS n FROM __ref("orders")'
+ ),
+ },
+ expected_error_fragments=(
+ "[P007] SQL hook 'record_rows' on model 'orders' reads model 'order_count'",
+ "[P007] inline SQL hook pre_hooks[0] on model 'orders' reads model "
+ "'order_rollup', which depends on 'orders'",
+ "use @@CTX:destination.qualified for 'orders' itself",
+ ),
+ ),
),
ids=lambda case: case.description,
)
@@ -152,6 +252,16 @@ def test_given_hook_reading_its_own_dependant_when_compiling_then_it_reports_p00
"reads unknown model 'missing_customers'",
),
),
+ HookReadErrorTestCase(
+ description="inline SQL hook reading an unknown model",
+ files={
+ "models/marts/orders.sql": gate_model(
+ sql="SELECT 1 AS order_id",
+ header=inline_sql_hook_header('SELECT * FROM __ref("missing_customers")'),
+ ),
+ },
+ expected_error_fragments=("references unknown model 'missing_customers'",),
+ ),
),
ids=lambda case: case.description,
)
diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py
index b670e4d119..f40a347736 100644
--- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py
+++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py
@@ -125,3 +125,12 @@ class ApplyDeferredTargetsTestCase:
selected_names: tuple[str, ...]
expected_model_qualified: dict[str, str | None]
expected_seed_qualified: dict[str, str | None] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class HookSqlResolutionTestCase:
+ description: str
+ pre_hooks: object
+ post_hooks: object
+ expected_pre_hooks: object
+ expected_post_hooks: object
diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py
new file mode 100644
index 0000000000..b9b7b95d33
--- /dev/null
+++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py
@@ -0,0 +1,85 @@
+"""Tests for resolving relation references written in model SQL hooks."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+import pytest
+
+from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter
+from sqlbuild.compiler.compile.models import CompiledModel
+from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry
+from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_hook_entries
+from sqlbuild.compiler.planner.models import ModelPlanContext
+from sqlbuild.spec.contracts.models import SourceEntry
+from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import (
+ HookSqlResolutionTestCase,
+)
+from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve.helpers import (
+ build_cursor_intrinsic_model,
+ build_empty_model_plan_context,
+ build_target,
+)
+
+_PYTHON_HOOK: PythonHookEntry = PythonHookEntry(name="refresh_lookup", kwargs={})
+_CONTEXT: ModelPlanContext = replace(
+ build_empty_model_plan_context(),
+ model_locations={"orders": build_target("staging.orders", "orders")},
+ seed_locations={"country_codes": build_target("seeds.country_codes", "country_codes")},
+ source_map={"raw_orders": SourceEntry(name="raw_orders", schema="raw_prod", table="orders")},
+)
+
+
+@pytest.mark.parametrize(
+ "test_case",
+ [
+ HookSqlResolutionTestCase(
+ description="inline hook model reference resolves to the planned location",
+ pre_hooks=[SqlHookEntry(statement='DELETE FROM __ref("orders")'), _PYTHON_HOOK],
+ post_hooks=None,
+ expected_pre_hooks=[SqlHookEntry(statement="DELETE FROM staging.orders"), _PYTHON_HOOK],
+ expected_post_hooks=None,
+ ),
+ HookSqlResolutionTestCase(
+ description="named hook source and seed references use the planned read entries",
+ pre_hooks=None,
+ post_hooks=(
+ SqlHookEntry(
+ statement='INSERT INTO t SELECT * FROM __source("raw_orders"), '
+ '__seed("country_codes")',
+ name="record_rows",
+ ),
+ ),
+ expected_pre_hooks=None,
+ expected_post_hooks=(
+ SqlHookEntry(
+ statement="INSERT INTO t SELECT * FROM raw_prod.orders, seeds.country_codes",
+ name="record_rows",
+ ),
+ ),
+ ),
+ ],
+ ids=lambda case: case.description,
+)
+def test_given_sql_hook_references_when_planning_then_they_resolve_like_model_sql(
+ test_case: HookSqlResolutionTestCase,
+) -> None:
+ model: CompiledModel = build_cursor_intrinsic_model(
+ config_values={"pre_hooks": test_case.pre_hooks, "post_hooks": test_case.post_hooks}
+ )
+
+ pre_hooks, post_hooks = resolve_model_hook_entries(
+ model=model,
+ adapter=DuckDbAdapter(),
+ context=_CONTEXT,
+ external_sql_reference_resolver=None,
+ )
+
+ assert (pre_hooks, post_hooks) == (
+ test_case.expected_pre_hooks,
+ test_case.expected_post_hooks,
+ )
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-vv"])
diff --git a/website/src/content/docs/docs/concepts/models/hooks/python.mdx b/website/src/content/docs/docs/concepts/models/hooks/python.mdx
index 048dc7eb7f..872516e359 100644
--- a/website/src/content/docs/docs/concepts/models/hooks/python.mdx
+++ b/website/src/content/docs/docs/concepts/models/hooks/python.mdx
@@ -63,6 +63,9 @@ separate build from the resources it reads, the hook reads their existing relati
Declaring reads does not change who may call the hook: the model must still see the hook under the
usual [scope rules](/docs/concepts/models/hooks#project-layout).
+SQL hooks declare reads by referencing resources in their SQL; see
+[Read models, sources, and seeds](/docs/concepts/models/hooks/sql#read-models-sources-and-seeds).
+
Literal project relation names in `ctx.query()` and `ctx.execute_sql()` SQL fail to compile, and
dynamically built names produce run-time warnings; see
[Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names).
diff --git a/website/src/content/docs/docs/concepts/models/hooks/sql.mdx b/website/src/content/docs/docs/concepts/models/hooks/sql.mdx
index 716ef7c03a..f134a538c5 100644
--- a/website/src/content/docs/docs/concepts/models/hooks/sql.mdx
+++ b/website/src/content/docs/docs/concepts/models/hooks/sql.mdx
@@ -151,6 +151,46 @@ LANGUAGE SQL
AS 'SELECT 1'
```
+## Read models, sources, and seeds
+
+SQL hooks reference other resources the same way model SQL does, with `__ref("...")`,
+`__source("...")`, and `__seed("...")`, written directly or passed as typed macro arguments:
+
+```sql
+MODEL (
+ materialized table,
+ pre_hooks [
+ inline_sql('DELETE FROM @@CTX:destination.qualified WHERE order_id IN (SELECT order_id FROM __ref("cancelled_orders"))'),
+ ],
+ post_hooks [sql("record_counts")],
+);
+
+SELECT order_id FROM __ref("stg_orders")
+```
+
+**`models/marts/_sqlbuild/_hooks/sql/record_counts.sql`**
+
+```sql
+HOOK ();
+
+@count_rows(__ref("stg_orders"))
+```
+
+At plan time each reference resolves exactly like one in model SQL: adapter quoting, source
+deferral, `--defer`, and target overrides all apply. Every reference is also a hook read, with the
+same rules as [declared Python hook reads](/docs/concepts/models/hooks/python#declared-reads):
+
+- The model that runs the hook waits for the resources it reads when both are in the build. Reads
+ do not change selection; in a partial build the hook reads the existing relations.
+- A hook cannot read a resource built from its own model; that fails to compile with `P007`. Use
+ `@@CTX:destination.qualified` for the hook's own model.
+- Referencing an unknown model, source, or seed fails to compile.
+- A reference passed to a macro is a read even when the macro does not render it. A reference a
+ macro writes into its output instead fails with `P006`, as in model SQL.
+
+The model must still see a named hook under the usual
+[scope rules](/docs/concepts/models/hooks#project-layout).
+
## Validation and diagnostics
SQLBuild strictly validates hook resource syntax, arguments, interpolation, macros, metadata, and non-empty payloads. It does not classify executable statement kinds or implement vendor SQL grammar. Each rendered hook payload is passed to the adapter in one `execute` call; whether a driver accepts multiple statements or client-side batch separators is adapter and warehouse behavior. Use separate hook entries when portable ordering between statements matters.